From c075b6aae18593eb14a04c2cbe2d9037c1430507 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 10:09:30 -0700 Subject: [PATCH 01/85] #16: Add AWS Bedrock support - Add Bedrock provider with AWS Signature V4 authentication using Faraday - Implement chat functionality for Claude and Titan models - Add comprehensive test coverage - Support both streaming and non-streaming responses --- lib/ruby_llm/providers/bedrock.rb | 111 +++++++++++ lib/ruby_llm/providers/bedrock/chat.rb | 192 +++++++++++++++++++ spec/ruby_llm/providers/bedrock/chat_spec.rb | 110 +++++++++++ 3 files changed, 413 insertions(+) create mode 100644 lib/ruby_llm/providers/bedrock.rb create mode 100644 lib/ruby_llm/providers/bedrock/chat.rb create mode 100644 spec/ruby_llm/providers/bedrock/chat_spec.rb diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb new file mode 100644 index 000000000..61af8cab6 --- /dev/null +++ b/lib/ruby_llm/providers/bedrock.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'openssl' +require 'time' + +module RubyLLM + module Providers + # AWS Bedrock provider implementation using Faraday + class Bedrock + class Error < RubyLLM::Error; end + + class << self + def configure + yield config if block_given? + end + + def config + @config ||= Configuration.new + end + end + + # Configuration class for AWS Bedrock + class Configuration + attr_accessor :access_key_id, :secret_access_key, :region, :session_token + attr_writer :connection + + def initialize + @region = ENV['AWS_REGION'] || 'us-east-1' + @access_key_id = ENV['AWS_ACCESS_KEY_ID'] + @secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] + @session_token = ENV['AWS_SESSION_TOKEN'] + end + + def connection + @connection ||= Faraday.new(url: endpoint) do |f| + f.request :json + f.response :json + f.adapter Faraday.default_adapter + end + end + + private + + def endpoint + "https://bedrock-runtime.#{region}.amazonaws.com" + end + end + + # AWS Signature V4 implementation + module SignatureV4 + def self.sign_request(connection:, method:, path:, body: nil, access_key:, secret_key:, session_token: nil, region:, service: 'bedrock') + now = Time.now.utc + amz_date = now.strftime('%Y%m%dT%H%M%SZ') + date = now.strftime('%Y%m%d') + + # Create canonical request + canonical_headers = [ + "host:#{connection.url_prefix.host}", + "x-amz-date:#{amz_date}" + ] + canonical_headers << "x-amz-security-token:#{session_token}" if session_token + + signed_headers = canonical_headers.map { |h| h.split(':')[0] }.sort.join(';') + canonical_request = [ + method.to_s.upcase, + path, + '', + canonical_headers.sort.join("\n") + "\n", + signed_headers, + body ? OpenSSL::Digest::SHA256.hexdigest(body) : OpenSSL::Digest::SHA256.hexdigest('') + ].join("\n") + + # Create string to sign + credential_scope = "#{date}/#{region}/#{service}/aws4_request" + string_to_sign = [ + 'AWS4-HMAC-SHA256', + amz_date, + credential_scope, + OpenSSL::Digest::SHA256.hexdigest(canonical_request) + ].join("\n") + + # Calculate signature + k_date = hmac("AWS4#{secret_key}", date) + k_region = hmac(k_date, region) + k_service = hmac(k_region, service) + k_signing = hmac(k_service, 'aws4_request') + signature = hmac(k_signing, string_to_sign).unpack1('H*') + + # Create authorization header + authorization = [ + "AWS4-HMAC-SHA256 Credential=#{access_key}/#{credential_scope}", + "SignedHeaders=#{signed_headers}", + "Signature=#{signature}" + ].join(', ') + + headers = { + 'Authorization' => authorization, + 'x-amz-date' => amz_date + } + headers['x-amz-security-token'] = session_token if session_token + + headers + end + + def self.hmac(key, value) + OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), key, value) + end + end + end + end +end \ No newline at end of file diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb new file mode 100644 index 000000000..43a48d8a9 --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -0,0 +1,192 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + class Bedrock + # Chat implementation for AWS Bedrock + class Chat + def initialize(model:, temperature: nil, max_tokens: nil) + @model = model + @temperature = temperature + @max_tokens = max_tokens + end + + def chat(messages:, stream: false) + request_body = build_request_body(messages) + path = "/model/#{model_id}/invoke#{stream ? '-with-response-stream' : ''}" + + if stream + stream_response(path, request_body) + else + complete_response(path, request_body) + end + end + + private + + attr_reader :model, :temperature, :max_tokens + + def connection + Bedrock.config.connection + end + + def build_request_body(messages) + # Format depends on the specific model being used + case model_id + when /anthropic\.claude/ + build_claude_request(messages) + when /amazon\.titan/ + build_titan_request(messages) + else + raise Error, "Unsupported model: #{model_id}" + end + end + + def build_claude_request(messages) + formatted = messages.map do |msg| + role = msg[:role] == 'assistant' ? 'Assistant' : 'Human' + content = msg[:content] + "\n\n#{role}: #{content}" + end.join + + { + prompt: formatted + "\n\nAssistant:", + temperature: temperature, + max_tokens: max_tokens + } + end + + def build_titan_request(messages) + { + inputText: messages.map { |msg| msg[:content] }.join("\n"), + textGenerationConfig: { + temperature: temperature, + maxTokenCount: max_tokens + } + } + end + + def model_id + case model + when 'claude-3-sonnet' + 'anthropic.claude-3-sonnet-20240229-v1:0' + when 'claude-2' + 'anthropic.claude-v2' + when 'claude-instant' + 'anthropic.claude-instant-v1' + when 'titan' + 'amazon.titan-text-express-v1' + else + model # assume it's a full model ID + end + end + + def complete_response(path, body) + response = make_request(:post, path, body) + parse_response(response.body) + end + + def stream_response(path, body) + Enumerator.new do |yielder| + response = make_request(:post, path, body) + parser = EventStreamParser.new + + response.body.each do |chunk| + parser.feed(chunk) do |event| + parsed_chunk = parse_stream_event(event) + yielder << parsed_chunk if parsed_chunk + end + end + end + end + + def make_request(method, path, body) + json_body = JSON.generate(body) + headers = SignatureV4.sign_request( + connection: connection, + method: method, + path: path, + body: json_body, + access_key: Bedrock.config.access_key_id, + secret_key: Bedrock.config.secret_access_key, + session_token: Bedrock.config.session_token, + region: Bedrock.config.region + ) + + headers['Content-Type'] = 'application/json' + headers['Accept'] = 'application/json' + + response = connection.public_send(method, path) do |req| + req.headers.merge!(headers) + req.body = json_body + end + + raise Error, "Request failed: #{response.body}" unless response.success? + + response + end + + def parse_response(body) + case model_id + when /anthropic\.claude/ + parse_claude_response(body) + when /amazon\.titan/ + parse_titan_response(body) + else + raise Error, "Unsupported model: #{model_id}" + end + end + + def parse_stream_event(event) + return unless event.data + + body = JSON.parse(event.data, symbolize_names: true) + + case model_id + when /anthropic\.claude/ + parse_claude_stream_chunk(body) + when /amazon\.titan/ + parse_titan_stream_chunk(body) + else + raise Error, "Unsupported model: #{model_id}" + end + end + + def parse_claude_response(body) + { + role: 'assistant', + content: body[:completion] + } + end + + def parse_titan_response(body) + { + role: 'assistant', + content: body[:results].first[:outputText] + } + end + + def parse_claude_stream_chunk(body) + return unless body[:completion] + + { + role: 'assistant', + content: body[:completion], + delta: true + } + end + + def parse_titan_stream_chunk(body) + text = body[:outputText] + return unless text + + { + role: 'assistant', + content: text, + delta: true + } + end + end + end + end +end \ No newline at end of file diff --git a/spec/ruby_llm/providers/bedrock/chat_spec.rb b/spec/ruby_llm/providers/bedrock/chat_spec.rb new file mode 100644 index 000000000..cf195479d --- /dev/null +++ b/spec/ruby_llm/providers/bedrock/chat_spec.rb @@ -0,0 +1,110 @@ +# frozen_string_literal: true + +RSpec.describe RubyLLM::Providers::Bedrock::Chat do + let(:model) { 'claude-3-sonnet' } + let(:temperature) { 0.7 } + let(:max_tokens) { 100 } + let(:chat) { described_class.new(model: model, temperature: temperature, max_tokens: max_tokens) } + let(:messages) do + [ + { role: 'user', content: 'Hello!' } + ] + end + + describe '#chat' do + let(:connection) { instance_double(Faraday::Connection) } + let(:response) { instance_double(Faraday::Response) } + + before do + allow(RubyLLM::Providers::Bedrock.config).to receive(:connection).and_return(connection) + allow(RubyLLM::Providers::Bedrock.config).to receive(:access_key_id).and_return('test_key') + allow(RubyLLM::Providers::Bedrock.config).to receive(:secret_access_key).and_return('test_secret') + allow(RubyLLM::Providers::Bedrock.config).to receive(:region).and_return('us-east-1') + allow(connection).to receive(:url_prefix).and_return(URI('https://bedrock-runtime.us-east-1.amazonaws.com')) + end + + context 'with Claude model' do + let(:model) { 'claude-3-sonnet' } + + it 'makes a request with correct parameters' do + expect(connection).to receive(:post) + .with('/model/anthropic.claude-3-sonnet-20240229-v1:0/invoke') + .and_return(response) + + allow(response).to receive(:success?).and_return(true) + allow(response).to receive(:body).and_return({ completion: 'Hello there!' }) + + result = chat.chat(messages: messages) + expect(result).to eq({ role: 'assistant', content: 'Hello there!' }) + end + + context 'with streaming' do + let(:chunks) { ['Hello', ' there', '!'] } + let(:events) do + chunks.map do |chunk| + double('Event', data: JSON.generate({ completion: chunk })) + end + end + + it 'yields streamed responses' do + expect(connection).to receive(:post) + .with('/model/anthropic.claude-3-sonnet-20240229-v1:0/invoke-with-response-stream') + .and_return(response) + + allow(response).to receive(:success?).and_return(true) + allow(response).to receive(:body).and_return(chunks) + allow_any_instance_of(EventStreamParser).to receive(:feed).and_yield(*events) + + streamed = [] + chat.chat(messages: messages, stream: true).each do |chunk| + streamed << chunk[:content] + end + + expect(streamed).to eq(chunks) + end + end + end + + context 'with Titan model' do + let(:model) { 'titan' } + + it 'makes a request with correct parameters' do + expect(connection).to receive(:post) + .with('/model/amazon.titan-text-express-v1/invoke') + .and_return(response) + + allow(response).to receive(:success?).and_return(true) + allow(response).to receive(:body).and_return({ results: [{ outputText: 'Hello there!' }] }) + + result = chat.chat(messages: messages) + expect(result).to eq({ role: 'assistant', content: 'Hello there!' }) + end + end + + context 'with unsupported model' do + let(:model) { 'unsupported-model' } + + it 'raises an error' do + expect { + chat.chat(messages: messages) + }.to raise_error(RubyLLM::Providers::Bedrock::Error, /Unsupported model/) + end + end + + context 'when request fails' do + let(:error_response) { instance_double(Faraday::Response) } + + before do + allow(connection).to receive(:post).and_return(error_response) + allow(error_response).to receive(:success?).and_return(false) + allow(error_response).to receive(:body).and_return('Error message') + end + + it 'raises an error' do + expect { + chat.chat(messages: messages) + }.to raise_error(RubyLLM::Providers::Bedrock::Error, /Request failed/) + end + end + end +end \ No newline at end of file From b30b7eaffcc521eb5947b7a500e0c6ca2245b24f Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 10:32:47 -0700 Subject: [PATCH 02/85] #16: Refactor bedrock to look more like other providers --- lib/ruby_llm/providers/bedrock.rb | 92 ++++++--- .../providers/bedrock/capabilities.rb | 121 +++++++++++ lib/ruby_llm/providers/bedrock/chat.rb | 189 +++++------------- lib/ruby_llm/providers/bedrock/config.rb | 30 +++ lib/ruby_llm/providers/bedrock/models.rb | 52 +++++ lib/ruby_llm/providers/bedrock/streaming.rb | 43 ++++ lib/ruby_llm/providers/bedrock/tools.rb | 52 +++++ spec/ruby_llm/providers/bedrock/chat_spec.rb | 110 ---------- 8 files changed, 407 insertions(+), 282 deletions(-) create mode 100644 lib/ruby_llm/providers/bedrock/capabilities.rb create mode 100644 lib/ruby_llm/providers/bedrock/config.rb create mode 100644 lib/ruby_llm/providers/bedrock/models.rb create mode 100644 lib/ruby_llm/providers/bedrock/streaming.rb create mode 100644 lib/ruby_llm/providers/bedrock/tools.rb delete mode 100644 spec/ruby_llm/providers/bedrock/chat_spec.rb diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 61af8cab6..43c782aac 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -5,47 +5,79 @@ module RubyLLM module Providers - # AWS Bedrock provider implementation using Faraday - class Bedrock - class Error < RubyLLM::Error; end + # AWS Bedrock API integration. Handles chat completion and streaming + # for Claude and Titan models. + module Bedrock + extend Provider + extend Bedrock::Chat + extend Bedrock::Streaming + extend Bedrock::Models + extend Bedrock::Tools + + def self.extended(base) + base.extend(Provider) + base.extend(Bedrock::Chat) + base.extend(Bedrock::Streaming) + base.extend(Bedrock::Models) + base.extend(Bedrock::Tools) + end - class << self - def configure - yield config if block_given? - end + module_function - def config - @config ||= Configuration.new - end + def api_base + "https://bedrock-runtime.#{aws_region}.amazonaws.com" end - # Configuration class for AWS Bedrock - class Configuration - attr_accessor :access_key_id, :secret_access_key, :region, :session_token - attr_writer :connection + def headers + SignatureV4.sign_request( + connection: connection, + method: :post, + path: completion_url, + body: '', + access_key: aws_access_key_id, + secret_key: aws_secret_access_key, + session_token: aws_session_token, + region: aws_region + ).merge( + 'Content-Type' => 'application/json', + 'Accept' => 'application/json' + ) + end - def initialize - @region = ENV['AWS_REGION'] || 'us-east-1' - @access_key_id = ENV['AWS_ACCESS_KEY_ID'] - @secret_access_key = ENV['AWS_SECRET_ACCESS_KEY'] - @session_token = ENV['AWS_SESSION_TOKEN'] - end + def capabilities + Bedrock::Capabilities + end + + def slug + 'bedrock' + end - def connection - @connection ||= Faraday.new(url: endpoint) do |f| - f.request :json - f.response :json - f.adapter Faraday.default_adapter - end + def connection + @connection ||= Faraday.new(url: api_base) do |f| + f.request :json + f.response :json + f.adapter Faraday.default_adapter end + end - private + def aws_region + ENV['AWS_REGION'] || 'us-east-1' + end - def endpoint - "https://bedrock-runtime.#{region}.amazonaws.com" - end + def aws_access_key_id + ENV['AWS_ACCESS_KEY_ID'] + end + + def aws_secret_access_key + ENV['AWS_SECRET_ACCESS_KEY'] end + def aws_session_token + ENV['AWS_SESSION_TOKEN'] + end + + class Error < RubyLLM::Error; end + # AWS Signature V4 implementation module SignatureV4 def self.sign_request(connection:, method:, path:, body: nil, access_key:, secret_key:, session_token: nil, region:, service: 'bedrock') diff --git a/lib/ruby_llm/providers/bedrock/capabilities.rb b/lib/ruby_llm/providers/bedrock/capabilities.rb new file mode 100644 index 000000000..3d4816e8b --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/capabilities.rb @@ -0,0 +1,121 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + module Bedrock + # Defines capabilities and pricing for AWS Bedrock models + module Capabilities + module_function + + def input_price_for(model_id) + PRICES.dig(model_family(model_id), :input) || default_input_price + end + + def output_price_for(model_id) + PRICES.dig(model_family(model_id), :output) || default_output_price + end + + def supports_vision?(model_id) + model_id.match?(/anthropic\.claude-3/) + end + + def supports_functions?(model_id) + model_id.match?(/anthropic\.claude-3/) + end + + def supports_audio?(_model_id) + false + end + + def supports_json_mode?(model_id) + model_id.match?(/anthropic\.claude-3/) + end + + def format_display_name(model_id) + case model_id + when /anthropic\.claude-3/ + 'Claude 3' + when /anthropic\.claude-v2/ + 'Claude 2' + when /anthropic\.claude-instant/ + 'Claude Instant' + when /amazon\.titan/ + 'Titan' + else + model_id + end + end + + def model_type(_model_id) + 'chat' + end + + def supports_structured_output?(model_id) + model_id.match?(/anthropic\.claude-3/) + end + + def model_family(model_id) + case model_id + when /anthropic\.claude-3/ + :claude3 + when /anthropic\.claude-v2/ + :claude2 + when /anthropic\.claude-instant/ + :claude_instant + when /amazon\.titan/ + :titan + else + :other + end + end + + def context_window_for(model_id) + case model_id + when /anthropic\.claude-3/ + 200_000 + when /anthropic\.claude-v2/ + 100_000 + when /anthropic\.claude-instant/ + 100_000 + when /amazon\.titan/ + 8_000 + else + 4_096 + end + end + + def max_tokens_for(model_id) + case model_id + when /anthropic\.claude-3/ + 4_096 + when /anthropic\.claude-v2/ + 4_096 + when /anthropic\.claude-instant/ + 2_048 + when /amazon\.titan/ + 4_096 + else + 2_048 + end + end + + private + + PRICES = { + claude3: { input: 15.0, output: 75.0 }, + claude2: { input: 8.0, output: 24.0 }, + claude_instant: { input: 0.8, output: 2.4 }, + titan: { input: 0.1, output: 0.2 } + }.freeze + + def default_input_price + 0.1 + end + + def default_output_price + 0.2 + end + end + end + end +end \ No newline at end of file diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index 43a48d8a9..85c86041d 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -2,71 +2,78 @@ module RubyLLM module Providers - class Bedrock - # Chat implementation for AWS Bedrock - class Chat - def initialize(model:, temperature: nil, max_tokens: nil) - @model = model - @temperature = temperature - @max_tokens = max_tokens - end - - def chat(messages:, stream: false) - request_body = build_request_body(messages) - path = "/model/#{model_id}/invoke#{stream ? '-with-response-stream' : ''}" - - if stream - stream_response(path, request_body) - else - complete_response(path, request_body) - end - end - - private - - attr_reader :model, :temperature, :max_tokens + module Bedrock + # Chat methods for the AWS Bedrock API implementation + module Chat + module_function - def connection - Bedrock.config.connection + def completion_url + 'model' end - def build_request_body(messages) + def render_payload(messages, tools:, temperature:, model:, stream: false) # Format depends on the specific model being used - case model_id + case model_id_for(model) when /anthropic\.claude/ - build_claude_request(messages) + build_claude_request(messages, temperature) when /amazon\.titan/ - build_titan_request(messages) + build_titan_request(messages, temperature) else - raise Error, "Unsupported model: #{model_id}" + raise Error, "Unsupported model: #{model}" end end - def build_claude_request(messages) + def parse_completion_response(response) + data = response.body + return if data.empty? + + Message.new( + role: :assistant, + content: extract_content(data), + input_tokens: data.dig('usage', 'prompt_tokens'), + output_tokens: data.dig('usage', 'completion_tokens'), + model_id: data['model'] + ) + end + + private + + def build_claude_request(messages, temperature) formatted = messages.map do |msg| - role = msg[:role] == 'assistant' ? 'Assistant' : 'Human' - content = msg[:content] + role = msg.role == :assistant ? 'Assistant' : 'Human' + content = msg.content "\n\n#{role}: #{content}" end.join { prompt: formatted + "\n\nAssistant:", temperature: temperature, - max_tokens: max_tokens + max_tokens: max_tokens_for(messages.first&.model_id) } end - def build_titan_request(messages) + def build_titan_request(messages, temperature) { - inputText: messages.map { |msg| msg[:content] }.join("\n"), + inputText: messages.map { |msg| msg.content }.join("\n"), textGenerationConfig: { temperature: temperature, - maxTokenCount: max_tokens + maxTokenCount: max_tokens_for(messages.first&.model_id) } } end - def model_id + def extract_content(data) + case data + when /anthropic\.claude/ + data[:completion] + when /amazon\.titan/ + data.dig(:results, 0, :outputText) + else + raise Error, "Unsupported model: #{data['model']}" + end + end + + def model_id_for(model) case model when 'claude-3-sonnet' 'anthropic.claude-3-sonnet-20240229-v1:0' @@ -81,110 +88,8 @@ def model_id end end - def complete_response(path, body) - response = make_request(:post, path, body) - parse_response(response.body) - end - - def stream_response(path, body) - Enumerator.new do |yielder| - response = make_request(:post, path, body) - parser = EventStreamParser.new - - response.body.each do |chunk| - parser.feed(chunk) do |event| - parsed_chunk = parse_stream_event(event) - yielder << parsed_chunk if parsed_chunk - end - end - end - end - - def make_request(method, path, body) - json_body = JSON.generate(body) - headers = SignatureV4.sign_request( - connection: connection, - method: method, - path: path, - body: json_body, - access_key: Bedrock.config.access_key_id, - secret_key: Bedrock.config.secret_access_key, - session_token: Bedrock.config.session_token, - region: Bedrock.config.region - ) - - headers['Content-Type'] = 'application/json' - headers['Accept'] = 'application/json' - - response = connection.public_send(method, path) do |req| - req.headers.merge!(headers) - req.body = json_body - end - - raise Error, "Request failed: #{response.body}" unless response.success? - - response - end - - def parse_response(body) - case model_id - when /anthropic\.claude/ - parse_claude_response(body) - when /amazon\.titan/ - parse_titan_response(body) - else - raise Error, "Unsupported model: #{model_id}" - end - end - - def parse_stream_event(event) - return unless event.data - - body = JSON.parse(event.data, symbolize_names: true) - - case model_id - when /anthropic\.claude/ - parse_claude_stream_chunk(body) - when /amazon\.titan/ - parse_titan_stream_chunk(body) - else - raise Error, "Unsupported model: #{model_id}" - end - end - - def parse_claude_response(body) - { - role: 'assistant', - content: body[:completion] - } - end - - def parse_titan_response(body) - { - role: 'assistant', - content: body[:results].first[:outputText] - } - end - - def parse_claude_stream_chunk(body) - return unless body[:completion] - - { - role: 'assistant', - content: body[:completion], - delta: true - } - end - - def parse_titan_stream_chunk(body) - text = body[:outputText] - return unless text - - { - role: 'assistant', - content: text, - delta: true - } + def max_tokens_for(model_id) + Models.find(model_id)&.max_tokens end end end diff --git a/lib/ruby_llm/providers/bedrock/config.rb b/lib/ruby_llm/providers/bedrock/config.rb new file mode 100644 index 000000000..f1a40871f --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/config.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + class Bedrock + # Configuration for AWS Bedrock provider + class Config + attr_reader :access_key_id, :secret_access_key, :session_token, :region, :connection + + def initialize(access_key_id:, secret_access_key:, region:, session_token: nil) + @access_key_id = access_key_id + @secret_access_key = secret_access_key + @session_token = session_token + @region = region + @connection = build_connection + end + + private + + def build_connection + Faraday.new(url: "https://bedrock-runtime.#{region}.amazonaws.com") do |f| + f.request :json + f.response :json, content_type: /\bjson$/ + f.adapter Faraday.default_adapter + end + end + end + end + end +end \ No newline at end of file diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb new file mode 100644 index 000000000..6122486d7 --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + module Bedrock + # Models methods for the AWS Bedrock API implementation + module Models + module_function + + def models_url + 'foundation-models' + end + + def parse_list_models_response(response) + data = response.body['modelSummaries'] || [] + data.map do |model| + model_id = model['modelId'] + { + id: model_id, + created_at: nil, + display_name: model['modelName'] || capabilities.format_display_name(model_id), + provider: 'bedrock', + context_window: capabilities.context_window_for(model_id), + max_tokens: capabilities.max_tokens_for(model_id), + type: capabilities.model_type(model_id), + family: capabilities.model_family(model_id).to_s, + supports_vision: capabilities.supports_vision?(model_id), + supports_functions: capabilities.supports_functions?(model_id), + supports_json_mode: capabilities.supports_json_mode?(model_id), + input_price_per_million: capabilities.input_price_for(model_id), + output_price_per_million: capabilities.output_price_for(model_id), + metadata: { + provider_name: model['providerName'], + customizations_supported: model['customizationsSupported'] || [], + inference_configurations: model['inferenceTypesSupported'] || [], + response_streaming_supported: model['responseStreamingSupported'] || false, + input_modalities: model['inputModalities'] || [], + output_modalities: model['outputModalities'] || [] + } + } + end + end + + private + + def capabilities + Bedrock::Capabilities + end + end + end + end +end \ No newline at end of file diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb new file mode 100644 index 000000000..9cb178875 --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + module Bedrock + # Streaming methods for the AWS Bedrock API implementation + module Streaming + module_function + + def stream_url + 'model' + end + + def handle_stream(&block) + to_json_stream do |data| + block.call( + Chunk.new( + role: :assistant, + model_id: data['model'], + content: extract_content(data), + input_tokens: data.dig('usage', 'prompt_tokens'), + output_tokens: data.dig('usage', 'completion_tokens') + ) + ) + end + end + + private + + def extract_content(data) + case data + when /anthropic\.claude/ + data[:completion] + when /amazon\.titan/ + data.dig(:results, 0, :outputText) + else + raise Error, "Unsupported model: #{data['model']}" + end + end + end + end + end +end \ No newline at end of file diff --git a/lib/ruby_llm/providers/bedrock/tools.rb b/lib/ruby_llm/providers/bedrock/tools.rb new file mode 100644 index 000000000..62d62c4a2 --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/tools.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + module Bedrock + # Tools methods for the AWS Bedrock API implementation + module Tools + module_function + + def tool_for(tool) + { + type: 'function', + function: { + name: tool.name, + description: tool.description, + parameters: tool.parameters + } + } + end + + def parse_tool_calls(tool_calls, parse_arguments: true) + return {} unless tool_calls + + tool_calls.each_with_object({}) do |call, hash| + hash[call['id']] = ToolCall.new( + id: call['id'], + name: call['function']['name'], + arguments: parse_arguments ? parse_arguments(call['function']['arguments']) : call['function']['arguments'] + ) + end + end + + private + + def parse_arguments(arguments) + return {} unless arguments + + case arguments + when String + JSON.parse(arguments) + when Hash + arguments + else + {} + end + rescue JSON::ParserError + {} + end + end + end + end +end \ No newline at end of file diff --git a/spec/ruby_llm/providers/bedrock/chat_spec.rb b/spec/ruby_llm/providers/bedrock/chat_spec.rb deleted file mode 100644 index cf195479d..000000000 --- a/spec/ruby_llm/providers/bedrock/chat_spec.rb +++ /dev/null @@ -1,110 +0,0 @@ -# frozen_string_literal: true - -RSpec.describe RubyLLM::Providers::Bedrock::Chat do - let(:model) { 'claude-3-sonnet' } - let(:temperature) { 0.7 } - let(:max_tokens) { 100 } - let(:chat) { described_class.new(model: model, temperature: temperature, max_tokens: max_tokens) } - let(:messages) do - [ - { role: 'user', content: 'Hello!' } - ] - end - - describe '#chat' do - let(:connection) { instance_double(Faraday::Connection) } - let(:response) { instance_double(Faraday::Response) } - - before do - allow(RubyLLM::Providers::Bedrock.config).to receive(:connection).and_return(connection) - allow(RubyLLM::Providers::Bedrock.config).to receive(:access_key_id).and_return('test_key') - allow(RubyLLM::Providers::Bedrock.config).to receive(:secret_access_key).and_return('test_secret') - allow(RubyLLM::Providers::Bedrock.config).to receive(:region).and_return('us-east-1') - allow(connection).to receive(:url_prefix).and_return(URI('https://bedrock-runtime.us-east-1.amazonaws.com')) - end - - context 'with Claude model' do - let(:model) { 'claude-3-sonnet' } - - it 'makes a request with correct parameters' do - expect(connection).to receive(:post) - .with('/model/anthropic.claude-3-sonnet-20240229-v1:0/invoke') - .and_return(response) - - allow(response).to receive(:success?).and_return(true) - allow(response).to receive(:body).and_return({ completion: 'Hello there!' }) - - result = chat.chat(messages: messages) - expect(result).to eq({ role: 'assistant', content: 'Hello there!' }) - end - - context 'with streaming' do - let(:chunks) { ['Hello', ' there', '!'] } - let(:events) do - chunks.map do |chunk| - double('Event', data: JSON.generate({ completion: chunk })) - end - end - - it 'yields streamed responses' do - expect(connection).to receive(:post) - .with('/model/anthropic.claude-3-sonnet-20240229-v1:0/invoke-with-response-stream') - .and_return(response) - - allow(response).to receive(:success?).and_return(true) - allow(response).to receive(:body).and_return(chunks) - allow_any_instance_of(EventStreamParser).to receive(:feed).and_yield(*events) - - streamed = [] - chat.chat(messages: messages, stream: true).each do |chunk| - streamed << chunk[:content] - end - - expect(streamed).to eq(chunks) - end - end - end - - context 'with Titan model' do - let(:model) { 'titan' } - - it 'makes a request with correct parameters' do - expect(connection).to receive(:post) - .with('/model/amazon.titan-text-express-v1/invoke') - .and_return(response) - - allow(response).to receive(:success?).and_return(true) - allow(response).to receive(:body).and_return({ results: [{ outputText: 'Hello there!' }] }) - - result = chat.chat(messages: messages) - expect(result).to eq({ role: 'assistant', content: 'Hello there!' }) - end - end - - context 'with unsupported model' do - let(:model) { 'unsupported-model' } - - it 'raises an error' do - expect { - chat.chat(messages: messages) - }.to raise_error(RubyLLM::Providers::Bedrock::Error, /Unsupported model/) - end - end - - context 'when request fails' do - let(:error_response) { instance_double(Faraday::Response) } - - before do - allow(connection).to receive(:post).and_return(error_response) - allow(error_response).to receive(:success?).and_return(false) - allow(error_response).to receive(:body).and_return('Error message') - end - - it 'raises an error' do - expect { - chat.chat(messages: messages) - }.to raise_error(RubyLLM::Providers::Bedrock::Error, /Request failed/) - end - end - end -end \ No newline at end of file From 88ef94da7da7b97bf5f914fdadf4e6b9dd1088d9 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 12:35:47 -0700 Subject: [PATCH 03/85] #16 Get bedrock models --- lib/ruby_llm.rb | 12 +- lib/ruby_llm/models.json | 3291 ++++++++++------- lib/ruby_llm/models.rb | 4 +- lib/ruby_llm/providers/bedrock.rb | 19 +- .../providers/bedrock/capabilities.rb | 218 +- lib/ruby_llm/providers/bedrock/chat.rb | 67 +- lib/ruby_llm/providers/bedrock/models.rb | 24 +- lib/ruby_llm/providers/bedrock/streaming.rb | 7 +- 8 files changed, 2145 insertions(+), 1497 deletions(-) diff --git a/lib/ruby_llm.rb b/lib/ruby_llm.rb index 0563e13e6..0b3f0a51b 100644 --- a/lib/ruby_llm.rb +++ b/lib/ruby_llm.rb @@ -15,7 +15,8 @@ 'llm' => 'LLM', 'openai' => 'OpenAI', 'api' => 'API', - 'deepseek' => 'DeepSeek' + 'deepseek' => 'DeepSeek', + 'bedrock' => 'Bedrock' ) loader.setup @@ -64,10 +65,11 @@ def logger end end -RubyLLM::Provider.register :openai, RubyLLM::Providers::OpenAI -RubyLLM::Provider.register :anthropic, RubyLLM::Providers::Anthropic -RubyLLM::Provider.register :gemini, RubyLLM::Providers::Gemini -RubyLLM::Provider.register :deepseek, RubyLLM::Providers::DeepSeek +# RubyLLM::Provider.register :openai, RubyLLM::Providers::OpenAI +# RubyLLM::Provider.register :anthropic, RubyLLM::Providers::Anthropic +# RubyLLM::Provider.register :gemini, RubyLLM::Providers::Gemini +# RubyLLM::Provider.register :deepseek, RubyLLM::Providers::DeepSeek +RubyLLM::Provider.register :bedrock, RubyLLM::Providers::Bedrock if defined?(Rails::Railtie) require 'ruby_llm/railtie' diff --git a/lib/ruby_llm/models.json b/lib/ruby_llm/models.json index 386e2ad70..24970c2a3 100644 --- a/lib/ruby_llm/models.json +++ b/lib/ruby_llm/models.json @@ -1,2065 +1,2596 @@ [ { - "id": "aqa", + "id": "amazon.nova-lite-v1:0", "created_at": null, - "display_name": "Model that performs Attributed Question Answering.", - "provider": "gemini", - "context_window": 7168, - "max_tokens": 1024, + "display_name": "Nova Lite", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "aqa", + "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, "input_price_per_million": 0.0, "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Model trained to return answers to questions that are grounded in provided sources, along with estimating answerable probability.", - "input_token_limit": 7168, - "output_token_limit": 1024, - "supported_generation_methods": [ - "generateAnswer" + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE", + "VIDEO" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "babbage-002", - "created_at": "2023-08-21T18:16:55+02:00", - "display_name": "Babbage 002", - "provider": "openai", - "context_window": 16385, - "max_tokens": 16384, + "id": "amazon.nova-micro-v1:0", + "created_at": null, + "display_name": "Nova Micro", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "babbage", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "chat-bison-001", + "id": "amazon.nova-pro-v1:0", "created_at": null, - "display_name": "PaLM 2 Chat (Legacy)", - "provider": "gemini", + "display_name": "Nova Pro", + "provider": "bedrock", "context_window": 4096, - "max_tokens": 1024, + "max_tokens": 4096, "type": "chat", "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "A legacy text-only model optimized for chat conversations", - "input_token_limit": 4096, - "output_token_limit": 1024, - "supported_generation_methods": [ - "generateMessage", - "countMessageTokens" + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE", + "VIDEO" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "chatgpt-4o-latest", - "created_at": "2024-08-13T04:12:11+02:00", - "display_name": "ChatGPT-4o Latest", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "amazon.rerank-v1:0", + "created_at": null, + "display_name": "Rerank 1.0", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "claude-2.0", - "created_at": "2023-07-11T00:00:00Z", - "display_name": "Claude 2.0", - "provider": "anthropic", - "context_window": 200000, + "id": "amazon.titan-embed-g1-text-02", + "created_at": null, + "display_name": "Titan Text Embeddings v2", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude2", + "family": "titan", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 3.0, - "output_price_per_million": 15.0, - "metadata": {} + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "EMBEDDING" + ] + } }, { - "id": "claude-2.1", - "created_at": "2023-11-21T00:00:00Z", - "display_name": "Claude 2.1", - "provider": "anthropic", - "context_window": 200000, + "id": "amazon.titan-embed-image-v1", + "created_at": null, + "display_name": "Titan Multimodal Embeddings G1", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude2", + "family": "titan", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 3.0, - "output_price_per_million": 15.0, - "metadata": {} - }, - { - "id": "claude-3-5-haiku-20241022", - "created_at": "2024-10-22T00:00:00Z", - "display_name": "Claude 3.5 Haiku", - "provider": "anthropic", - "context_window": 200000, - "max_tokens": 8192, - "type": "chat", - "family": "claude35_haiku", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.8, - "output_price_per_million": 4.0, - "metadata": {} - }, - { - "id": "claude-3-5-sonnet-20240620", - "created_at": "2024-06-20T00:00:00Z", - "display_name": "Claude 3.5 Sonnet (Old)", - "provider": "anthropic", - "context_window": 200000, - "max_tokens": 8192, - "type": "chat", - "family": "claude35_sonnet", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 3.0, - "output_price_per_million": 15.0, - "metadata": {} - }, - { - "id": "claude-3-5-sonnet-20241022", - "created_at": "2024-10-22T00:00:00Z", - "display_name": "Claude 3.5 Sonnet (New)", - "provider": "anthropic", - "context_window": 200000, - "max_tokens": 8192, - "type": "chat", - "family": "claude35_sonnet", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 3.0, - "output_price_per_million": 15.0, - "metadata": {} - }, - { - "id": "claude-3-7-sonnet-20250219", - "created_at": "2025-02-24T00:00:00Z", - "display_name": "Claude 3.7 Sonnet", - "provider": "anthropic", - "context_window": 200000, - "max_tokens": 8192, - "type": "chat", - "family": "claude37_sonnet", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 3.0, - "output_price_per_million": 15.0, - "metadata": {} + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "EMBEDDING" + ] + } }, { - "id": "claude-3-haiku-20240307", - "created_at": "2024-03-07T00:00:00Z", - "display_name": "Claude 3 Haiku", - "provider": "anthropic", - "context_window": 200000, + "id": "amazon.titan-embed-image-v1:0", + "created_at": null, + "display_name": "Titan Multimodal Embeddings G1", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3_haiku", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.25, - "output_price_per_million": 1.25, - "metadata": {} + "family": "titan", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Amazon", + "customizations_supported": [ + "FINE_TUNING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "EMBEDDING" + ] + } }, { - "id": "claude-3-opus-20240229", - "created_at": "2024-02-29T00:00:00Z", - "display_name": "Claude 3 Opus", - "provider": "anthropic", - "context_window": 200000, + "id": "amazon.titan-embed-text-v1", + "created_at": null, + "display_name": "Titan Embeddings G1 - Text", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3_opus", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 15.0, - "output_price_per_million": 75.0, - "metadata": {} + "family": "titan", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "EMBEDDING" + ] + } }, { - "id": "claude-3-sonnet-20240229", - "created_at": "2024-02-29T00:00:00Z", - "display_name": "Claude 3 Sonnet", - "provider": "anthropic", - "context_window": 200000, + "id": "amazon.titan-embed-text-v1:2:8k", + "created_at": null, + "display_name": "Titan Embeddings G1 - Text", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3_sonnet", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 3.0, - "output_price_per_million": 15.0, - "metadata": {} + "family": "titan", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "EMBEDDING" + ] + } }, { - "id": "dall-e-2", - "created_at": "2023-11-01T01:22:57+01:00", - "display_name": "DALL-E-2", - "provider": "openai", + "id": "amazon.titan-embed-text-v2:0", + "created_at": null, + "display_name": "Titan Text Embeddings V2", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "image", - "family": "dalle2", + "type": "chat", + "family": "titan", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "EMBEDDING" + ] } }, { - "id": "dall-e-3", - "created_at": "2023-10-31T21:46:29+01:00", - "display_name": "DALL-E-3", - "provider": "openai", + "id": "amazon.titan-image-generator-v1", + "created_at": null, + "display_name": "Titan Image Generator G1", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "image", - "family": "dalle3", + "type": "chat", + "family": "titan", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "IMAGE" + ] } }, { - "id": "davinci-002", - "created_at": "2023-08-21T18:11:41+02:00", - "display_name": "Davinci 002", - "provider": "openai", - "context_window": 16385, - "max_tokens": 16384, + "id": "amazon.titan-image-generator-v1:0", + "created_at": null, + "display_name": "Titan Image Generator G1", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "davinci", + "family": "titan", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Amazon", + "customizations_supported": [ + "FINE_TUNING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "IMAGE" + ] } }, { - "id": "deepseek-chat", + "id": "amazon.titan-image-generator-v2:0", "created_at": null, - "display_name": "DeepSeek V3", - "provider": "deepseek", - "context_window": 64000, - "max_tokens": 8192, + "display_name": "Titan Image Generator G1 v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "chat", + "family": "titan", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.27, - "output_price_per_million": 1.1, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "deepseek" + "provider_name": "Amazon", + "customizations_supported": [ + "FINE_TUNING" + ], + "inference_configurations": [ + "PROVISIONED", + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "IMAGE" + ] } }, { - "id": "deepseek-reasoner", + "id": "amazon.titan-text-express-v1", "created_at": null, - "display_name": "DeepSeek R1", - "provider": "deepseek", - "context_window": 64000, - "max_tokens": 8192, + "display_name": "Titan Text G1 - Express", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "reasoner", + "family": "titan", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.55, - "output_price_per_million": 2.19, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "deepseek" + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "embedding-001", + "id": "amazon.titan-text-express-v1:0:8k", "created_at": null, - "display_name": "Embedding 001", - "provider": "gemini", - "context_window": 2048, - "max_tokens": 1, - "type": "embedding", - "family": "embedding1", + "display_name": "Titan Text G1 - Express", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "titan", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, "input_price_per_million": 0.0, "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Obtain a distributed representation of a text.", - "input_token_limit": 2048, - "output_token_limit": 1, - "supported_generation_methods": [ - "embedContent" + "provider_name": "Amazon", + "customizations_supported": [ + "FINE_TUNING", + "CONTINUED_PRE_TRAINING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "embedding-gecko-001", + "id": "amazon.titan-text-lite-v1", "created_at": null, - "display_name": "Embedding Gecko", - "provider": "gemini", - "context_window": 1024, - "max_tokens": 1, - "type": "embedding", - "family": "other", + "display_name": "Titan Text G1 - Lite", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "titan", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, "input_price_per_million": 0.0, "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Obtain a distributed representation of a text.", - "input_token_limit": 1024, - "output_token_limit": 1, - "supported_generation_methods": [ - "embedText", - "countTextTokens" + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.0-pro-vision-latest", + "id": "amazon.titan-text-lite-v1:0:4k", "created_at": null, - "display_name": "Gemini 1.0 Pro Vision", - "provider": "gemini", - "context_window": 12288, + "display_name": "Titan Text G1 - Lite", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gemini10_pro", + "family": "titan", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "The original Gemini 1.0 Pro Vision model version which was optimized for image understanding. Gemini 1.0 Pro Vision was deprecated on July 12, 2024. Move to a newer Gemini version.", - "input_token_limit": 12288, - "output_token_limit": 4096, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "provider_name": "Amazon", + "customizations_supported": [ + "FINE_TUNING", + "CONTINUED_PRE_TRAINING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash", + "id": "amazon.titan-tg1-large", "created_at": null, - "display_name": "Gemini 1.5 Flash", - "provider": "gemini", - "context_window": 1000000, - "max_tokens": 8192, + "display_name": "Titan Text Large", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.15, - "output_price_per_million": 0.6, + "family": "titan", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Alias that points to the most recent stable version of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks.", - "input_token_limit": 1000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "provider_name": "Amazon", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash-001", + "id": "anthropic.claude-3-5-haiku-20241022-v1:0", "created_at": null, - "display_name": "Gemini 1.5 Flash 001", - "provider": "gemini", - "context_window": 1000000, - "max_tokens": 8192, + "display_name": "Claude 3.5 Haiku", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.15, - "output_price_per_million": 0.6, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Stable version of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks, released in May of 2024.", - "input_token_limit": 1000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens", - "createCachedContent" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash-001-tuning", + "id": "anthropic.claude-3-5-sonnet-20240620-v1:0", "created_at": null, - "display_name": "Gemini 1.5 Flash 001 Tuning", - "provider": "gemini", - "context_window": 16384, - "max_tokens": 8192, + "display_name": "Claude 3.5 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.15, - "output_price_per_million": 0.6, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Version of Gemini 1.5 Flash that supports tuning, our fast and versatile multimodal model for scaling across diverse tasks, released in May of 2024.", - "input_token_limit": 16384, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens", - "createTunedModel" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash-002", + "id": "anthropic.claude-3-5-sonnet-20240620-v1:0:18k", "created_at": null, - "display_name": "Gemini 1.5 Flash 002", - "provider": "gemini", - "context_window": 1000000, - "max_tokens": 8192, + "display_name": "Claude 3.5 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.15, - "output_price_per_million": 0.6, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "002", - "description": "Stable version of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks, released in September of 2024.", - "input_token_limit": 1000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens", - "createCachedContent" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash-8b", + "id": "anthropic.claude-3-5-sonnet-20240620-v1:0:200k", "created_at": null, - "display_name": "Gemini 1.5 Flash-8B", - "provider": "gemini", - "context_window": 1000000, - "max_tokens": 8192, + "display_name": "Claude 3.5 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash_8b", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Stable version of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model, released in October of 2024.", - "input_token_limit": 1000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "createCachedContent", - "generateContent", - "countTokens" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash-8b-001", + "id": "anthropic.claude-3-5-sonnet-20240620-v1:0:51k", "created_at": null, - "display_name": "Gemini 1.5 Flash-8B 001", - "provider": "gemini", - "context_window": 1000000, - "max_tokens": 8192, + "display_name": "Claude 3.5 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash_8b", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Stable version of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model, released in October of 2024.", - "input_token_limit": 1000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "createCachedContent", - "generateContent", - "countTokens" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash-8b-exp-0827", + "id": "anthropic.claude-3-5-sonnet-20241022-v2:0", "created_at": null, - "display_name": "Gemini 1.5 Flash 8B Experimental 0827", - "provider": "gemini", - "context_window": 1000000, - "max_tokens": 8192, + "display_name": "Claude 3.5 Sonnet v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash_8b", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Experimental release (August 27th, 2024) of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model. Replaced by Gemini-1.5-flash-8b-001 (stable).", - "input_token_limit": 1000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash-8b-exp-0924", + "id": "anthropic.claude-3-5-sonnet-20241022-v2:0:18k", "created_at": null, - "display_name": "Gemini 1.5 Flash 8B Experimental 0924", - "provider": "gemini", - "context_window": 1000000, - "max_tokens": 8192, + "display_name": "Claude 3.5 Sonnet v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash_8b", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Experimental release (September 24th, 2024) of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model. Replaced by Gemini-1.5-flash-8b-001 (stable).", - "input_token_limit": 1000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash-8b-latest", + "id": "anthropic.claude-3-5-sonnet-20241022-v2:0:200k", "created_at": null, - "display_name": "Gemini 1.5 Flash-8B Latest", - "provider": "gemini", - "context_window": 1000000, - "max_tokens": 8192, + "display_name": "Claude 3.5 Sonnet v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash_8b", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Alias that points to the most recent production (non-experimental) release of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model, released in October of 2024.", - "input_token_limit": 1000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "createCachedContent", - "generateContent", - "countTokens" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-flash-latest", + "id": "anthropic.claude-3-5-sonnet-20241022-v2:0:51k", "created_at": null, - "display_name": "Gemini 1.5 Flash Latest", - "provider": "gemini", - "context_window": 1000000, - "max_tokens": 8192, + "display_name": "Claude 3.5 Sonnet v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_flash", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.15, - "output_price_per_million": 0.6, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Alias that points to the most recent production (non-experimental) release of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks.", - "input_token_limit": 1000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-pro", + "id": "anthropic.claude-3-7-sonnet-20250219-v1:0", "created_at": null, - "display_name": "Gemini 1.5 Pro", - "provider": "gemini", - "context_window": 2000000, - "max_tokens": 8192, + "display_name": "Claude 3.7 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_pro", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 2.5, - "output_price_per_million": 10.0, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "version": "001", - "description": "Stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens, released in May of 2024.", - "input_token_limit": 2000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-pro-001", + "id": "anthropic.claude-3-haiku-20240307-v1:0", "created_at": null, - "display_name": "Gemini 1.5 Pro 001", - "provider": "gemini", - "context_window": 2000000, - "max_tokens": 8192, + "display_name": "Claude 3 Haiku", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_pro", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 2.5, - "output_price_per_million": 10.0, - "metadata": { - "version": "001", - "description": "Stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens, released in May of 2024.", - "input_token_limit": 2000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens", - "createCachedContent" + "input_price_per_million": 0.0005, + "output_price_per_million": 0.0025, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-pro-002", + "id": "anthropic.claude-3-haiku-20240307-v1:0:200k", "created_at": null, - "display_name": "Gemini 1.5 Pro 002", - "provider": "gemini", - "context_window": 2000000, - "max_tokens": 8192, + "display_name": "Claude 3 Haiku", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_pro", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 2.5, - "output_price_per_million": 10.0, - "metadata": { - "version": "002", - "description": "Stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens, released in September of 2024.", - "input_token_limit": 2000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens", - "createCachedContent" + "input_price_per_million": 0.0005, + "output_price_per_million": 0.0025, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [ + "FINE_TUNING", + "DISTILLATION" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-1.5-pro-latest", + "id": "anthropic.claude-3-haiku-20240307-v1:0:48k", "created_at": null, - "display_name": "Gemini 1.5 Pro Latest", - "provider": "gemini", - "context_window": 2000000, - "max_tokens": 8192, + "display_name": "Claude 3 Haiku", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini15_pro", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 2.5, - "output_price_per_million": 10.0, - "metadata": { - "version": "001", - "description": "Alias that points to the most recent production (non-experimental) release of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens.", - "input_token_limit": 2000000, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "input_price_per_million": 0.0005, + "output_price_per_million": 0.0025, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-flash", + "id": "anthropic.claude-3-opus-20240229-v1:0", "created_at": null, - "display_name": "Gemini 2.0 Flash", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 8192, + "display_name": "Claude 3 Opus", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini20_flash", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.1, - "output_price_per_million": 0.4, - "metadata": { - "version": "2.0", - "description": "Gemini 2.0 Flash", - "input_token_limit": 1048576, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "input_price_per_million": 0.015, + "output_price_per_million": 0.075, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-flash-001", + "id": "anthropic.claude-3-opus-20240229-v1:0:12k", "created_at": null, - "display_name": "Gemini 2.0 Flash 001", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 8192, + "display_name": "Claude 3 Opus", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini20_flash", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.1, - "output_price_per_million": 0.4, - "metadata": { - "version": "2.0", - "description": "Stable version of Gemini 2.0 Flash, our fast and versatile multimodal model for scaling across diverse tasks, released in January of 2025.", - "input_token_limit": 1048576, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "input_price_per_million": 0.015, + "output_price_per_million": 0.075, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-flash-exp", + "id": "anthropic.claude-3-opus-20240229-v1:0:200k", "created_at": null, - "display_name": "Gemini 2.0 Flash Experimental", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 8192, + "display_name": "Claude 3 Opus", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini20_flash", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.1, - "output_price_per_million": 0.4, - "metadata": { - "version": "2.0", - "description": "Gemini 2.0 Flash Experimental", - "input_token_limit": 1048576, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens", - "bidiGenerateContent" + "input_price_per_million": 0.015, + "output_price_per_million": 0.075, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-flash-lite", + "id": "anthropic.claude-3-opus-20240229-v1:0:28k", "created_at": null, - "display_name": "Gemini 2.0 Flash-Lite", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 8192, + "display_name": "Claude 3 Opus", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini20_flash_lite", + "family": "claude3", "supports_vision": true, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "2.0", - "description": "Gemini 2.0 Flash-Lite", - "input_token_limit": 1048576, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.015, + "output_price_per_million": 0.075, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-flash-lite-001", + "id": "anthropic.claude-3-sonnet-20240229-v1:0", "created_at": null, - "display_name": "Gemini 2.0 Flash-Lite 001", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 8192, + "display_name": "Claude 3 Sonnet", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini20_flash_lite", + "family": "claude3", "supports_vision": true, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "2.0", - "description": "Stable version of Gemini 2.0 Flash Lite", - "input_token_limit": 1048576, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.003, + "output_price_per_million": 0.015, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-flash-lite-preview", + "id": "anthropic.claude-3-sonnet-20240229-v1:0:200k", "created_at": null, - "display_name": "Gemini 2.0 Flash-Lite Preview", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 8192, - "type": "chat", - "family": "gemini20_flash_lite", - "supports_vision": true, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "preview-02-05", - "description": "Preview release (February 5th, 2025) of Gemini 2.0 Flash Lite", - "input_token_limit": 1048576, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" - ] - } - }, - { - "id": "gemini-2.0-flash-lite-preview-02-05", - "created_at": null, - "display_name": "Gemini 2.0 Flash-Lite Preview 02-05", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 8192, - "type": "chat", - "family": "gemini20_flash_lite", - "supports_vision": true, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "preview-02-05", - "description": "Preview release (February 5th, 2025) of Gemini 2.0 Flash Lite", - "input_token_limit": 1048576, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" - ] - } - }, - { - "id": "gemini-2.0-flash-thinking-exp", - "created_at": null, - "display_name": "Gemini 2.0 Flash Thinking Experimental 01-21", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 65536, + "display_name": "Claude 3 Sonnet", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini20_flash", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.1, - "output_price_per_million": 0.4, - "metadata": { - "version": "2.0-exp-01-21", - "description": "Experimental release (January 21st, 2025) of Gemini 2.0 Flash Thinking", - "input_token_limit": 1048576, - "output_token_limit": 65536, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "input_price_per_million": 0.003, + "output_price_per_million": 0.015, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-flash-thinking-exp-01-21", + "id": "anthropic.claude-3-sonnet-20240229-v1:0:28k", "created_at": null, - "display_name": "Gemini 2.0 Flash Thinking Experimental 01-21", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 65536, + "display_name": "Claude 3 Sonnet", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, "type": "chat", - "family": "gemini20_flash", + "family": "claude3", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.1, - "output_price_per_million": 0.4, - "metadata": { - "version": "2.0-exp-01-21", - "description": "Experimental release (January 21st, 2025) of Gemini 2.0 Flash Thinking", - "input_token_limit": 1048576, - "output_token_limit": 65536, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "input_price_per_million": 0.003, + "output_price_per_million": 0.015, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-flash-thinking-exp-1219", + "id": "anthropic.claude-instant-v1", "created_at": null, - "display_name": "Gemini 2.0 Flash Thinking Experimental", - "provider": "gemini", - "context_window": 1048576, - "max_tokens": 65536, + "display_name": "Claude Instant", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gemini20_flash", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.1, - "output_price_per_million": 0.4, - "metadata": { - "version": "2.0", - "description": "Gemini 2.0 Flash Thinking Experimental", - "input_token_limit": 1048576, - "output_token_limit": 65536, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "family": "claude_instant", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0008, + "output_price_per_million": 0.0024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-pro-exp", + "id": "anthropic.claude-instant-v1:2:100k", "created_at": null, - "display_name": "Gemini 2.0 Pro Experimental", - "provider": "gemini", - "context_window": 2097152, - "max_tokens": 8192, + "display_name": "Claude Instant", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "other", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "2.0", - "description": "Experimental release (February 5th, 2025) of Gemini 2.0 Pro", - "input_token_limit": 2097152, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "family": "claude_instant", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0008, + "output_price_per_million": 0.0024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-2.0-pro-exp-02-05", + "id": "anthropic.claude-v2", "created_at": null, - "display_name": "Gemini 2.0 Pro Experimental 02-05", - "provider": "gemini", - "context_window": 2097152, - "max_tokens": 8192, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "other", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "2.0", - "description": "Experimental release (February 5th, 2025) of Gemini 2.0 Pro", - "input_token_limit": 2097152, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "family": "claude2", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-exp-1206", + "id": "anthropic.claude-v2:0:100k", "created_at": null, - "display_name": "Gemini Experimental 1206", - "provider": "gemini", - "context_window": 2097152, - "max_tokens": 8192, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "claude2", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "2.0", - "description": "Experimental release (February 5th, 2025) of Gemini 2.0 Pro", - "input_token_limit": 2097152, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gemini-pro-vision", + "id": "anthropic.claude-v2:0:18k", "created_at": null, - "display_name": "Gemini 1.0 Pro Vision", - "provider": "gemini", - "context_window": 12288, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "claude2", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "001", - "description": "The original Gemini 1.0 Pro Vision model version which was optimized for image understanding. Gemini 1.0 Pro Vision was deprecated on July 12, 2024. Move to a newer Gemini version.", - "input_token_limit": 12288, - "output_token_limit": 4096, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "gpt-3.5-turbo", - "created_at": "2023-02-28T19:56:42+01:00", - "display_name": "GPT-3.5-Turbo", - "provider": "openai", - "context_window": 16385, + "id": "anthropic.claude-v2:1", + "created_at": null, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt35", + "family": "claude2", "supports_vision": false, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "openai" + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-3.5-turbo-0125", - "created_at": "2024-01-23T23:19:18+01:00", - "display_name": "GPT-3.5-Turbo 0125", - "provider": "openai", + "id": "anthropic.claude-v2:1:18k", + "created_at": null, + "display_name": "Claude", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt35", + "family": "claude2", "supports_vision": false, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-3.5-turbo-1106", - "created_at": "2023-11-02T22:15:48+01:00", - "display_name": "GPT-3.5-Turbo 1106", - "provider": "openai", + "id": "anthropic.claude-v2:1:200k", + "created_at": null, + "display_name": "Claude", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt35", + "family": "claude2", "supports_vision": false, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-3.5-turbo-16k", - "created_at": "2023-05-11T00:35:02+02:00", - "display_name": "GPT-3.5-Turbo 16k", - "provider": "openai", - "context_window": 16385, + "id": "cohere.command-light-text-v14", + "created_at": null, + "display_name": "Command Light", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt35", + "family": "other", "supports_vision": false, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "openai-internal" + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0003, + "output_price_per_million": 0.0006, + "metadata": { + "provider_name": "Cohere", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-3.5-turbo-instruct", - "created_at": "2023-08-24T20:23:47+02:00", - "display_name": "GPT-3.5-Turbo Instruct", - "provider": "openai", + "id": "cohere.command-light-text-v14:7:4k", + "created_at": null, + "display_name": "Command Light", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt35_instruct", + "family": "other", "supports_vision": false, "supports_functions": false, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" + "supports_json_mode": false, + "input_price_per_million": 0.0003, + "output_price_per_million": 0.0006, + "metadata": { + "provider_name": "Cohere", + "customizations_supported": [ + "FINE_TUNING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-3.5-turbo-instruct-0914", - "created_at": "2023-09-07T23:34:32+02:00", - "display_name": "GPT-3.5-Turbo Instruct 0914", - "provider": "openai", + "id": "cohere.command-r-plus-v1:0", + "created_at": null, + "display_name": "Command R+", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt35_instruct", + "family": "other", "supports_vision": false, "supports_functions": false, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "supports_json_mode": false, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Cohere", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4", - "created_at": "2023-06-27T18:13:31+02:00", - "display_name": "GPT-4", - "provider": "openai", + "id": "cohere.command-r-v1:0", + "created_at": null, + "display_name": "Command R", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt4", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "openai" - } - }, - { - "id": "gpt-4-0125-preview", - "created_at": "2024-01-23T20:20:12+01:00", - "display_name": "GPT-4-0125 Preview", - "provider": "openai", - "context_window": 8192, - "max_tokens": 8192, - "type": "chat", - "family": "gpt4", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Cohere", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4-0613", - "created_at": "2023-06-12T18:54:56+02:00", - "display_name": "GPT-4-0613", - "provider": "openai", - "context_window": 8192, - "max_tokens": 8192, + "id": "cohere.command-text-v14", + "created_at": null, + "display_name": "Command", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "openai" + "input_price_per_million": 0.0015, + "output_price_per_million": 0.002, + "metadata": { + "provider_name": "Cohere", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4-1106-preview", - "created_at": "2023-11-02T21:33:26+01:00", - "display_name": "GPT-4-1106 Preview", - "provider": "openai", + "id": "cohere.command-text-v14:7:4k", + "created_at": null, + "display_name": "Command", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt4", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" - } - }, - { - "id": "gpt-4-turbo", - "created_at": "2024-04-06T01:57:21+02:00", - "display_name": "GPT-4-Turbo", - "provider": "openai", - "context_window": 128000, - "max_tokens": 4096, - "type": "chat", - "family": "gpt4_turbo", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" + "family": "other", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0015, + "output_price_per_million": 0.002, + "metadata": { + "provider_name": "Cohere", + "customizations_supported": [ + "FINE_TUNING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4-turbo-2024-04-09", - "created_at": "2024-04-08T20:41:17+02:00", - "display_name": "GPT-4-Turbo 20240409", - "provider": "openai", - "context_window": 128000, + "id": "cohere.embed-english-v3", + "created_at": null, + "display_name": "Embed English", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt4_turbo", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "family": "other", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Cohere", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "EMBEDDING" + ] } }, { - "id": "gpt-4-turbo-preview", - "created_at": "2024-01-23T20:22:57+01:00", - "display_name": "GPT-4-Turbo Preview", - "provider": "openai", - "context_window": 128000, + "id": "cohere.embed-english-v3:0:512", + "created_at": null, + "display_name": "Embed English", + "provider": "bedrock", + "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt4_turbo", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "family": "other", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Cohere", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "EMBEDDING" + ] } }, { - "id": "gpt-4.5-preview", - "created_at": "2025-02-27T03:24:19+01:00", - "display_name": "GPT-4.5 Preview", - "provider": "openai", + "id": "cohere.embed-multilingual-v3", + "created_at": null, + "display_name": "Embed Multilingual", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt4", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Cohere", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "EMBEDDING" + ] } }, { - "id": "gpt-4.5-preview-2025-02-27", - "created_at": "2025-02-27T03:28:24+01:00", - "display_name": "GPT-4.5 Preview 20250227", - "provider": "openai", + "id": "cohere.embed-multilingual-v3:0:512", + "created_at": null, + "display_name": "Embed Multilingual", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "gpt4", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Cohere", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "EMBEDDING" + ] } }, { - "id": "gpt-4o", - "created_at": "2024-05-10T20:50:49+02:00", - "display_name": "GPT-4o", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "cohere.rerank-v3-5:0", + "created_at": null, + "display_name": "Rerank 3.5", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Cohere", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-2024-05-13", - "created_at": "2024-05-10T21:08:52+02:00", - "display_name": "GPT-4o 20240513", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "deepseek.r1-v1:0", + "created_at": null, + "display_name": "DeepSeek-R1", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "DeepSeek", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-2024-08-06", - "created_at": "2024-08-05T01:38:39+02:00", - "display_name": "GPT-4o 20240806", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "luma.ray-v2:0", + "created_at": null, + "display_name": "Ray v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Luma AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "VIDEO" + ] } }, { - "id": "gpt-4o-2024-11-20", - "created_at": "2025-02-12T04:39:03+01:00", - "display_name": "GPT-4o 20241120", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-1-405b-instruct-v1:0", + "created_at": null, + "display_name": "Llama 3.1 405B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-audio-preview", - "created_at": "2024-09-27T20:07:23+02:00", - "display_name": "GPT-4o-Audio Preview", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-1-70b-instruct-v1:0", + "created_at": null, + "display_name": "Llama 3.1 70B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_audio", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" + "input_price_per_million": 0.00195, + "output_price_per_million": 0.00256, + "metadata": { + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-audio-preview-2024-10-01", - "created_at": "2024-09-27T00:17:22+02:00", - "display_name": "GPT-4o-Audio Preview 20241001", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-1-70b-instruct-v1:0:128k", + "created_at": null, + "display_name": "Llama 3.1 70B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_audio", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" + "input_price_per_million": 0.00195, + "output_price_per_million": 0.00256, + "metadata": { + "provider_name": "Meta", + "customizations_supported": [ + "FINE_TUNING", + "DISTILLATION" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-audio-preview-2024-12-17", - "created_at": "2024-12-12T21:10:39+01:00", - "display_name": "GPT-4o-Audio Preview 20241217", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-1-8b-instruct-v1:0", + "created_at": null, + "display_name": "Llama 3.1 8B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_audio", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-mini", - "created_at": "2024-07-17T01:32:21+02:00", - "display_name": "GPT-4o-Mini", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-1-8b-instruct-v1:0:128k", + "created_at": null, + "display_name": "Llama 3.1 8B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_mini", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [ + "FINE_TUNING", + "DISTILLATION" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-mini-2024-07-18", - "created_at": "2024-07-17T01:31:57+02:00", - "display_name": "GPT-4o-Mini 20240718", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-2-11b-instruct-v1:0", + "created_at": null, + "display_name": "Llama 3.2 11B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_mini", + "family": "other", "supports_vision": true, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-mini-audio-preview", - "created_at": "2024-12-16T23:17:04+01:00", - "display_name": "GPT-4o-Mini Audio Preview", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-2-11b-instruct-v1:0:128k", + "created_at": null, + "display_name": "Llama 3.2 11B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_mini_audio", + "family": "other", "supports_vision": true, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [ + "FINE_TUNING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-mini-audio-preview-2024-12-17", - "created_at": "2024-12-13T19:52:00+01:00", - "display_name": "GPT-4o-Mini Audio Preview 20241217", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-2-1b-instruct-v1:0", + "created_at": null, + "display_name": "Llama 3.2 1B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_mini_audio", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-mini-realtime-preview", - "created_at": "2024-12-16T23:16:20+01:00", - "display_name": "GPT-4o-Mini Realtime Preview", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-2-1b-instruct-v1:0:128k", + "created_at": null, + "display_name": "Llama 3.2 1B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_mini_realtime", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [ + "FINE_TUNING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-mini-realtime-preview-2024-12-17", - "created_at": "2024-12-13T18:56:41+01:00", - "display_name": "GPT-4o-Mini Realtime Preview 20241217", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-2-3b-instruct-v1:0", + "created_at": null, + "display_name": "Llama 3.2 3B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_mini_realtime", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-realtime-preview", - "created_at": "2024-09-30T03:33:18+02:00", - "display_name": "GPT-4o-Realtime Preview", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-2-3b-instruct-v1:0:128k", + "created_at": null, + "display_name": "Llama 3.2 3B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_realtime", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [ + "FINE_TUNING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-realtime-preview-2024-10-01", - "created_at": "2024-09-24T00:49:26+02:00", - "display_name": "GPT-4o-Realtime Preview 20241001", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-2-90b-instruct-v1:0", + "created_at": null, + "display_name": "Llama 3.2 90B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_realtime", + "family": "other", "supports_vision": true, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "gpt-4o-realtime-preview-2024-12-17", - "created_at": "2024-12-11T20:30:30+01:00", - "display_name": "GPT-4o-Realtime Preview 20241217", - "provider": "openai", - "context_window": 128000, - "max_tokens": 16384, + "id": "meta.llama3-2-90b-instruct-v1:0:128k", + "created_at": null, + "display_name": "Llama 3.2 90B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", - "family": "gpt4o_realtime", + "family": "other", "supports_vision": true, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [ + "FINE_TUNING" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "imagen-3.0-generate-002", + "id": "meta.llama3-3-70b-instruct-v1:0", "created_at": null, - "display_name": "Imagen 3.0 002 model", - "provider": "gemini", - "context_window": 480, - "max_tokens": 8192, - "type": "image", + "display_name": "Llama 3.3 70B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "002", - "description": "Vertex served Imagen 3.0 002 model", - "input_token_limit": 480, - "output_token_limit": 8192, - "supported_generation_methods": [ - "predict" + "input_price_per_million": 0.00195, + "output_price_per_million": 0.00256, + "metadata": { + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "learnlm-1.5-pro-experimental", + "id": "meta.llama3-70b-instruct-v1:0", "created_at": null, - "display_name": "LearnLM 1.5 Pro Experimental", - "provider": "gemini", - "context_window": 32767, - "max_tokens": 8192, + "display_name": "Llama 3 70B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "001", - "description": "Alias that points to the most recent stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens.", - "input_token_limit": 32767, - "output_token_limit": 8192, - "supported_generation_methods": [ - "generateContent", - "countTokens" + "input_price_per_million": 0.00195, + "output_price_per_million": 0.00256, + "metadata": { + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "o1-mini", - "created_at": "2024-09-06T20:56:48+02:00", - "display_name": "O1-Mini", - "provider": "openai", - "context_window": 128000, - "max_tokens": 4096, - "type": "chat", - "family": "o1_mini", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" - } - }, - { - "id": "o1-mini-2024-09-12", - "created_at": "2024-09-06T20:56:19+02:00", - "display_name": "O1-Mini 20240912", - "provider": "openai", - "context_window": 128000, - "max_tokens": 65536, - "type": "chat", - "family": "o1_mini", - "supports_vision": true, - "supports_functions": true, - "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" - } - }, - { - "id": "o1-preview", - "created_at": "2024-09-06T20:54:57+02:00", - "display_name": "O1-Preview", - "provider": "openai", + "id": "meta.llama3-8b-instruct-v1:0", + "created_at": null, + "display_name": "Llama 3 8B Instruct", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "o1", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Meta", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "o1-preview-2024-09-12", - "created_at": "2024-09-06T20:54:25+02:00", - "display_name": "O1-Preview 20240912", - "provider": "openai", + "id": "mistral.mistral-7b-instruct-v0:2", + "created_at": null, + "display_name": "Mistral 7B Instruct", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "o1", - "supports_vision": true, - "supports_functions": true, + "family": "other", + "supports_vision": false, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" + "input_price_per_million": 0.0002, + "output_price_per_million": 0.0002, + "metadata": { + "provider_name": "Mistral AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "omni-moderation-2024-09-26", - "created_at": "2024-11-27T20:07:46+01:00", - "display_name": "Omni-Moderation 20240926", - "provider": "openai", + "id": "mistral.mistral-large-2402-v1:0", + "created_at": null, + "display_name": "Mistral Large (24.02)", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "moderation", - "family": "moderation", + "type": "chat", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" + "input_price_per_million": 0.002, + "output_price_per_million": 0.002, + "metadata": { + "provider_name": "Mistral AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "omni-moderation-latest", - "created_at": "2024-11-15T17:47:45+01:00", - "display_name": "Omni-Moderation Latest", - "provider": "openai", + "id": "mistral.mistral-large-2407-v1:0", + "created_at": null, + "display_name": "Mistral Large (24.07)", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "moderation", - "family": "moderation", + "type": "chat", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" + "input_price_per_million": 0.002, + "output_price_per_million": 0.002, + "metadata": { + "provider_name": "Mistral AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] } }, { - "id": "text-bison-001", + "id": "mistral.mixtral-8x7b-instruct-v0:1", "created_at": null, - "display_name": "PaLM 2 (Legacy)", - "provider": "gemini", - "context_window": 8196, - "max_tokens": 1024, + "display_name": "Mixtral 8x7B Instruct", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, "type": "chat", "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.075, - "output_price_per_million": 0.3, - "metadata": { - "version": "001", - "description": "A legacy model that understands text and generates text as an output", - "input_token_limit": 8196, - "output_token_limit": 1024, - "supported_generation_methods": [ - "generateText", - "countTextTokens", - "createTunedTextModel" + "input_price_per_million": 0.0007, + "output_price_per_million": 0.0007, + "metadata": { + "provider_name": "Mistral AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" ] } }, { - "id": "text-embedding-004", + "id": "stability.sd3-5-large-v1:0", "created_at": null, - "display_name": "Text Embedding 004", - "provider": "gemini", - "context_window": 2048, - "max_tokens": 1, - "type": "embedding", - "family": "embedding4", + "display_name": "Stable Diffusion 3.5 Large", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, "input_price_per_million": 0.0, "output_price_per_million": 0.0, "metadata": { - "version": "004", - "description": "Obtain a distributed representation of a text.", - "input_token_limit": 2048, - "output_token_limit": 1, - "supported_generation_methods": [ - "embedContent" + "provider_name": "Stability AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "IMAGE" ] } }, { - "id": "text-embedding-3-large", - "created_at": "2024-01-22T20:53:00+01:00", - "display_name": "Text Embedding 3 Large", - "provider": "openai", - "context_window": 4096, - "max_tokens": 4096, - "type": "embedding", - "family": "embedding3_large", - "supports_vision": false, - "supports_functions": true, - "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, - "metadata": { - "object": "model", - "owned_by": "system" - } - }, - { - "id": "text-embedding-3-small", - "created_at": "2024-01-22T19:43:17+01:00", - "display_name": "Text Embedding 3 Small", - "provider": "openai", + "id": "stability.sd3-large-v1:0", + "created_at": null, + "display_name": "SD3 Large 1.0", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "embedding", - "family": "embedding3_small", + "type": "chat", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Stability AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "IMAGE" + ] } }, { - "id": "text-embedding-ada-002", - "created_at": "2022-12-16T20:01:39+01:00", - "display_name": "Text Embedding Ada 002", - "provider": "openai", + "id": "stability.stable-diffusion-xl-v1", + "created_at": null, + "display_name": "SDXL 1.0", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "embedding", - "family": "embedding2", + "type": "chat", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "openai-internal" + "provider_name": "Stability AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "IMAGE" + ] } }, { - "id": "tts-1", - "created_at": "2023-04-19T23:49:11+02:00", - "display_name": "TTS-1", - "provider": "openai", + "id": "stability.stable-diffusion-xl-v1:0", + "created_at": null, + "display_name": "SDXL 1.0", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "audio", - "family": "tts1", + "type": "chat", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "openai-internal" + "provider_name": "Stability AI", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "IMAGE" + ] } }, { - "id": "tts-1-1106", - "created_at": "2023-11-04T00:14:01+01:00", - "display_name": "TTS-1 1106", - "provider": "openai", + "id": "stability.stable-image-core-v1:0", + "created_at": null, + "display_name": "Stable Image Core 1.0", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "audio", - "family": "tts1", + "type": "chat", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Stability AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "IMAGE" + ] } }, { - "id": "tts-1-hd", - "created_at": "2023-11-03T22:13:35+01:00", - "display_name": "TTS-1 HD", - "provider": "openai", + "id": "stability.stable-image-core-v1:1", + "created_at": null, + "display_name": "Stable Image Core 1.0", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "audio", - "family": "tts1_hd", + "type": "chat", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Stability AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "IMAGE" + ] } }, { - "id": "tts-1-hd-1106", - "created_at": "2023-11-04T00:18:53+01:00", - "display_name": "TTS-1 HD 1106", - "provider": "openai", + "id": "stability.stable-image-ultra-v1:0", + "created_at": null, + "display_name": "Stable Image Ultra 1.0", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "audio", - "family": "tts1_hd", + "type": "chat", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "system" + "provider_name": "Stability AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "IMAGE" + ] } }, { - "id": "whisper-1", - "created_at": "2023-02-27T22:13:04+01:00", - "display_name": "Whisper 1", - "provider": "openai", + "id": "stability.stable-image-ultra-v1:1", + "created_at": null, + "display_name": "Stable Image Ultra 1.0", + "provider": "bedrock", "context_window": 4096, "max_tokens": 4096, - "type": "audio", - "family": "whisper1", + "type": "chat", + "family": "other", "supports_vision": false, - "supports_functions": true, + "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.5, - "output_price_per_million": 1.5, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, "metadata": { - "object": "model", - "owned_by": "openai-internal" + "provider_name": "Stability AI", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": false, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "IMAGE" + ] } } ] \ No newline at end of file diff --git a/lib/ruby_llm/models.rb b/lib/ruby_llm/models.rb index bea9bad16..571c21718 100644 --- a/lib/ruby_llm/models.rb +++ b/lib/ruby_llm/models.rb @@ -103,8 +103,8 @@ def by_provider(provider) # Instance method to refresh models def refresh! self.class.refresh! - # Return self for method chaining - self + # Return instance for method chaining + self.class.instance end end end diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 43c782aac..413fc8270 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -25,14 +25,15 @@ def self.extended(base) module_function def api_base - "https://bedrock-runtime.#{aws_region}.amazonaws.com" + "https://bedrock.#{aws_region}.amazonaws.com" + #"https://bedrock-runtime.#{aws_region}.amazonaws.com" end - def headers + def headers(method: :post, path: nil, model_id: nil) SignatureV4.sign_request( connection: connection, - method: :post, - path: completion_url, + method: method, + path: path || completion_url, body: '', access_key: aws_access_key_id, secret_key: aws_secret_access_key, @@ -52,14 +53,6 @@ def slug 'bedrock' end - def connection - @connection ||= Faraday.new(url: api_base) do |f| - f.request :json - f.response :json - f.adapter Faraday.default_adapter - end - end - def aws_region ENV['AWS_REGION'] || 'us-east-1' end @@ -95,7 +88,7 @@ def self.sign_request(connection:, method:, path:, body: nil, access_key:, secre signed_headers = canonical_headers.map { |h| h.split(':')[0] }.sort.join(';') canonical_request = [ method.to_s.upcase, - path, + "/#{path}", '', canonical_headers.sort.join("\n") + "\n", signed_headers, diff --git a/lib/ruby_llm/providers/bedrock/capabilities.rb b/lib/ruby_llm/providers/bedrock/capabilities.rb index 3d4816e8b..7f4847f1f 100644 --- a/lib/ruby_llm/providers/bedrock/capabilities.rb +++ b/lib/ruby_llm/providers/bedrock/capabilities.rb @@ -7,16 +7,183 @@ module Bedrock module Capabilities module_function - def input_price_for(model_id) - PRICES.dig(model_family(model_id), :input) || default_input_price + # Default prices per 1000 input tokens + DEFAULT_INPUT_PRICE = 0.0 + DEFAULT_OUTPUT_PRICE = 0.0 + + ANTHROPIC_CLAUDE_3_OPUS_INPUT_PRICE = 0.015 + ANTHROPIC_CLAUDE_3_OPUS_OUTPUT_PRICE = 0.075 + + ANTHROPIC_CLAUDE_3_SONNET_INPUT_PRICE = 0.003 + ANTHROPIC_CLAUDE_3_SONNET_OUTPUT_PRICE = 0.015 + + ANTHROPIC_CLAUDE_3_HAIKU_INPUT_PRICE = 0.0005 + ANTHROPIC_CLAUDE_3_HAIKU_OUTPUT_PRICE = 0.0025 + + ANTHROPIC_CLAUDE_2_INPUT_PRICE = 0.008 + ANTHROPIC_CLAUDE_2_OUTPUT_PRICE = 0.024 + + ANTHROPIC_CLAUDE_INSTANT_INPUT_PRICE = 0.0008 + ANTHROPIC_CLAUDE_INSTANT_OUTPUT_PRICE = 0.0024 + + COHERE_COMMAND_INPUT_PRICE = 0.0015 + COHERE_COMMAND_OUTPUT_PRICE = 0.0020 + + COHERE_COMMAND_LIGHT_INPUT_PRICE = 0.0003 + COHERE_COMMAND_LIGHT_OUTPUT_PRICE = 0.0006 + + META_LLAMA_70B_INPUT_PRICE = 0.00195 + META_LLAMA_70B_OUTPUT_PRICE = 0.00256 + + META_LLAMA_13B_INPUT_PRICE = 0.00075 + META_LLAMA_13B_OUTPUT_PRICE = 0.001 + + META_LLAMA_7B_INPUT_PRICE = 0.0004 + META_LLAMA_7B_OUTPUT_PRICE = 0.0005 + + MISTRAL_7B_INPUT_PRICE = 0.0002 + MISTRAL_7B_OUTPUT_PRICE = 0.0002 + + MISTRAL_MIXTRAL_INPUT_PRICE = 0.0007 + MISTRAL_MIXTRAL_OUTPUT_PRICE = 0.0007 + + MISTRAL_LARGE_INPUT_PRICE = 0.002 + MISTRAL_LARGE_OUTPUT_PRICE = 0.002 + + def self.input_price_for(model_id) + case model_id + when /anthropic\.claude-3-opus/ + ANTHROPIC_CLAUDE_3_OPUS_INPUT_PRICE + when /anthropic\.claude-3-sonnet/ + ANTHROPIC_CLAUDE_3_SONNET_INPUT_PRICE + when /anthropic\.claude-3-haiku/ + ANTHROPIC_CLAUDE_3_HAIKU_INPUT_PRICE + when /anthropic\.claude-v2/ + ANTHROPIC_CLAUDE_2_INPUT_PRICE + when /anthropic\.claude-instant/ + ANTHROPIC_CLAUDE_INSTANT_INPUT_PRICE + when /cohere\.command-text/ + COHERE_COMMAND_INPUT_PRICE + when /cohere\.command-light/ + COHERE_COMMAND_LIGHT_INPUT_PRICE + when /meta\.llama.*70b/i + META_LLAMA_70B_INPUT_PRICE + when /meta\.llama.*13b/i + META_LLAMA_13B_INPUT_PRICE + when /meta\.llama.*7b/i + META_LLAMA_7B_INPUT_PRICE + when /mistral\.mistral-7b/ + MISTRAL_7B_INPUT_PRICE + when /mistral\.mixtral/ + MISTRAL_MIXTRAL_INPUT_PRICE + when /mistral\.mistral-large/ + MISTRAL_LARGE_INPUT_PRICE + else + DEFAULT_INPUT_PRICE + end end - def output_price_for(model_id) - PRICES.dig(model_family(model_id), :output) || default_output_price + def self.output_price_for(model_id) + case model_id + when /anthropic\.claude-3-opus/ + ANTHROPIC_CLAUDE_3_OPUS_OUTPUT_PRICE + when /anthropic\.claude-3-sonnet/ + ANTHROPIC_CLAUDE_3_SONNET_OUTPUT_PRICE + when /anthropic\.claude-3-haiku/ + ANTHROPIC_CLAUDE_3_HAIKU_OUTPUT_PRICE + when /anthropic\.claude-v2/ + ANTHROPIC_CLAUDE_2_OUTPUT_PRICE + when /anthropic\.claude-instant/ + ANTHROPIC_CLAUDE_INSTANT_OUTPUT_PRICE + when /cohere\.command-text/ + COHERE_COMMAND_OUTPUT_PRICE + when /cohere\.command-light/ + COHERE_COMMAND_LIGHT_OUTPUT_PRICE + when /meta\.llama.*70b/i + META_LLAMA_70B_OUTPUT_PRICE + when /meta\.llama.*13b/i + META_LLAMA_13B_OUTPUT_PRICE + when /meta\.llama.*7b/i + META_LLAMA_7B_OUTPUT_PRICE + when /mistral\.mistral-7b/ + MISTRAL_7B_OUTPUT_PRICE + when /mistral\.mixtral/ + MISTRAL_MIXTRAL_OUTPUT_PRICE + when /mistral\.mistral-large/ + MISTRAL_LARGE_OUTPUT_PRICE + else + DEFAULT_OUTPUT_PRICE + end end - def supports_vision?(model_id) - model_id.match?(/anthropic\.claude-3/) + def self.supports_chat?(model_id) + case model_id + when /anthropic\.claude/, /cohere\.command/, /meta\.llama/, /mistral\./ + true + else + false + end + end + + def self.supports_streaming?(model_id) + case model_id + when /anthropic\.claude/, /cohere\.command/, /meta\.llama/, /mistral\./ + true + else + false + end + end + + def self.supports_images?(model_id) + case model_id + when /anthropic\.claude-3/, /meta\.llama3-2-(?:11b|90b)/ + true + else + false + end + end + + def self.context_window_for(model_id) + case model_id + when /anthropic\.claude-3-opus/ + 200_000 + when /anthropic\.claude-3-sonnet/ + 200_000 + when /anthropic\.claude-3-haiku/ + 200_000 + when /anthropic\.claude-2/ + 100_000 + when /meta\.llama/ + 4_096 + else + 4_096 + end + end + + def self.supports_vision?(model_id) + case model_id + when /anthropic\.claude-3/, /meta\.llama3-2-(?:11b|90b)/ + true + else + false + end + end + + def self.max_tokens_for(model_id) + case model_id + when /anthropic\.claude-3/ + 4096 + when /anthropic\.claude-v2/ + 4096 + when /anthropic\.claude-instant/ + 4096 + when /meta\.llama/ + 4096 + when /mistral\./ + 4096 + else + 4096 + end end def supports_functions?(model_id) @@ -69,45 +236,6 @@ def model_family(model_id) end end - def context_window_for(model_id) - case model_id - when /anthropic\.claude-3/ - 200_000 - when /anthropic\.claude-v2/ - 100_000 - when /anthropic\.claude-instant/ - 100_000 - when /amazon\.titan/ - 8_000 - else - 4_096 - end - end - - def max_tokens_for(model_id) - case model_id - when /anthropic\.claude-3/ - 4_096 - when /anthropic\.claude-v2/ - 4_096 - when /anthropic\.claude-instant/ - 2_048 - when /amazon\.titan/ - 4_096 - else - 2_048 - end - end - - private - - PRICES = { - claude3: { input: 15.0, output: 75.0 }, - claude2: { input: 8.0, output: 24.0 }, - claude_instant: { input: 0.8, output: 2.4 }, - titan: { input: 0.1, output: 0.2 } - }.freeze - def default_input_price 0.1 end diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index 85c86041d..48d2e7e0b 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -7,17 +7,17 @@ module Bedrock module Chat module_function - def completion_url - 'model' + def model_id + @model_id end def render_payload(messages, tools:, temperature:, model:, stream: false) - # Format depends on the specific model being used - case model_id_for(model) + @model_id = model + case model when /anthropic\.claude/ - build_claude_request(messages, temperature) + build_claude_request(messages, temperature, model) when /amazon\.titan/ - build_titan_request(messages, temperature) + build_titan_request(messages, temperature, model) else raise Error, "Unsupported model: #{model}" end @@ -25,7 +25,8 @@ def render_payload(messages, tools:, temperature:, model:, stream: false) def parse_completion_response(response) data = response.body - return if data.empty? + data = JSON.parse(data) if data.is_a?(String) + return if data.nil? || data.empty? Message.new( role: :assistant, @@ -38,59 +39,51 @@ def parse_completion_response(response) private - def build_claude_request(messages, temperature) + def build_claude_request(messages, temperature, model_id) formatted = messages.map do |msg| - role = msg.role == :assistant ? 'Assistant' : 'Human' - content = msg.content - "\n\n#{role}: #{content}" - end.join + { + role: msg.role == :assistant ? "assistant" : "user", + content: msg.content + } + end { - prompt: formatted + "\n\nAssistant:", + anthropic_version: "bedrock-2023-05-31", + messages: formatted, temperature: temperature, - max_tokens: max_tokens_for(messages.first&.model_id) + max_tokens: max_tokens_for(model_id) } end - def build_titan_request(messages, temperature) + def build_titan_request(messages, temperature, model_id) { inputText: messages.map { |msg| msg.content }.join("\n"), textGenerationConfig: { temperature: temperature, - maxTokenCount: max_tokens_for(messages.first&.model_id) + maxTokenCount: max_tokens_for(model_id) } } end def extract_content(data) case data - when /anthropic\.claude/ - data[:completion] - when /amazon\.titan/ - data.dig(:results, 0, :outputText) + when Hash + if data.key?('completion') + data['completion'] + elsif data.dig('results', 0, 'outputText') + data.dig('results', 0, 'outputText') + else + raise Error, "Unexpected response format: #{data.keys}" + end else - raise Error, "Unsupported model: #{data['model']}" - end - end - - def model_id_for(model) - case model - when 'claude-3-sonnet' - 'anthropic.claude-3-sonnet-20240229-v1:0' - when 'claude-2' - 'anthropic.claude-v2' - when 'claude-instant' - 'anthropic.claude-instant-v1' - when 'titan' - 'amazon.titan-text-express-v1' - else - model # assume it's a full model ID + raise Error, "Unexpected response type: #{data.class}" end end def max_tokens_for(model_id) - Models.find(model_id)&.max_tokens + RubyLLM.models.find(model_id)&.max_tokens end + end end end diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index 6122486d7..262a94354 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -5,21 +5,31 @@ module Providers module Bedrock # Models methods for the AWS Bedrock API implementation module Models + def list_models + response = connection.get(models_url) do |req| + req.headers.merge! headers(method: :get, path: models_url) + end + + parse_list_models_response(response, slug, capabilities) + end + module_function def models_url 'foundation-models' end - def parse_list_models_response(response) + def parse_list_models_response(response, slug, capabilities) + puts "Got response from Bedrock" + pp response.body data = response.body['modelSummaries'] || [] data.map do |model| model_id = model['modelId'] - { + ModelInfo.new( id: model_id, created_at: nil, display_name: model['modelName'] || capabilities.format_display_name(model_id), - provider: 'bedrock', + provider: slug, context_window: capabilities.context_window_for(model_id), max_tokens: capabilities.max_tokens_for(model_id), type: capabilities.model_type(model_id), @@ -37,15 +47,9 @@ def parse_list_models_response(response) input_modalities: model['inputModalities'] || [], output_modalities: model['outputModalities'] || [] } - } + ) end end - - private - - def capabilities - Bedrock::Capabilities - end end end end diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index 9cb178875..db138ddcd 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -7,10 +7,6 @@ module Bedrock module Streaming module_function - def stream_url - 'model' - end - def handle_stream(&block) to_json_stream do |data| block.call( @@ -34,9 +30,10 @@ def extract_content(data) when /amazon\.titan/ data.dig(:results, 0, :outputText) else - raise Error, "Unsupported model: #{data['model']}" + raise Error, data.inspect end end + end end end From 2d9136d6ad3fbf39d21a9dd8e35da98c8c1e3b43 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 13:49:46 -0700 Subject: [PATCH 04/85] #16: Remove debug prints --- lib/ruby_llm/providers/bedrock/models.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index 262a94354..d64df5933 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -20,8 +20,6 @@ def models_url end def parse_list_models_response(response, slug, capabilities) - puts "Got response from Bedrock" - pp response.body data = response.body['modelSummaries'] || [] data.map do |model| model_id = model['modelId'] From 4ea740535acffc3b90b021755ea9d13398919784 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 13:54:02 -0700 Subject: [PATCH 05/85] #16: Move signature to separate file --- lib/ruby_llm/providers/bedrock.rb | 64 +------------------ lib/ruby_llm/providers/bedrock/signature.rb | 70 +++++++++++++++++++++ 2 files changed, 73 insertions(+), 61 deletions(-) create mode 100644 lib/ruby_llm/providers/bedrock/signature.rb diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 413fc8270..4521e2220 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -13,6 +13,7 @@ module Bedrock extend Bedrock::Streaming extend Bedrock::Models extend Bedrock::Tools + extend Bedrock::Signature def self.extended(base) base.extend(Provider) @@ -20,6 +21,7 @@ def self.extended(base) base.extend(Bedrock::Streaming) base.extend(Bedrock::Models) base.extend(Bedrock::Tools) + base.extend(Bedrock::Signature) end module_function @@ -30,7 +32,7 @@ def api_base end def headers(method: :post, path: nil, model_id: nil) - SignatureV4.sign_request( + Signature.sign_request( connection: connection, method: method, path: path || completion_url, @@ -71,66 +73,6 @@ def aws_session_token class Error < RubyLLM::Error; end - # AWS Signature V4 implementation - module SignatureV4 - def self.sign_request(connection:, method:, path:, body: nil, access_key:, secret_key:, session_token: nil, region:, service: 'bedrock') - now = Time.now.utc - amz_date = now.strftime('%Y%m%dT%H%M%SZ') - date = now.strftime('%Y%m%d') - - # Create canonical request - canonical_headers = [ - "host:#{connection.url_prefix.host}", - "x-amz-date:#{amz_date}" - ] - canonical_headers << "x-amz-security-token:#{session_token}" if session_token - - signed_headers = canonical_headers.map { |h| h.split(':')[0] }.sort.join(';') - canonical_request = [ - method.to_s.upcase, - "/#{path}", - '', - canonical_headers.sort.join("\n") + "\n", - signed_headers, - body ? OpenSSL::Digest::SHA256.hexdigest(body) : OpenSSL::Digest::SHA256.hexdigest('') - ].join("\n") - - # Create string to sign - credential_scope = "#{date}/#{region}/#{service}/aws4_request" - string_to_sign = [ - 'AWS4-HMAC-SHA256', - amz_date, - credential_scope, - OpenSSL::Digest::SHA256.hexdigest(canonical_request) - ].join("\n") - - # Calculate signature - k_date = hmac("AWS4#{secret_key}", date) - k_region = hmac(k_date, region) - k_service = hmac(k_region, service) - k_signing = hmac(k_service, 'aws4_request') - signature = hmac(k_signing, string_to_sign).unpack1('H*') - - # Create authorization header - authorization = [ - "AWS4-HMAC-SHA256 Credential=#{access_key}/#{credential_scope}", - "SignedHeaders=#{signed_headers}", - "Signature=#{signature}" - ].join(', ') - - headers = { - 'Authorization' => authorization, - 'x-amz-date' => amz_date - } - headers['x-amz-security-token'] = session_token if session_token - - headers - end - - def self.hmac(key, value) - OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), key, value) - end - end end end end \ No newline at end of file diff --git a/lib/ruby_llm/providers/bedrock/signature.rb b/lib/ruby_llm/providers/bedrock/signature.rb new file mode 100644 index 000000000..24d3c67ea --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/signature.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + module Bedrock + # AWS Signature V4 implementation + module Signature + module_function + + def self.sign_request(connection:, method:, path:, body: nil, access_key:, secret_key:, session_token: nil, region:, service: 'bedrock') + now = Time.now.utc + amz_date = now.strftime('%Y%m%dT%H%M%SZ') + date = now.strftime('%Y%m%d') + + # Create canonical request + canonical_headers = [ + "host:#{connection.url_prefix.host}", + "x-amz-date:#{amz_date}" + ] + canonical_headers << "x-amz-security-token:#{session_token}" if session_token + + signed_headers = canonical_headers.map { |h| h.split(':')[0] }.sort.join(';') + canonical_request = [ + method.to_s.upcase, + "/#{path}", + '', + canonical_headers.sort.join("\n") + "\n", + signed_headers, + body ? OpenSSL::Digest::SHA256.hexdigest(body) : OpenSSL::Digest::SHA256.hexdigest('') + ].join("\n") + + # Create string to sign + credential_scope = "#{date}/#{region}/#{service}/aws4_request" + string_to_sign = [ + 'AWS4-HMAC-SHA256', + amz_date, + credential_scope, + OpenSSL::Digest::SHA256.hexdigest(canonical_request) + ].join("\n") + + # Calculate signature + k_date = hmac("AWS4#{secret_key}", date) + k_region = hmac(k_date, region) + k_service = hmac(k_region, service) + k_signing = hmac(k_service, 'aws4_request') + signature = hmac(k_signing, string_to_sign).unpack1('H*') + + # Create authorization header + authorization = [ + "AWS4-HMAC-SHA256 Credential=#{access_key}/#{credential_scope}", + "SignedHeaders=#{signed_headers}", + "Signature=#{signature}" + ].join(', ') + + headers = { + 'Authorization' => authorization, + 'x-amz-date' => amz_date + } + headers['x-amz-security-token'] = session_token if session_token + + headers + end + + def self.hmac(key, value) + OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), key, value) + end + end + end + end +end From 1a0430369348b4157529546167443bdfb5157109 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 14:47:29 -0700 Subject: [PATCH 06/85] #16: Bring in signing code from aws-sdk-ruby https://github.com/aws/aws-sdk-ruby/blob/version-3/gems/aws-sigv4/lib/aws-sigv4/ --- lib/ruby_llm/providers/bedrock.rb | 41 +- lib/ruby_llm/providers/bedrock/chat.rb | 20 +- lib/ruby_llm/providers/bedrock/config.rb | 30 - lib/ruby_llm/providers/bedrock/models.rb | 3 + lib/ruby_llm/providers/bedrock/signature.rb | 70 -- lib/ruby_llm/providers/bedrock/signing.rb | 856 ++++++++++++++++++++ 6 files changed, 900 insertions(+), 120 deletions(-) delete mode 100644 lib/ruby_llm/providers/bedrock/config.rb delete mode 100644 lib/ruby_llm/providers/bedrock/signature.rb create mode 100644 lib/ruby_llm/providers/bedrock/signing.rb diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 4521e2220..870022571 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -13,7 +13,7 @@ module Bedrock extend Bedrock::Streaming extend Bedrock::Models extend Bedrock::Tools - extend Bedrock::Signature + extend Bedrock::Signing def self.extended(base) base.extend(Provider) @@ -21,27 +21,40 @@ def self.extended(base) base.extend(Bedrock::Streaming) base.extend(Bedrock::Models) base.extend(Bedrock::Tools) - base.extend(Bedrock::Signature) + base.extend(Bedrock::Signing) end module_function def api_base - "https://bedrock.#{aws_region}.amazonaws.com" - #"https://bedrock-runtime.#{aws_region}.amazonaws.com" + @api_base ||= "https://bedrock-runtime.#{aws_region}.amazonaws.com" end - def headers(method: :post, path: nil, model_id: nil) - Signature.sign_request( + def post(url, payload) + connection.post url, payload do |req| + req.headers.merge! headers(method: :post, path: "#{connection.url_prefix}#{url}", body: payload.to_json) + + yield req if block_given? + end + end + + def headers(method: :post, path: nil, body: nil) + signer = Signing::Signer.new({ + access_key_id: aws_access_key_id, + secret_access_key: aws_secret_access_key, + session_token: aws_session_token, + region: aws_region, + service: 'bedrock', + }) + request = { connection: connection, - method: method, - path: path || completion_url, - body: '', - access_key: aws_access_key_id, - secret_key: aws_secret_access_key, - session_token: aws_session_token, - region: aws_region - ).merge( + http_method: method, + url: path || completion_url, + body: body || '', + } + signature = signer.sign_request(request) + + signature.headers.merge( 'Content-Type' => 'application/json', 'Accept' => 'application/json' ) diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index 48d2e7e0b..f251b72cf 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -7,6 +7,10 @@ module Bedrock module Chat module_function + def completion_url + "model/#{model_id}/invoke-with-response-stream" + end + def model_id @model_id end @@ -41,15 +45,19 @@ def parse_completion_response(response) def build_claude_request(messages, temperature, model_id) formatted = messages.map do |msg| - { - role: msg.role == :assistant ? "assistant" : "user", - content: msg.content - } - end + role = msg.role == :assistant ? 'Assistant' : 'Human' + content = msg.content + "\n\n#{role}: #{content}" + end.join { anthropic_version: "bedrock-2023-05-31", - messages: formatted, + messages: [ + { + role: "user", + content: formatted + } + ], temperature: temperature, max_tokens: max_tokens_for(model_id) } diff --git a/lib/ruby_llm/providers/bedrock/config.rb b/lib/ruby_llm/providers/bedrock/config.rb deleted file mode 100644 index f1a40871f..000000000 --- a/lib/ruby_llm/providers/bedrock/config.rb +++ /dev/null @@ -1,30 +0,0 @@ -# frozen_string_literal: true - -module RubyLLM - module Providers - class Bedrock - # Configuration for AWS Bedrock provider - class Config - attr_reader :access_key_id, :secret_access_key, :session_token, :region, :connection - - def initialize(access_key_id:, secret_access_key:, region:, session_token: nil) - @access_key_id = access_key_id - @secret_access_key = secret_access_key - @session_token = session_token - @region = region - @connection = build_connection - end - - private - - def build_connection - Faraday.new(url: "https://bedrock-runtime.#{region}.amazonaws.com") do |f| - f.request :json - f.response :json, content_type: /\bjson$/ - f.adapter Faraday.default_adapter - end - end - end - end - end -end \ No newline at end of file diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index d64df5933..e39079c7d 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -6,9 +6,12 @@ module Bedrock # Models methods for the AWS Bedrock API implementation module Models def list_models + @connection = nil # reset connection since base url is different + @api_base = "https://bedrock.#{aws_region}.amazonaws.com" response = connection.get(models_url) do |req| req.headers.merge! headers(method: :get, path: models_url) end + @connection = nil # reset connection since base url is different parse_list_models_response(response, slug, capabilities) end diff --git a/lib/ruby_llm/providers/bedrock/signature.rb b/lib/ruby_llm/providers/bedrock/signature.rb deleted file mode 100644 index 24d3c67ea..000000000 --- a/lib/ruby_llm/providers/bedrock/signature.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -module RubyLLM - module Providers - module Bedrock - # AWS Signature V4 implementation - module Signature - module_function - - def self.sign_request(connection:, method:, path:, body: nil, access_key:, secret_key:, session_token: nil, region:, service: 'bedrock') - now = Time.now.utc - amz_date = now.strftime('%Y%m%dT%H%M%SZ') - date = now.strftime('%Y%m%d') - - # Create canonical request - canonical_headers = [ - "host:#{connection.url_prefix.host}", - "x-amz-date:#{amz_date}" - ] - canonical_headers << "x-amz-security-token:#{session_token}" if session_token - - signed_headers = canonical_headers.map { |h| h.split(':')[0] }.sort.join(';') - canonical_request = [ - method.to_s.upcase, - "/#{path}", - '', - canonical_headers.sort.join("\n") + "\n", - signed_headers, - body ? OpenSSL::Digest::SHA256.hexdigest(body) : OpenSSL::Digest::SHA256.hexdigest('') - ].join("\n") - - # Create string to sign - credential_scope = "#{date}/#{region}/#{service}/aws4_request" - string_to_sign = [ - 'AWS4-HMAC-SHA256', - amz_date, - credential_scope, - OpenSSL::Digest::SHA256.hexdigest(canonical_request) - ].join("\n") - - # Calculate signature - k_date = hmac("AWS4#{secret_key}", date) - k_region = hmac(k_date, region) - k_service = hmac(k_region, service) - k_signing = hmac(k_service, 'aws4_request') - signature = hmac(k_signing, string_to_sign).unpack1('H*') - - # Create authorization header - authorization = [ - "AWS4-HMAC-SHA256 Credential=#{access_key}/#{credential_scope}", - "SignedHeaders=#{signed_headers}", - "Signature=#{signature}" - ].join(', ') - - headers = { - 'Authorization' => authorization, - 'x-amz-date' => amz_date - } - headers['x-amz-security-token'] = session_token if session_token - - headers - end - - def self.hmac(key, value) - OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), key, value) - end - end - end - end -end diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb new file mode 100644 index 000000000..2d49c14ef --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -0,0 +1,856 @@ +# frozen_string_literal: true + +require 'openssl' +require 'tempfile' +require 'time' +require 'uri' +require 'set' +require 'cgi' +require 'pathname' + +module RubyLLM + module Providers + module Bedrock + module Signing + + # Utility class for creating AWS signature version 4 signature. This class + # provides two methods for generating signatures: + # + # * {#sign_request} - Computes a signature of the given request, returning + # the hash of headers that should be applied to the request. + # + # * {#presign_url} - Computes a presigned request with an expiration. + # By default, the body of this request is not signed and the request + # expires in 15 minutes. + # + # ## Configuration + # + # To use the signer, you need to specify the service, region, and credentials. + # The service name is normally the endpoint prefix to an AWS service. For + # example: + # + # ec2.us-west-1.amazonaws.com => ec2 + # + # The region is normally the second portion of the endpoint, following + # the service name. + # + # ec2.us-west-1.amazonaws.com => us-west-1 + # + # It is important to have the correct service and region name, or the + # signature will be invalid. + # + # ## Credentials + # + # The signer requires credentials. You can configure the signer + # with static credentials: + # + # signer = Aws::Sigv4::Signer.new( + # service: 's3', + # region: 'us-east-1', + # # static credentials + # access_key_id: 'akid', + # secret_access_key: 'secret' + # ) + # + # You can also provide refreshing credentials via the `:credentials_provider`. + # If you are using the AWS SDK for Ruby, you can use any of the credential + # classes: + # + # signer = Aws::Sigv4::Signer.new( + # service: 's3', + # region: 'us-east-1', + # credentials_provider: Aws::InstanceProfileCredentials.new + # ) + # + # Other AWS SDK for Ruby classes that can be provided via `:credentials_provider`: + # + # * `Aws::Credentials` + # * `Aws::SharedCredentials` + # * `Aws::InstanceProfileCredentials` + # * `Aws::AssumeRoleCredentials` + # * `Aws::ECSCredentials` + # + # A credential provider is any object that responds to `#credentials` + # returning another object that responds to `#access_key_id`, `#secret_access_key`, + # and `#session_token`. + module Errors + class MissingCredentialsError < ArgumentError + def initialize(msg = nil) + super(msg || <<-MSG.strip) +missing credentials, provide credentials with one of the following options: + - :access_key_id and :secret_access_key + - :credentials + - :credentials_provider + MSG + end + end + + class MissingRegionError < ArgumentError + def initialize(*args) + super("missing required option :region") + end + end + end + class Signature + + # @api private + def initialize(options) + options.each_pair do |attr_name, attr_value| + send("#{attr_name}=", attr_value) + end + end + + # @return [Hash] A hash of headers that should + # be applied to the HTTP request. Header keys are lower + # cased strings and may include the following: + # + # * 'host' + # * 'x-amz-date' + # * 'x-amz-security-token' + # * 'x-amz-content-sha256' + # * 'authorization' + # + attr_accessor :headers + + # @return [String] For debugging purposes. + attr_accessor :canonical_request + + # @return [String] For debugging purposes. + attr_accessor :string_to_sign + + # @return [String] For debugging purposes. + attr_accessor :content_sha256 + + # @return [String] For debugging purposes. + attr_accessor :signature + + # @return [Hash] Internal data for debugging purposes. + attr_accessor :extra + end + class Credentials + + # @option options [required, String] :access_key_id + # @option options [required, String] :secret_access_key + # @option options [String, nil] :session_token (nil) + def initialize(options = {}) + if options[:access_key_id] && options[:secret_access_key] + @access_key_id = options[:access_key_id] + @secret_access_key = options[:secret_access_key] + @session_token = options[:session_token] + else + msg = "expected both :access_key_id and :secret_access_key options" + raise ArgumentError, msg + end + end + + # @return [String] + attr_reader :access_key_id + + # @return [String] + attr_reader :secret_access_key + + # @return [String, nil] + attr_reader :session_token + + # @return [Boolean] Returns `true` if the access key id and secret + # access key are both set. + def set? + !access_key_id.nil? && + !access_key_id.empty? && + !secret_access_key.nil? && + !secret_access_key.empty? + end + end + # Users that wish to configure static credentials can use the + # `:access_key_id` and `:secret_access_key` constructor options. + # @api private + class StaticCredentialsProvider + + # @option options [Credentials] :credentials + # @option options [String] :access_key_id + # @option options [String] :secret_access_key + # @option options [String] :session_token (nil) + def initialize(options = {}) + @credentials = options[:credentials] ? + options[:credentials] : + Credentials.new(options) + end + + # @return [Credentials] + attr_reader :credentials + + # @return [Boolean] + def set? + !!credentials && credentials.set? + end + end + + class Signer + # @overload initialize(service:, region:, access_key_id:, secret_access_key:, session_token:nil, **options) + # @param [String] :service The service signing name, e.g. 's3'. + # @param [String] :region The region name, e.g. 'us-east-1'. When signing + # with sigv4a, this should be a comma separated list of regions. + # @param [String] :access_key_id + # @param [String] :secret_access_key + # @param [String] :session_token (nil) + # + # @overload initialize(service:, region:, credentials:, **options) + # @param [String] :service The service signing name, e.g. 's3'. + # @param [String] :region The region name, e.g. 'us-east-1'. When signing + # with sigv4a, this should be a comma separated list of regions. + # @param [Credentials] :credentials Any object that responds to the following + # methods: + # + # * `#access_key_id` => String + # * `#secret_access_key` => String + # * `#session_token` => String, nil + # * `#set?` => Boolean + # + # @overload initialize(service:, region:, credentials_provider:, **options) + # @param [String] :service The service signing name, e.g. 's3'. + # @param [String] :region The region name, e.g. 'us-east-1'. When signing + # with sigv4a, this should be a comma separated list of regions. + # @param [#credentials] :credentials_provider An object that responds + # to `#credentials`, returning an object that responds to the following + # methods: + # + # * `#access_key_id` => String + # * `#secret_access_key` => String + # * `#session_token` => String, nil + # * `#set?` => Boolean + # + # @option options [Array] :unsigned_headers ([]) A list of + # headers that should not be signed. This is useful when a proxy + # modifies headers, such as 'User-Agent', invalidating a signature. + # + # @option options [Boolean] :uri_escape_path (true) When `true`, + # the request URI path is uri-escaped as part of computing the canonical + # request string. This is required for every service, except Amazon S3, + # as of late 2016. + # + # @option options [Boolean] :apply_checksum_header (true) When `true`, + # the computed content checksum is returned in the hash of signature + # headers. This is required for AWS Glacier, and optional for + # every other AWS service as of late 2016. + # + # @option options [Symbol] :signing_algorithm (:sigv4) The + # algorithm to use for signing. + # + # @option options [Boolean] :omit_session_token (false) + # (Supported only when `aws-crt` is available) If `true`, + # then security token is added to the final signing result, + # but is treated as "unsigned" and does not contribute + # to the authorization signature. + # + # @option options [Boolean] :normalize_path (true) When `true`, the + # uri paths will be normalized when building the canonical request. + def initialize(options = {}) + @service = extract_service(options) + @region = extract_region(options) + @credentials_provider = extract_credentials_provider(options) + @unsigned_headers = Set.new((options.fetch(:unsigned_headers, [])).map(&:downcase)) + @unsigned_headers << 'authorization' + @unsigned_headers << 'x-amzn-trace-id' + @unsigned_headers << 'expect' + @uri_escape_path = options.fetch(:uri_escape_path, true) + @apply_checksum_header = options.fetch(:apply_checksum_header, true) + @signing_algorithm = options.fetch(:signing_algorithm, :sigv4) + @normalize_path = options.fetch(:normalize_path, true) + @omit_session_token = options.fetch(:omit_session_token, false) + end + + # @return [String] + attr_reader :service + + # @return [String] + attr_reader :region + + # @return [#credentials] Returns an object that responds to + # `#credentials`, returning an object that responds to the following + # methods: + # + # * `#access_key_id` => String + # * `#secret_access_key` => String + # * `#session_token` => String, nil + # * `#set?` => Boolean + # + attr_reader :credentials_provider + + # @return [Set] Returns a set of header names that should not be signed. + # All header names have been downcased. + attr_reader :unsigned_headers + + # @return [Boolean] When `true` the `x-amz-content-sha256` header will be signed and + # returned in the signature headers. + attr_reader :apply_checksum_header + + # Computes a version 4 signature signature. Returns the resultant + # signature as a hash of headers to apply to your HTTP request. The given + # request is not modified. + # + # signature = signer.sign_request( + # http_method: 'PUT', + # url: 'https://domain.com', + # headers: { + # 'Abc' => 'xyz', + # }, + # body: 'body' # String or IO object + # ) + # + # # Apply the following hash of headers to your HTTP request + # signature.headers['host'] + # signature.headers['x-amz-date'] + # signature.headers['x-amz-security-token'] + # signature.headers['x-amz-content-sha256'] + # signature.headers['authorization'] + # + # In addition to computing the signature headers, the canonicalized + # request, string to sign and content sha256 checksum are also available. + # These values are useful for debugging signature errors returned by AWS. + # + # signature.canonical_request #=> "..." + # signature.string_to_sign #=> "..." + # signature.content_sha256 #=> "..." + # + # @param [Hash] request + # + # @option request [required, String] :http_method One of + # 'GET', 'HEAD', 'PUT', 'POST', 'PATCH', or 'DELETE' + # + # @option request [required, String, URI::HTTP, URI::HTTPS] :url + # The request URI. Must be a valid HTTP or HTTPS URI. + # + # @option request [optional, Hash] :headers ({}) A hash of headers + # to sign. If the 'X-Amz-Content-Sha256' header is set, the `:body` + # is optional and will not be read. + # + # @option request [optional, String, IO] :body ('') The HTTP request body. + # A sha256 checksum is computed of the body unless the + # 'X-Amz-Content-Sha256' header is set. + # + # @return [Signature] Return an instance of {Signature} that has + # a `#headers` method. The headers must be applied to your request. + # + def sign_request(request) + creds, _ = fetch_credentials + + http_method = extract_http_method(request) + url = extract_url(request) + Signer.normalize_path(url) if @normalize_path + headers = downcase_headers(request[:headers]) + + datetime = headers['x-amz-date'] + datetime ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ") + date = datetime[0, 8] + + content_sha256 = headers['x-amz-content-sha256'] + content_sha256 ||= sha256_hexdigest(request[:body] || '') + + sigv4_headers = {} + sigv4_headers['host'] = headers['host'] || host(url) + sigv4_headers['x-amz-date'] = datetime + if creds.session_token && !@omit_session_token + if @signing_algorithm == 'sigv4-s3express'.to_sym + sigv4_headers['x-amz-s3session-token'] = creds.session_token + else + sigv4_headers['x-amz-security-token'] = creds.session_token + end + end + + sigv4_headers['x-amz-content-sha256'] ||= content_sha256 if @apply_checksum_header + + if @signing_algorithm == :sigv4a && @region && !@region.empty? + sigv4_headers['x-amz-region-set'] = @region + end + headers = headers.merge(sigv4_headers) # merge so we do not modify given headers hash + + algorithm = sts_algorithm + + # compute signature parts + creq = canonical_request(http_method, url, headers, content_sha256) + sts = string_to_sign(datetime, creq, algorithm) + + sig = + if @signing_algorithm == :sigv4a + asymmetric_signature(creds, sts) + else + signature(creds.secret_access_key, date, sts) + end + + algorithm = sts_algorithm + + # apply signature + sigv4_headers['authorization'] = [ + "#{algorithm} Credential=#{credential(creds, date)}", + "SignedHeaders=#{signed_headers(headers)}", + "Signature=#{sig}", + ].join(', ') + + # skip signing the session token, but include it in the headers + if creds.session_token && @omit_session_token + sigv4_headers['x-amz-security-token'] = creds.session_token + end + + # Returning the signature components. + Signature.new( + headers: sigv4_headers, + string_to_sign: sts, + canonical_request: creq, + content_sha256: content_sha256, + signature: sig + ) + end + + # Signs a URL with query authentication. Using query parameters + # to authenticate requests is useful when you want to express a + # request entirely in a URL. This method is also referred as + # presigning a URL. + # + # See [Authenticating Requests: Using Query Parameters (AWS Signature Version 4)](http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html) for more information. + # + # To generate a presigned URL, you must provide a HTTP URI and + # the http method. + # + # url = signer.presign_url( + # http_method: 'GET', + # url: 'https://my-bucket.s3-us-east-1.amazonaws.com/key', + # expires_in: 60 + # ) + # + # By default, signatures are valid for 15 minutes. You can specify + # the number of seconds for the URL to expire in. + # + # url = signer.presign_url( + # http_method: 'GET', + # url: 'https://my-bucket.s3-us-east-1.amazonaws.com/key', + # expires_in: 3600 # one hour + # ) + # + # You can provide a hash of headers that you plan to send with the + # request. Every 'X-Amz-*' header you plan to send with the request + # **must** be provided, or the signature is invalid. Other headers + # are optional, but should be provided for security reasons. + # + # url = signer.presign_url( + # http_method: 'PUT', + # url: 'https://my-bucket.s3-us-east-1.amazonaws.com/key', + # headers: { + # 'X-Amz-Meta-Custom' => 'metadata' + # } + # ) + # + # @option options [required, String] :http_method The HTTP request method, + # e.g. 'GET', 'HEAD', 'PUT', 'POST', 'PATCH', or 'DELETE'. + # + # @option options [required, String, URI::HTTP, URI::HTTPS] :url + # The URI to sign. + # + # @option options [Hash] :headers ({}) Headers that should + # be signed and sent along with the request. All x-amz-* + # headers must be present during signing. Other + # headers are optional. + # + # @option options [Integer] :expires_in (900) + # How long the presigned URL should be valid for. Defaults + # to 15 minutes (900 seconds). + # + # @option options [optional, String, IO] :body + # If the `:body` is set, then a SHA256 hexdigest will be computed of the body. + # If `:body_digest` is set, this option is ignored. If neither are set, then + # the `:body_digest` will be computed of the empty string. + # + # @option options [optional, String] :body_digest + # The SHA256 hexdigest of the request body. If you wish to send the presigned + # request without signing the body, you can pass 'UNSIGNED-PAYLOAD' as the + # `:body_digest` in place of passing `:body`. + # + # @option options [Time] :time (Time.now) Time of the signature. + # You should only set this value for testing. + # + # @return [HTTPS::URI, HTTP::URI] + # + def presign_url(options) + creds, expiration = fetch_credentials + + http_method = extract_http_method(options) + url = extract_url(options) + Signer.normalize_path(url) if @normalize_path + + headers = downcase_headers(options[:headers]) + headers['host'] ||= host(url) + + datetime = headers['x-amz-date'] + datetime ||= (options[:time] || Time.now).utc.strftime("%Y%m%dT%H%M%SZ") + date = datetime[0, 8] + + content_sha256 = headers['x-amz-content-sha256'] + content_sha256 ||= options[:body_digest] + content_sha256 ||= sha256_hexdigest(options[:body] || '') + + algorithm = sts_algorithm + + params = {} + params['X-Amz-Algorithm'] = algorithm + params['X-Amz-Credential'] = credential(creds, date) + params['X-Amz-Date'] = datetime + params['X-Amz-Expires'] = presigned_url_expiration(options, expiration, Time.strptime(datetime, "%Y%m%dT%H%M%S%Z")).to_s + if creds.session_token + if @signing_algorithm == 'sigv4-s3express'.to_sym + params['X-Amz-S3session-Token'] = creds.session_token + else + params['X-Amz-Security-Token'] = creds.session_token + end + end + params['X-Amz-SignedHeaders'] = signed_headers(headers) + + if @signing_algorithm == :sigv4a && @region + params['X-Amz-Region-Set'] = @region + end + + params = params.map do |key, value| + "#{uri_escape(key)}=#{uri_escape(value)}" + end.join('&') + + if url.query + url.query += '&' + params + else + url.query = params + end + + creq = canonical_request(http_method, url, headers, content_sha256) + sts = string_to_sign(datetime, creq, algorithm) + signature = + if @signing_algorithm == :sigv4a + asymmetric_signature(creds, sts) + else + signature(creds.secret_access_key, date, sts) + end + url.query += '&X-Amz-Signature=' + signature + url + end + + private + + def sts_algorithm + @signing_algorithm == :sigv4a ? 'AWS4-ECDSA-P256-SHA256' : 'AWS4-HMAC-SHA256' + end + + def canonical_request(http_method, url, headers, content_sha256) + [ + http_method, + path(url), + normalized_querystring(url.query || ''), + canonical_headers(headers) + "\n", + signed_headers(headers), + content_sha256, + ].join("\n") + end + + def string_to_sign(datetime, canonical_request, algorithm) + [ + algorithm, + datetime, + credential_scope(datetime[0, 8]), + sha256_hexdigest(canonical_request), + ].join("\n") + end + + def credential_scope(date) + [ + date, + (@region unless @signing_algorithm == :sigv4a), + @service, + 'aws4_request' + ].compact.join('/') + end + + def credential(credentials, date) + "#{credentials.access_key_id}/#{credential_scope(date)}" + end + + def signature(secret_access_key, date, string_to_sign) + k_date = hmac("AWS4" + secret_access_key, date) + k_region = hmac(k_date, @region) + k_service = hmac(k_region, @service) + k_credentials = hmac(k_service, 'aws4_request') + hexhmac(k_credentials, string_to_sign) + end + + def asymmetric_signature(creds, string_to_sign) + ec, _ = Aws::Sigv4::AsymmetricCredentials.derive_asymmetric_key( + creds.access_key_id, creds.secret_access_key + ) + sts_digest = OpenSSL::Digest::SHA256.digest(string_to_sign) + s = ec.dsa_sign_asn1(sts_digest) + + Digest.hexencode(s) + end + + # Comparing to original signature v4 algorithm, + # returned signature is a binary string instread of + # hex-encoded string. (Since ':chunk-signature' requires + # 'bytes' type) + # + # Note: + # converting signature from binary string to hex-encoded + # string is handled at #sign_event instead. (Will be used + # as next prior signature for event signing) + def event_signature(secret_access_key, date, string_to_sign) + k_date = hmac("AWS4" + secret_access_key, date) + k_region = hmac(k_date, @region) + k_service = hmac(k_region, @service) + k_credentials = hmac(k_service, 'aws4_request') + hmac(k_credentials, string_to_sign) + end + + def path(url) + path = url.path + path = '/' if path == '' + if @uri_escape_path + uri_escape_path(path) + else + path + end + end + + def normalized_querystring(querystring) + params = querystring.split('&') + params = params.map { |p| p.match(/=/) ? p : p + '=' } + # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + # Sort the parameter names by character code point in ascending order. + # Parameters with duplicate names should be sorted by value. + # + # Default sort <=> in JRuby will swap members + # occasionally when <=> is 0 (considered still sorted), but this + # causes our normalized query string to not match the sent querystring. + # When names match, we then sort by their values. When values also + # match then we sort by their original order + params.each.with_index.sort do |a, b| + a, a_offset = a + b, b_offset = b + a_name, a_value = a.split('=') + b_name, b_value = b.split('=') + if a_name == b_name + if a_value == b_value + a_offset <=> b_offset + else + a_value <=> b_value + end + else + a_name <=> b_name + end + end.map(&:first).join('&') + end + + def signed_headers(headers) + headers.inject([]) do |signed_headers, (header, _)| + if @unsigned_headers.include?(header) + signed_headers + else + signed_headers << header + end + end.sort.join(';') + end + + def canonical_headers(headers) + headers = headers.inject([]) do |hdrs, (k, v)| + if @unsigned_headers.include?(k) + hdrs + else + hdrs << [k, v] + end + end + headers = headers.sort_by(&:first) + headers.map { |k, v| "#{k}:#{canonical_header_value(v.to_s)}" }.join("\n") + end + + def canonical_header_value(value) + value.gsub(/\s+/, ' ').strip + end + + def host(uri) + # Handles known and unknown URI schemes; default_port nil when unknown. + if uri.default_port == uri.port + uri.host + else + "#{uri.host}:#{uri.port}" + end + end + + # @param [File, Tempfile, IO#read, String] value + # @return [String] + def sha256_hexdigest(value) + if (File === value || Tempfile === value) && !value.path.nil? && File.exist?(value.path) + OpenSSL::Digest::SHA256.file(value).hexdigest + elsif value.respond_to?(:read) + sha256 = OpenSSL::Digest::SHA256.new + loop do + chunk = value.read(1024 * 1024) # 1MB + break unless chunk + sha256.update(chunk) + end + value.rewind + sha256.hexdigest + else + OpenSSL::Digest::SHA256.hexdigest(value) + end + end + + def hmac(key, value) + OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), key, value) + end + + def hexhmac(key, value) + OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha256'), key, value) + end + + def extract_service(options) + if options[:service] + options[:service] + else + msg = "missing required option :service" + raise ArgumentError, msg + end + end + + def extract_region(options) + if options[:region] + options[:region] + else + raise Errors::MissingRegionError + end + end + + def extract_credentials_provider(options) + if options[:credentials_provider] + options[:credentials_provider] + elsif options.key?(:credentials) || options.key?(:access_key_id) + StaticCredentialsProvider.new(options) + else + raise Errors::MissingCredentialsError + end + end + + def extract_http_method(request) + if request[:http_method] + request[:http_method].upcase + else + msg = "missing required option :http_method" + raise ArgumentError, msg + end + end + + def extract_url(request) + if request[:url] + URI.parse(request[:url].to_s) + else + msg = "missing required option :url" + raise ArgumentError, msg + end + end + + def downcase_headers(headers) + (headers || {}).to_hash.inject({}) do |hash, (key, value)| + hash[key.downcase] = value + hash + end + end + + def extract_expires_in(options) + case options[:expires_in] + when nil then 900 + when Integer then options[:expires_in] + else + msg = "expected :expires_in to be a number of seconds" + raise ArgumentError, msg + end + end + + def uri_escape(string) + self.class.uri_escape(string) + end + + def uri_escape_path(string) + self.class.uri_escape_path(string) + end + + def fetch_credentials + credentials = @credentials_provider.credentials + if credentials_set?(credentials) + expiration = nil + if @credentials_provider.respond_to?(:expiration) + expiration = @credentials_provider.expiration + end + [credentials, expiration] + else + raise Errors::MissingCredentialsError, + 'unable to sign request without credentials set' + end + end + + # Returns true if credentials are set (not nil or empty) + # Credentials may not implement the Credentials interface + # and may just be credential like Client response objects + # (eg those returned by sts#assume_role) + def credentials_set?(credentials) + !credentials.access_key_id.nil? && + !credentials.access_key_id.empty? && + !credentials.secret_access_key.nil? && + !credentials.secret_access_key.empty? + end + + def presigned_url_expiration(options, expiration, datetime) + expires_in = extract_expires_in(options) + return expires_in unless expiration + + expiration_seconds = (expiration - datetime).to_i + # In the static stability case, credentials may expire in the past + # but still be valid. For those cases, use the user configured + # expires_in and ingore expiration. + if expiration_seconds <= 0 + expires_in + else + [expires_in, expiration_seconds].min + end + end + + class << self + + # Kept for backwards compatability + # Always return false since we are not using crt signing functionality + def use_crt? + false + end + + # @api private + def uri_escape_path(path) + path.gsub(/[^\/]+/) { |part| uri_escape(part) } + end + + # @api private + def uri_escape(string) + if string.nil? + nil + else + CGI.escape(string.encode('UTF-8')).gsub('+', '%20').gsub('%7E', '~') + end + end + + # @api private + def normalize_path(uri) + normalized_path = Pathname.new(uri.path).cleanpath.to_s + # Pathname is probably not correct to use. Empty paths will + # resolve to "." and should be disregarded + normalized_path = '' if normalized_path == '.' + # Ensure trailing slashes are correctly preserved + if uri.path.end_with?('/') && !normalized_path.end_with?('/') + normalized_path << '/' + end + uri.path = normalized_path + end + end + end + end + end + end +end From 6bf2a4aa3c1c00d8f488176e1a859b9147c759f6 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 14:48:53 -0700 Subject: [PATCH 07/85] #16: Remove some unneeded methods --- lib/ruby_llm/providers/bedrock/signing.rb | 145 ---------------------- 1 file changed, 145 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 2d49c14ef..027d9d7e7 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -401,134 +401,6 @@ def sign_request(request) ) end - # Signs a URL with query authentication. Using query parameters - # to authenticate requests is useful when you want to express a - # request entirely in a URL. This method is also referred as - # presigning a URL. - # - # See [Authenticating Requests: Using Query Parameters (AWS Signature Version 4)](http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html) for more information. - # - # To generate a presigned URL, you must provide a HTTP URI and - # the http method. - # - # url = signer.presign_url( - # http_method: 'GET', - # url: 'https://my-bucket.s3-us-east-1.amazonaws.com/key', - # expires_in: 60 - # ) - # - # By default, signatures are valid for 15 minutes. You can specify - # the number of seconds for the URL to expire in. - # - # url = signer.presign_url( - # http_method: 'GET', - # url: 'https://my-bucket.s3-us-east-1.amazonaws.com/key', - # expires_in: 3600 # one hour - # ) - # - # You can provide a hash of headers that you plan to send with the - # request. Every 'X-Amz-*' header you plan to send with the request - # **must** be provided, or the signature is invalid. Other headers - # are optional, but should be provided for security reasons. - # - # url = signer.presign_url( - # http_method: 'PUT', - # url: 'https://my-bucket.s3-us-east-1.amazonaws.com/key', - # headers: { - # 'X-Amz-Meta-Custom' => 'metadata' - # } - # ) - # - # @option options [required, String] :http_method The HTTP request method, - # e.g. 'GET', 'HEAD', 'PUT', 'POST', 'PATCH', or 'DELETE'. - # - # @option options [required, String, URI::HTTP, URI::HTTPS] :url - # The URI to sign. - # - # @option options [Hash] :headers ({}) Headers that should - # be signed and sent along with the request. All x-amz-* - # headers must be present during signing. Other - # headers are optional. - # - # @option options [Integer] :expires_in (900) - # How long the presigned URL should be valid for. Defaults - # to 15 minutes (900 seconds). - # - # @option options [optional, String, IO] :body - # If the `:body` is set, then a SHA256 hexdigest will be computed of the body. - # If `:body_digest` is set, this option is ignored. If neither are set, then - # the `:body_digest` will be computed of the empty string. - # - # @option options [optional, String] :body_digest - # The SHA256 hexdigest of the request body. If you wish to send the presigned - # request without signing the body, you can pass 'UNSIGNED-PAYLOAD' as the - # `:body_digest` in place of passing `:body`. - # - # @option options [Time] :time (Time.now) Time of the signature. - # You should only set this value for testing. - # - # @return [HTTPS::URI, HTTP::URI] - # - def presign_url(options) - creds, expiration = fetch_credentials - - http_method = extract_http_method(options) - url = extract_url(options) - Signer.normalize_path(url) if @normalize_path - - headers = downcase_headers(options[:headers]) - headers['host'] ||= host(url) - - datetime = headers['x-amz-date'] - datetime ||= (options[:time] || Time.now).utc.strftime("%Y%m%dT%H%M%SZ") - date = datetime[0, 8] - - content_sha256 = headers['x-amz-content-sha256'] - content_sha256 ||= options[:body_digest] - content_sha256 ||= sha256_hexdigest(options[:body] || '') - - algorithm = sts_algorithm - - params = {} - params['X-Amz-Algorithm'] = algorithm - params['X-Amz-Credential'] = credential(creds, date) - params['X-Amz-Date'] = datetime - params['X-Amz-Expires'] = presigned_url_expiration(options, expiration, Time.strptime(datetime, "%Y%m%dT%H%M%S%Z")).to_s - if creds.session_token - if @signing_algorithm == 'sigv4-s3express'.to_sym - params['X-Amz-S3session-Token'] = creds.session_token - else - params['X-Amz-Security-Token'] = creds.session_token - end - end - params['X-Amz-SignedHeaders'] = signed_headers(headers) - - if @signing_algorithm == :sigv4a && @region - params['X-Amz-Region-Set'] = @region - end - - params = params.map do |key, value| - "#{uri_escape(key)}=#{uri_escape(value)}" - end.join('&') - - if url.query - url.query += '&' + params - else - url.query = params - end - - creq = canonical_request(http_method, url, headers, content_sha256) - sts = string_to_sign(datetime, creq, algorithm) - signature = - if @signing_algorithm == :sigv4a - asymmetric_signature(creds, sts) - else - signature(creds.secret_access_key, date, sts) - end - url.query += '&X-Amz-Signature=' + signature - url - end - private def sts_algorithm @@ -586,23 +458,6 @@ def asymmetric_signature(creds, string_to_sign) Digest.hexencode(s) end - # Comparing to original signature v4 algorithm, - # returned signature is a binary string instread of - # hex-encoded string. (Since ':chunk-signature' requires - # 'bytes' type) - # - # Note: - # converting signature from binary string to hex-encoded - # string is handled at #sign_event instead. (Will be used - # as next prior signature for event signing) - def event_signature(secret_access_key, date, string_to_sign) - k_date = hmac("AWS4" + secret_access_key, date) - k_region = hmac(k_date, @region) - k_service = hmac(k_region, @service) - k_credentials = hmac(k_service, 'aws4_request') - hmac(k_credentials, string_to_sign) - end - def path(url) path = url.path path = '/' if path == '' From 641c9161fa7ef231130ea58726a955145faf5862 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 15:15:21 -0700 Subject: [PATCH 08/85] #16: Get completion working --- lib/ruby_llm/providers/bedrock.rb | 12 ++++++++---- lib/ruby_llm/providers/bedrock/chat.rb | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 870022571..b392ddfec 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -32,13 +32,15 @@ def api_base def post(url, payload) connection.post url, payload do |req| - req.headers.merge! headers(method: :post, path: "#{connection.url_prefix}#{url}", body: payload.to_json) - + req.headers.merge! headers(method: :post, + path: "#{connection.url_prefix}#{url}", + body: payload.to_json, + streaming: block_given?) yield req if block_given? end end - def headers(method: :post, path: nil, body: nil) + def headers(method: :post, path: nil, body: nil, streaming: false) signer = Signing::Signer.new({ access_key_id: aws_access_key_id, secret_access_key: aws_secret_access_key, @@ -54,9 +56,11 @@ def headers(method: :post, path: nil, body: nil) } signature = signer.sign_request(request) + accept_header = streaming ? 'application/vnd.amazon.eventstream' : 'application/json' + signature.headers.merge( 'Content-Type' => 'application/json', - 'Accept' => 'application/json' + 'Accept' => accept_header ) end diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index f251b72cf..da9e976e9 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -8,7 +8,7 @@ module Chat module_function def completion_url - "model/#{model_id}/invoke-with-response-stream" + "model/#{model_id}/invoke" end def model_id From 5bf4b1fc17566c8109425bcf0d00f11069fd1057 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 16:22:51 -0700 Subject: [PATCH 09/85] #16: Get streaming working --- lib/ruby_llm/providers/bedrock.rb | 23 +- lib/ruby_llm/providers/bedrock/decoder.rb | 224 ++++++++++++++++++++ lib/ruby_llm/providers/bedrock/streaming.rb | 141 ++++++++++-- 3 files changed, 359 insertions(+), 29 deletions(-) create mode 100644 lib/ruby_llm/providers/bedrock/decoder.rb diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index b392ddfec..f9e944102 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -42,17 +42,17 @@ def post(url, payload) def headers(method: :post, path: nil, body: nil, streaming: false) signer = Signing::Signer.new({ - access_key_id: aws_access_key_id, - secret_access_key: aws_secret_access_key, - session_token: aws_session_token, - region: aws_region, - service: 'bedrock', - }) + access_key_id: aws_access_key_id, + secret_access_key: aws_secret_access_key, + session_token: aws_session_token, + region: aws_region, + service: 'bedrock' + }) request = { connection: connection, http_method: method, url: path || completion_url, - body: body || '', + body: body || '' } signature = signer.sign_request(request) @@ -77,19 +77,18 @@ def aws_region end def aws_access_key_id - ENV['AWS_ACCESS_KEY_ID'] + ENV.fetch('AWS_ACCESS_KEY_ID', nil) end def aws_secret_access_key - ENV['AWS_SECRET_ACCESS_KEY'] + ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) end def aws_session_token - ENV['AWS_SESSION_TOKEN'] + ENV.fetch('AWS_SESSION_TOKEN', nil) end class Error < RubyLLM::Error; end - end end -end \ No newline at end of file +end diff --git a/lib/ruby_llm/providers/bedrock/decoder.rb b/lib/ruby_llm/providers/bedrock/decoder.rb new file mode 100644 index 000000000..93b692fec --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/decoder.rb @@ -0,0 +1,224 @@ +# frozen_string_literal: true + +require 'stringio' +require 'tempfile' +require 'zlib' + +module RubyLLM + module Providers + module Bedrock + # Decoder for AWS EventStream format used by Bedrock streaming responses + class Decoder + include Enumerable + + ONE_MEGABYTE = 1024 * 1024 + private_constant :ONE_MEGABYTE + + # bytes of prelude part, including 4 bytes of + # total message length, headers length and crc checksum of prelude + PRELUDE_LENGTH = 12 + private_constant :PRELUDE_LENGTH + + # 4 bytes message crc checksum + CRC32_LENGTH = 4 + private_constant :CRC32_LENGTH + + # @param [Hash] options The initialization options. + # @option options [Boolean] :format (true) When `false` it + # disables user-friendly formatting for message header values + # including timestamp and uuid etc. + def initialize(options = {}) + @format = options.fetch(:format, true) + @message_buffer = '' + end + + # Decodes messages from a binary stream + # + # @param [IO#read] io An IO-like object + # that responds to `#read` + # + # @yieldparam [Message] message + # @return [Enumerable, nil] Returns a new Enumerable + # containing decoded messages if no block is given + def decode(io, &block) + raw_message = io.read + decoded_message = decode_message(raw_message) + return wrap_as_enumerator(decoded_message) unless block_given? + # fetch message only + raw_event, _eof = decoded_message + block.call(raw_event) + end + + # Decodes a single message from a chunk of string + # + # @param [String] chunk A chunk of string to be decoded, + # chunk can contain partial event message to multiple event messages + # When not provided, decode data from #message_buffer + # + # @return [Array] Returns single decoded message + # and boolean pair, the boolean flag indicates whether this chunk + # has been fully consumed, unused data is tracked at #message_buffer + def decode_chunk(chunk = nil) + @message_buffer = [@message_buffer, chunk].pack('a*a*') if chunk + decode_message(@message_buffer) + end + + private + + # exposed via object.send for testing + attr_reader :message_buffer + + def wrap_as_enumerator(decoded_message) + Enumerator.new do |yielder| + yielder << decoded_message + end + end + + def decode_message(raw_message) + # incomplete message prelude received + return [nil, true] if raw_message.bytesize < PRELUDE_LENGTH + + prelude, content = raw_message.unpack("a#{PRELUDE_LENGTH}a*") + + # decode prelude + total_length, header_length = decode_prelude(prelude) + + # incomplete message received, leave it in the buffer + return [nil, true] if raw_message.bytesize < total_length + + content, checksum, remaining = content.unpack("a#{total_length - PRELUDE_LENGTH - CRC32_LENGTH}Na*") + unless Zlib.crc32([prelude, content].pack('a*a*')) == checksum + raise Error, "Message checksum error" + end + + # decode headers and payload + headers, payload = decode_context(content, header_length) + + @message_buffer = remaining + + [Message.new(headers: headers, payload: payload), remaining.empty?] + end + + def decode_prelude(prelude) + # prelude contains length of message and headers, + # followed with CRC checksum of itself + content, checksum = prelude.unpack("a#{PRELUDE_LENGTH - CRC32_LENGTH}N") + raise Error, "Prelude checksum error" unless Zlib.crc32(content) == checksum + content.unpack('N*') + end + + def decode_context(content, header_length) + encoded_header, encoded_payload = content.unpack("a#{header_length}a*") + [ + extract_headers(encoded_header), + extract_payload(encoded_payload) + ] + end + + def extract_headers(buffer) + scanner = buffer + headers = {} + until scanner.bytesize == 0 + # header key + key_length, scanner = scanner.unpack('Ca*') + key, scanner = scanner.unpack("a#{key_length}a*") + + # header value + type_index, scanner = scanner.unpack('Ca*') + value_type = Types.types[type_index] + unpack_pattern, value_length = Types.pattern[value_type] + value = if !!unpack_pattern == unpack_pattern + # boolean types won't have value specified + unpack_pattern + else + value_length, scanner = scanner.unpack('S>a*') unless value_length + unpacked_value, scanner = scanner.unpack("#{unpack_pattern || "a#{value_length}"}a*") + unpacked_value + end + + headers[key] = HeaderValue.new( + format: @format, + value: value, + type: value_type + ) + end + headers + end + + def extract_payload(encoded) + encoded.bytesize <= ONE_MEGABYTE ? + payload_stringio(encoded) : + payload_tempfile(encoded) + end + + def payload_stringio(encoded) + StringIO.new(encoded) + end + + def payload_tempfile(encoded) + payload = Tempfile.new + payload.binmode + payload.write(encoded) + payload.rewind + payload + end + + # Simple message class to hold decoded data + class Message + attr_reader :headers, :payload + + def initialize(headers:, payload:) + @headers = headers + @payload = payload + end + end + + # Header value wrapper + class HeaderValue + attr_reader :value, :type + + def initialize(format:, value:, type:) + @format = format + @value = value + @type = type + end + end + + # Types module for header value types + module Types + def self.types + { + 0 => :true, + 1 => :false, + 2 => :byte, + 3 => :short, + 4 => :integer, + 5 => :long, + 6 => :bytes, + 7 => :string, + 8 => :timestamp, + 9 => :uuid + } + end + + def self.pattern + { + true: [true, 0], + false: [false, 0], + byte: ['c', 1], + short: ['s>', 2], + integer: ['l>', 4], + long: ['q>', 8], + bytes: [nil, nil], + string: [nil, nil], + timestamp: ['q>', 8], + uuid: ['H32', 16] + } + end + end + + class Error < StandardError; end + end + end + end +end diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index db138ddcd..f9b969222 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true +require 'base64' module RubyLLM module Providers @@ -7,17 +8,109 @@ module Bedrock module Streaming module_function + def stream_url + "model/#{model_id}/invoke-with-response-stream" + end + def handle_stream(&block) - to_json_stream do |data| - block.call( - Chunk.new( - role: :assistant, - model_id: data['model'], - content: extract_content(data), - input_tokens: data.dig('usage', 'prompt_tokens'), - output_tokens: data.dig('usage', 'completion_tokens') - ) - ) + decoder = Decoder.new + buffer = String.new + accumulated_content = "" + + proc do |chunk, _bytes, env| + if env && env.status != 200 + # Accumulate error chunks + buffer << chunk + begin + error_data = JSON.parse(buffer) + error_response = env.merge(body: error_data) + ErrorMiddleware.parse_error(provider: self, response: error_response) + rescue JSON::ParserError + # Keep accumulating if we don't have complete JSON yet + RubyLLM.logger.debug "Accumulating error chunk: #{chunk}" + end + else + # Process the chunk using the AWS EventStream decoder + begin + message, is_complete = decoder.decode_chunk(chunk) + + if message + payload = message.payload.read + parsed_data = nil + + begin + if payload.is_a?(String) + # Try parsing as JSON + json_data = JSON.parse(payload) + + # Handle Base64 encoded bytes + if json_data.is_a?(Hash) && json_data["bytes"] + # Decode the Base64 string + decoded_bytes = Base64.strict_decode64(json_data["bytes"]) + # Parse the decoded JSON + parsed_data = JSON.parse(decoded_bytes) + else + # Handle normal JSON data + parsed_data = json_data + end + end + rescue JSON::ParserError => e + RubyLLM.logger.debug "Failed to parse payload as JSON: #{e.message}" + next + rescue StandardError => e + RubyLLM.logger.debug "Error processing payload: #{e.message}" + next + end + + next if parsed_data.nil? + + # Extract content based on the event type + content = extract_streaming_content(parsed_data) + + # Only emit a chunk if there's content to emit + unless content.nil? || content.empty? + accumulated_content += content + + block.call( + Chunk.new( + role: :assistant, + model_id: parsed_data.dig('message', 'model') || @model_id, + content: content, + input_tokens: parsed_data.dig('message', 'usage', 'input_tokens'), + output_tokens: parsed_data.dig('message', 'usage', 'output_tokens') + ) + ) + end + end + rescue StandardError => e + RubyLLM.logger.debug "Error processing chunk: #{e.message}" + next + end + end + end + end + + def extract_streaming_content(data) + if data.is_a?(Hash) + case data['type'] + when 'message_start' + # No content yet in message_start + "" + when 'content_block_start' + # Initial content block, might have some text + data.dig('content_block', 'text').to_s + when 'content_block_delta' + # Incremental content updates + data.dig('delta', 'text').to_s + when 'message_delta' + # Might contain updates to usage stats, but no new content + "" + else + # Fall back to the existing extract_content method for other formats + extract_content(data) + end + else + extract_content(data) end end @@ -25,16 +118,30 @@ def handle_stream(&block) def extract_content(data) case data - when /anthropic\.claude/ - data[:completion] - when /amazon\.titan/ - data.dig(:results, 0, :outputText) + when Hash + if data.key?('completion') + data['completion'] + elsif data.dig('results', 0, 'outputText') + data.dig('results', 0, 'outputText') + elsif data.key?('content') + data['content'].is_a?(Array) ? data['content'].map { |item| item['text'] }.join('') : data['content'] + elsif data.key?('content_block') && data['content_block'].key?('text') + # Handle the newly decoded JSON structure + data['content_block']['text'] + else + nil + end + when Array + if data[0] == 'bytes' + data[1] + else + nil + end else - raise Error, data.inspect + nil end end - end end end -end \ No newline at end of file +end From 82158b249583be1fda8cf757c1ba8734a350847f Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 16:26:29 -0700 Subject: [PATCH 10/85] #16: Remove references to Titan - We only need Claude right now --- lib/ruby_llm/providers/bedrock.rb | 2 +- lib/ruby_llm/providers/bedrock/chat.rb | 12 ------------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index f9e944102..31bafe7b0 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -6,7 +6,7 @@ module RubyLLM module Providers # AWS Bedrock API integration. Handles chat completion and streaming - # for Claude and Titan models. + # for Claude models. module Bedrock extend Provider extend Bedrock::Chat diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index da9e976e9..92eac83cd 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -20,8 +20,6 @@ def render_payload(messages, tools:, temperature:, model:, stream: false) case model when /anthropic\.claude/ build_claude_request(messages, temperature, model) - when /amazon\.titan/ - build_titan_request(messages, temperature, model) else raise Error, "Unsupported model: #{model}" end @@ -63,16 +61,6 @@ def build_claude_request(messages, temperature, model_id) } end - def build_titan_request(messages, temperature, model_id) - { - inputText: messages.map { |msg| msg.content }.join("\n"), - textGenerationConfig: { - temperature: temperature, - maxTokenCount: max_tokens_for(model_id) - } - } - end - def extract_content(data) case data when Hash From ba937c8d04a99029a1c5b7fda17140060c5669cc Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 16:44:17 -0700 Subject: [PATCH 11/85] #16: Restore original models list --- lib/ruby_llm/models.json | 3297 ++++++++++++++++---------------------- 1 file changed, 1383 insertions(+), 1914 deletions(-) diff --git a/lib/ruby_llm/models.json b/lib/ruby_llm/models.json index 24970c2a3..386e2ad70 100644 --- a/lib/ruby_llm/models.json +++ b/lib/ruby_llm/models.json @@ -1,2596 +1,2065 @@ [ { - "id": "amazon.nova-lite-v1:0", + "id": "aqa", "created_at": null, - "display_name": "Nova Lite", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Model that performs Attributed Question Answering.", + "provider": "gemini", + "context_window": 7168, + "max_tokens": 1024, "type": "chat", - "family": "other", + "family": "aqa", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, "input_price_per_million": 0.0, "output_price_per_million": 0.0, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE", - "VIDEO" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Model trained to return answers to questions that are grounded in provided sources, along with estimating answerable probability.", + "input_token_limit": 7168, + "output_token_limit": 1024, + "supported_generation_methods": [ + "generateAnswer" ] } }, { - "id": "amazon.nova-micro-v1:0", - "created_at": null, - "display_name": "Nova Micro", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "babbage-002", + "created_at": "2023-08-21T18:16:55+02:00", + "display_name": "Babbage 002", + "provider": "openai", + "context_window": 16385, + "max_tokens": 16384, "type": "chat", - "family": "other", + "family": "babbage", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "amazon.nova-pro-v1:0", + "id": "chat-bison-001", "created_at": null, - "display_name": "Nova Pro", - "provider": "bedrock", + "display_name": "PaLM 2 Chat (Legacy)", + "provider": "gemini", "context_window": 4096, - "max_tokens": 4096, + "max_tokens": 1024, "type": "chat", "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE", - "VIDEO" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "A legacy text-only model optimized for chat conversations", + "input_token_limit": 4096, + "output_token_limit": 1024, + "supported_generation_methods": [ + "generateMessage", + "countMessageTokens" ] } }, { - "id": "amazon.rerank-v1:0", - "created_at": null, - "display_name": "Rerank 1.0", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "chatgpt-4o-latest", + "created_at": "2024-08-13T04:12:11+02:00", + "display_name": "ChatGPT-4o Latest", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "gpt4o", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "amazon.titan-embed-g1-text-02", - "created_at": null, - "display_name": "Titan Text Embeddings v2", - "provider": "bedrock", - "context_window": 4096, + "id": "claude-2.0", + "created_at": "2023-07-11T00:00:00Z", + "display_name": "Claude 2.0", + "provider": "anthropic", + "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "titan", + "family": "claude2", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, - "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "EMBEDDING" - ] - } + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, + "metadata": {} }, { - "id": "amazon.titan-embed-image-v1", - "created_at": null, - "display_name": "Titan Multimodal Embeddings G1", - "provider": "bedrock", - "context_window": 4096, + "id": "claude-2.1", + "created_at": "2023-11-21T00:00:00Z", + "display_name": "Claude 2.1", + "provider": "anthropic", + "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "titan", + "family": "claude2", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, - "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "EMBEDDING" - ] - } + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, + "metadata": {} }, { - "id": "amazon.titan-embed-image-v1:0", - "created_at": null, - "display_name": "Titan Multimodal Embeddings G1", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "claude-3-5-haiku-20241022", + "created_at": "2024-10-22T00:00:00Z", + "display_name": "Claude 3.5 Haiku", + "provider": "anthropic", + "context_window": 200000, + "max_tokens": 8192, "type": "chat", - "family": "titan", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, - "metadata": { - "provider_name": "Amazon", - "customizations_supported": [ - "FINE_TUNING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "EMBEDDING" - ] - } + "family": "claude35_haiku", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.8, + "output_price_per_million": 4.0, + "metadata": {} }, { - "id": "amazon.titan-embed-text-v1", - "created_at": null, - "display_name": "Titan Embeddings G1 - Text", - "provider": "bedrock", - "context_window": 4096, + "id": "claude-3-5-sonnet-20240620", + "created_at": "2024-06-20T00:00:00Z", + "display_name": "Claude 3.5 Sonnet (Old)", + "provider": "anthropic", + "context_window": 200000, + "max_tokens": 8192, + "type": "chat", + "family": "claude35_sonnet", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, + "metadata": {} + }, + { + "id": "claude-3-5-sonnet-20241022", + "created_at": "2024-10-22T00:00:00Z", + "display_name": "Claude 3.5 Sonnet (New)", + "provider": "anthropic", + "context_window": 200000, + "max_tokens": 8192, + "type": "chat", + "family": "claude35_sonnet", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, + "metadata": {} + }, + { + "id": "claude-3-7-sonnet-20250219", + "created_at": "2025-02-24T00:00:00Z", + "display_name": "Claude 3.7 Sonnet", + "provider": "anthropic", + "context_window": 200000, + "max_tokens": 8192, + "type": "chat", + "family": "claude37_sonnet", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, + "metadata": {} + }, + { + "id": "claude-3-haiku-20240307", + "created_at": "2024-03-07T00:00:00Z", + "display_name": "Claude 3 Haiku", + "provider": "anthropic", + "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "titan", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, - "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "EMBEDDING" - ] - } + "family": "claude3_haiku", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.25, + "output_price_per_million": 1.25, + "metadata": {} }, { - "id": "amazon.titan-embed-text-v1:2:8k", - "created_at": null, - "display_name": "Titan Embeddings G1 - Text", - "provider": "bedrock", - "context_window": 4096, + "id": "claude-3-opus-20240229", + "created_at": "2024-02-29T00:00:00Z", + "display_name": "Claude 3 Opus", + "provider": "anthropic", + "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "titan", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, - "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "EMBEDDING" - ] - } + "family": "claude3_opus", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 15.0, + "output_price_per_million": 75.0, + "metadata": {} }, { - "id": "amazon.titan-embed-text-v2:0", - "created_at": null, - "display_name": "Titan Text Embeddings V2", - "provider": "bedrock", - "context_window": 4096, + "id": "claude-3-sonnet-20240229", + "created_at": "2024-02-29T00:00:00Z", + "display_name": "Claude 3 Sonnet", + "provider": "anthropic", + "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "titan", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, - "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "EMBEDDING" - ] - } + "family": "claude3_sonnet", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, + "metadata": {} }, { - "id": "amazon.titan-image-generator-v1", - "created_at": null, - "display_name": "Titan Image Generator G1", - "provider": "bedrock", + "id": "dall-e-2", + "created_at": "2023-11-01T01:22:57+01:00", + "display_name": "DALL-E-2", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "titan", + "type": "image", + "family": "dalle2", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "amazon.titan-image-generator-v1:0", - "created_at": null, - "display_name": "Titan Image Generator G1", - "provider": "bedrock", + "id": "dall-e-3", + "created_at": "2023-10-31T21:46:29+01:00", + "display_name": "DALL-E-3", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "titan", + "type": "image", + "family": "dalle3", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [ - "FINE_TUNING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "amazon.titan-image-generator-v2:0", - "created_at": null, - "display_name": "Titan Image Generator G1 v2", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "davinci-002", + "created_at": "2023-08-21T18:11:41+02:00", + "display_name": "Davinci 002", + "provider": "openai", + "context_window": 16385, + "max_tokens": 16384, "type": "chat", - "family": "titan", + "family": "davinci", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [ - "FINE_TUNING" - ], - "inference_configurations": [ - "PROVISIONED", - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "amazon.titan-text-express-v1", + "id": "deepseek-chat", "created_at": null, - "display_name": "Titan Text G1 - Express", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "DeepSeek V3", + "provider": "deepseek", + "context_window": 64000, + "max_tokens": 8192, "type": "chat", - "family": "titan", + "family": "chat", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.27, + "output_price_per_million": 1.1, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "deepseek" } }, { - "id": "amazon.titan-text-express-v1:0:8k", + "id": "deepseek-reasoner", "created_at": null, - "display_name": "Titan Text G1 - Express", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "DeepSeek R1", + "provider": "deepseek", + "context_window": 64000, + "max_tokens": 8192, "type": "chat", - "family": "titan", + "family": "reasoner", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.55, + "output_price_per_million": 2.19, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [ - "FINE_TUNING", - "CONTINUED_PRE_TRAINING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "deepseek" } }, { - "id": "amazon.titan-text-lite-v1", + "id": "embedding-001", "created_at": null, - "display_name": "Titan Text G1 - Lite", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, - "type": "chat", - "family": "titan", + "display_name": "Embedding 001", + "provider": "gemini", + "context_window": 2048, + "max_tokens": 1, + "type": "embedding", + "family": "embedding1", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, "input_price_per_million": 0.0, "output_price_per_million": 0.0, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Obtain a distributed representation of a text.", + "input_token_limit": 2048, + "output_token_limit": 1, + "supported_generation_methods": [ + "embedContent" ] } }, { - "id": "amazon.titan-text-lite-v1:0:4k", + "id": "embedding-gecko-001", "created_at": null, - "display_name": "Titan Text G1 - Lite", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, - "type": "chat", - "family": "titan", + "display_name": "Embedding Gecko", + "provider": "gemini", + "context_window": 1024, + "max_tokens": 1, + "type": "embedding", + "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, "input_price_per_million": 0.0, "output_price_per_million": 0.0, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [ - "FINE_TUNING", - "CONTINUED_PRE_TRAINING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Obtain a distributed representation of a text.", + "input_token_limit": 1024, + "output_token_limit": 1, + "supported_generation_methods": [ + "embedText", + "countTextTokens" ] } }, { - "id": "amazon.titan-tg1-large", + "id": "gemini-1.0-pro-vision-latest", "created_at": null, - "display_name": "Titan Text Large", - "provider": "bedrock", - "context_window": 4096, + "display_name": "Gemini 1.0 Pro Vision", + "provider": "gemini", + "context_window": 12288, "max_tokens": 4096, "type": "chat", - "family": "titan", + "family": "gemini10_pro", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Amazon", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "The original Gemini 1.0 Pro Vision model version which was optimized for image understanding. Gemini 1.0 Pro Vision was deprecated on July 12, 2024. Move to a newer Gemini version.", + "input_token_limit": 12288, + "output_token_limit": 4096, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-5-haiku-20241022-v1:0", + "id": "gemini-1.5-flash", "created_at": null, - "display_name": "Claude 3.5 Haiku", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash", + "provider": "gemini", + "context_window": 1000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.15, + "output_price_per_million": 0.6, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Alias that points to the most recent stable version of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks.", + "input_token_limit": 1000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "id": "gemini-1.5-flash-001", "created_at": null, - "display_name": "Claude 3.5 Sonnet", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash 001", + "provider": "gemini", + "context_window": 1000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.15, + "output_price_per_million": 0.6, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Stable version of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks, released in May of 2024.", + "input_token_limit": 1000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens", + "createCachedContent" ] } }, { - "id": "anthropic.claude-3-5-sonnet-20240620-v1:0:18k", + "id": "gemini-1.5-flash-001-tuning", "created_at": null, - "display_name": "Claude 3.5 Sonnet", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash 001 Tuning", + "provider": "gemini", + "context_window": 16384, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.15, + "output_price_per_million": 0.6, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Version of Gemini 1.5 Flash that supports tuning, our fast and versatile multimodal model for scaling across diverse tasks, released in May of 2024.", + "input_token_limit": 16384, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens", + "createTunedModel" ] } }, { - "id": "anthropic.claude-3-5-sonnet-20240620-v1:0:200k", + "id": "gemini-1.5-flash-002", "created_at": null, - "display_name": "Claude 3.5 Sonnet", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash 002", + "provider": "gemini", + "context_window": 1000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.15, + "output_price_per_million": 0.6, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "002", + "description": "Stable version of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks, released in September of 2024.", + "input_token_limit": 1000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens", + "createCachedContent" ] } }, { - "id": "anthropic.claude-3-5-sonnet-20240620-v1:0:51k", + "id": "gemini-1.5-flash-8b", "created_at": null, - "display_name": "Claude 3.5 Sonnet", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash-8B", + "provider": "gemini", + "context_window": 1000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash_8b", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Stable version of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model, released in October of 2024.", + "input_token_limit": 1000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "createCachedContent", + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "id": "gemini-1.5-flash-8b-001", "created_at": null, - "display_name": "Claude 3.5 Sonnet v2", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash-8B 001", + "provider": "gemini", + "context_window": 1000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash_8b", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Stable version of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model, released in October of 2024.", + "input_token_limit": 1000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "createCachedContent", + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-5-sonnet-20241022-v2:0:18k", + "id": "gemini-1.5-flash-8b-exp-0827", "created_at": null, - "display_name": "Claude 3.5 Sonnet v2", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash 8B Experimental 0827", + "provider": "gemini", + "context_window": 1000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash_8b", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Experimental release (August 27th, 2024) of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model. Replaced by Gemini-1.5-flash-8b-001 (stable).", + "input_token_limit": 1000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-5-sonnet-20241022-v2:0:200k", + "id": "gemini-1.5-flash-8b-exp-0924", "created_at": null, - "display_name": "Claude 3.5 Sonnet v2", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash 8B Experimental 0924", + "provider": "gemini", + "context_window": 1000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash_8b", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Experimental release (September 24th, 2024) of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model. Replaced by Gemini-1.5-flash-8b-001 (stable).", + "input_token_limit": 1000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-5-sonnet-20241022-v2:0:51k", + "id": "gemini-1.5-flash-8b-latest", "created_at": null, - "display_name": "Claude 3.5 Sonnet v2", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash-8B Latest", + "provider": "gemini", + "context_window": 1000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash_8b", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Alias that points to the most recent production (non-experimental) release of Gemini 1.5 Flash-8B, our smallest and most cost effective Flash model, released in October of 2024.", + "input_token_limit": 1000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "createCachedContent", + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-7-sonnet-20250219-v1:0", + "id": "gemini-1.5-flash-latest", "created_at": null, - "display_name": "Claude 3.7 Sonnet", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Flash Latest", + "provider": "gemini", + "context_window": 1000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_flash", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.15, + "output_price_per_million": 0.6, "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "version": "001", + "description": "Alias that points to the most recent production (non-experimental) release of Gemini 1.5 Flash, our fast and versatile multimodal model for scaling across diverse tasks.", + "input_token_limit": 1000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-haiku-20240307-v1:0", + "id": "gemini-1.5-pro", "created_at": null, - "display_name": "Claude 3 Haiku", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Pro", + "provider": "gemini", + "context_window": 2000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_pro", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0005, - "output_price_per_million": 0.0025, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 2.5, + "output_price_per_million": 10.0, + "metadata": { + "version": "001", + "description": "Stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens, released in May of 2024.", + "input_token_limit": 2000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-haiku-20240307-v1:0:200k", + "id": "gemini-1.5-pro-001", "created_at": null, - "display_name": "Claude 3 Haiku", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Pro 001", + "provider": "gemini", + "context_window": 2000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_pro", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0005, - "output_price_per_million": 0.0025, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [ - "FINE_TUNING", - "DISTILLATION" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 2.5, + "output_price_per_million": 10.0, + "metadata": { + "version": "001", + "description": "Stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens, released in May of 2024.", + "input_token_limit": 2000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens", + "createCachedContent" ] } }, { - "id": "anthropic.claude-3-haiku-20240307-v1:0:48k", + "id": "gemini-1.5-pro-002", "created_at": null, - "display_name": "Claude 3 Haiku", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Pro 002", + "provider": "gemini", + "context_window": 2000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_pro", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0005, - "output_price_per_million": 0.0025, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 2.5, + "output_price_per_million": 10.0, + "metadata": { + "version": "002", + "description": "Stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens, released in September of 2024.", + "input_token_limit": 2000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens", + "createCachedContent" ] } }, { - "id": "anthropic.claude-3-opus-20240229-v1:0", + "id": "gemini-1.5-pro-latest", "created_at": null, - "display_name": "Claude 3 Opus", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 1.5 Pro Latest", + "provider": "gemini", + "context_window": 2000000, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini15_pro", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.015, - "output_price_per_million": 0.075, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 2.5, + "output_price_per_million": 10.0, + "metadata": { + "version": "001", + "description": "Alias that points to the most recent production (non-experimental) release of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens.", + "input_token_limit": 2000000, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-opus-20240229-v1:0:12k", + "id": "gemini-2.0-flash", "created_at": null, - "display_name": "Claude 3 Opus", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini20_flash", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.015, - "output_price_per_million": 0.075, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 0.1, + "output_price_per_million": 0.4, + "metadata": { + "version": "2.0", + "description": "Gemini 2.0 Flash", + "input_token_limit": 1048576, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-opus-20240229-v1:0:200k", + "id": "gemini-2.0-flash-001", "created_at": null, - "display_name": "Claude 3 Opus", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash 001", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini20_flash", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.015, - "output_price_per_million": 0.075, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 0.1, + "output_price_per_million": 0.4, + "metadata": { + "version": "2.0", + "description": "Stable version of Gemini 2.0 Flash, our fast and versatile multimodal model for scaling across diverse tasks, released in January of 2025.", + "input_token_limit": 1048576, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-opus-20240229-v1:0:28k", + "id": "gemini-2.0-flash-exp", "created_at": null, - "display_name": "Claude 3 Opus", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash Experimental", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini20_flash", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.015, - "output_price_per_million": 0.075, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 0.1, + "output_price_per_million": 0.4, + "metadata": { + "version": "2.0", + "description": "Gemini 2.0 Flash Experimental", + "input_token_limit": 1048576, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens", + "bidiGenerateContent" ] } }, { - "id": "anthropic.claude-3-sonnet-20240229-v1:0", + "id": "gemini-2.0-flash-lite", "created_at": null, - "display_name": "Claude 3 Sonnet", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash-Lite", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini20_flash_lite", "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.003, - "output_price_per_million": 0.015, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "2.0", + "description": "Gemini 2.0 Flash-Lite", + "input_token_limit": 1048576, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-3-sonnet-20240229-v1:0:200k", + "id": "gemini-2.0-flash-lite-001", "created_at": null, - "display_name": "Claude 3 Sonnet", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash-Lite 001", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini20_flash_lite", "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.003, - "output_price_per_million": 0.015, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" - ] - } + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "2.0", + "description": "Stable version of Gemini 2.0 Flash Lite", + "input_token_limit": 1048576, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" + ] + } }, { - "id": "anthropic.claude-3-sonnet-20240229-v1:0:28k", + "id": "gemini-2.0-flash-lite-preview", "created_at": null, - "display_name": "Claude 3 Sonnet", - "provider": "bedrock", - "context_window": 200000, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash-Lite Preview", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 8192, "type": "chat", - "family": "claude3", + "family": "gemini20_flash_lite", "supports_vision": true, - "supports_functions": true, - "supports_json_mode": true, - "input_price_per_million": 0.003, - "output_price_per_million": 0.015, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "preview-02-05", + "description": "Preview release (February 5th, 2025) of Gemini 2.0 Flash Lite", + "input_token_limit": 1048576, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-instant-v1", + "id": "gemini-2.0-flash-lite-preview-02-05", "created_at": null, - "display_name": "Claude Instant", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash-Lite Preview 02-05", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 8192, "type": "chat", - "family": "claude_instant", - "supports_vision": false, + "family": "gemini20_flash_lite", + "supports_vision": true, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.0008, - "output_price_per_million": 0.0024, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "preview-02-05", + "description": "Preview release (February 5th, 2025) of Gemini 2.0 Flash Lite", + "input_token_limit": 1048576, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-instant-v1:2:100k", + "id": "gemini-2.0-flash-thinking-exp", "created_at": null, - "display_name": "Claude Instant", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash Thinking Experimental 01-21", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 65536, "type": "chat", - "family": "claude_instant", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0008, - "output_price_per_million": 0.0024, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "family": "gemini20_flash", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.1, + "output_price_per_million": 0.4, + "metadata": { + "version": "2.0-exp-01-21", + "description": "Experimental release (January 21st, 2025) of Gemini 2.0 Flash Thinking", + "input_token_limit": 1048576, + "output_token_limit": 65536, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-v2", + "id": "gemini-2.0-flash-thinking-exp-01-21", "created_at": null, - "display_name": "Claude", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash Thinking Experimental 01-21", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 65536, "type": "chat", - "family": "claude2", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "family": "gemini20_flash", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.1, + "output_price_per_million": 0.4, + "metadata": { + "version": "2.0-exp-01-21", + "description": "Experimental release (January 21st, 2025) of Gemini 2.0 Flash Thinking", + "input_token_limit": 1048576, + "output_token_limit": 65536, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-v2:0:100k", + "id": "gemini-2.0-flash-thinking-exp-1219", "created_at": null, - "display_name": "Claude", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Flash Thinking Experimental", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 65536, "type": "chat", - "family": "claude2", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "family": "gemini20_flash", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.1, + "output_price_per_million": 0.4, + "metadata": { + "version": "2.0", + "description": "Gemini 2.0 Flash Thinking Experimental", + "input_token_limit": 1048576, + "output_token_limit": 65536, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-v2:0:18k", + "id": "gemini-2.0-pro-exp", "created_at": null, - "display_name": "Claude", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Pro Experimental", + "provider": "gemini", + "context_window": 2097152, + "max_tokens": 8192, "type": "chat", - "family": "claude2", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "family": "other", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "2.0", + "description": "Experimental release (February 5th, 2025) of Gemini 2.0 Pro", + "input_token_limit": 2097152, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-v2:1", + "id": "gemini-2.0-pro-exp-02-05", "created_at": null, - "display_name": "Claude", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini 2.0 Pro Experimental 02-05", + "provider": "gemini", + "context_window": 2097152, + "max_tokens": 8192, "type": "chat", - "family": "claude2", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "family": "other", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "2.0", + "description": "Experimental release (February 5th, 2025) of Gemini 2.0 Pro", + "input_token_limit": 2097152, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-v2:1:18k", + "id": "gemini-exp-1206", "created_at": null, - "display_name": "Claude", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "Gemini Experimental 1206", + "provider": "gemini", + "context_window": 2097152, + "max_tokens": 8192, "type": "chat", - "family": "claude2", + "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "2.0", + "description": "Experimental release (February 5th, 2025) of Gemini 2.0 Pro", + "input_token_limit": 2097152, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "anthropic.claude-v2:1:200k", + "id": "gemini-pro-vision", "created_at": null, - "display_name": "Claude", - "provider": "bedrock", - "context_window": 4096, + "display_name": "Gemini 1.0 Pro Vision", + "provider": "gemini", + "context_window": 12288, "max_tokens": 4096, "type": "chat", - "family": "claude2", + "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, - "metadata": { - "provider_name": "Anthropic", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "001", + "description": "The original Gemini 1.0 Pro Vision model version which was optimized for image understanding. Gemini 1.0 Pro Vision was deprecated on July 12, 2024. Move to a newer Gemini version.", + "input_token_limit": 12288, + "output_token_limit": 4096, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "cohere.command-light-text-v14", - "created_at": null, - "display_name": "Command Light", - "provider": "bedrock", - "context_window": 4096, + "id": "gpt-3.5-turbo", + "created_at": "2023-02-28T19:56:42+01:00", + "display_name": "GPT-3.5-Turbo", + "provider": "openai", + "context_window": 16385, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "gpt35", "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0003, - "output_price_per_million": 0.0006, - "metadata": { - "provider_name": "Cohere", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "openai" } }, { - "id": "cohere.command-light-text-v14:7:4k", - "created_at": null, - "display_name": "Command Light", - "provider": "bedrock", + "id": "gpt-3.5-turbo-0125", + "created_at": "2024-01-23T23:19:18+01:00", + "display_name": "GPT-3.5-Turbo 0125", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "gpt35", "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0003, - "output_price_per_million": 0.0006, - "metadata": { - "provider_name": "Cohere", - "customizations_supported": [ - "FINE_TUNING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" } }, { - "id": "cohere.command-r-plus-v1:0", - "created_at": null, - "display_name": "Command R+", - "provider": "bedrock", + "id": "gpt-3.5-turbo-1106", + "created_at": "2023-11-02T22:15:48+01:00", + "display_name": "GPT-3.5-Turbo 1106", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "gpt35", "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Cohere", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "cohere.command-r-v1:0", - "created_at": null, - "display_name": "Command R", - "provider": "bedrock", - "context_window": 4096, + "id": "gpt-3.5-turbo-16k", + "created_at": "2023-05-11T00:35:02+02:00", + "display_name": "GPT-3.5-Turbo 16k", + "provider": "openai", + "context_window": 16385, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "gpt35", "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Cohere", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "openai-internal" } }, { - "id": "cohere.command-text-v14", - "created_at": null, - "display_name": "Command", - "provider": "bedrock", + "id": "gpt-3.5-turbo-instruct", + "created_at": "2023-08-24T20:23:47+02:00", + "display_name": "GPT-3.5-Turbo Instruct", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "gpt35_instruct", "supports_vision": false, "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0015, - "output_price_per_million": 0.002, - "metadata": { - "provider_name": "Cohere", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" } }, { - "id": "cohere.command-text-v14:7:4k", - "created_at": null, - "display_name": "Command", - "provider": "bedrock", + "id": "gpt-3.5-turbo-instruct-0914", + "created_at": "2023-09-07T23:34:32+02:00", + "display_name": "GPT-3.5-Turbo Instruct 0914", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "gpt35_instruct", "supports_vision": false, "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0015, - "output_price_per_million": 0.002, - "metadata": { - "provider_name": "Cohere", - "customizations_supported": [ - "FINE_TUNING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" } }, { - "id": "cohere.embed-english-v3", - "created_at": null, - "display_name": "Embed English", - "provider": "bedrock", + "id": "gpt-4", + "created_at": "2023-06-27T18:13:31+02:00", + "display_name": "GPT-4", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "gpt4", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Cohere", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "EMBEDDING" - ] + "object": "model", + "owned_by": "openai" } }, { - "id": "cohere.embed-english-v3:0:512", - "created_at": null, - "display_name": "Embed English", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4-0125-preview", + "created_at": "2024-01-23T20:20:12+01:00", + "display_name": "GPT-4-0125 Preview", + "provider": "openai", + "context_window": 8192, + "max_tokens": 8192, "type": "chat", - "family": "other", + "family": "gpt4", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" + } + }, + { + "id": "gpt-4-0613", + "created_at": "2023-06-12T18:54:56+02:00", + "display_name": "GPT-4-0613", + "provider": "openai", + "context_window": 8192, + "max_tokens": 8192, + "type": "chat", + "family": "gpt4", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Cohere", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "EMBEDDING" - ] + "object": "model", + "owned_by": "openai" } }, { - "id": "cohere.embed-multilingual-v3", - "created_at": null, - "display_name": "Embed Multilingual", - "provider": "bedrock", + "id": "gpt-4-1106-preview", + "created_at": "2023-11-02T21:33:26+01:00", + "display_name": "GPT-4-1106 Preview", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "family": "gpt4", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Cohere", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "EMBEDDING" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "cohere.embed-multilingual-v3:0:512", - "created_at": null, - "display_name": "Embed Multilingual", - "provider": "bedrock", - "context_window": 4096, + "id": "gpt-4-turbo", + "created_at": "2024-04-06T01:57:21+02:00", + "display_name": "GPT-4-Turbo", + "provider": "openai", + "context_window": 128000, "max_tokens": 4096, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "family": "gpt4_turbo", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Cohere", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "EMBEDDING" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "cohere.rerank-v3-5:0", - "created_at": null, - "display_name": "Rerank 3.5", - "provider": "bedrock", - "context_window": 4096, + "id": "gpt-4-turbo-2024-04-09", + "created_at": "2024-04-08T20:41:17+02:00", + "display_name": "GPT-4-Turbo 20240409", + "provider": "openai", + "context_window": 128000, "max_tokens": 4096, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "family": "gpt4_turbo", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Cohere", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "deepseek.r1-v1:0", - "created_at": null, - "display_name": "DeepSeek-R1", - "provider": "bedrock", - "context_window": 4096, + "id": "gpt-4-turbo-preview", + "created_at": "2024-01-23T20:22:57+01:00", + "display_name": "GPT-4-Turbo Preview", + "provider": "openai", + "context_window": 128000, "max_tokens": 4096, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, - "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "family": "gpt4_turbo", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "DeepSeek", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "luma.ray-v2:0", - "created_at": null, - "display_name": "Ray v2", - "provider": "bedrock", + "id": "gpt-4.5-preview", + "created_at": "2025-02-27T03:24:19+01:00", + "display_name": "GPT-4.5 Preview", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "gpt4", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Luma AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "VIDEO" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-1-405b-instruct-v1:0", - "created_at": null, - "display_name": "Llama 3.1 405B Instruct", - "provider": "bedrock", + "id": "gpt-4.5-preview-2025-02-27", + "created_at": "2025-02-27T03:28:24+01:00", + "display_name": "GPT-4.5 Preview 20250227", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", + "family": "gpt4", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-1-70b-instruct-v1:0", - "created_at": null, - "display_name": "Llama 3.1 70B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o", + "created_at": "2024-05-10T20:50:49+02:00", + "display_name": "GPT-4o", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "gpt4o", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.00195, - "output_price_per_million": 0.00256, - "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-1-70b-instruct-v1:0:128k", - "created_at": null, - "display_name": "Llama 3.1 70B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-2024-05-13", + "created_at": "2024-05-10T21:08:52+02:00", + "display_name": "GPT-4o 20240513", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "gpt4o", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.00195, - "output_price_per_million": 0.00256, - "metadata": { - "provider_name": "Meta", - "customizations_supported": [ - "FINE_TUNING", - "DISTILLATION" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-1-8b-instruct-v1:0", - "created_at": null, - "display_name": "Llama 3.1 8B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-2024-08-06", + "created_at": "2024-08-05T01:38:39+02:00", + "display_name": "GPT-4o 20240806", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "gpt4o", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-1-8b-instruct-v1:0:128k", - "created_at": null, - "display_name": "Llama 3.1 8B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-2024-11-20", + "created_at": "2025-02-12T04:39:03+01:00", + "display_name": "GPT-4o 20241120", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "gpt4o", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [ - "FINE_TUNING", - "DISTILLATION" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-2-11b-instruct-v1:0", - "created_at": null, - "display_name": "Llama 3.2 11B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-audio-preview", + "created_at": "2024-09-27T20:07:23+02:00", + "display_name": "GPT-4o-Audio Preview", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", + "family": "gpt4o_audio", "supports_vision": true, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-2-11b-instruct-v1:0:128k", - "created_at": null, - "display_name": "Llama 3.2 11B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-audio-preview-2024-10-01", + "created_at": "2024-09-27T00:17:22+02:00", + "display_name": "GPT-4o-Audio Preview 20241001", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", + "family": "gpt4o_audio", "supports_vision": true, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [ - "FINE_TUNING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-2-1b-instruct-v1:0", - "created_at": null, - "display_name": "Llama 3.2 1B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-audio-preview-2024-12-17", + "created_at": "2024-12-12T21:10:39+01:00", + "display_name": "GPT-4o-Audio Preview 20241217", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "gpt4o_audio", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-2-1b-instruct-v1:0:128k", - "created_at": null, - "display_name": "Llama 3.2 1B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-mini", + "created_at": "2024-07-17T01:32:21+02:00", + "display_name": "GPT-4o-Mini", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "gpt4o_mini", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [ - "FINE_TUNING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-2-3b-instruct-v1:0", - "created_at": null, - "display_name": "Llama 3.2 3B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-mini-2024-07-18", + "created_at": "2024-07-17T01:31:57+02:00", + "display_name": "GPT-4o-Mini 20240718", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "gpt4o_mini", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-2-3b-instruct-v1:0:128k", - "created_at": null, - "display_name": "Llama 3.2 3B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-mini-audio-preview", + "created_at": "2024-12-16T23:17:04+01:00", + "display_name": "GPT-4o-Mini Audio Preview", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "gpt4o_mini_audio", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [ - "FINE_TUNING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-2-90b-instruct-v1:0", - "created_at": null, - "display_name": "Llama 3.2 90B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-mini-audio-preview-2024-12-17", + "created_at": "2024-12-13T19:52:00+01:00", + "display_name": "GPT-4o-Mini Audio Preview 20241217", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", + "family": "gpt4o_mini_audio", "supports_vision": true, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-2-90b-instruct-v1:0:128k", - "created_at": null, - "display_name": "Llama 3.2 90B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-mini-realtime-preview", + "created_at": "2024-12-16T23:16:20+01:00", + "display_name": "GPT-4o-Mini Realtime Preview", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", - "family": "other", + "family": "gpt4o_mini_realtime", "supports_vision": true, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [ - "FINE_TUNING" - ], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "meta.llama3-3-70b-instruct-v1:0", - "created_at": null, - "display_name": "Llama 3.3 70B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "id": "gpt-4o-mini-realtime-preview-2024-12-17", + "created_at": "2024-12-13T18:56:41+01:00", + "display_name": "GPT-4o-Mini Realtime Preview 20241217", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, "type": "chat", + "family": "gpt4o_mini_realtime", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": false, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" + } + }, + { + "id": "gpt-4o-realtime-preview", + "created_at": "2024-09-30T03:33:18+02:00", + "display_name": "GPT-4o-Realtime Preview", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, + "type": "chat", + "family": "gpt4o_realtime", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": false, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" + } + }, + { + "id": "gpt-4o-realtime-preview-2024-10-01", + "created_at": "2024-09-24T00:49:26+02:00", + "display_name": "GPT-4o-Realtime Preview 20241001", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, + "type": "chat", + "family": "gpt4o_realtime", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": false, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" + } + }, + { + "id": "gpt-4o-realtime-preview-2024-12-17", + "created_at": "2024-12-11T20:30:30+01:00", + "display_name": "GPT-4o-Realtime Preview 20241217", + "provider": "openai", + "context_window": 128000, + "max_tokens": 16384, + "type": "chat", + "family": "gpt4o_realtime", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": false, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" + } + }, + { + "id": "imagen-3.0-generate-002", + "created_at": null, + "display_name": "Imagen 3.0 002 model", + "provider": "gemini", + "context_window": 480, + "max_tokens": 8192, + "type": "image", "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.00195, - "output_price_per_million": 0.00256, - "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "INFERENCE_PROFILE" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "002", + "description": "Vertex served Imagen 3.0 002 model", + "input_token_limit": 480, + "output_token_limit": 8192, + "supported_generation_methods": [ + "predict" ] } }, { - "id": "meta.llama3-70b-instruct-v1:0", + "id": "learnlm-1.5-pro-experimental", "created_at": null, - "display_name": "Llama 3 70B Instruct", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, + "display_name": "LearnLM 1.5 Pro Experimental", + "provider": "gemini", + "context_window": 32767, + "max_tokens": 8192, "type": "chat", "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.00195, - "output_price_per_million": 0.00256, - "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "001", + "description": "Alias that points to the most recent stable version of Gemini 1.5 Pro, our mid-size multimodal model that supports up to 2 million tokens.", + "input_token_limit": 32767, + "output_token_limit": 8192, + "supported_generation_methods": [ + "generateContent", + "countTokens" ] } }, { - "id": "meta.llama3-8b-instruct-v1:0", - "created_at": null, - "display_name": "Llama 3 8B Instruct", - "provider": "bedrock", - "context_window": 4096, + "id": "o1-mini", + "created_at": "2024-09-06T20:56:48+02:00", + "display_name": "O1-Mini", + "provider": "openai", + "context_window": 128000, "max_tokens": 4096, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "o1_mini", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Meta", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "mistral.mistral-7b-instruct-v0:2", - "created_at": null, - "display_name": "Mistral 7B Instruct", - "provider": "bedrock", + "id": "o1-mini-2024-09-12", + "created_at": "2024-09-06T20:56:19+02:00", + "display_name": "O1-Mini 20240912", + "provider": "openai", + "context_window": 128000, + "max_tokens": 65536, + "type": "chat", + "family": "o1_mini", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": false, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" + } + }, + { + "id": "o1-preview", + "created_at": "2024-09-06T20:54:57+02:00", + "display_name": "O1-Preview", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "o1", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0002, - "output_price_per_million": 0.0002, - "metadata": { - "provider_name": "Mistral AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" } }, { - "id": "mistral.mistral-large-2402-v1:0", - "created_at": null, - "display_name": "Mistral Large (24.02)", - "provider": "bedrock", + "id": "o1-preview-2024-09-12", + "created_at": "2024-09-06T20:54:25+02:00", + "display_name": "O1-Preview 20240912", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "other", - "supports_vision": false, - "supports_functions": false, + "family": "o1", + "supports_vision": true, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.002, - "output_price_per_million": 0.002, - "metadata": { - "provider_name": "Mistral AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" } }, { - "id": "mistral.mistral-large-2407-v1:0", - "created_at": null, - "display_name": "Mistral Large (24.07)", - "provider": "bedrock", + "id": "omni-moderation-2024-09-26", + "created_at": "2024-11-27T20:07:46+01:00", + "display_name": "Omni-Moderation 20240926", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "other", + "type": "moderation", + "family": "moderation", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.002, - "output_price_per_million": 0.002, - "metadata": { - "provider_name": "Mistral AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" - ] + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" } }, { - "id": "mistral.mixtral-8x7b-instruct-v0:1", - "created_at": null, - "display_name": "Mixtral 8x7B Instruct", - "provider": "bedrock", + "id": "omni-moderation-latest", + "created_at": "2024-11-15T17:47:45+01:00", + "display_name": "Omni-Moderation Latest", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, + "type": "moderation", + "family": "moderation", + "supports_vision": false, + "supports_functions": true, + "supports_json_mode": false, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "system" + } + }, + { + "id": "text-bison-001", + "created_at": null, + "display_name": "PaLM 2 (Legacy)", + "provider": "gemini", + "context_window": 8196, + "max_tokens": 1024, "type": "chat", "family": "other", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.0007, - "output_price_per_million": 0.0007, - "metadata": { - "provider_name": "Mistral AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": true, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "TEXT" + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "001", + "description": "A legacy model that understands text and generates text as an output", + "input_token_limit": 8196, + "output_token_limit": 1024, + "supported_generation_methods": [ + "generateText", + "countTextTokens", + "createTunedTextModel" ] } }, { - "id": "stability.sd3-5-large-v1:0", + "id": "text-embedding-004", "created_at": null, - "display_name": "Stable Diffusion 3.5 Large", - "provider": "bedrock", - "context_window": 4096, - "max_tokens": 4096, - "type": "chat", - "family": "other", + "display_name": "Text Embedding 004", + "provider": "gemini", + "context_window": 2048, + "max_tokens": 1, + "type": "embedding", + "family": "embedding4", "supports_vision": false, "supports_functions": false, "supports_json_mode": false, "input_price_per_million": 0.0, "output_price_per_million": 0.0, "metadata": { - "provider_name": "Stability AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "IMAGE" + "version": "004", + "description": "Obtain a distributed representation of a text.", + "input_token_limit": 2048, + "output_token_limit": 1, + "supported_generation_methods": [ + "embedContent" ] } }, { - "id": "stability.sd3-large-v1:0", - "created_at": null, - "display_name": "SD3 Large 1.0", - "provider": "bedrock", + "id": "text-embedding-3-large", + "created_at": "2024-01-22T20:53:00+01:00", + "display_name": "Text Embedding 3 Large", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "other", + "type": "embedding", + "family": "embedding3_large", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Stability AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "stability.stable-diffusion-xl-v1", - "created_at": null, - "display_name": "SDXL 1.0", - "provider": "bedrock", + "id": "text-embedding-3-small", + "created_at": "2024-01-22T19:43:17+01:00", + "display_name": "Text Embedding 3 Small", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "other", + "type": "embedding", + "family": "embedding3_small", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Stability AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "stability.stable-diffusion-xl-v1:0", - "created_at": null, - "display_name": "SDXL 1.0", - "provider": "bedrock", + "id": "text-embedding-ada-002", + "created_at": "2022-12-16T20:01:39+01:00", + "display_name": "Text Embedding Ada 002", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "other", + "type": "embedding", + "family": "embedding2", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Stability AI", - "customizations_supported": [], - "inference_configurations": [ - "PROVISIONED" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT", - "IMAGE" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "openai-internal" } }, { - "id": "stability.stable-image-core-v1:0", - "created_at": null, - "display_name": "Stable Image Core 1.0", - "provider": "bedrock", + "id": "tts-1", + "created_at": "2023-04-19T23:49:11+02:00", + "display_name": "TTS-1", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "other", + "type": "audio", + "family": "tts1", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Stability AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "openai-internal" } }, { - "id": "stability.stable-image-core-v1:1", - "created_at": null, - "display_name": "Stable Image Core 1.0", - "provider": "bedrock", + "id": "tts-1-1106", + "created_at": "2023-11-04T00:14:01+01:00", + "display_name": "TTS-1 1106", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "other", + "type": "audio", + "family": "tts1", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Stability AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "stability.stable-image-ultra-v1:0", - "created_at": null, - "display_name": "Stable Image Ultra 1.0", - "provider": "bedrock", + "id": "tts-1-hd", + "created_at": "2023-11-03T22:13:35+01:00", + "display_name": "TTS-1 HD", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "other", + "type": "audio", + "family": "tts1_hd", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Stability AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "system" } }, { - "id": "stability.stable-image-ultra-v1:1", - "created_at": null, - "display_name": "Stable Image Ultra 1.0", - "provider": "bedrock", + "id": "tts-1-hd-1106", + "created_at": "2023-11-04T00:18:53+01:00", + "display_name": "TTS-1 HD 1106", + "provider": "openai", "context_window": 4096, "max_tokens": 4096, - "type": "chat", - "family": "other", + "type": "audio", + "family": "tts1_hd", "supports_vision": false, - "supports_functions": false, + "supports_functions": true, "supports_json_mode": false, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, "metadata": { - "provider_name": "Stability AI", - "customizations_supported": [], - "inference_configurations": [ - "ON_DEMAND" - ], - "response_streaming_supported": false, - "input_modalities": [ - "TEXT" - ], - "output_modalities": [ - "IMAGE" - ] + "object": "model", + "owned_by": "system" + } + }, + { + "id": "whisper-1", + "created_at": "2023-02-27T22:13:04+01:00", + "display_name": "Whisper 1", + "provider": "openai", + "context_window": 4096, + "max_tokens": 4096, + "type": "audio", + "family": "whisper1", + "supports_vision": false, + "supports_functions": true, + "supports_json_mode": false, + "input_price_per_million": 0.5, + "output_price_per_million": 1.5, + "metadata": { + "object": "model", + "owned_by": "openai-internal" } } ] \ No newline at end of file From 1bcb8a0f97413ffb963c04a5bdd1402aa64ee3cc Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 17:08:01 -0700 Subject: [PATCH 12/85] #16: Bring in bedrock models --- lib/ruby_llm.rb | 8 +- lib/ruby_llm/configuration.rb | 4 + lib/ruby_llm/error.rb | 3 + lib/ruby_llm/models.json | 835 +++++++++++++++++++++++ lib/ruby_llm/models.rb | 23 + lib/ruby_llm/providers/bedrock.rb | 8 +- lib/ruby_llm/providers/bedrock/models.rb | 5 +- lib/tasks/models.rake | 89 ++- 8 files changed, 947 insertions(+), 28 deletions(-) diff --git a/lib/ruby_llm.rb b/lib/ruby_llm.rb index 0b3f0a51b..b3f3d4592 100644 --- a/lib/ruby_llm.rb +++ b/lib/ruby_llm.rb @@ -65,10 +65,10 @@ def logger end end -# RubyLLM::Provider.register :openai, RubyLLM::Providers::OpenAI -# RubyLLM::Provider.register :anthropic, RubyLLM::Providers::Anthropic -# RubyLLM::Provider.register :gemini, RubyLLM::Providers::Gemini -# RubyLLM::Provider.register :deepseek, RubyLLM::Providers::DeepSeek +RubyLLM::Provider.register :openai, RubyLLM::Providers::OpenAI +RubyLLM::Provider.register :anthropic, RubyLLM::Providers::Anthropic +RubyLLM::Provider.register :gemini, RubyLLM::Providers::Gemini +RubyLLM::Provider.register :deepseek, RubyLLM::Providers::DeepSeek RubyLLM::Provider.register :bedrock, RubyLLM::Providers::Bedrock if defined?(Rails::Railtie) diff --git a/lib/ruby_llm/configuration.rb b/lib/ruby_llm/configuration.rb index 72a878aae..a5a41542b 100644 --- a/lib/ruby_llm/configuration.rb +++ b/lib/ruby_llm/configuration.rb @@ -14,6 +14,10 @@ class Configuration :anthropic_api_key, :gemini_api_key, :deepseek_api_key, + :bedrock_api_key, + :bedrock_secret_key, + :bedrock_region, + :bedrock_session_token, :default_model, :default_embedding_model, :default_image_model, diff --git a/lib/ruby_llm/error.rb b/lib/ruby_llm/error.rb index 0deb07ffa..970cbbbe1 100644 --- a/lib/ruby_llm/error.rb +++ b/lib/ruby_llm/error.rb @@ -28,6 +28,7 @@ class ServiceUnavailableError < Error; end class BadRequestError < Error; end class RateLimitError < Error; end class ServerError < Error; end + class ForbiddenError < Error; end # Faraday middleware that maps provider-specific API errors to RubyLLM errors. # Uses provider's parse_error method to extract meaningful error messages. @@ -56,6 +57,8 @@ def parse_error(provider:, response:) # rubocop:disable Metrics/CyclomaticComple raise UnauthorizedError.new(response, message || 'Invalid API key - check your credentials') when 402 raise PaymentRequiredError.new(response, message || 'Payment required - please top up your account') + when 403 + raise ForbiddenError.new(response, message || 'Forbidden - you do not have permission to access this resource') when 429 raise RateLimitError.new(response, message || 'Rate limit exceeded - please wait a moment') when 500 diff --git a/lib/ruby_llm/models.json b/lib/ruby_llm/models.json index 386e2ad70..2e4fc4c46 100644 --- a/lib/ruby_llm/models.json +++ b/lib/ruby_llm/models.json @@ -1,4 +1,839 @@ [ + { + "id": "anthropic.claude-3-5-haiku-20241022-v1:0", + "created_at": null, + "display_name": "Claude 3.5 Haiku", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-5-sonnet-20240620-v1:0", + "created_at": null, + "display_name": "Claude 3.5 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-5-sonnet-20240620-v1:0:18k", + "created_at": null, + "display_name": "Claude 3.5 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-5-sonnet-20240620-v1:0:200k", + "created_at": null, + "display_name": "Claude 3.5 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-5-sonnet-20240620-v1:0:51k", + "created_at": null, + "display_name": "Claude 3.5 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-5-sonnet-20241022-v2:0", + "created_at": null, + "display_name": "Claude 3.5 Sonnet v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-5-sonnet-20241022-v2:0:18k", + "created_at": null, + "display_name": "Claude 3.5 Sonnet v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-5-sonnet-20241022-v2:0:200k", + "created_at": null, + "display_name": "Claude 3.5 Sonnet v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-5-sonnet-20241022-v2:0:51k", + "created_at": null, + "display_name": "Claude 3.5 Sonnet v2", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-7-sonnet-20250219-v1:0", + "created_at": null, + "display_name": "Claude 3.7 Sonnet", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0, + "output_price_per_million": 0.0, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "INFERENCE_PROFILE" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-haiku-20240307-v1:0", + "created_at": null, + "display_name": "Claude 3 Haiku", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0005, + "output_price_per_million": 0.0025, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-haiku-20240307-v1:0:200k", + "created_at": null, + "display_name": "Claude 3 Haiku", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0005, + "output_price_per_million": 0.0025, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [ + "FINE_TUNING", + "DISTILLATION" + ], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-haiku-20240307-v1:0:48k", + "created_at": null, + "display_name": "Claude 3 Haiku", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.0005, + "output_price_per_million": 0.0025, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-opus-20240229-v1:0", + "created_at": null, + "display_name": "Claude 3 Opus", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.015, + "output_price_per_million": 0.075, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-opus-20240229-v1:0:12k", + "created_at": null, + "display_name": "Claude 3 Opus", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.015, + "output_price_per_million": 0.075, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-opus-20240229-v1:0:200k", + "created_at": null, + "display_name": "Claude 3 Opus", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.015, + "output_price_per_million": 0.075, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-opus-20240229-v1:0:28k", + "created_at": null, + "display_name": "Claude 3 Opus", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.015, + "output_price_per_million": 0.075, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-sonnet-20240229-v1:0", + "created_at": null, + "display_name": "Claude 3 Sonnet", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.003, + "output_price_per_million": 0.015, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-sonnet-20240229-v1:0:200k", + "created_at": null, + "display_name": "Claude 3 Sonnet", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.003, + "output_price_per_million": 0.015, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-3-sonnet-20240229-v1:0:28k", + "created_at": null, + "display_name": "Claude 3 Sonnet", + "provider": "bedrock", + "context_window": 200000, + "max_tokens": 4096, + "type": "chat", + "family": "claude3", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.003, + "output_price_per_million": 0.015, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT", + "IMAGE" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-instant-v1", + "created_at": null, + "display_name": "Claude Instant", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude_instant", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0008, + "output_price_per_million": 0.0024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-instant-v1:2:100k", + "created_at": null, + "display_name": "Claude Instant", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude_instant", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.0008, + "output_price_per_million": 0.0024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-v2", + "created_at": null, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude2", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-v2:0:100k", + "created_at": null, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude2", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-v2:0:18k", + "created_at": null, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude2", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-v2:1", + "created_at": null, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude2", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "ON_DEMAND" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-v2:1:18k", + "created_at": null, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude2", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] + } + }, + { + "id": "anthropic.claude-v2:1:200k", + "created_at": null, + "display_name": "Claude", + "provider": "bedrock", + "context_window": 4096, + "max_tokens": 4096, + "type": "chat", + "family": "claude2", + "supports_vision": false, + "supports_functions": false, + "supports_json_mode": false, + "input_price_per_million": 0.008, + "output_price_per_million": 0.024, + "metadata": { + "provider_name": "Anthropic", + "customizations_supported": [], + "inference_configurations": [ + "PROVISIONED" + ], + "response_streaming_supported": true, + "input_modalities": [ + "TEXT" + ], + "output_modalities": [ + "TEXT" + ] + } + }, { "id": "aqa", "created_at": null, diff --git a/lib/ruby_llm/models.rb b/lib/ruby_llm/models.rb index 571c21718..af3cd6044 100644 --- a/lib/ruby_llm/models.rb +++ b/lib/ruby_llm/models.rb @@ -26,6 +26,22 @@ def self.refresh! @instance = new(models) end + # Class method to refresh models for a specific provider + def self.refresh_provider!(provider_name) + # Get existing models from the instance + existing_models = instance.all + + # Find the specified provider and get its new models + provider = RubyLLM.providers.find { |p| p.to_s.downcase == "rubyllm::providers::#{provider_name.downcase}" } + new_provider_models = Array(provider&.list_models) + + # Replace models only for the specified provider + merged_models = existing_models.reject { |m| m.provider == provider_name.to_s } + new_provider_models + merged_models = merged_models.sort_by(&:id) + + @instance = new(merged_models) + end + # Delegate class methods to the singleton instance class << self def method_missing(method, ...) @@ -106,5 +122,12 @@ def refresh! # Return instance for method chaining self.class.instance end + + # Instance method to refresh models for a specific provider + def refresh_provider!(provider) + self.class.refresh_provider!(provider) + # Return instance for method chaining + self.class.instance + end end end diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 31bafe7b0..90e594d9e 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -73,19 +73,19 @@ def slug end def aws_region - ENV['AWS_REGION'] || 'us-east-1' + RubyLLM.config.bedrock_region end def aws_access_key_id - ENV.fetch('AWS_ACCESS_KEY_ID', nil) + RubyLLM.config.bedrock_api_key end def aws_secret_access_key - ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) + RubyLLM.config.bedrock_secret_key end def aws_session_token - ENV.fetch('AWS_SESSION_TOKEN', nil) + RubyLLM.config.bedrock_session_token end class Error < RubyLLM::Error; end diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index e39079c7d..da38e771c 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -9,7 +9,8 @@ def list_models @connection = nil # reset connection since base url is different @api_base = "https://bedrock.#{aws_region}.amazonaws.com" response = connection.get(models_url) do |req| - req.headers.merge! headers(method: :get, path: models_url) + req.headers.merge! headers(method: :get, + path: "#{connection.url_prefix}#{models_url}") end @connection = nil # reset connection since base url is different @@ -24,6 +25,8 @@ def models_url def parse_list_models_response(response, slug, capabilities) data = response.body['modelSummaries'] || [] + + data = data.filter { |model| model['modelId'].include?('claude') } data.map do |model| model_id = model['modelId'] ModelInfo.new( diff --git a/lib/tasks/models.rake b/lib/tasks/models.rake index 73a34ee0d..a82f14bab 100644 --- a/lib/tasks/models.rake +++ b/lib/tasks/models.rake @@ -61,32 +61,83 @@ def http_client end end -namespace :models do # rubocop:disable Metrics/BlockLength - desc 'Update available models from providers' - task :update do - require 'ruby_llm' +def configure_providers + # Configure API keys + RubyLLM.configure do |config| + config.openai_api_key = ENV.fetch('OPENAI_API_KEY', nil) + config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY', nil) + config.gemini_api_key = ENV.fetch('GEMINI_API_KEY', nil) + config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) + config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) + config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) + end +end - # Configure API keys - RubyLLM.configure do |config| - config.openai_api_key = ENV.fetch('OPENAI_API_KEY') - config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY') - config.gemini_api_key = ENV.fetch('GEMINI_API_KEY') - config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY') +namespace :models do + namespace :update do + desc 'Update all available models from providers' + task :all do + require 'ruby_llm' + configure_providers + + # Refresh all models + models = RubyLLM.models.refresh!.all + + # Write to models.json + File.write(File.expand_path('../ruby_llm/models.json', __dir__), JSON.pretty_generate(models.map(&:to_h))) + + puts "Updated models.json with #{models.size} models:" + RubyLLM::Provider.providers.each do |provider_sym, provider_module| + provider_name = provider_module.to_s.split('::').last + provider_models = models.select { |m| m.provider == provider_sym.to_s } + puts "#{provider_name} models: #{provider_models.size}" + end end - # Refresh models (now returns self instead of models array) - models = RubyLLM.models.refresh!.all - # Write to models.json - File.write(File.expand_path('../ruby_llm/models.json', __dir__), JSON.pretty_generate(models.map(&:to_h))) + # Define a task that will create provider-specific tasks at runtime + task :setup do + require 'ruby_llm' + + # Create a task for each provider + RubyLLM::Provider.providers.keys.each do |provider| + puts "Setting up task for provider: #{provider}" + + task_name = "models:update:#{provider}" + Rake::Task.define_task(task_name => :setup) do + puts "Running task for provider: #{provider}" + configure_providers + begin + models = RubyLLM.models.refresh_provider!(provider.to_s).all + # Write to models.json + File.write(File.expand_path('../ruby_llm/models.json', __dir__), JSON.pretty_generate(models.map(&:to_h))) + + provider_models = models.select { |m| m.provider == provider.to_s } + puts "#{provider} models: #{provider_models.size}" + rescue => e + puts "Error refreshing models for #{provider}: #{e.message}" + puts e.backtrace + end + end - puts "Updated models.json with #{models.size} models:" - RubyLLM::Provider.providers.each do |provider_sym, provider_module| - provider_name = provider_module.to_s.split('::').last - provider_models = models.select { |m| m.provider == provider_sym.to_s } - puts "#{provider_name} models: #{provider_models.size}" + # Add description for the task + Rake.application.last_description = "Update models for #{provider} provider" + end end + + # Make sure setup runs before any potential provider tasks + task :bedrock => :setup + task :openai => :setup + task :anthropic => :setup + task :gemini => :setup + task :deepseek => :setup end + # Make models:update an alias for models:update:all for backward compatibility + desc 'Update all available models from providers' + task update: 'update:all' + desc 'Update model capabilities modules by scraping provider documentation (use PROVIDER=name to update only one)' task :update_capabilities do # rubocop:disable Metrics/BlockLength # Check if a specific provider was requested From 728bbf8abb937aebc3e2cafc73be6f5ebb963f3f Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 17:11:16 -0700 Subject: [PATCH 13/85] 16: Refactor capabilities for consistency --- .../providers/bedrock/capabilities.rb | 300 ++++++++---------- 1 file changed, 137 insertions(+), 163 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/capabilities.rb b/lib/ruby_llm/providers/bedrock/capabilities.rb index 7f4847f1f..0c154666f 100644 --- a/lib/ruby_llm/providers/bedrock/capabilities.rb +++ b/lib/ruby_llm/providers/bedrock/capabilities.rb @@ -3,120 +3,56 @@ module RubyLLM module Providers module Bedrock - # Defines capabilities and pricing for AWS Bedrock models - module Capabilities + # Determines capabilities and pricing for AWS Bedrock models + module Capabilities # rubocop:disable Metrics/ModuleLength module_function - # Default prices per 1000 input tokens - DEFAULT_INPUT_PRICE = 0.0 - DEFAULT_OUTPUT_PRICE = 0.0 - - ANTHROPIC_CLAUDE_3_OPUS_INPUT_PRICE = 0.015 - ANTHROPIC_CLAUDE_3_OPUS_OUTPUT_PRICE = 0.075 - - ANTHROPIC_CLAUDE_3_SONNET_INPUT_PRICE = 0.003 - ANTHROPIC_CLAUDE_3_SONNET_OUTPUT_PRICE = 0.015 - - ANTHROPIC_CLAUDE_3_HAIKU_INPUT_PRICE = 0.0005 - ANTHROPIC_CLAUDE_3_HAIKU_OUTPUT_PRICE = 0.0025 - - ANTHROPIC_CLAUDE_2_INPUT_PRICE = 0.008 - ANTHROPIC_CLAUDE_2_OUTPUT_PRICE = 0.024 - - ANTHROPIC_CLAUDE_INSTANT_INPUT_PRICE = 0.0008 - ANTHROPIC_CLAUDE_INSTANT_OUTPUT_PRICE = 0.0024 - - COHERE_COMMAND_INPUT_PRICE = 0.0015 - COHERE_COMMAND_OUTPUT_PRICE = 0.0020 - - COHERE_COMMAND_LIGHT_INPUT_PRICE = 0.0003 - COHERE_COMMAND_LIGHT_OUTPUT_PRICE = 0.0006 - - META_LLAMA_70B_INPUT_PRICE = 0.00195 - META_LLAMA_70B_OUTPUT_PRICE = 0.00256 - - META_LLAMA_13B_INPUT_PRICE = 0.00075 - META_LLAMA_13B_OUTPUT_PRICE = 0.001 - - META_LLAMA_7B_INPUT_PRICE = 0.0004 - META_LLAMA_7B_OUTPUT_PRICE = 0.0005 - - MISTRAL_7B_INPUT_PRICE = 0.0002 - MISTRAL_7B_OUTPUT_PRICE = 0.0002 - - MISTRAL_MIXTRAL_INPUT_PRICE = 0.0007 - MISTRAL_MIXTRAL_OUTPUT_PRICE = 0.0007 - - MISTRAL_LARGE_INPUT_PRICE = 0.002 - MISTRAL_LARGE_OUTPUT_PRICE = 0.002 - - def self.input_price_for(model_id) + # Returns the context window size for the given model ID + # @param model_id [String] the model identifier + # @return [Integer] the context window size in tokens + def context_window_for(model_id) case model_id - when /anthropic\.claude-3-opus/ - ANTHROPIC_CLAUDE_3_OPUS_INPUT_PRICE - when /anthropic\.claude-3-sonnet/ - ANTHROPIC_CLAUDE_3_SONNET_INPUT_PRICE - when /anthropic\.claude-3-haiku/ - ANTHROPIC_CLAUDE_3_HAIKU_INPUT_PRICE - when /anthropic\.claude-v2/ - ANTHROPIC_CLAUDE_2_INPUT_PRICE - when /anthropic\.claude-instant/ - ANTHROPIC_CLAUDE_INSTANT_INPUT_PRICE - when /cohere\.command-text/ - COHERE_COMMAND_INPUT_PRICE - when /cohere\.command-light/ - COHERE_COMMAND_LIGHT_INPUT_PRICE - when /meta\.llama.*70b/i - META_LLAMA_70B_INPUT_PRICE - when /meta\.llama.*13b/i - META_LLAMA_13B_INPUT_PRICE - when /meta\.llama.*7b/i - META_LLAMA_7B_INPUT_PRICE - when /mistral\.mistral-7b/ - MISTRAL_7B_INPUT_PRICE - when /mistral\.mixtral/ - MISTRAL_MIXTRAL_INPUT_PRICE - when /mistral\.mistral-large/ - MISTRAL_LARGE_INPUT_PRICE - else - DEFAULT_INPUT_PRICE + when /anthropic\.claude-3-opus/ then 200_000 + when /anthropic\.claude-3-sonnet/ then 200_000 + when /anthropic\.claude-3-haiku/ then 200_000 + when /anthropic\.claude-2/ then 100_000 + when /meta\.llama/ then 4_096 + else 4_096 end end - def self.output_price_for(model_id) + # Returns the maximum output tokens for the given model ID + # @param model_id [String] the model identifier + # @return [Integer] the maximum output tokens + def max_tokens_for(model_id) case model_id - when /anthropic\.claude-3-opus/ - ANTHROPIC_CLAUDE_3_OPUS_OUTPUT_PRICE - when /anthropic\.claude-3-sonnet/ - ANTHROPIC_CLAUDE_3_SONNET_OUTPUT_PRICE - when /anthropic\.claude-3-haiku/ - ANTHROPIC_CLAUDE_3_HAIKU_OUTPUT_PRICE - when /anthropic\.claude-v2/ - ANTHROPIC_CLAUDE_2_OUTPUT_PRICE - when /anthropic\.claude-instant/ - ANTHROPIC_CLAUDE_INSTANT_OUTPUT_PRICE - when /cohere\.command-text/ - COHERE_COMMAND_OUTPUT_PRICE - when /cohere\.command-light/ - COHERE_COMMAND_LIGHT_OUTPUT_PRICE - when /meta\.llama.*70b/i - META_LLAMA_70B_OUTPUT_PRICE - when /meta\.llama.*13b/i - META_LLAMA_13B_OUTPUT_PRICE - when /meta\.llama.*7b/i - META_LLAMA_7B_OUTPUT_PRICE - when /mistral\.mistral-7b/ - MISTRAL_7B_OUTPUT_PRICE - when /mistral\.mixtral/ - MISTRAL_MIXTRAL_OUTPUT_PRICE - when /mistral\.mistral-large/ - MISTRAL_LARGE_OUTPUT_PRICE - else - DEFAULT_OUTPUT_PRICE + when /anthropic\.claude-3/ then 4096 + when /anthropic\.claude-v2/ then 4096 + when /anthropic\.claude-instant/ then 4096 + when /meta\.llama/ then 4096 + when /mistral\./ then 4096 + else 4096 end end - def self.supports_chat?(model_id) + # Returns the input price per million tokens for the given model ID + # @param model_id [String] the model identifier + # @return [Float] the price per million tokens for input + def input_price_for(model_id) + PRICES.dig(model_family(model_id), :input) || default_input_price + end + + # Returns the output price per million tokens for the given model ID + # @param model_id [String] the model identifier + # @return [Float] the price per million tokens for output + def output_price_for(model_id) + PRICES.dig(model_family(model_id), :output) || default_output_price + end + + # Determines if the model supports chat capabilities + # @param model_id [String] the model identifier + # @return [Boolean] true if the model supports chat + def supports_chat?(model_id) case model_id when /anthropic\.claude/, /cohere\.command/, /meta\.llama/, /mistral\./ true @@ -125,7 +61,10 @@ def self.supports_chat?(model_id) end end - def self.supports_streaming?(model_id) + # Determines if the model supports streaming capabilities + # @param model_id [String] the model identifier + # @return [Boolean] true if the model supports streaming + def supports_streaming?(model_id) case model_id when /anthropic\.claude/, /cohere\.command/, /meta\.llama/, /mistral\./ true @@ -134,7 +73,10 @@ def self.supports_streaming?(model_id) end end - def self.supports_images?(model_id) + # Determines if the model supports image input/output + # @param model_id [String] the model identifier + # @return [Boolean] true if the model supports images + def supports_images?(model_id) case model_id when /anthropic\.claude-3/, /meta\.llama3-2-(?:11b|90b)/ true @@ -143,24 +85,10 @@ def self.supports_images?(model_id) end end - def self.context_window_for(model_id) - case model_id - when /anthropic\.claude-3-opus/ - 200_000 - when /anthropic\.claude-3-sonnet/ - 200_000 - when /anthropic\.claude-3-haiku/ - 200_000 - when /anthropic\.claude-2/ - 100_000 - when /meta\.llama/ - 4_096 - else - 4_096 - end - end - - def self.supports_vision?(model_id) + # Determines if the model supports vision capabilities + # @param model_id [String] the model identifier + # @return [Boolean] true if the model supports vision + def supports_vision?(model_id) case model_id when /anthropic\.claude-3/, /meta\.llama3-2-(?:11b|90b)/ true @@ -169,80 +97,126 @@ def self.supports_vision?(model_id) end end - def self.max_tokens_for(model_id) - case model_id - when /anthropic\.claude-3/ - 4096 - when /anthropic\.claude-v2/ - 4096 - when /anthropic\.claude-instant/ - 4096 - when /meta\.llama/ - 4096 - when /mistral\./ - 4096 - else - 4096 - end - end - + # Determines if the model supports function calling + # @param model_id [String] the model identifier + # @return [Boolean] true if the model supports functions def supports_functions?(model_id) model_id.match?(/anthropic\.claude-3/) end + # Determines if the model supports audio input/output + # @param model_id [String] the model identifier + # @return [Boolean] true if the model supports audio def supports_audio?(_model_id) false end + # Determines if the model supports JSON mode + # @param model_id [String] the model identifier + # @return [Boolean] true if the model supports JSON mode def supports_json_mode?(model_id) model_id.match?(/anthropic\.claude-3/) end + # Formats the model ID into a human-readable display name + # @param model_id [String] the model identifier + # @return [String] the formatted display name def format_display_name(model_id) - case model_id - when /anthropic\.claude-3/ - 'Claude 3' - when /anthropic\.claude-v2/ - 'Claude 2' - when /anthropic\.claude-instant/ - 'Claude Instant' - when /amazon\.titan/ - 'Titan' - else - model_id - end + model_id.then { |id| humanize(id) } + .then { |name| apply_special_formatting(name) } end + # Determines the type of model + # @param model_id [String] the model identifier + # @return [String] the model type (chat, embedding, image, audio) def model_type(_model_id) 'chat' end + # Determines if the model supports structured output + # @param model_id [String] the model identifier + # @return [Boolean] true if the model supports structured output def supports_structured_output?(model_id) model_id.match?(/anthropic\.claude-3/) end + # Determines the model family for pricing and capability lookup + # @param model_id [String] the model identifier + # @return [Symbol] the model family identifier def model_family(model_id) case model_id - when /anthropic\.claude-3/ - :claude3 - when /anthropic\.claude-v2/ - :claude2 - when /anthropic\.claude-instant/ - :claude_instant - when /amazon\.titan/ - :titan - else - :other + when /anthropic\.claude-3-opus/ then :claude3_opus + when /anthropic\.claude-3-sonnet/ then :claude3_sonnet + when /anthropic\.claude-3-haiku/ then :claude3_haiku + when /anthropic\.claude-v2/ then :claude2 + when /anthropic\.claude-instant/ then :claude_instant + when /cohere\.command-text/ then :cohere_command + when /cohere\.command-light/ then :cohere_command_light + when /meta\.llama.*70b/i then :llama_70b + when /meta\.llama.*13b/i then :llama_13b + when /meta\.llama.*7b/i then :llama_7b + when /mistral\.mistral-7b/ then :mistral_7b + when /mistral\.mixtral/ then :mistral_mixtral + when /mistral\.mistral-large/ then :mistral_large + else :other end end + # Pricing information for Bedrock models (per million tokens) + PRICES = { + claude3_opus: { input: 15.0, output: 75.0 }, + claude3_sonnet: { input: 3.0, output: 15.0 }, + claude3_haiku: { input: 0.5, output: 2.5 }, + claude2: { input: 8.0, output: 24.0 }, + claude_instant: { input: 0.8, output: 2.4 }, + cohere_command: { input: 1.5, output: 2.0 }, + cohere_command_light: { input: 0.3, output: 0.6 }, + llama_70b: { input: 1.95, output: 2.56 }, + llama_13b: { input: 0.75, output: 1.0 }, + llama_7b: { input: 0.4, output: 0.5 }, + mistral_7b: { input: 0.2, output: 0.2 }, + mistral_mixtral: { input: 0.7, output: 0.7 }, + mistral_large: { input: 2.0, output: 2.0 } + }.freeze + + # Default input price when model-specific pricing is not available + # @return [Float] the default price per million tokens def default_input_price 0.1 end + # Default output price when model-specific pricing is not available + # @return [Float] the default price per million tokens def default_output_price 0.2 end + + private + + # Converts a model ID to a human-readable format + # @param id [String] the model identifier + # @return [String] the humanized model name + def humanize(id) + id.tr('-', ' ') + .split('.') + .last + .split + .map(&:capitalize) + .join(' ') + end + + # Applies special formatting rules to model names + # @param name [String] the humanized model name + # @return [String] the specially formatted model name + def apply_special_formatting(name) + name + .gsub(/Claude (\d)/, 'Claude \1') + .gsub(/Claude Instant/, 'Claude Instant') + .gsub(/Llama (\d+)b/i, 'Llama-\1B') + .gsub(/Mistral (\d+)b/i, 'Mistral-\1B') + .gsub(/Command Light/, 'Command-Light') + .gsub(/Command Text/, 'Command') + end end end end From 09aab0a38b92a777cfbfa6cefeb5292a11e3e781 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 17:13:51 -0700 Subject: [PATCH 14/85] 16: Remove some unneeded code --- lib/ruby_llm/providers/bedrock/chat.rb | 15 --------------- lib/ruby_llm/providers/bedrock/streaming.rb | 6 ------ 2 files changed, 21 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index 92eac83cd..c34e9a88d 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -61,21 +61,6 @@ def build_claude_request(messages, temperature, model_id) } end - def extract_content(data) - case data - when Hash - if data.key?('completion') - data['completion'] - elsif data.dig('results', 0, 'outputText') - data.dig('results', 0, 'outputText') - else - raise Error, "Unexpected response format: #{data.keys}" - end - else - raise Error, "Unexpected response type: #{data.class}" - end - end - def max_tokens_for(model_id) RubyLLM.models.find(model_id)&.max_tokens end diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index f9b969222..1dc5d51a6 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -131,12 +131,6 @@ def extract_content(data) else nil end - when Array - if data[0] == 'bytes' - data[1] - else - nil - end else nil end From 600151bd34b603d78d9ca45e60a1dde90733412b Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 19 Mar 2025 17:25:45 -0700 Subject: [PATCH 15/85] #16: Update claude family and pricing --- lib/ruby_llm/models.json | 152 +++++++++--------- .../providers/bedrock/capabilities.rb | 58 ++----- 2 files changed, 86 insertions(+), 124 deletions(-) diff --git a/lib/ruby_llm/models.json b/lib/ruby_llm/models.json index 2e4fc4c46..c00a93c1e 100644 --- a/lib/ruby_llm/models.json +++ b/lib/ruby_llm/models.json @@ -7,12 +7,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_5_haiku", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 0.8, + "output_price_per_million": 4.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -37,12 +37,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -67,12 +67,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -97,12 +97,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -127,12 +127,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -157,12 +157,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -187,12 +187,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -217,12 +217,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -247,12 +247,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -277,12 +277,12 @@ "context_window": 4096, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0, - "output_price_per_million": 0.0, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -307,12 +307,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_haiku", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0005, - "output_price_per_million": 0.0025, + "input_price_per_million": 0.25, + "output_price_per_million": 1.25, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -337,12 +337,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_haiku", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0005, - "output_price_per_million": 0.0025, + "input_price_per_million": 0.25, + "output_price_per_million": 1.25, "metadata": { "provider_name": "Anthropic", "customizations_supported": [ @@ -370,12 +370,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_haiku", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.0005, - "output_price_per_million": 0.0025, + "input_price_per_million": 0.25, + "output_price_per_million": 1.25, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -400,12 +400,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_opus", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.015, - "output_price_per_million": 0.075, + "input_price_per_million": 15.0, + "output_price_per_million": 75.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -430,12 +430,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_opus", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.015, - "output_price_per_million": 0.075, + "input_price_per_million": 15.0, + "output_price_per_million": 75.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -460,12 +460,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_opus", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.015, - "output_price_per_million": 0.075, + "input_price_per_million": 15.0, + "output_price_per_million": 75.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -490,12 +490,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_opus", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.015, - "output_price_per_million": 0.075, + "input_price_per_million": 15.0, + "output_price_per_million": 75.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -520,12 +520,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.003, - "output_price_per_million": 0.015, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -550,12 +550,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.003, - "output_price_per_million": 0.015, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -580,12 +580,12 @@ "context_window": 200000, "max_tokens": 4096, "type": "chat", - "family": "claude3", + "family": "claude3_sonnet", "supports_vision": true, "supports_functions": true, "supports_json_mode": true, - "input_price_per_million": 0.003, - "output_price_per_million": 0.015, + "input_price_per_million": 3.0, + "output_price_per_million": 15.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -614,8 +614,8 @@ "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.0008, - "output_price_per_million": 0.0024, + "input_price_per_million": 0.8, + "output_price_per_million": 2.4, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -643,8 +643,8 @@ "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.0008, - "output_price_per_million": 0.0024, + "input_price_per_million": 0.8, + "output_price_per_million": 2.4, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -672,8 +672,8 @@ "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, + "input_price_per_million": 8.0, + "output_price_per_million": 24.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -701,8 +701,8 @@ "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, + "input_price_per_million": 8.0, + "output_price_per_million": 24.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -730,8 +730,8 @@ "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, + "input_price_per_million": 8.0, + "output_price_per_million": 24.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -759,8 +759,8 @@ "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, + "input_price_per_million": 8.0, + "output_price_per_million": 24.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -788,8 +788,8 @@ "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, + "input_price_per_million": 8.0, + "output_price_per_million": 24.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], @@ -817,8 +817,8 @@ "supports_vision": false, "supports_functions": false, "supports_json_mode": false, - "input_price_per_million": 0.008, - "output_price_per_million": 0.024, + "input_price_per_million": 8.0, + "output_price_per_million": 24.0, "metadata": { "provider_name": "Anthropic", "customizations_supported": [], diff --git a/lib/ruby_llm/providers/bedrock/capabilities.rb b/lib/ruby_llm/providers/bedrock/capabilities.rb index 0c154666f..81f2d19b3 100644 --- a/lib/ruby_llm/providers/bedrock/capabilities.rb +++ b/lib/ruby_llm/providers/bedrock/capabilities.rb @@ -16,7 +16,6 @@ def context_window_for(model_id) when /anthropic\.claude-3-sonnet/ then 200_000 when /anthropic\.claude-3-haiku/ then 200_000 when /anthropic\.claude-2/ then 100_000 - when /meta\.llama/ then 4_096 else 4_096 end end @@ -29,8 +28,6 @@ def max_tokens_for(model_id) when /anthropic\.claude-3/ then 4096 when /anthropic\.claude-v2/ then 4096 when /anthropic\.claude-instant/ then 4096 - when /meta\.llama/ then 4096 - when /mistral\./ then 4096 else 4096 end end @@ -53,24 +50,14 @@ def output_price_for(model_id) # @param model_id [String] the model identifier # @return [Boolean] true if the model supports chat def supports_chat?(model_id) - case model_id - when /anthropic\.claude/, /cohere\.command/, /meta\.llama/, /mistral\./ - true - else - false - end + model_id.match?(/anthropic\.claude/) end # Determines if the model supports streaming capabilities # @param model_id [String] the model identifier # @return [Boolean] true if the model supports streaming def supports_streaming?(model_id) - case model_id - when /anthropic\.claude/, /cohere\.command/, /meta\.llama/, /mistral\./ - true - else - false - end + model_id.match?(/anthropic\.claude/) end # Determines if the model supports image input/output @@ -78,7 +65,7 @@ def supports_streaming?(model_id) # @return [Boolean] true if the model supports images def supports_images?(model_id) case model_id - when /anthropic\.claude-3/, /meta\.llama3-2-(?:11b|90b)/ + when /anthropic\.claude-3/ true else false @@ -90,7 +77,7 @@ def supports_images?(model_id) # @return [Boolean] true if the model supports vision def supports_vision?(model_id) case model_id - when /anthropic\.claude-3/, /meta\.llama3-2-(?:11b|90b)/ + when /anthropic\.claude-3/ true else false @@ -123,7 +110,6 @@ def supports_json_mode?(model_id) # @return [String] the formatted display name def format_display_name(model_id) model_id.then { |id| humanize(id) } - .then { |name| apply_special_formatting(name) } end # Determines the type of model @@ -147,17 +133,12 @@ def model_family(model_id) case model_id when /anthropic\.claude-3-opus/ then :claude3_opus when /anthropic\.claude-3-sonnet/ then :claude3_sonnet + when /anthropic\.claude-3-5-sonnet/ then :claude3_sonnet + when /anthropic\.claude-3-7-sonnet/ then :claude3_sonnet when /anthropic\.claude-3-haiku/ then :claude3_haiku + when /anthropic\.claude-3-5-haiku/ then :claude3_5_haiku when /anthropic\.claude-v2/ then :claude2 when /anthropic\.claude-instant/ then :claude_instant - when /cohere\.command-text/ then :cohere_command - when /cohere\.command-light/ then :cohere_command_light - when /meta\.llama.*70b/i then :llama_70b - when /meta\.llama.*13b/i then :llama_13b - when /meta\.llama.*7b/i then :llama_7b - when /mistral\.mistral-7b/ then :mistral_7b - when /mistral\.mixtral/ then :mistral_mixtral - when /mistral\.mistral-large/ then :mistral_large else :other end end @@ -166,17 +147,10 @@ def model_family(model_id) PRICES = { claude3_opus: { input: 15.0, output: 75.0 }, claude3_sonnet: { input: 3.0, output: 15.0 }, - claude3_haiku: { input: 0.5, output: 2.5 }, + claude3_haiku: { input: 0.25, output: 1.25 }, + claude3_5_haiku: { input: 0.8, output: 4.0 }, claude2: { input: 8.0, output: 24.0 }, - claude_instant: { input: 0.8, output: 2.4 }, - cohere_command: { input: 1.5, output: 2.0 }, - cohere_command_light: { input: 0.3, output: 0.6 }, - llama_70b: { input: 1.95, output: 2.56 }, - llama_13b: { input: 0.75, output: 1.0 }, - llama_7b: { input: 0.4, output: 0.5 }, - mistral_7b: { input: 0.2, output: 0.2 }, - mistral_mixtral: { input: 0.7, output: 0.7 }, - mistral_large: { input: 2.0, output: 2.0 } + claude_instant: { input: 0.8, output: 2.4 } }.freeze # Default input price when model-specific pricing is not available @@ -205,18 +179,6 @@ def humanize(id) .join(' ') end - # Applies special formatting rules to model names - # @param name [String] the humanized model name - # @return [String] the specially formatted model name - def apply_special_formatting(name) - name - .gsub(/Claude (\d)/, 'Claude \1') - .gsub(/Claude Instant/, 'Claude Instant') - .gsub(/Llama (\d+)b/i, 'Llama-\1B') - .gsub(/Mistral (\d+)b/i, 'Mistral-\1B') - .gsub(/Command Light/, 'Command-Light') - .gsub(/Command Text/, 'Command') - end end end end From 88c330438476ceb3bf6a8afbabf67efa6f75286c Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sat, 22 Mar 2025 22:02:37 -0700 Subject: [PATCH 16/85] Take a stab at an initial aliases file --- lib/ruby_llm/aliases.json | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 lib/ruby_llm/aliases.json diff --git a/lib/ruby_llm/aliases.json b/lib/ruby_llm/aliases.json new file mode 100644 index 000000000..f6cc8f089 --- /dev/null +++ b/lib/ruby_llm/aliases.json @@ -0,0 +1,38 @@ +{ + "claude-3-5-haiku": { + "anthropic": "claude-3-5-haiku-20241022", + "bedrock": "anthropic.claude-3-5-haiku-20241022-v1:0" + }, + "claude-3-5-sonnet-v2": { + "anthropic": "claude-3-5-sonnet-20241022", + "bedrock": "anthropic.claude-3-5-sonnet-20241022-v2:0" + }, + "claude-3-5-sonnet-v1": { + "anthropic": "claude-3-5-sonnet-20240620", + "bedrock": "anthropic.claude-3-5-sonnet-20240620-v1:0" + }, + "claude-3-7-sonnet": { + "anthropic": "claude-3-7-sonnet-20250219", + "bedrock": "anthropic.claude-3-7-sonnet-20250219-v1:0" + }, + "claude-3-haiku": { + "anthropic": "claude-3-haiku-20240307", + "bedrock": "anthropic.claude-3-haiku-20240307-v1:0" + }, + "claude-3-opus": { + "anthropic": "claude-3-opus-20240229", + "bedrock": "anthropic.claude-3-opus-20240229-v1:0" + }, + "claude-3-sonnet": { + "anthropic": "claude-3-sonnet-20240229", + "bedrock": "anthropic.claude-3-sonnet-20240229-v1:0" + }, + "claude-2.0": { + "anthropic": "claude-2.0", + "bedrock": "anthropic.claude-2.0" + }, + "claude-2.1": { + "anthropic": "claude-2.1", + "bedrock": "anthropic.claude-2.1" + } +} \ No newline at end of file From c2a9d2870bcc2d47737a0bdae61f21f78e09ba6f Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 15:13:49 -0700 Subject: [PATCH 17/85] Merge aliases.json --- lib/ruby_llm/aliases.json | 57 ++++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 15 deletions(-) diff --git a/lib/ruby_llm/aliases.json b/lib/ruby_llm/aliases.json index ad2b618a9..67f203a01 100644 --- a/lib/ruby_llm/aliases.json +++ b/lib/ruby_llm/aliases.json @@ -1,24 +1,16 @@ { - "claude-3-5-haiku": { - "anthropic": "claude-3-5-haiku-20241022", - "bedrock": "anthropic.claude-3-5-haiku-20241022-v1:0" - }, - "claude-3-5-sonnet-v2": { + "claude-3-5-sonnet": { "anthropic": "claude-3-5-sonnet-20241022", "bedrock": "anthropic.claude-3-5-sonnet-20241022-v2:0" }, - "claude-3-5-sonnet-v1": { - "anthropic": "claude-3-5-sonnet-20240620", - "bedrock": "anthropic.claude-3-5-sonnet-20240620-v1:0" + "claude-3-5-haiku": { + "anthropic": "claude-3-5-haiku-20241022", + "bedrock": "anthropic.claude-3-5-haiku-20241022-v1:0" }, "claude-3-7-sonnet": { "anthropic": "claude-3-7-sonnet-20250219", "bedrock": "anthropic.claude-3-7-sonnet-20250219-v1:0" }, - "claude-3-haiku": { - "anthropic": "claude-3-haiku-20240307", - "bedrock": "anthropic.claude-3-haiku-20240307-v1:0" - }, "claude-3-opus": { "anthropic": "claude-3-opus-20240229", "bedrock": "anthropic.claude-3-opus-20240229-v1:0" @@ -27,12 +19,47 @@ "anthropic": "claude-3-sonnet-20240229", "bedrock": "anthropic.claude-3-sonnet-20240229-v1:0" }, - "claude-2.0": { + "claude-3-haiku": { + "anthropic": "claude-3-haiku-20240307", + "bedrock": "anthropic.claude-3-haiku-20240307-v1:0" + }, + "claude-3": { + "anthropic": "claude-3-sonnet-20240229", + "bedrock": "anthropic.claude-3-sonnet-20240229-v1:0" + }, + "claude-2": { "anthropic": "claude-2.0", "bedrock": "anthropic.claude-2.0" }, - "claude-2.1": { + "claude-2-1": { "anthropic": "claude-2.1", "bedrock": "anthropic.claude-2.1" + }, + "gpt-4o": { + "openai": "gpt-4o-2024-11-20" + }, + "gpt-4o-mini": { + "openai": "gpt-4o-mini-2024-07-18" + }, + "gpt-4-turbo": { + "openai": "gpt-4-turbo-2024-04-09" + }, + "gemini-1.5-flash": { + "gemini": "gemini-1.5-flash-002" + }, + "gemini-1.5-flash-8b": { + "gemini": "gemini-1.5-flash-8b-001" + }, + "gemini-1.5-pro": { + "gemini": "gemini-1.5-pro-002" + }, + "gemini-2.0-flash": { + "gemini": "gemini-2.0-flash-001" + }, + "o1": { + "openai": "o1-2024-12-17" + }, + "o3-mini": { + "openai": "o3-mini-2025-01-31" } -} +} \ No newline at end of file From b96505bfbb7137cc24a2b4cbc14643ee3ccfda0a Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 16:17:40 -0700 Subject: [PATCH 18/85] Fix some small bedrock payload render issues --- lib/ruby_llm/providers/bedrock/chat.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index c34e9a88d..631583517 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -18,10 +18,10 @@ def model_id def render_payload(messages, tools:, temperature:, model:, stream: false) @model_id = model case model - when /anthropic\.claude/ + when /claude/ build_claude_request(messages, temperature, model) else - raise Error, "Unsupported model: #{model}" + raise Error, nil, "Unsupported model: #{model}" end end From 50684b16a97cf7e64c31fe86b877245a16128a45 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 16:18:17 -0700 Subject: [PATCH 19/85] Support switching model based on provider --- lib/ruby_llm/chat.rb | 10 ++++++---- lib/ruby_llm/models.rb | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/lib/ruby_llm/chat.rb b/lib/ruby_llm/chat.rb index 4827e9789..891aa5ba7 100644 --- a/lib/ruby_llm/chat.rb +++ b/lib/ruby_llm/chat.rb @@ -11,12 +11,13 @@ module RubyLLM class Chat include Enumerable - attr_reader :model, :messages, :tools + attr_reader :model, :model_alias, :messages, :tools def initialize(model: nil, provider: nil) model_id = model || RubyLLM.config.default_model - self.model = model_id self.provider = provider if provider + self.model = model_id + @model_alias = model_id @temperature = 0.7 @messages = [] @tools = {} @@ -49,8 +50,8 @@ def with_tools(*tools) end def model=(model_id) - @model = Models.find model_id - @provider = Models.provider_for model_id + @provider ||= Models.provider_for model_id + @model = Models.find model_id, @provider.slug end def provider=(provider_slug) @@ -60,6 +61,7 @@ def provider=(provider_slug) def with_provider(provider_slug) self.provider = provider_slug + self.model = model_alias self end diff --git a/lib/ruby_llm/models.rb b/lib/ruby_llm/models.rb index ee9a9daf6..de1f718f9 100644 --- a/lib/ruby_llm/models.rb +++ b/lib/ruby_llm/models.rb @@ -73,13 +73,13 @@ def each(&) end # Find a specific model by ID - def find(model_id) + def find(model_id, provider_slug = nil) # Try exact match first exact_match = all.find { |m| m.id == model_id } return exact_match if exact_match # Try to resolve via alias - resolved_id = Aliases.resolve(model_id) + resolved_id = Aliases.resolve(model_id, provider_slug) if resolved_id != model_id alias_match = all.find { |m| m.id == resolved_id } return alias_match if alias_match From 12029e8e8299e4d234d5ebc55716e623d5e9b988 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 16:29:31 -0700 Subject: [PATCH 20/85] Add doc on bedrock usage --- docs/guides/bedrock.md | 111 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) create mode 100644 docs/guides/bedrock.md diff --git a/docs/guides/bedrock.md b/docs/guides/bedrock.md new file mode 100644 index 000000000..8f6cfa9a0 --- /dev/null +++ b/docs/guides/bedrock.md @@ -0,0 +1,111 @@ +--- +layout: default +title: Using AWS Bedrock +parent: Guides +nav_order: 10 +permalink: /guides/bedrock +--- + +# Using AWS Bedrock + +RubyLLM supports AWS Bedrock as a model provider, giving you access to Claude models (and eventually others) through your AWS infrastructure. This guide explains how to configure and use AWS Bedrock with RubyLLM. + +## Configuration + +To use AWS Bedrock, you'll need to configure RubyLLM with your AWS credentials. There are several ways to do this: + +### Using AWS STS (Security Token Service) + +If you're using temporary credentials through AWS STS, you can configure RubyLLM like this: + +```ruby +require 'aws-sdk-core' +require 'ruby_llm' + +# Get credentials from STS +sts_client = Aws::STS::Client.new +creds = sts_client.get_session_token + +RubyLLM.configure do |config| + config.bedrock_api_key = creds.credentials.access_key_id + config.bedrock_secret_key = creds.credentials.secret_access_key + config.bedrock_session_token = creds.credentials.session_token + config.bedrock_region = 'us-west-2' # Specify your desired AWS region +end +``` + +### Using Environment Variables + +You can also configure AWS credentials through environment variables: + +```bash +export AWS_ACCESS_KEY_ID=your_access_key +export AWS_SECRET_ACCESS_KEY=your_secret_key +export AWS_SESSION_TOKEN=your_session_token # If using temporary credentials +export AWS_REGION=us-west-2 +``` +```ruby +RubyLLM.configure do |config| + config.bedrock_api_key = ENV['AWS_ACCESS_KEY_ID'] + config.bedrock_secret_key = ENV['AWS_SECRET_ACCESS_KEY'] + config.bedrock_session_token = ENV['AWS_SESSION_TOKEN'] # If using temporary credentials + config.bedrock_region = ENV['AWS_REGION'] +end +``` + +## Using Bedrock Models + +There are two ways to specify that you want to use Bedrock as your provider: + +### Method 1: Specify Provider at Initialization + +```ruby +chat = RubyLLM.chat( + model: 'claude-3-5-sonnet', + provider: :bedrock +) + +response = chat.ask('Hello, how are you?') +puts response.content +``` + +### Method 2: Use the Provider Chain Method + +```ruby +chat = RubyLLM.chat( + model: 'claude-3-5-sonnet' +).with_provider(:bedrock) + +response = chat.ask('Hello, how are you?') +puts response.content +``` + +## Available Models + +AWS Bedrock provides access to various models. You can list available Bedrock models using: + +```ruby +bedrock_models = RubyLLM.models.by_provider('bedrock') +bedrock_models.each do |model| + puts "#{model.id} (#{model.display_name})" +end +``` + +## Best Practices + +When using AWS Bedrock with RubyLLM: + +1. **Credential Management** - Always follow AWS best practices for credential management +2. **Region Selection** - Choose the AWS region closest to your application for best performance +3. **Error Handling** - Handle AWS-specific errors appropriately in your application +4. **Cost Monitoring** - Monitor your AWS Bedrock usage through AWS Console or CloudWatch + +## Troubleshooting + +Common issues and solutions: + +- **Authentication Errors**: Ensure your AWS credentials are properly configured and have the necessary permissions for Bedrock +- **Region Issues**: Verify that Bedrock is available in your selected AWS region +- **Token Expiration**: When using temporary credentials, make sure to refresh them before they expire + +For more information about AWS Bedrock, visit the [AWS Bedrock documentation](https://docs.aws.amazon.com/bedrock). From f4901c01a9a92036bd395741a2256cf3c99037b9 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 16:30:44 -0700 Subject: [PATCH 21/85] Improve var name in example --- docs/guides/bedrock.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/guides/bedrock.md b/docs/guides/bedrock.md index 8f6cfa9a0..778c301e8 100644 --- a/docs/guides/bedrock.md +++ b/docs/guides/bedrock.md @@ -24,12 +24,12 @@ require 'ruby_llm' # Get credentials from STS sts_client = Aws::STS::Client.new -creds = sts_client.get_session_token +session_token = sts_client.get_session_token RubyLLM.configure do |config| - config.bedrock_api_key = creds.credentials.access_key_id - config.bedrock_secret_key = creds.credentials.secret_access_key - config.bedrock_session_token = creds.credentials.session_token + config.bedrock_api_key = session_token.credentials.access_key_id + config.bedrock_secret_key = session_token.credentials.secret_access_key + config.bedrock_session_token = session_token.credentials.session_token config.bedrock_region = 'us-west-2' # Specify your desired AWS region end ``` From 3ac131334c1caaf084392d87008ca882ceaa684a Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 17:06:53 -0700 Subject: [PATCH 22/85] Add us. to Claude 3.7 model ID to use cross-region inference profile --- lib/ruby_llm/aliases.json | 2 +- lib/ruby_llm/models.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ruby_llm/aliases.json b/lib/ruby_llm/aliases.json index 67f203a01..18b9a5707 100644 --- a/lib/ruby_llm/aliases.json +++ b/lib/ruby_llm/aliases.json @@ -9,7 +9,7 @@ }, "claude-3-7-sonnet": { "anthropic": "claude-3-7-sonnet-20250219", - "bedrock": "anthropic.claude-3-7-sonnet-20250219-v1:0" + "bedrock": "us.anthropic.claude-3-7-sonnet-20250219-v1:0" }, "claude-3-opus": { "anthropic": "claude-3-opus-20240229", diff --git a/lib/ruby_llm/models.json b/lib/ruby_llm/models.json index f6cdf64aa..f47f7b60a 100644 --- a/lib/ruby_llm/models.json +++ b/lib/ruby_llm/models.json @@ -270,7 +270,7 @@ } }, { - "id": "anthropic.claude-3-7-sonnet-20250219-v1:0", + "id": "us.anthropic.claude-3-7-sonnet-20250219-v1:0", "created_at": null, "display_name": "Claude 3.7 Sonnet", "provider": "bedrock", From 91ece398e0e9add965f30a903f17ea111ab781cb Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 17:13:43 -0700 Subject: [PATCH 23/85] Undo rake changes that should probably be a separate PR --- lib/tasks/models.rake | 66 +------------------------------------------ 1 file changed, 1 insertion(+), 65 deletions(-) diff --git a/lib/tasks/models.rake b/lib/tasks/models.rake index 8d536a7b2..3bbe31496 100644 --- a/lib/tasks/models.rake +++ b/lib/tasks/models.rake @@ -88,70 +88,6 @@ namespace :models do # rubocop:disable Metrics/BlockLength puts "#{provider_name} models: #{provider_models.size}" end end -end - -namespace :models do - namespace :update do - desc 'Update all available models from providers' - task :all do - require 'ruby_llm' - configure_providers - - # Refresh all models - models = RubyLLM.models.refresh!.all - - # Write to models.json - File.write(File.expand_path('../ruby_llm/models.json', __dir__), JSON.pretty_generate(models.map(&:to_h))) - - puts "Updated models.json with #{models.size} models:" - RubyLLM::Provider.providers.each do |provider_sym, provider_module| - provider_name = provider_module.to_s.split('::').last - provider_models = models.select { |m| m.provider == provider_sym.to_s } - puts "#{provider_name} models: #{provider_models.size}" - end - end - - # Define a task that will create provider-specific tasks at runtime - task :setup do - require 'ruby_llm' - - # Create a task for each provider - RubyLLM::Provider.providers.keys.each do |provider| - puts "Setting up task for provider: #{provider}" - - task_name = "models:update:#{provider}" - Rake::Task.define_task(task_name => :setup) do - puts "Running task for provider: #{provider}" - configure_providers - begin - models = RubyLLM.models.refresh_provider!(provider.to_s).all - # Write to models.json - File.write(File.expand_path('../ruby_llm/models.json', __dir__), JSON.pretty_generate(models.map(&:to_h))) - - provider_models = models.select { |m| m.provider == provider.to_s } - puts "#{provider} models: #{provider_models.size}" - rescue => e - puts "Error refreshing models for #{provider}: #{e.message}" - puts e.backtrace - end - end - - # Add description for the task - Rake.application.last_description = "Update models for #{provider} provider" - end - end - - # Make sure setup runs before any potential provider tasks - task :bedrock => :setup - task :openai => :setup - task :anthropic => :setup - task :gemini => :setup - task :deepseek => :setup - end - - # Make models:update an alias for models:update:all for backward compatibility - desc 'Update all available models from providers' - task update: 'update:all' desc 'Update model capabilities modules by scraping provider documentation (use PROVIDER=name to update only one)' task :update_capabilities do # rubocop:disable Metrics/BlockLength @@ -273,4 +209,4 @@ namespace :models do puts "Done! Don't forget to review the generated code and run the tests." end -end +end \ No newline at end of file From 6d46985cc0471009255fa28400b534a2595844fe Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 17:14:24 -0700 Subject: [PATCH 24/85] Blank line --- lib/tasks/models.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/models.rake b/lib/tasks/models.rake index 3bbe31496..f3f46672c 100644 --- a/lib/tasks/models.rake +++ b/lib/tasks/models.rake @@ -209,4 +209,4 @@ namespace :models do # rubocop:disable Metrics/BlockLength puts "Done! Don't forget to review the generated code and run the tests." end -end \ No newline at end of file +end From b07687bda2a5340b8fc2cb43e566248e1f874aac Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 17:18:30 -0700 Subject: [PATCH 25/85] bundle exec rubocop -A --- lib/ruby_llm/error.rb | 3 +- .../providers/bedrock/capabilities.rb | 8 +- lib/ruby_llm/providers/bedrock/chat.rb | 7 +- lib/ruby_llm/providers/bedrock/decoder.rb | 40 ++++---- lib/ruby_llm/providers/bedrock/models.rb | 2 +- lib/ruby_llm/providers/bedrock/signing.rb | 94 ++++++++----------- lib/ruby_llm/providers/bedrock/streaming.rb | 27 +++--- lib/ruby_llm/providers/bedrock/tools.rb | 2 +- 8 files changed, 82 insertions(+), 101 deletions(-) diff --git a/lib/ruby_llm/error.rb b/lib/ruby_llm/error.rb index 970cbbbe1..e886078ca 100644 --- a/lib/ruby_llm/error.rb +++ b/lib/ruby_llm/error.rb @@ -58,7 +58,8 @@ def parse_error(provider:, response:) # rubocop:disable Metrics/CyclomaticComple when 402 raise PaymentRequiredError.new(response, message || 'Payment required - please top up your account') when 403 - raise ForbiddenError.new(response, message || 'Forbidden - you do not have permission to access this resource') + raise ForbiddenError.new(response, + message || 'Forbidden - you do not have permission to access this resource') when 429 raise RateLimitError.new(response, message || 'Rate limit exceeded - please wait a moment') when 500 diff --git a/lib/ruby_llm/providers/bedrock/capabilities.rb b/lib/ruby_llm/providers/bedrock/capabilities.rb index 81f2d19b3..fd7666df5 100644 --- a/lib/ruby_llm/providers/bedrock/capabilities.rb +++ b/lib/ruby_llm/providers/bedrock/capabilities.rb @@ -25,11 +25,8 @@ def context_window_for(model_id) # @return [Integer] the maximum output tokens def max_tokens_for(model_id) case model_id - when /anthropic\.claude-3/ then 4096 - when /anthropic\.claude-v2/ then 4096 - when /anthropic\.claude-instant/ then 4096 - else 4096 end +4096 end # Returns the input price per million tokens for the given model ID @@ -178,8 +175,7 @@ def humanize(id) .map(&:capitalize) .join(' ') end - end end end -end \ No newline at end of file +end diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index 631583517..dca8d67e3 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -49,10 +49,10 @@ def build_claude_request(messages, temperature, model_id) end.join { - anthropic_version: "bedrock-2023-05-31", + anthropic_version: 'bedrock-2023-05-31', messages: [ { - role: "user", + role: 'user', content: formatted } ], @@ -64,8 +64,7 @@ def build_claude_request(messages, temperature, model_id) def max_tokens_for(model_id) RubyLLM.models.find(model_id)&.max_tokens end - end end end -end \ No newline at end of file +end diff --git a/lib/ruby_llm/providers/bedrock/decoder.rb b/lib/ruby_llm/providers/bedrock/decoder.rb index 93b692fec..356b07348 100644 --- a/lib/ruby_llm/providers/bedrock/decoder.rb +++ b/lib/ruby_llm/providers/bedrock/decoder.rb @@ -44,6 +44,7 @@ def decode(io, &block) raw_message = io.read decoded_message = decode_message(raw_message) return wrap_as_enumerator(decoded_message) unless block_given? + # fetch message only raw_event, _eof = decoded_message block.call(raw_event) @@ -87,9 +88,7 @@ def decode_message(raw_message) return [nil, true] if raw_message.bytesize < total_length content, checksum, remaining = content.unpack("a#{total_length - PRELUDE_LENGTH - CRC32_LENGTH}Na*") - unless Zlib.crc32([prelude, content].pack('a*a*')) == checksum - raise Error, "Message checksum error" - end + raise Error, 'Message checksum error' unless Zlib.crc32([prelude, content].pack('a*a*')) == checksum # decode headers and payload headers, payload = decode_context(content, header_length) @@ -103,7 +102,8 @@ def decode_prelude(prelude) # prelude contains length of message and headers, # followed with CRC checksum of itself content, checksum = prelude.unpack("a#{PRELUDE_LENGTH - CRC32_LENGTH}N") - raise Error, "Prelude checksum error" unless Zlib.crc32(content) == checksum + raise Error, 'Prelude checksum error' unless Zlib.crc32(content) == checksum + content.unpack('N*') end @@ -118,7 +118,7 @@ def decode_context(content, header_length) def extract_headers(buffer) scanner = buffer headers = {} - until scanner.bytesize == 0 + until scanner.bytesize.zero? # header key key_length, scanner = scanner.unpack('Ca*') key, scanner = scanner.unpack("a#{key_length}a*") @@ -127,14 +127,14 @@ def extract_headers(buffer) type_index, scanner = scanner.unpack('Ca*') value_type = Types.types[type_index] unpack_pattern, value_length = Types.pattern[value_type] - value = if !!unpack_pattern == unpack_pattern - # boolean types won't have value specified - unpack_pattern - else - value_length, scanner = scanner.unpack('S>a*') unless value_length - unpacked_value, scanner = scanner.unpack("#{unpack_pattern || "a#{value_length}"}a*") - unpacked_value - end + value = if !unpack_pattern.nil? == unpack_pattern + # boolean types won't have value specified + unpack_pattern + else + value_length, scanner = scanner.unpack('S>a*') unless value_length + unpacked_value, scanner = scanner.unpack("#{unpack_pattern || "a#{value_length}"}a*") + unpacked_value + end headers[key] = HeaderValue.new( format: @format, @@ -146,9 +146,11 @@ def extract_headers(buffer) end def extract_payload(encoded) - encoded.bytesize <= ONE_MEGABYTE ? - payload_stringio(encoded) : + if encoded.bytesize <= ONE_MEGABYTE + payload_stringio(encoded) + else payload_tempfile(encoded) + end end def payload_stringio(encoded) @@ -188,8 +190,8 @@ def initialize(format:, value:, type:) module Types def self.types { - 0 => :true, - 1 => :false, + 0 => true, + 1 => false, 2 => :byte, 3 => :short, 4 => :integer, @@ -203,8 +205,8 @@ def self.types def self.pattern { - true: [true, 0], - false: [false, 0], + true => [true, 0], + false => [false, 0], byte: ['c', 1], short: ['s>', 2], integer: ['l>', 4], diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index da38e771c..3412cadc4 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -57,4 +57,4 @@ def parse_list_models_response(response, slug, capabilities) end end end -end \ No newline at end of file +end diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 027d9d7e7..f543e61e0 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -12,7 +12,6 @@ module RubyLLM module Providers module Bedrock module Signing - # Utility class for creating AWS signature version 4 signature. This class # provides two methods for generating signatures: # @@ -76,23 +75,23 @@ module Signing module Errors class MissingCredentialsError < ArgumentError def initialize(msg = nil) - super(msg || <<-MSG.strip) -missing credentials, provide credentials with one of the following options: - - :access_key_id and :secret_access_key - - :credentials - - :credentials_provider + super(msg || <<~MSG.strip) + missing credentials, provide credentials with one of the following options: + - :access_key_id and :secret_access_key + - :credentials + - :credentials_provider MSG end end class MissingRegionError < ArgumentError - def initialize(*args) - super("missing required option :region") + def initialize(*_args) + super('missing required option :region') end end end - class Signature + class Signature # @api private def initialize(options) options.each_pair do |attr_name, attr_value| @@ -127,8 +126,8 @@ def initialize(options) # @return [Hash] Internal data for debugging purposes. attr_accessor :extra end - class Credentials + class Credentials # @option options [required, String] :access_key_id # @option options [required, String] :secret_access_key # @option options [String, nil] :session_token (nil) @@ -138,7 +137,7 @@ def initialize(options = {}) @secret_access_key = options[:secret_access_key] @session_token = options[:session_token] else - msg = "expected both :access_key_id and :secret_access_key options" + msg = 'expected both :access_key_id and :secret_access_key options' raise ArgumentError, msg end end @@ -161,19 +160,17 @@ def set? !secret_access_key.empty? end end + # Users that wish to configure static credentials can use the # `:access_key_id` and `:secret_access_key` constructor options. # @api private class StaticCredentialsProvider - # @option options [Credentials] :credentials # @option options [String] :access_key_id # @option options [String] :secret_access_key # @option options [String] :session_token (nil) def initialize(options = {}) - @credentials = options[:credentials] ? - options[:credentials] : - Credentials.new(options) + @credentials = options[:credentials] || Credentials.new(options) end # @return [Credentials] @@ -248,7 +245,7 @@ def initialize(options = {}) @service = extract_service(options) @region = extract_region(options) @credentials_provider = extract_credentials_provider(options) - @unsigned_headers = Set.new((options.fetch(:unsigned_headers, [])).map(&:downcase)) + @unsigned_headers = Set.new(options.fetch(:unsigned_headers, []).map(&:downcase)) @unsigned_headers << 'authorization' @unsigned_headers << 'x-amzn-trace-id' @unsigned_headers << 'expect' @@ -332,7 +329,7 @@ def initialize(options = {}) # a `#headers` method. The headers must be applied to your request. # def sign_request(request) - creds, _ = fetch_credentials + creds, = fetch_credentials http_method = extract_http_method(request) url = extract_url(request) @@ -340,7 +337,7 @@ def sign_request(request) headers = downcase_headers(request[:headers]) datetime = headers['x-amz-date'] - datetime ||= Time.now.utc.strftime("%Y%m%dT%H%M%SZ") + datetime ||= Time.now.utc.strftime('%Y%m%dT%H%M%SZ') date = datetime[0, 8] content_sha256 = headers['x-amz-content-sha256'] @@ -350,7 +347,7 @@ def sign_request(request) sigv4_headers['host'] = headers['host'] || host(url) sigv4_headers['x-amz-date'] = datetime if creds.session_token && !@omit_session_token - if @signing_algorithm == 'sigv4-s3express'.to_sym + if @signing_algorithm == :'sigv4-s3express' sigv4_headers['x-amz-s3session-token'] = creds.session_token else sigv4_headers['x-amz-security-token'] = creds.session_token @@ -359,9 +356,7 @@ def sign_request(request) sigv4_headers['x-amz-content-sha256'] ||= content_sha256 if @apply_checksum_header - if @signing_algorithm == :sigv4a && @region && !@region.empty? - sigv4_headers['x-amz-region-set'] = @region - end + sigv4_headers['x-amz-region-set'] = @region if @signing_algorithm == :sigv4a && @region && !@region.empty? headers = headers.merge(sigv4_headers) # merge so we do not modify given headers hash algorithm = sts_algorithm @@ -383,13 +378,11 @@ def sign_request(request) sigv4_headers['authorization'] = [ "#{algorithm} Credential=#{credential(creds, date)}", "SignedHeaders=#{signed_headers(headers)}", - "Signature=#{sig}", + "Signature=#{sig}" ].join(', ') # skip signing the session token, but include it in the headers - if creds.session_token && @omit_session_token - sigv4_headers['x-amz-security-token'] = creds.session_token - end + sigv4_headers['x-amz-security-token'] = creds.session_token if creds.session_token && @omit_session_token # Returning the signature components. Signature.new( @@ -412,9 +405,9 @@ def canonical_request(http_method, url, headers, content_sha256) http_method, path(url), normalized_querystring(url.query || ''), - canonical_headers(headers) + "\n", + "#{canonical_headers(headers)}\n", signed_headers(headers), - content_sha256, + content_sha256 ].join("\n") end @@ -423,7 +416,7 @@ def string_to_sign(datetime, canonical_request, algorithm) algorithm, datetime, credential_scope(datetime[0, 8]), - sha256_hexdigest(canonical_request), + sha256_hexdigest(canonical_request) ].join("\n") end @@ -441,7 +434,7 @@ def credential(credentials, date) end def signature(secret_access_key, date, string_to_sign) - k_date = hmac("AWS4" + secret_access_key, date) + k_date = hmac("AWS4#{secret_access_key}", date) k_region = hmac(k_date, @region) k_service = hmac(k_region, @service) k_credentials = hmac(k_service, 'aws4_request') @@ -449,7 +442,7 @@ def signature(secret_access_key, date, string_to_sign) end def asymmetric_signature(creds, string_to_sign) - ec, _ = Aws::Sigv4::AsymmetricCredentials.derive_asymmetric_key( + ec, = Aws::Sigv4::AsymmetricCredentials.derive_asymmetric_key( creds.access_key_id, creds.secret_access_key ) sts_digest = OpenSSL::Digest::SHA256.digest(string_to_sign) @@ -470,7 +463,7 @@ def path(url) def normalized_querystring(querystring) params = querystring.split('&') - params = params.map { |p| p.match(/=/) ? p : p + '=' } + params = params.map { |p| p.match(/=/) ? p : "#{p}=" } # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html # Sort the parameter names by character code point in ascending order. # Parameters with duplicate names should be sorted by value. @@ -535,13 +528,14 @@ def host(uri) # @param [File, Tempfile, IO#read, String] value # @return [String] def sha256_hexdigest(value) - if (File === value || Tempfile === value) && !value.path.nil? && File.exist?(value.path) + if (value.is_a?(File) || value.is_a?(Tempfile)) && !value.path.nil? && File.exist?(value.path) OpenSSL::Digest::SHA256.file(value).hexdigest elsif value.respond_to?(:read) - sha256 = OpenSSL::Digest::SHA256.new + sha256 = OpenSSL::Digest.new('SHA256') loop do chunk = value.read(1024 * 1024) # 1MB break unless chunk + sha256.update(chunk) end value.rewind @@ -563,17 +557,15 @@ def extract_service(options) if options[:service] options[:service] else - msg = "missing required option :service" + msg = 'missing required option :service' raise ArgumentError, msg end end def extract_region(options) - if options[:region] - options[:region] - else - raise Errors::MissingRegionError - end + raise Errors::MissingRegionError unless options[:region] + + options[:region] end def extract_credentials_provider(options) @@ -590,7 +582,7 @@ def extract_http_method(request) if request[:http_method] request[:http_method].upcase else - msg = "missing required option :http_method" + msg = 'missing required option :http_method' raise ArgumentError, msg end end @@ -599,16 +591,13 @@ def extract_url(request) if request[:url] URI.parse(request[:url].to_s) else - msg = "missing required option :url" + msg = 'missing required option :url' raise ArgumentError, msg end end def downcase_headers(headers) - (headers || {}).to_hash.inject({}) do |hash, (key, value)| - hash[key.downcase] = value - hash - end + (headers || {}).to_hash.transform_keys(&:downcase) end def extract_expires_in(options) @@ -616,7 +605,7 @@ def extract_expires_in(options) when nil then 900 when Integer then options[:expires_in] else - msg = "expected :expires_in to be a number of seconds" + msg = 'expected :expires_in to be a number of seconds' raise ArgumentError, msg end end @@ -633,9 +622,7 @@ def fetch_credentials credentials = @credentials_provider.credentials if credentials_set?(credentials) expiration = nil - if @credentials_provider.respond_to?(:expiration) - expiration = @credentials_provider.expiration - end + expiration = @credentials_provider.expiration if @credentials_provider.respond_to?(:expiration) [credentials, expiration] else raise Errors::MissingCredentialsError, @@ -670,7 +657,6 @@ def presigned_url_expiration(options, expiration, datetime) end class << self - # Kept for backwards compatability # Always return false since we are not using crt signing functionality def use_crt? @@ -679,7 +665,7 @@ def use_crt? # @api private def uri_escape_path(path) - path.gsub(/[^\/]+/) { |part| uri_escape(part) } + path.gsub(%r{[^/]+}) { |part| uri_escape(part) } end # @api private @@ -698,9 +684,7 @@ def normalize_path(uri) # resolve to "." and should be disregarded normalized_path = '' if normalized_path == '.' # Ensure trailing slashes are correctly preserved - if uri.path.end_with?('/') && !normalized_path.end_with?('/') - normalized_path << '/' - end + normalized_path << '/' if uri.path.end_with?('/') && !normalized_path.end_with?('/') uri.path = normalized_path end end diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index 1dc5d51a6..02e040f4f 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -1,4 +1,5 @@ # frozen_string_literal: true + require 'base64' module RubyLLM @@ -15,7 +16,7 @@ def stream_url def handle_stream(&block) decoder = Decoder.new buffer = String.new - accumulated_content = "" + accumulated_content = '' proc do |chunk, _bytes, env| if env && env.status != 200 @@ -32,21 +33,21 @@ def handle_stream(&block) else # Process the chunk using the AWS EventStream decoder begin - message, is_complete = decoder.decode_chunk(chunk) - + message, = decoder.decode_chunk(chunk) + if message payload = message.payload.read parsed_data = nil - + begin if payload.is_a?(String) # Try parsing as JSON json_data = JSON.parse(payload) - + # Handle Base64 encoded bytes - if json_data.is_a?(Hash) && json_data["bytes"] + if json_data.is_a?(Hash) && json_data['bytes'] # Decode the Base64 string - decoded_bytes = Base64.strict_decode64(json_data["bytes"]) + decoded_bytes = Base64.strict_decode64(json_data['bytes']) # Parse the decoded JSON parsed_data = JSON.parse(decoded_bytes) else @@ -66,11 +67,11 @@ def handle_stream(&block) # Extract content based on the event type content = extract_streaming_content(parsed_data) - + # Only emit a chunk if there's content to emit unless content.nil? || content.empty? accumulated_content += content - + block.call( Chunk.new( role: :assistant, @@ -95,7 +96,7 @@ def extract_streaming_content(data) case data['type'] when 'message_start' # No content yet in message_start - "" + '' when 'content_block_start' # Initial content block, might have some text data.dig('content_block', 'text').to_s @@ -104,7 +105,7 @@ def extract_streaming_content(data) data.dig('delta', 'text').to_s when 'message_delta' # Might contain updates to usage stats, but no new content - "" + '' else # Fall back to the existing extract_content method for other formats extract_content(data) @@ -124,15 +125,13 @@ def extract_content(data) elsif data.dig('results', 0, 'outputText') data.dig('results', 0, 'outputText') elsif data.key?('content') - data['content'].is_a?(Array) ? data['content'].map { |item| item['text'] }.join('') : data['content'] + data['content'].is_a?(Array) ? data['content'].map { |item| item['text'] }.join : data['content'] elsif data.key?('content_block') && data['content_block'].key?('text') # Handle the newly decoded JSON structure data['content_block']['text'] else nil end - else - nil end end end diff --git a/lib/ruby_llm/providers/bedrock/tools.rb b/lib/ruby_llm/providers/bedrock/tools.rb index 62d62c4a2..afadda9f5 100644 --- a/lib/ruby_llm/providers/bedrock/tools.rb +++ b/lib/ruby_llm/providers/bedrock/tools.rb @@ -49,4 +49,4 @@ def parse_arguments(arguments) end end end -end \ No newline at end of file +end From fa54026b50377c7a3fc86b57cc837ab547a04bee Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Sun, 23 Mar 2025 17:24:18 -0700 Subject: [PATCH 26/85] Resolve simpler rubocop offenses --- lib/ruby_llm/providers/bedrock/capabilities.rb | 8 +++----- lib/ruby_llm/providers/bedrock/signing.rb | 5 +++++ lib/ruby_llm/providers/bedrock/streaming.rb | 2 -- lib/ruby_llm/providers/bedrock/tools.rb | 3 ++- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/capabilities.rb b/lib/ruby_llm/providers/bedrock/capabilities.rb index fd7666df5..d081d870d 100644 --- a/lib/ruby_llm/providers/bedrock/capabilities.rb +++ b/lib/ruby_llm/providers/bedrock/capabilities.rb @@ -4,7 +4,7 @@ module RubyLLM module Providers module Bedrock # Determines capabilities and pricing for AWS Bedrock models - module Capabilities # rubocop:disable Metrics/ModuleLength + module Capabilities module_function # Returns the context window size for the given model ID @@ -23,10 +23,8 @@ def context_window_for(model_id) # Returns the maximum output tokens for the given model ID # @param model_id [String] the model identifier # @return [Integer] the maximum output tokens - def max_tokens_for(model_id) - case model_id - end -4096 + def max_tokens_for(_model_id) + 4096 end # Returns the input price per million tokens for the given model ID diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index f543e61e0..e1b3846b0 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -73,6 +73,7 @@ module Signing # returning another object that responds to `#access_key_id`, `#secret_access_key`, # and `#session_token`. module Errors + # Error raised when AWS credentials are missing or incomplete class MissingCredentialsError < ArgumentError def initialize(msg = nil) super(msg || <<~MSG.strip) @@ -84,6 +85,7 @@ def initialize(msg = nil) end end + # Error raised when AWS region is not specified class MissingRegionError < ArgumentError def initialize(*_args) super('missing required option :region') @@ -91,6 +93,7 @@ def initialize(*_args) end end + # Represents a signature for AWS request signing class Signature # @api private def initialize(options) @@ -127,6 +130,7 @@ def initialize(options) attr_accessor :extra end + # Manages AWS credentials for authentication class Credentials # @option options [required, String] :access_key_id # @option options [required, String] :secret_access_key @@ -182,6 +186,7 @@ def set? end end + # Handles AWS request signing using SigV4 or SigV4a class Signer # @overload initialize(service:, region:, access_key_id:, secret_access_key:, session_token:nil, **options) # @param [String] :service The service signing name, e.g. 's3'. diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index 02e040f4f..c02306252 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -129,8 +129,6 @@ def extract_content(data) elsif data.key?('content_block') && data['content_block'].key?('text') # Handle the newly decoded JSON structure data['content_block']['text'] - else - nil end end end diff --git a/lib/ruby_llm/providers/bedrock/tools.rb b/lib/ruby_llm/providers/bedrock/tools.rb index afadda9f5..ad4d70a16 100644 --- a/lib/ruby_llm/providers/bedrock/tools.rb +++ b/lib/ruby_llm/providers/bedrock/tools.rb @@ -22,10 +22,11 @@ def parse_tool_calls(tool_calls, parse_arguments: true) return {} unless tool_calls tool_calls.each_with_object({}) do |call, hash| + function_args = call['function']['arguments'] hash[call['id']] = ToolCall.new( id: call['id'], name: call['function']['name'], - arguments: parse_arguments ? parse_arguments(call['function']['arguments']) : call['function']['arguments'] + arguments: parse_arguments ? parse_arguments(function_args) : function_args ) end end From e9d99e2bb9ac36edc60b56e8e7030eb5bd7b0710 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 09:23:52 -0700 Subject: [PATCH 27/85] Remove unneeded change after merge --- lib/ruby_llm/chat.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ruby_llm/chat.rb b/lib/ruby_llm/chat.rb index 67b7ecc26..9c6d29419 100644 --- a/lib/ruby_llm/chat.rb +++ b/lib/ruby_llm/chat.rb @@ -11,7 +11,7 @@ module RubyLLM class Chat include Enumerable - attr_reader :model, :model_alias, :messages, :tools + attr_reader :model, :messages, :tools def initialize(model: nil, provider: nil) model_id = model || RubyLLM.config.default_model From a02c1ec1adb0d0019c8132d348e0c85755b71857 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 09:25:03 -0700 Subject: [PATCH 28/85] Remove unneeded guide --- docs/guides/bedrock.md | 111 ----------------------------------------- 1 file changed, 111 deletions(-) delete mode 100644 docs/guides/bedrock.md diff --git a/docs/guides/bedrock.md b/docs/guides/bedrock.md deleted file mode 100644 index 778c301e8..000000000 --- a/docs/guides/bedrock.md +++ /dev/null @@ -1,111 +0,0 @@ ---- -layout: default -title: Using AWS Bedrock -parent: Guides -nav_order: 10 -permalink: /guides/bedrock ---- - -# Using AWS Bedrock - -RubyLLM supports AWS Bedrock as a model provider, giving you access to Claude models (and eventually others) through your AWS infrastructure. This guide explains how to configure and use AWS Bedrock with RubyLLM. - -## Configuration - -To use AWS Bedrock, you'll need to configure RubyLLM with your AWS credentials. There are several ways to do this: - -### Using AWS STS (Security Token Service) - -If you're using temporary credentials through AWS STS, you can configure RubyLLM like this: - -```ruby -require 'aws-sdk-core' -require 'ruby_llm' - -# Get credentials from STS -sts_client = Aws::STS::Client.new -session_token = sts_client.get_session_token - -RubyLLM.configure do |config| - config.bedrock_api_key = session_token.credentials.access_key_id - config.bedrock_secret_key = session_token.credentials.secret_access_key - config.bedrock_session_token = session_token.credentials.session_token - config.bedrock_region = 'us-west-2' # Specify your desired AWS region -end -``` - -### Using Environment Variables - -You can also configure AWS credentials through environment variables: - -```bash -export AWS_ACCESS_KEY_ID=your_access_key -export AWS_SECRET_ACCESS_KEY=your_secret_key -export AWS_SESSION_TOKEN=your_session_token # If using temporary credentials -export AWS_REGION=us-west-2 -``` -```ruby -RubyLLM.configure do |config| - config.bedrock_api_key = ENV['AWS_ACCESS_KEY_ID'] - config.bedrock_secret_key = ENV['AWS_SECRET_ACCESS_KEY'] - config.bedrock_session_token = ENV['AWS_SESSION_TOKEN'] # If using temporary credentials - config.bedrock_region = ENV['AWS_REGION'] -end -``` - -## Using Bedrock Models - -There are two ways to specify that you want to use Bedrock as your provider: - -### Method 1: Specify Provider at Initialization - -```ruby -chat = RubyLLM.chat( - model: 'claude-3-5-sonnet', - provider: :bedrock -) - -response = chat.ask('Hello, how are you?') -puts response.content -``` - -### Method 2: Use the Provider Chain Method - -```ruby -chat = RubyLLM.chat( - model: 'claude-3-5-sonnet' -).with_provider(:bedrock) - -response = chat.ask('Hello, how are you?') -puts response.content -``` - -## Available Models - -AWS Bedrock provides access to various models. You can list available Bedrock models using: - -```ruby -bedrock_models = RubyLLM.models.by_provider('bedrock') -bedrock_models.each do |model| - puts "#{model.id} (#{model.display_name})" -end -``` - -## Best Practices - -When using AWS Bedrock with RubyLLM: - -1. **Credential Management** - Always follow AWS best practices for credential management -2. **Region Selection** - Choose the AWS region closest to your application for best performance -3. **Error Handling** - Handle AWS-specific errors appropriately in your application -4. **Cost Monitoring** - Monitor your AWS Bedrock usage through AWS Console or CloudWatch - -## Troubleshooting - -Common issues and solutions: - -- **Authentication Errors**: Ensure your AWS credentials are properly configured and have the necessary permissions for Bedrock -- **Region Issues**: Verify that Bedrock is available in your selected AWS region -- **Token Expiration**: When using temporary credentials, make sure to refresh them before they expire - -For more information about AWS Bedrock, visit the [AWS Bedrock documentation](https://docs.aws.amazon.com/bedrock). From 683604cba72a15133644789c363aeb12030dcbc3 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 09:28:31 -0700 Subject: [PATCH 29/85] Add new optional keys to config --- README.md | 12 ++++++++---- bin/console | 4 ++++ docs/guides/getting-started.md | 13 +++++++++---- docs/guides/rails.md | 12 ++++++++---- docs/installation.md | 12 ++++++++---- spec/spec_helper.rb | 4 ++++ 6 files changed, 41 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f7da4bbd9..6225f735a 100644 --- a/README.md +++ b/README.md @@ -99,10 +99,14 @@ Configure with your API keys: ```ruby RubyLLM.configure do |config| - config.openai_api_key = ENV['OPENAI_API_KEY'] - config.anthropic_api_key = ENV['ANTHROPIC_API_KEY'] - config.gemini_api_key = ENV['GEMINI_API_KEY'] - config.deepseek_api_key = ENV['DEEPSEEK_API_KEY'] + config.openai_api_key = ENV.fetch('OPENAI_API_KEY', nil) + config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY', nil) + config.gemini_api_key = ENV.fetch('GEMINI_API_KEY', nil) + config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) + config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) + config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end ``` diff --git a/bin/console b/bin/console index e43cf3d8a..f3c2b322a 100755 --- a/bin/console +++ b/bin/console @@ -12,6 +12,10 @@ RubyLLM.configure do |config| config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY', nil) config.gemini_api_key = ENV.fetch('GEMINI_API_KEY', nil) config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) + config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) + config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end IRB.start(__FILE__) diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index ff6cb1c0b..4428f908b 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -26,10 +26,15 @@ require 'ruby_llm' RubyLLM.configure do |config| # Add the API keys you have available - config.openai_api_key = ENV['OPENAI_API_KEY'] - config.anthropic_api_key = ENV['ANTHROPIC_API_KEY'] - config.gemini_api_key = ENV['GEMINI_API_KEY'] - config.deepseek_api_key = ENV['DEEPSEEK_API_KEY'] + config.openai_api_key = ENV.fetch('OPENAI_API_KEY', nil) + config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY', nil) + config.gemini_api_key = ENV.fetch('GEMINI_API_KEY', nil) + config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) + config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) + config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) + end ``` diff --git a/docs/guides/rails.md b/docs/guides/rails.md index ec36e4070..8cf783355 100644 --- a/docs/guides/rails.md +++ b/docs/guides/rails.md @@ -92,10 +92,14 @@ In an initializer (e.g., `config/initializers/ruby_llm.rb`): ```ruby RubyLLM.configure do |config| - config.openai_api_key = ENV['OPENAI_API_KEY'] - config.anthropic_api_key = ENV['ANTHROPIC_API_KEY'] - config.gemini_api_key = ENV['GEMINI_API_KEY'] - config.deepseek_api_key = ENV['DEEPSEEK_API_KEY'] + config.openai_api_key = ENV.fetch('OPENAI_API_KEY', nil) + config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY', nil) + config.gemini_api_key = ENV.fetch('GEMINI_API_KEY', nil) + config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) + config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) + config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end ``` diff --git a/docs/installation.md b/docs/installation.md index aac4f32fb..510977c22 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -51,10 +51,14 @@ require 'ruby_llm' RubyLLM.configure do |config| # Required: At least one API key - config.openai_api_key = ENV['OPENAI_API_KEY'] - config.anthropic_api_key = ENV['ANTHROPIC_API_KEY'] - config.gemini_api_key = ENV['GEMINI_API_KEY'] - config.deepseek_api_key = ENV['DEEPSEEK_API_KEY'] + config.openai_api_key = ENV.fetch('OPENAI_API_KEY', nil) + config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY', nil) + config.gemini_api_key = ENV.fetch('GEMINI_API_KEY', nil) + config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) + config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) + config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) # Optional: Set default models config.default_model = 'gpt-4o-mini' # Default chat model diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 26aea5dfd..f6b14d419 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -88,6 +88,10 @@ config.anthropic_api_key = ENV.fetch('ANTHROPIC_API_KEY', 'test') config.gemini_api_key = ENV.fetch('GEMINI_API_KEY', 'test') config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', 'test') + config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', 'test') + config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', 'test') + config.bedrock_region = ENV.fetch('AWS_REGION', 'test') + config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', 'test') config.max_retries = 50 end end From fd1a77703995e050003171b296d05ce183f5cf0f Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 09:29:51 -0700 Subject: [PATCH 30/85] Remove as we will not extend Bedrock elsewhere --- lib/ruby_llm/providers/bedrock.rb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 90e594d9e..dbf64974b 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -15,15 +15,6 @@ module Bedrock extend Bedrock::Tools extend Bedrock::Signing - def self.extended(base) - base.extend(Provider) - base.extend(Bedrock::Chat) - base.extend(Bedrock::Streaming) - base.extend(Bedrock::Models) - base.extend(Bedrock::Tools) - base.extend(Bedrock::Signing) - end - module_function def api_base From ed5319883c8e2c99261ff11e58b4b63a66bbbb0f Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 10:15:18 -0700 Subject: [PATCH 31/85] Add config requirements and remove extra error class --- lib/ruby_llm/providers/bedrock.rb | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index dbf64974b..0619a50d8 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -79,7 +79,9 @@ def aws_session_token RubyLLM.config.bedrock_session_token end - class Error < RubyLLM::Error; end + def configuration_requirements + %i[bedrock_api_key bedrock_secret_key bedrock_region] + end end end end From e2cbbf4121ed9359afcfa5466035ee4ea7eb0d28 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 10:17:14 -0700 Subject: [PATCH 32/85] Add bedrock call to cassettes for models_refresh tests --- ...nd_returns_a_chainable_models_instance.yml | 124 ++++++++++++++++++ ...ls_refresh_works_as_a_class_method_too.yml | 124 ++++++++++++++++++ 2 files changed, 248 insertions(+) diff --git a/spec/fixtures/vcr_cassettes/models_refresh_updates_models_and_returns_a_chainable_models_instance.yml b/spec/fixtures/vcr_cassettes/models_refresh_updates_models_and_returns_a_chainable_models_instance.yml index 507916bbc..4208a097b 100644 --- a/spec/fixtures/vcr_cassettes/models_refresh_updates_models_and_returns_a_chainable_models_instance.yml +++ b/spec/fixtures/vcr_cassettes/models_refresh_updates_models_and_returns_a_chainable_models_instance.yml @@ -1227,4 +1227,128 @@ http_interactions: encoding: ASCII-8BIT string: '{"object":"list","data":[{"id":"deepseek-chat","object":"model","owned_by":"deepseek"},{"id":"deepseek-reasoner","object":"model","owned_by":"deepseek"}]}' recorded_at: Sat, 22 Mar 2025 11:07:37 GMT +- request: + method: get + uri: https://bedrock.us-west-2.amazonaws.com/foundation-models + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T171540Z + X-Amz-Security-Token: + - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i + X-Amz-Content-Sha256: + - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + Authorization: + - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=e4a35761abf5be047cdd3c2b4383e1f3b30031acb59a6254960f1ef94b1a939a + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 17:14:59 GMT + Content-Type: + - application/json + Content-Length: + - '45249' + Connection: + - keep-alive + X-Amzn-Requestid: + - '08c81519-bf5d-4c0f-8dad-e1fe0e12c7fa' + body: + encoding: UTF-8 + string: '{"modelSummaries":[{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-tg1-large","modelId":"amazon.titan-tg1-large","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text Large","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-g1-text-02","modelId":"amazon.titan-embed-g1-text-02","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text Embeddings v2","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":["FINE_TUNING","CONTINUED_PRE_TRAINING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-text-lite-v1:0:4k","modelId":"amazon.titan-text-lite-v1:0:4k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text G1 - Lite","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-text-lite-v1","modelId":"amazon.titan-text-lite-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text G1 - Lite","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING","CONTINUED_PRE_TRAINING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-text-express-v1:0:8k","modelId":"amazon.titan-text-express-v1:0:8k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text G1 - Express","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-text-express-v1","modelId":"amazon.titan-text-express-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text G1 - Express","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE","VIDEO"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.nova-pro-v1:0","modelId":"amazon.nova-pro-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Nova + Pro","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE","VIDEO"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.nova-lite-v1:0","modelId":"amazon.nova-lite-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Nova + Lite","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.nova-micro-v1:0","modelId":"amazon.nova-micro-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Nova + Micro","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-text-v1:2:8k","modelId":"amazon.titan-embed-text-v1:2:8k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Embeddings G1 - Text","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-text-v1","modelId":"amazon.titan-embed-text-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Embeddings G1 - Text","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-text-v2:0","modelId":"amazon.titan-embed-text-v2:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text Embeddings V2","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":false},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-image-v1:0","modelId":"amazon.titan-embed-image-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Multimodal Embeddings G1","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-image-v1","modelId":"amazon.titan-embed-image-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Multimodal Embeddings G1","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-image-generator-v1:0","modelId":"amazon.titan-image-generator-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Image Generator G1","outputModalities":["IMAGE"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-image-generator-v1","modelId":"amazon.titan-image-generator-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Image Generator G1","outputModalities":["IMAGE"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED","ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-image-generator-v2:0","modelId":"amazon.titan-image-generator-v2:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Image Generator G1 v2","outputModalities":["IMAGE"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0","modelId":"amazon.rerank-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Rerank + 1.0","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-diffusion-xl-v1:0","modelId":"stability.stable-diffusion-xl-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"SDXL + 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-diffusion-xl-v1","modelId":"stability.stable-diffusion-xl-v1","modelLifecycle":{"status":"LEGACY"},"modelName":"SDXL + 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.sd3-large-v1:0","modelId":"stability.sd3-large-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"SD3 + Large 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.sd3-5-large-v1:0","modelId":"stability.sd3-5-large-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Stable + Diffusion 3.5 Large","outputModalities":["IMAGE"],"providerName":"Stability + AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-image-core-v1:0","modelId":"stability.stable-image-core-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"Stable + Image Core 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-image-core-v1:1","modelId":"stability.stable-image-core-v1:1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Stable + Image Core 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-image-ultra-v1:0","modelId":"stability.stable-image-ultra-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"Stable + Image Ultra 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-image-ultra-v1:1","modelId":"stability.stable-image-ultra-v1:1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Stable + Image Ultra 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20241022-v2:0:18k","modelId":"anthropic.claude-3-5-sonnet-20241022-v2:0:18k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet v2","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20241022-v2:0:51k","modelId":"anthropic.claude-3-5-sonnet-20241022-v2:0:51k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet v2","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20241022-v2:0:200k","modelId":"anthropic.claude-3-5-sonnet-20241022-v2:0:200k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet v2","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20241022-v2:0","modelId":"anthropic.claude-3-5-sonnet-20241022-v2:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet v2","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0","modelId":"anthropic.claude-3-7-sonnet-20250219-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.7 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-haiku-20241022-v1:0","modelId":"anthropic.claude-3-5-haiku-20241022-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Haiku","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-instant-v1:2:100k","modelId":"anthropic.claude-instant-v1:2:100k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + Instant","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-instant-v1","modelId":"anthropic.claude-instant-v1","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + Instant","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:0:18k","modelId":"anthropic.claude-v2:0:18k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:0:100k","modelId":"anthropic.claude-v2:0:100k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:1:18k","modelId":"anthropic.claude-v2:1:18k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:1:200k","modelId":"anthropic.claude-v2:1:200k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:1","modelId":"anthropic.claude-v2:1","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2","modelId":"anthropic.claude-v2","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-sonnet-20240229-v1:0:28k","modelId":"anthropic.claude-3-sonnet-20240229-v1:0:28k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + 3 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-sonnet-20240229-v1:0:200k","modelId":"anthropic.claude-3-sonnet-20240229-v1:0:200k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + 3 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-sonnet-20240229-v1:0","modelId":"anthropic.claude-3-sonnet-20240229-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + 3 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-haiku-20240307-v1:0:48k","modelId":"anthropic.claude-3-haiku-20240307-v1:0:48k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Haiku","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING","DISTILLATION"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-haiku-20240307-v1:0:200k","modelId":"anthropic.claude-3-haiku-20240307-v1:0:200k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Haiku","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-haiku-20240307-v1:0","modelId":"anthropic.claude-3-haiku-20240307-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Haiku","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-opus-20240229-v1:0:12k","modelId":"anthropic.claude-3-opus-20240229-v1:0:12k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Opus","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-opus-20240229-v1:0:28k","modelId":"anthropic.claude-3-opus-20240229-v1:0:28k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Opus","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-opus-20240229-v1:0:200k","modelId":"anthropic.claude-3-opus-20240229-v1:0:200k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Opus","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-opus-20240229-v1:0","modelId":"anthropic.claude-3-opus-20240229-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Opus","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20240620-v1:0:18k","modelId":"anthropic.claude-3-5-sonnet-20240620-v1:0:18k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20240620-v1:0:51k","modelId":"anthropic.claude-3-5-sonnet-20240620-v1:0:51k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20240620-v1:0:200k","modelId":"anthropic.claude-3-5-sonnet-20240620-v1:0:200k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND","INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20240620-v1:0","modelId":"anthropic.claude-3-5-sonnet-20240620-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-text-v14:7:4k","modelId":"cohere.command-text-v14:7:4k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-text-v14","modelId":"cohere.command-text-v14","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-r-v1:0","modelId":"cohere.command-r-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command + R","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-r-plus-v1:0","modelId":"cohere.command-r-plus-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command + R+","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-light-text-v14:7:4k","modelId":"cohere.command-light-text-v14:7:4k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command + Light","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-light-text-v14","modelId":"cohere.command-light-text-v14","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command + Light","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-english-v3:0:512","modelId":"cohere.embed-english-v3:0:512","modelLifecycle":{"status":"ACTIVE"},"modelName":"Embed + English","outputModalities":["EMBEDDING"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-english-v3","modelId":"cohere.embed-english-v3","modelLifecycle":{"status":"ACTIVE"},"modelName":"Embed + English","outputModalities":["EMBEDDING"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-multilingual-v3:0:512","modelId":"cohere.embed-multilingual-v3:0:512","modelLifecycle":{"status":"ACTIVE"},"modelName":"Embed + Multilingual","outputModalities":["EMBEDDING"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-multilingual-v3","modelId":"cohere.embed-multilingual-v3","modelLifecycle":{"status":"ACTIVE"},"modelName":"Embed + Multilingual","outputModalities":["EMBEDDING"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.rerank-v3-5:0","modelId":"cohere.rerank-v3-5:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Rerank + 3.5","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/deepseek.r1-v1:0","modelId":"deepseek.r1-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"DeepSeek-R1","outputModalities":["TEXT"],"providerName":"DeepSeek","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-8b-instruct-v1:0","modelId":"meta.llama3-8b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3 8B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-70b-instruct-v1:0","modelId":"meta.llama3-70b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3 70B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING","DISTILLATION"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-8b-instruct-v1:0:128k","modelId":"meta.llama3-1-8b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 8B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-8b-instruct-v1:0","modelId":"meta.llama3-1-8b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 8B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING","DISTILLATION"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-70b-instruct-v1:0:128k","modelId":"meta.llama3-1-70b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 70B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-70b-instruct-v1:0","modelId":"meta.llama3-1-70b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 70B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-405b-instruct-v1:0","modelId":"meta.llama3-1-405b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 405B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-11b-instruct-v1:0:128k","modelId":"meta.llama3-2-11b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 11B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-11b-instruct-v1:0","modelId":"meta.llama3-2-11b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 11B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-90b-instruct-v1:0:128k","modelId":"meta.llama3-2-90b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 90B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-90b-instruct-v1:0","modelId":"meta.llama3-2-90b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 90B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-1b-instruct-v1:0:128k","modelId":"meta.llama3-2-1b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 1B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-1b-instruct-v1:0","modelId":"meta.llama3-2-1b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 1B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-3b-instruct-v1:0:128k","modelId":"meta.llama3-2-3b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 3B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-3b-instruct-v1:0","modelId":"meta.llama3-2-3b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 3B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-3-70b-instruct-v1:0","modelId":"meta.llama3-3-70b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.3 70B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/mistral.mistral-7b-instruct-v0:2","modelId":"mistral.mistral-7b-instruct-v0:2","modelLifecycle":{"status":"ACTIVE"},"modelName":"Mistral + 7B Instruct","outputModalities":["TEXT"],"providerName":"Mistral AI","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/mistral.mixtral-8x7b-instruct-v0:1","modelId":"mistral.mixtral-8x7b-instruct-v0:1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Mixtral + 8x7B Instruct","outputModalities":["TEXT"],"providerName":"Mistral AI","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/mistral.mistral-large-2402-v1:0","modelId":"mistral.mistral-large-2402-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Mistral + Large (24.02)","outputModalities":["TEXT"],"providerName":"Mistral AI","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/mistral.mistral-large-2407-v1:0","modelId":"mistral.mistral-large-2407-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Mistral + Large (24.07)","outputModalities":["TEXT"],"providerName":"Mistral AI","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/luma.ray-v2:0","modelId":"luma.ray-v2:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Ray + v2","outputModalities":["VIDEO"],"providerName":"Luma AI","responseStreamingSupported":false}]}' + recorded_at: Tue, 25 Mar 2025 17:15:40 GMT recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/models_refresh_works_as_a_class_method_too.yml b/spec/fixtures/vcr_cassettes/models_refresh_works_as_a_class_method_too.yml index 1739461ce..cbb284a6e 100644 --- a/spec/fixtures/vcr_cassettes/models_refresh_works_as_a_class_method_too.yml +++ b/spec/fixtures/vcr_cassettes/models_refresh_works_as_a_class_method_too.yml @@ -1227,4 +1227,128 @@ http_interactions: encoding: ASCII-8BIT string: '{"object":"list","data":[{"id":"deepseek-chat","object":"model","owned_by":"deepseek"},{"id":"deepseek-reasoner","object":"model","owned_by":"deepseek"}]}' recorded_at: Sat, 22 Mar 2025 11:07:38 GMT +- request: + method: get + uri: https://bedrock.us-west-2.amazonaws.com/foundation-models + body: + encoding: US-ASCII + string: '' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T171540Z + X-Amz-Security-Token: + - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i + X-Amz-Content-Sha256: + - e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 + Authorization: + - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=e4a35761abf5be047cdd3c2b4383e1f3b30031acb59a6254960f1ef94b1a939a + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 17:14:59 GMT + Content-Type: + - application/json + Content-Length: + - '45236' + Connection: + - keep-alive + X-Amzn-Requestid: + - 1d0095b0-65f3-4c04-82e3-8e4af99316cd + body: + encoding: UTF-8 + string: '{"modelSummaries":[{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-tg1-large","modelId":"amazon.titan-tg1-large","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text Large","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-g1-text-02","modelId":"amazon.titan-embed-g1-text-02","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text Embeddings v2","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":["FINE_TUNING","CONTINUED_PRE_TRAINING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-text-lite-v1:0:4k","modelId":"amazon.titan-text-lite-v1:0:4k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text G1 - Lite","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-text-lite-v1","modelId":"amazon.titan-text-lite-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text G1 - Lite","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING","CONTINUED_PRE_TRAINING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-text-express-v1:0:8k","modelId":"amazon.titan-text-express-v1:0:8k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text G1 - Express","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-text-express-v1","modelId":"amazon.titan-text-express-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text G1 - Express","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE","VIDEO"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.nova-pro-v1:0","modelId":"amazon.nova-pro-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Nova + Pro","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE","VIDEO"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.nova-lite-v1:0","modelId":"amazon.nova-lite-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Nova + Lite","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.nova-micro-v1:0","modelId":"amazon.nova-micro-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Nova + Micro","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-text-v1:2:8k","modelId":"amazon.titan-embed-text-v1:2:8k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Embeddings G1 - Text","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-text-v1","modelId":"amazon.titan-embed-text-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Embeddings G1 - Text","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-text-v2:0","modelId":"amazon.titan-embed-text-v2:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Text Embeddings V2","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":false},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-image-v1:0","modelId":"amazon.titan-embed-image-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Multimodal Embeddings G1","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-embed-image-v1","modelId":"amazon.titan-embed-image-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Multimodal Embeddings G1","outputModalities":["EMBEDDING"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-image-generator-v1:0","modelId":"amazon.titan-image-generator-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Image Generator G1","outputModalities":["IMAGE"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-image-generator-v1","modelId":"amazon.titan-image-generator-v1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Image Generator G1","outputModalities":["IMAGE"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED","ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.titan-image-generator-v2:0","modelId":"amazon.titan-image-generator-v2:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Titan + Image Generator G1 v2","outputModalities":["IMAGE"],"providerName":"Amazon","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/amazon.rerank-v1:0","modelId":"amazon.rerank-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Rerank + 1.0","outputModalities":["TEXT"],"providerName":"Amazon","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-diffusion-xl-v1:0","modelId":"stability.stable-diffusion-xl-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"SDXL + 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-diffusion-xl-v1","modelId":"stability.stable-diffusion-xl-v1","modelLifecycle":{"status":"LEGACY"},"modelName":"SDXL + 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.sd3-large-v1:0","modelId":"stability.sd3-large-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"SD3 + Large 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.sd3-5-large-v1:0","modelId":"stability.sd3-5-large-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Stable + Diffusion 3.5 Large","outputModalities":["IMAGE"],"providerName":"Stability + AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-image-core-v1:0","modelId":"stability.stable-image-core-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"Stable + Image Core 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-image-core-v1:1","modelId":"stability.stable-image-core-v1:1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Stable + Image Core 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-image-ultra-v1:0","modelId":"stability.stable-image-ultra-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"Stable + Image Ultra 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/stability.stable-image-ultra-v1:1","modelId":"stability.stable-image-ultra-v1:1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Stable + Image Ultra 1.0","outputModalities":["IMAGE"],"providerName":"Stability AI","responseStreamingSupported":null},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20241022-v2:0:18k","modelId":"anthropic.claude-3-5-sonnet-20241022-v2:0:18k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet v2","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20241022-v2:0:51k","modelId":"anthropic.claude-3-5-sonnet-20241022-v2:0:51k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet v2","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20241022-v2:0:200k","modelId":"anthropic.claude-3-5-sonnet-20241022-v2:0:200k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet v2","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20241022-v2:0","modelId":"anthropic.claude-3-5-sonnet-20241022-v2:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet v2","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-7-sonnet-20250219-v1:0","modelId":"anthropic.claude-3-7-sonnet-20250219-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.7 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":true},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-haiku-20241022-v1:0","modelId":"anthropic.claude-3-5-haiku-20241022-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Haiku","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-instant-v1:2:100k","modelId":"anthropic.claude-instant-v1:2:100k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + Instant","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-instant-v1","modelId":"anthropic.claude-instant-v1","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + Instant","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:0:18k","modelId":"anthropic.claude-v2:0:18k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:0:100k","modelId":"anthropic.claude-v2:0:100k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:1:18k","modelId":"anthropic.claude-v2:1:18k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:1:200k","modelId":"anthropic.claude-v2:1:200k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2:1","modelId":"anthropic.claude-v2:1","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-v2","modelId":"anthropic.claude-v2","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-sonnet-20240229-v1:0:28k","modelId":"anthropic.claude-3-sonnet-20240229-v1:0:28k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + 3 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-sonnet-20240229-v1:0:200k","modelId":"anthropic.claude-3-sonnet-20240229-v1:0:200k","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + 3 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-sonnet-20240229-v1:0","modelId":"anthropic.claude-3-sonnet-20240229-v1:0","modelLifecycle":{"status":"LEGACY"},"modelName":"Claude + 3 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-haiku-20240307-v1:0:48k","modelId":"anthropic.claude-3-haiku-20240307-v1:0:48k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Haiku","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING","DISTILLATION"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-haiku-20240307-v1:0:200k","modelId":"anthropic.claude-3-haiku-20240307-v1:0:200k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Haiku","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-haiku-20240307-v1:0","modelId":"anthropic.claude-3-haiku-20240307-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Haiku","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-opus-20240229-v1:0:12k","modelId":"anthropic.claude-3-opus-20240229-v1:0:12k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Opus","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-opus-20240229-v1:0:28k","modelId":"anthropic.claude-3-opus-20240229-v1:0:28k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Opus","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-opus-20240229-v1:0:200k","modelId":"anthropic.claude-3-opus-20240229-v1:0:200k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Opus","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-opus-20240229-v1:0","modelId":"anthropic.claude-3-opus-20240229-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3 Opus","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20240620-v1:0:18k","modelId":"anthropic.claude-3-5-sonnet-20240620-v1:0:18k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20240620-v1:0:51k","modelId":"anthropic.claude-3-5-sonnet-20240620-v1:0:51k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20240620-v1:0:200k","modelId":"anthropic.claude-3-5-sonnet-20240620-v1:0:200k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/anthropic.claude-3-5-sonnet-20240620-v1:0","modelId":"anthropic.claude-3-5-sonnet-20240620-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Claude + 3.5 Sonnet","outputModalities":["TEXT"],"providerName":"Anthropic","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-text-v14:7:4k","modelId":"cohere.command-text-v14:7:4k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-text-v14","modelId":"cohere.command-text-v14","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-r-v1:0","modelId":"cohere.command-r-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command + R","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-r-plus-v1:0","modelId":"cohere.command-r-plus-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command + R+","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-light-text-v14:7:4k","modelId":"cohere.command-light-text-v14:7:4k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command + Light","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.command-light-text-v14","modelId":"cohere.command-light-text-v14","modelLifecycle":{"status":"ACTIVE"},"modelName":"Command + Light","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-english-v3:0:512","modelId":"cohere.embed-english-v3:0:512","modelLifecycle":{"status":"ACTIVE"},"modelName":"Embed + English","outputModalities":["EMBEDDING"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-english-v3","modelId":"cohere.embed-english-v3","modelLifecycle":{"status":"ACTIVE"},"modelName":"Embed + English","outputModalities":["EMBEDDING"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-multilingual-v3:0:512","modelId":"cohere.embed-multilingual-v3:0:512","modelLifecycle":{"status":"ACTIVE"},"modelName":"Embed + Multilingual","outputModalities":["EMBEDDING"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.embed-multilingual-v3","modelId":"cohere.embed-multilingual-v3","modelLifecycle":{"status":"ACTIVE"},"modelName":"Embed + Multilingual","outputModalities":["EMBEDDING"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/cohere.rerank-v3-5:0","modelId":"cohere.rerank-v3-5:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Rerank + 3.5","outputModalities":["TEXT"],"providerName":"Cohere","responseStreamingSupported":false},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/deepseek.r1-v1:0","modelId":"deepseek.r1-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"DeepSeek-R1","outputModalities":["TEXT"],"providerName":"DeepSeek","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-8b-instruct-v1:0","modelId":"meta.llama3-8b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3 8B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-70b-instruct-v1:0","modelId":"meta.llama3-70b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3 70B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING","DISTILLATION"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-8b-instruct-v1:0:128k","modelId":"meta.llama3-1-8b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 8B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-8b-instruct-v1:0","modelId":"meta.llama3-1-8b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 8B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING","DISTILLATION"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-70b-instruct-v1:0:128k","modelId":"meta.llama3-1-70b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 70B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":true},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-70b-instruct-v1:0","modelId":"meta.llama3-1-70b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 70B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-1-405b-instruct-v1:0","modelId":"meta.llama3-1-405b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.1 405B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-11b-instruct-v1:0:128k","modelId":"meta.llama3-2-11b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 11B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-11b-instruct-v1:0","modelId":"meta.llama3-2-11b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 11B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-90b-instruct-v1:0:128k","modelId":"meta.llama3-2-90b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 90B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT","IMAGE"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-90b-instruct-v1:0","modelId":"meta.llama3-2-90b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 90B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-1b-instruct-v1:0:128k","modelId":"meta.llama3-2-1b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 1B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-1b-instruct-v1:0","modelId":"meta.llama3-2-1b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 1B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":["FINE_TUNING"],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["PROVISIONED"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-3b-instruct-v1:0:128k","modelId":"meta.llama3-2-3b-instruct-v1:0:128k","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 3B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-2-3b-instruct-v1:0","modelId":"meta.llama3-2-3b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.2 3B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["INFERENCE_PROFILE"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/meta.llama3-3-70b-instruct-v1:0","modelId":"meta.llama3-3-70b-instruct-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Llama + 3.3 70B Instruct","outputModalities":["TEXT"],"providerName":"Meta","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/mistral.mistral-7b-instruct-v0:2","modelId":"mistral.mistral-7b-instruct-v0:2","modelLifecycle":{"status":"ACTIVE"},"modelName":"Mistral + 7B Instruct","outputModalities":["TEXT"],"providerName":"Mistral AI","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/mistral.mixtral-8x7b-instruct-v0:1","modelId":"mistral.mixtral-8x7b-instruct-v0:1","modelLifecycle":{"status":"ACTIVE"},"modelName":"Mixtral + 8x7B Instruct","outputModalities":["TEXT"],"providerName":"Mistral AI","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/mistral.mistral-large-2402-v1:0","modelId":"mistral.mistral-large-2402-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Mistral + Large (24.02)","outputModalities":["TEXT"],"providerName":"Mistral AI","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":true,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/mistral.mistral-large-2407-v1:0","modelId":"mistral.mistral-large-2407-v1:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Mistral + Large (24.07)","outputModalities":["TEXT"],"providerName":"Mistral AI","responseStreamingSupported":true},{"customizationsSupported":[],"explicitPromptCaching":{"isSupported":false},"guardrailsSupported":false,"inferenceTypesSupported":["ON_DEMAND"],"inputModalities":["TEXT"],"intelligentPromptRouting":{"isSupported":false},"modelArn":"arn:aws:bedrock:us-west-2::foundation-model/luma.ray-v2:0","modelId":"luma.ray-v2:0","modelLifecycle":{"status":"ACTIVE"},"modelName":"Ray + v2","outputModalities":["VIDEO"],"providerName":"Luma AI","responseStreamingSupported":false}]}' + recorded_at: Tue, 25 Mar 2025 17:15:41 GMT recorded_with: VCR 6.3.1 From 108f752442c2bd25d857e705c35ac271fb883d7b Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 10:36:35 -0700 Subject: [PATCH 33/85] Add chat spec for bedrock --- lib/ruby_llm/providers/bedrock/chat.rb | 4 +- ...ku_can_handle_multi-turn_conversations.yml | 119 ++++++++++++++++++ ...-5-haiku_can_have_a_basic_conversation.yml | 55 ++++++++ spec/ruby_llm/chat_spec.rb | 15 +-- 4 files changed, 184 insertions(+), 9 deletions(-) create mode 100644 spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation.yml diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index dca8d67e3..cb1b67def 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -33,8 +33,8 @@ def parse_completion_response(response) Message.new( role: :assistant, content: extract_content(data), - input_tokens: data.dig('usage', 'prompt_tokens'), - output_tokens: data.dig('usage', 'completion_tokens'), + input_tokens: data.dig('usage', 'input_tokens'), + output_tokens: data.dig('usage', 'output_tokens'), model_id: data['model'] ) end diff --git a/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations.yml b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations.yml new file mode 100644 index 000000000..6fcc57be0 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations.yml @@ -0,0 +1,119 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"\n\nHuman: + Who was Ruby''s creator?"}],"temperature":0.7,"max_tokens":4096}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T173340Z + X-Amz-Security-Token: + - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i + X-Amz-Content-Sha256: + - c56a0aec87689b8f166eb4fb30443eed5b7a2264ff9c2e92f3bf062e913be0e0 + Authorization: + - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=7cfe5d1e8ff9b8a53e290de10e252bea3bf83e131f05f5c63160730ae25f1b67 + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 17:33:02 GMT + Content-Type: + - application/json + Content-Length: + - '635' + Connection: + - keep-alive + X-Amzn-Requestid: + - ec6af9e7-4d52-48ef-94cd-aa2015c467dd + X-Amzn-Bedrock-Invocation-Latency: + - '3915' + X-Amzn-Bedrock-Output-Token-Count: + - '87' + X-Amzn-Bedrock-Input-Token-Count: + - '17' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_01QQmcf4XLa4Cd6kiPxcmJV7","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I + apologize, but I cannot find a specific context about who Ruby''s creator + is from our previous conversation. However, I can tell you that Ruby is a + programming language created by Yukihiro Matsumoto (often called \"Matz\") + from Japan. He first released Ruby in 1995, designing it to be a dynamic, + object-oriented programming language that prioritizes programmer happiness + and productivity."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":17,"output_tokens":87}}' + recorded_at: Tue, 25 Mar 2025 17:33:44 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"\n\nHuman: + Who was Ruby''s creator?\n\nAssistant: I apologize, but I cannot find a specific + context about who Ruby''s creator is from our previous conversation. However, + I can tell you that Ruby is a programming language created by Yukihiro Matsumoto + (often called \"Matz\") from Japan. He first released Ruby in 1995, designing + it to be a dynamic, object-oriented programming language that prioritizes + programmer happiness and productivity.\n\nHuman: What year did he create Ruby?"}],"temperature":0.7,"max_tokens":4096}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T173344Z + X-Amz-Security-Token: + - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i + X-Amz-Content-Sha256: + - 2a7153f7766f359e70b001cf5f11256fc4021b7a87b1eebfeff6ee72863d5dff + Authorization: + - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=1110ac74ef6058078f7725f1cc7bad40aebb9613bc5b3e649f6b5d040c4c3f16 + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 17:33:03 GMT + Content-Type: + - application/json + Content-Length: + - '328' + Connection: + - keep-alive + X-Amzn-Requestid: + - 75ede0a1-054e-45de-813c-268940c554b5 + X-Amzn-Bedrock-Invocation-Latency: + - '743' + X-Amzn-Bedrock-Output-Token-Count: + - '25' + X-Amzn-Bedrock-Input-Token-Count: + - '114' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_01CVgzsoXEFjZHb7q9GpQWT8","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"According + to my previous response, Yukihiro Matsumoto first released Ruby in 1995."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":114,"output_tokens":25}}' + recorded_at: Tue, 25 Mar 2025 17:33:45 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation.yml b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation.yml new file mode 100644 index 000000000..b1ced5672 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation.yml @@ -0,0 +1,55 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"\n\nHuman: + What''s 2 + 2?"}],"temperature":0.7,"max_tokens":4096}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T173339Z + X-Amz-Security-Token: + - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i + X-Amz-Content-Sha256: + - 433228d8fa6dfe7b1ac9ba49a6cb06882cf07f05124baee8a2f48d38b4d7e855 + Authorization: + - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=8aaf70e624d3df533de47499fde8d1285474f77fddf7fa1e6d71c4c738528a0e + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 17:32:58 GMT + Content-Type: + - application/json + Content-Length: + - '245' + Connection: + - keep-alive + X-Amzn-Requestid: + - 763d4d4c-c80b-4361-8203-1366e8046222 + X-Amzn-Bedrock-Invocation-Latency: + - '838' + X-Amzn-Bedrock-Output-Token-Count: + - '5' + X-Amzn-Bedrock-Input-Token-Count: + - '20' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_01BfVD9JeTeEH5mcUYcBLkpp","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"4"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":20,"output_tokens":5}}' + recorded_at: Tue, 25 Mar 2025 17:33:40 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/ruby_llm/chat_spec.rb b/spec/ruby_llm/chat_spec.rb index 7ced4f959..58c05810b 100644 --- a/spec/ruby_llm/chat_spec.rb +++ b/spec/ruby_llm/chat_spec.rb @@ -8,13 +8,14 @@ describe 'basic chat functionality' do [ - 'claude-3-5-haiku-20241022', - 'gemini-2.0-flash', - 'deepseek-chat', - 'gpt-4o-mini' - ].each do |model| + ['claude-3-5-haiku-20241022', nil], + ['gemini-2.0-flash', nil], + ['deepseek-chat', nil], + ['gpt-4o-mini', nil], + ['claude-3-5-haiku', 'bedrock'], + ].each do |model, provider| it "#{model} can have a basic conversation" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations - chat = RubyLLM.chat(model: model) + chat = RubyLLM.chat(model: model, provider: provider) response = chat.ask("What's 2 + 2?") expect(response.content).to include('4') @@ -24,7 +25,7 @@ end it "#{model} can handle multi-turn conversations" do # rubocop:disable RSpec/MultipleExpectations - chat = RubyLLM.chat(model: model) + chat = RubyLLM.chat(model: model, provider: provider) first = chat.ask("Who was Ruby's creator?") expect(first.content).to include('Matz') From 8838b5f0498bc348d4deed2393273f20d0e5ce6c Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 11:50:29 -0700 Subject: [PATCH 34/85] Improve building claude messages --- lib/ruby_llm/providers/bedrock/chat.rb | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index cb1b67def..12052f2b6 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -42,20 +42,11 @@ def parse_completion_response(response) private def build_claude_request(messages, temperature, model_id) - formatted = messages.map do |msg| - role = msg.role == :assistant ? 'Assistant' : 'Human' - content = msg.content - "\n\n#{role}: #{content}" - end.join - { anthropic_version: 'bedrock-2023-05-31', - messages: [ - { - role: 'user', - content: formatted - } - ], + messages: messages.map do |msg| + { role: msg.role == :assistant ? 'assistant' : 'user', content: msg.content } + end, temperature: temperature, max_tokens: max_tokens_for(model_id) } From 74447a88d5912aa79f2d79827dd4212003a52ac5 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 13:43:02 -0700 Subject: [PATCH 35/85] Get streaming test working - Broke streaming outside of test --- lib/ruby_llm/providers/bedrock/streaming.rb | 159 +++++++++++++----- ...3-5-haiku_supports_streaming_responses.yml | 52 ++++++ spec/ruby_llm/chat_streaming_spec.rb | 13 +- spec/spec_helper.rb | 2 + 4 files changed, 177 insertions(+), 49 deletions(-) create mode 100644 spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses.yml diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index c02306252..d41b62142 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -31,61 +31,107 @@ def handle_stream(&block) RubyLLM.logger.debug "Accumulating error chunk: #{chunk}" end else - # Process the chunk using the AWS EventStream decoder begin - message, = decoder.decode_chunk(chunk) - - if message - payload = message.payload.read - parsed_data = nil - + # Process each event stream message in the chunk + offset = 0 + while offset < chunk.bytesize + # Read the prelude (total length + headers length) + break if chunk.bytesize - offset < 12 # Need at least prelude size + + total_length = chunk[offset...offset + 4].unpack('N').first + headers_length = chunk[offset + 4...offset + 8].unpack('N').first + + # Validate lengths to ensure they're reasonable + if total_length.nil? || headers_length.nil? || + total_length <= 0 || total_length > 1_000_000 || # Sanity check for message size + headers_length <= 0 || headers_length > total_length + RubyLLM.logger.debug "Invalid lengths detected, trying next potential message" + # Try to find the next message prelude marker + next_prelude = find_next_prelude(chunk, offset + 4) + offset = next_prelude || chunk.bytesize + next + end + + # Verify we have the complete message + message_end = offset + total_length + 4 # +4 for the message CRC + break if chunk.bytesize < message_end + + # Extract headers and payload + headers_end = offset + 12 + headers_length + payload_end = message_end - 4 # Subtract 4 bytes for message CRC + + # Safety check for valid positions + if headers_end >= payload_end || headers_end >= chunk.bytesize || payload_end > chunk.bytesize + RubyLLM.logger.debug "Invalid positions detected, trying next potential message" + # Try to find the next message prelude marker + next_prelude = find_next_prelude(chunk, offset + 4) + offset = next_prelude || chunk.bytesize + next + end + + # Get payload + payload = chunk[headers_end...payload_end] + + # Safety check for payload + if payload.nil? || payload.empty? + RubyLLM.logger.debug "Empty or nil payload detected, skipping chunk" + offset = message_end + next + end + + # Find valid JSON in the payload + json_start = payload.index('{') + json_end = payload.rindex('}') + + if json_start.nil? || json_end.nil? || json_start >= json_end + RubyLLM.logger.debug "No valid JSON found in payload, skipping chunk" + offset = message_end + next + end + + # Extract just the JSON portion + json_payload = payload[json_start..json_end] + begin - if payload.is_a?(String) - # Try parsing as JSON - json_data = JSON.parse(payload) - - # Handle Base64 encoded bytes - if json_data.is_a?(Hash) && json_data['bytes'] - # Decode the Base64 string - decoded_bytes = Base64.strict_decode64(json_data['bytes']) - # Parse the decoded JSON - parsed_data = JSON.parse(decoded_bytes) - else - # Handle normal JSON data - parsed_data = json_data + # Parse the JSON payload + json_data = JSON.parse(json_payload) + + # Handle Base64 encoded bytes + if json_data['bytes'] + decoded_bytes = Base64.strict_decode64(json_data['bytes']) + parsed_data = JSON.parse(decoded_bytes) + + # Extract content based on the event type + content = extract_streaming_content(parsed_data) + + # Only emit a chunk if there's content to emit + unless content.nil? || content.empty? + accumulated_content += content + + block.call( + Chunk.new( + role: :assistant, + model_id: parsed_data.dig('message', 'model') || @model_id, + content: content, + input_tokens: parsed_data.dig('message', 'usage', 'input_tokens'), + output_tokens: parsed_data.dig('message', 'usage', 'output_tokens') + ) + ) end end rescue JSON::ParserError => e RubyLLM.logger.debug "Failed to parse payload as JSON: #{e.message}" - next + RubyLLM.logger.debug "Attempted JSON payload: #{json_payload.inspect}" rescue StandardError => e RubyLLM.logger.debug "Error processing payload: #{e.message}" - next - end - - next if parsed_data.nil? - - # Extract content based on the event type - content = extract_streaming_content(parsed_data) - - # Only emit a chunk if there's content to emit - unless content.nil? || content.empty? - accumulated_content += content - - block.call( - Chunk.new( - role: :assistant, - model_id: parsed_data.dig('message', 'model') || @model_id, - content: content, - input_tokens: parsed_data.dig('message', 'usage', 'input_tokens'), - output_tokens: parsed_data.dig('message', 'usage', 'output_tokens') - ) - ) end + + # Move to next message + offset = message_end end rescue StandardError => e RubyLLM.logger.debug "Error processing chunk: #{e.message}" - next + RubyLLM.logger.debug "Chunk size: #{chunk.bytesize}" end end end @@ -132,6 +178,33 @@ def extract_content(data) end end end + + def split_event_stream_chunk(chunk) + # Find the position of the first '{' character which indicates start of JSON + json_start = chunk.index('{') + return [nil, nil] unless json_start + + header = chunk[0...json_start].strip + payload = chunk[json_start..-1] + + [header, payload] + end + + def find_next_prelude(chunk, start_offset) + # Look for potential message prelude by scanning for reasonable length values + (start_offset...(chunk.bytesize - 8)).each do |pos| + potential_total_length = chunk[pos...pos + 4].unpack('N').first + potential_headers_length = chunk[pos + 4...pos + 8].unpack('N').first + + # Check if these look like valid lengths + if potential_total_length && potential_headers_length && + potential_total_length > 0 && potential_total_length < 1_000_000 && + potential_headers_length > 0 && potential_headers_length < potential_total_length + return pos + end + end + nil + end end end end diff --git a/spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses.yml b/spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses.yml new file mode 100644 index 000000000..357acbf05 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses.yml @@ -0,0 +1,52 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke-with-response-stream + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"Count + from 1 to 3"}],"temperature":0.7,"max_tokens":4096}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T181243Z + X-Amz-Security-Token: + - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i + X-Amz-Content-Sha256: + - 7444002409115a1a6689bf49e7a862dedd9b424399df485afe093d31dbd9e873 + Authorization: + - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=a7a876c520661dd43232ab57fadf3f9102ff2ae37722dbff323c719b48c48f65 + Content-Type: + - application/json + Accept: + - application/vnd.amazon.eventstream + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 18:12:02 GMT + Content-Type: + - application/vnd.amazon.eventstream + Transfer-Encoding: + - chunked + Connection: + - keep-alive + X-Amzn-Requestid: + - 07114d7d-0375-4bf3-a5f8-abcee919319e + X-Amzn-Bedrock-Content-Type: + - application/json + body: + encoding: ASCII-8BIT + string: !binary |- + AAABwwAAAEvhxQbfCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHRnlkQ0lzSW0xbGMzTmhaMlVpT25zaWFXUWlPaUp0YzJkZlltUnlhMTh3TVVOdVFUVmFSM1pxZFZCNlVFeDZTR0ZEVFhremVYTWlMQ0owZVhCbElqb2liV1Z6YzJGblpTSXNJbkp2YkdVaU9pSmhjM05wYzNSaGJuUWlMQ0p0YjJSbGJDSTZJbU5zWVhWa1pTMHpMVFV0YUdGcGEzVXRNakF5TkRFd01qSWlMQ0pqYjI1MFpXNTBJanBiWFN3aWMzUnZjRjl5WldGemIyNGlPbTUxYkd3c0luTjBiM0JmYzJWeGRXVnVZMlVpT201MWJHd3NJblZ6WVdkbElqcDdJbWx1Y0hWMFgzUnZhMlZ1Y3lJNk1UVXNJbTkxZEhCMWRGOTBiMnRsYm5NaU9qRjlmWDA9IiwicCI6ImFiY2RlZmdoaSJ9G3dfmQAAAOMAAABL61j6fgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5emRHRnlkQ0lzSW1sdVpHVjRJam93TENKamIyNTBaVzUwWDJKc2IyTnJJanA3SW5SNWNHVWlPaUowWlhoMElpd2lkR1Y0ZENJNklpSjlmUT09IiwicCI6ImFiY2RlIn1qLPEUAAABAQAAAEtyEL1kCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUpJWlhKbEluMTkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSEkifVk6xqcAAAEAAAAAS09wlNQLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSW5jeUJqYjNWdWRHbHVaeUJtY205dElERWdkRzhnTXpvaWZYMD0iLCJwIjoiYWJjZGVmIn2rSl0OAAABIAAAAEuOsbvQCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUpjYmx4dU1WeHVNbHh1TXlKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxIn3+SA/UAAAAvgAAAEsr2/EECzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEc5d0lpd2lhVzVrWlhnaU9qQjkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQiJ9gGjF+gAAAT0AAABLFsHo4ws6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2liV1Z6YzJGblpWOWtaV3gwWVNJc0ltUmxiSFJoSWpwN0luTjBiM0JmY21WaGMyOXVJam9pWlc1a1gzUjFjbTRpTENKemRHOXdYM05sY1hWbGJtTmxJanB1ZFd4c2ZTd2lkWE5oWjJVaU9uc2liM1YwY0hWMFgzUnZhMlZ1Y3lJNk1qQjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2In1sT4BMAAABTQAAAEvvEwgsCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHOXdJaXdpWVcxaGVtOXVMV0psWkhKdlkyc3RhVzUyYjJOaGRHbHZiazFsZEhKcFkzTWlPbnNpYVc1d2RYUlViMnRsYmtOdmRXNTBJam94TlN3aWIzVjBjSFYwVkc5clpXNURiM1Z1ZENJNk1qQXNJbWx1ZG05allYUnBiMjVNWVhSbGJtTjVJam95TnpRNExDSm1hWEp6ZEVKNWRHVk1ZWFJsYm1ONUlqb3lNREV5ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzIn0wmND4 + recorded_at: Tue, 25 Mar 2025 18:12:46 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/ruby_llm/chat_streaming_spec.rb b/spec/ruby_llm/chat_streaming_spec.rb index a42bf9ca4..99e99c347 100644 --- a/spec/ruby_llm/chat_streaming_spec.rb +++ b/spec/ruby_llm/chat_streaming_spec.rb @@ -8,13 +8,14 @@ describe 'streaming responses' do [ - 'claude-3-5-haiku-20241022', - 'gemini-2.0-flash', - 'deepseek-chat', - 'gpt-4o-mini' - ].each do |model| + # ['claude-3-5-haiku-20241022', nil], + # ['gemini-2.0-flash', nil], + # ['deepseek-chat', nil], + # ['gpt-4o-mini', nil], + ['claude-3-5-haiku', 'bedrock'], + ].each do |model, provider| it "#{model} supports streaming responses" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations - chat = RubyLLM.chat(model: model) + chat = RubyLLM.chat(model: model, provider: provider) chunks = [] chat.ask('Count from 1 to 3') do |chunk| diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index f6b14d419..2cd1d8fb5 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -45,6 +45,8 @@ config.filter_sensitive_data('') { ENV.fetch('ANTHROPIC_API_KEY', nil) } config.filter_sensitive_data('') { ENV.fetch('GEMINI_API_KEY', nil) } config.filter_sensitive_data('') { ENV.fetch('DEEPSEEK_API_KEY', nil) } + config.filter_sensitive_data('') { ENV.fetch('AWS_ACCESS_KEY_ID', nil) } + config.filter_sensitive_data('') { ENV.fetch('AWS_SESSION_TOKEN', nil) } # Filter sensitive response headers config.filter_sensitive_data('') do |interaction| From 2420bf4fd0d52230d819c3bab93ac46cc4365749 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 13:50:35 -0700 Subject: [PATCH 36/85] Get streaming working with tests --- lib/ruby_llm/providers/bedrock/streaming.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index d41b62142..4207421b4 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -53,7 +53,7 @@ def handle_stream(&block) end # Verify we have the complete message - message_end = offset + total_length + 4 # +4 for the message CRC + message_end = offset + total_length break if chunk.bytesize < message_end # Extract headers and payload From 4cffcc5e2e2133dd1566d27a414e5f8526074056 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 15:25:25 -0700 Subject: [PATCH 37/85] Get tools working with Bedrock --- lib/ruby_llm/providers/bedrock.rb | 5 +- lib/ruby_llm/providers/bedrock/chat.rb | 65 +++- lib/ruby_llm/providers/bedrock/decoder.rb | 226 ----------- lib/ruby_llm/providers/bedrock/streaming.rb | 48 +-- lib/ruby_llm/providers/bedrock/tools.rb | 53 --- ...ku_can_handle_multi-turn_conversations.yml | 119 ------ ..._multi-turn_conversations_with_bedrock.yml | 120 ++++++ ...-5-haiku_can_have_a_basic_conversation.yml | 55 --- ...have_a_basic_conversation_with_bedrock.yml | 55 +++ ..._multi-turn_conversations_with_bedrock.yml | 245 ++++++++++++ ...e-3-5-haiku_can_use_tools_with_bedrock.yml | 117 ++++++ ...n_streaming_conversations_with_bedrock.yml | 365 ++++++++++++++++++ ...3-5-haiku_supports_streaming_responses.yml | 52 --- ...ports_streaming_responses_with_bedrock.yml | 52 +++ spec/ruby_llm/chat_spec.rb | 7 +- spec/ruby_llm/chat_streaming_spec.rb | 13 +- spec/ruby_llm/chat_tools_spec.rb | 22 +- 17 files changed, 1055 insertions(+), 564 deletions(-) delete mode 100644 lib/ruby_llm/providers/bedrock/decoder.rb delete mode 100644 lib/ruby_llm/providers/bedrock/tools.rb delete mode 100644 spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations_with_bedrock.yml delete mode 100644 spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation_with_bedrock.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_in_multi-turn_conversations_with_bedrock.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_with_bedrock.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_with_multi-turn_streaming_conversations_with_bedrock.yml delete mode 100644 spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses_with_bedrock.yml diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 0619a50d8..c2791751b 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -12,9 +12,12 @@ module Bedrock extend Bedrock::Chat extend Bedrock::Streaming extend Bedrock::Models - extend Bedrock::Tools extend Bedrock::Signing + # This provider currently only supports Anthropic models, so the tools/media implementation is shared + extend Anthropic::Media + extend Anthropic::Tools + module_function def api_base diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index 12052f2b6..e61dc2bcc 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -5,21 +5,18 @@ module Providers module Bedrock # Chat methods for the AWS Bedrock API implementation module Chat - module_function + private def completion_url - "model/#{model_id}/invoke" - end - - def model_id - @model_id + "model/#{@model_id}/invoke" end def render_payload(messages, tools:, temperature:, model:, stream: false) + # Hold model_id in instance variable for use in completion_url and stream_url @model_id = model case model when /claude/ - build_claude_request(messages, temperature, model) + build_claude_request(messages, tools:, temperature:, model:) else raise Error, nil, "Unsupported model: #{model}" end @@ -27,31 +24,65 @@ def render_payload(messages, tools:, temperature:, model:, stream: false) def parse_completion_response(response) data = response.body - data = JSON.parse(data) if data.is_a?(String) - return if data.nil? || data.empty? + content_blocks = data['content'] || [] + + text_content = extract_text_content(content_blocks) + tool_use = find_tool_use(content_blocks) + + build_message(data, text_content, tool_use) + end + def extract_text_content(blocks) + text_blocks = blocks.select { |c| c['type'] == 'text' } + text_blocks.map { |c| c['text'] }.join + end + + def build_message(data, content, tool_use) Message.new( role: :assistant, - content: extract_content(data), + content: content, + tool_calls: parse_tool_calls(tool_use), input_tokens: data.dig('usage', 'input_tokens'), output_tokens: data.dig('usage', 'output_tokens'), model_id: data['model'] ) end - private - - def build_claude_request(messages, temperature, model_id) + def build_claude_request(messages, tools:, temperature:, model:) { anthropic_version: 'bedrock-2023-05-31', - messages: messages.map do |msg| - { role: msg.role == :assistant ? 'assistant' : 'user', content: msg.content } - end, + messages: messages.map { |msg| format_message(msg) }, temperature: temperature, - max_tokens: max_tokens_for(model_id) + max_tokens: max_tokens_for(model) + }.tap do |payload| + payload[:tools] = tools.values.map { |t| function_for(t) } if tools.any? + end + end + + def format_message(msg) + if msg.tool_call? + format_tool_call(msg) + elsif msg.tool_result? + format_tool_result(msg) + else + format_basic_message(msg) + end + end + + def format_basic_message(msg) + { + role: convert_role(msg.role), + content: Anthropic::Media.format_content(msg.content) } end + def convert_role(role) + case role + when :tool, :user then 'user' + else 'assistant' + end + end + def max_tokens_for(model_id) RubyLLM.models.find(model_id)&.max_tokens end diff --git a/lib/ruby_llm/providers/bedrock/decoder.rb b/lib/ruby_llm/providers/bedrock/decoder.rb deleted file mode 100644 index 356b07348..000000000 --- a/lib/ruby_llm/providers/bedrock/decoder.rb +++ /dev/null @@ -1,226 +0,0 @@ -# frozen_string_literal: true - -require 'stringio' -require 'tempfile' -require 'zlib' - -module RubyLLM - module Providers - module Bedrock - # Decoder for AWS EventStream format used by Bedrock streaming responses - class Decoder - include Enumerable - - ONE_MEGABYTE = 1024 * 1024 - private_constant :ONE_MEGABYTE - - # bytes of prelude part, including 4 bytes of - # total message length, headers length and crc checksum of prelude - PRELUDE_LENGTH = 12 - private_constant :PRELUDE_LENGTH - - # 4 bytes message crc checksum - CRC32_LENGTH = 4 - private_constant :CRC32_LENGTH - - # @param [Hash] options The initialization options. - # @option options [Boolean] :format (true) When `false` it - # disables user-friendly formatting for message header values - # including timestamp and uuid etc. - def initialize(options = {}) - @format = options.fetch(:format, true) - @message_buffer = '' - end - - # Decodes messages from a binary stream - # - # @param [IO#read] io An IO-like object - # that responds to `#read` - # - # @yieldparam [Message] message - # @return [Enumerable, nil] Returns a new Enumerable - # containing decoded messages if no block is given - def decode(io, &block) - raw_message = io.read - decoded_message = decode_message(raw_message) - return wrap_as_enumerator(decoded_message) unless block_given? - - # fetch message only - raw_event, _eof = decoded_message - block.call(raw_event) - end - - # Decodes a single message from a chunk of string - # - # @param [String] chunk A chunk of string to be decoded, - # chunk can contain partial event message to multiple event messages - # When not provided, decode data from #message_buffer - # - # @return [Array] Returns single decoded message - # and boolean pair, the boolean flag indicates whether this chunk - # has been fully consumed, unused data is tracked at #message_buffer - def decode_chunk(chunk = nil) - @message_buffer = [@message_buffer, chunk].pack('a*a*') if chunk - decode_message(@message_buffer) - end - - private - - # exposed via object.send for testing - attr_reader :message_buffer - - def wrap_as_enumerator(decoded_message) - Enumerator.new do |yielder| - yielder << decoded_message - end - end - - def decode_message(raw_message) - # incomplete message prelude received - return [nil, true] if raw_message.bytesize < PRELUDE_LENGTH - - prelude, content = raw_message.unpack("a#{PRELUDE_LENGTH}a*") - - # decode prelude - total_length, header_length = decode_prelude(prelude) - - # incomplete message received, leave it in the buffer - return [nil, true] if raw_message.bytesize < total_length - - content, checksum, remaining = content.unpack("a#{total_length - PRELUDE_LENGTH - CRC32_LENGTH}Na*") - raise Error, 'Message checksum error' unless Zlib.crc32([prelude, content].pack('a*a*')) == checksum - - # decode headers and payload - headers, payload = decode_context(content, header_length) - - @message_buffer = remaining - - [Message.new(headers: headers, payload: payload), remaining.empty?] - end - - def decode_prelude(prelude) - # prelude contains length of message and headers, - # followed with CRC checksum of itself - content, checksum = prelude.unpack("a#{PRELUDE_LENGTH - CRC32_LENGTH}N") - raise Error, 'Prelude checksum error' unless Zlib.crc32(content) == checksum - - content.unpack('N*') - end - - def decode_context(content, header_length) - encoded_header, encoded_payload = content.unpack("a#{header_length}a*") - [ - extract_headers(encoded_header), - extract_payload(encoded_payload) - ] - end - - def extract_headers(buffer) - scanner = buffer - headers = {} - until scanner.bytesize.zero? - # header key - key_length, scanner = scanner.unpack('Ca*') - key, scanner = scanner.unpack("a#{key_length}a*") - - # header value - type_index, scanner = scanner.unpack('Ca*') - value_type = Types.types[type_index] - unpack_pattern, value_length = Types.pattern[value_type] - value = if !unpack_pattern.nil? == unpack_pattern - # boolean types won't have value specified - unpack_pattern - else - value_length, scanner = scanner.unpack('S>a*') unless value_length - unpacked_value, scanner = scanner.unpack("#{unpack_pattern || "a#{value_length}"}a*") - unpacked_value - end - - headers[key] = HeaderValue.new( - format: @format, - value: value, - type: value_type - ) - end - headers - end - - def extract_payload(encoded) - if encoded.bytesize <= ONE_MEGABYTE - payload_stringio(encoded) - else - payload_tempfile(encoded) - end - end - - def payload_stringio(encoded) - StringIO.new(encoded) - end - - def payload_tempfile(encoded) - payload = Tempfile.new - payload.binmode - payload.write(encoded) - payload.rewind - payload - end - - # Simple message class to hold decoded data - class Message - attr_reader :headers, :payload - - def initialize(headers:, payload:) - @headers = headers - @payload = payload - end - end - - # Header value wrapper - class HeaderValue - attr_reader :value, :type - - def initialize(format:, value:, type:) - @format = format - @value = value - @type = type - end - end - - # Types module for header value types - module Types - def self.types - { - 0 => true, - 1 => false, - 2 => :byte, - 3 => :short, - 4 => :integer, - 5 => :long, - 6 => :bytes, - 7 => :string, - 8 => :timestamp, - 9 => :uuid - } - end - - def self.pattern - { - true => [true, 0], - false => [false, 0], - byte: ['c', 1], - short: ['s>', 2], - integer: ['l>', 4], - long: ['q>', 8], - bytes: [nil, nil], - string: [nil, nil], - timestamp: ['q>', 8], - uuid: ['H32', 16] - } - end - end - - class Error < StandardError; end - end - end - end -end diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index 4207421b4..c572da8d2 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -10,17 +10,15 @@ module Streaming module_function def stream_url - "model/#{model_id}/invoke-with-response-stream" + "model/#{@model_id}/invoke-with-response-stream" end def handle_stream(&block) - decoder = Decoder.new - buffer = String.new - accumulated_content = '' proc do |chunk, _bytes, env| if env && env.status != 200 # Accumulate error chunks + buffer = String.new buffer << chunk begin error_data = JSON.parse(buffer) @@ -99,26 +97,20 @@ def handle_stream(&block) # Handle Base64 encoded bytes if json_data['bytes'] decoded_bytes = Base64.strict_decode64(json_data['bytes']) - parsed_data = JSON.parse(decoded_bytes) - - # Extract content based on the event type - content = extract_streaming_content(parsed_data) - - # Only emit a chunk if there's content to emit - unless content.nil? || content.empty? - accumulated_content += content - - block.call( - Chunk.new( - role: :assistant, - model_id: parsed_data.dig('message', 'model') || @model_id, - content: content, - input_tokens: parsed_data.dig('message', 'usage', 'input_tokens'), - output_tokens: parsed_data.dig('message', 'usage', 'output_tokens') - ) + data = JSON.parse(decoded_bytes) + + block.call( + Chunk.new( + role: :assistant, + model_id: data.dig('message', 'model') || @model_id, + content: extract_streaming_content(data), + input_tokens: extract_input_tokens(data), + output_tokens: extract_output_tokens(data), + tool_calls: extract_tool_calls(data) ) - end + ) end + rescue JSON::ParserError => e RubyLLM.logger.debug "Failed to parse payload as JSON: #{e.message}" RubyLLM.logger.debug "Attempted JSON payload: #{json_payload.inspect}" @@ -137,6 +129,10 @@ def handle_stream(&block) end end + def json_delta?(data) + data['type'] == 'content_block_delta' && data.dig('delta', 'type') == 'input_json_delta' + end + def extract_streaming_content(data) if data.is_a?(Hash) case data['type'] @@ -163,6 +159,14 @@ def extract_streaming_content(data) private + def extract_input_tokens(data) + data.dig('message', 'usage', 'input_tokens') + end + + def extract_output_tokens(data) + data.dig('message', 'usage', 'output_tokens') || data.dig('usage', 'output_tokens') + end + def extract_content(data) case data when Hash diff --git a/lib/ruby_llm/providers/bedrock/tools.rb b/lib/ruby_llm/providers/bedrock/tools.rb deleted file mode 100644 index ad4d70a16..000000000 --- a/lib/ruby_llm/providers/bedrock/tools.rb +++ /dev/null @@ -1,53 +0,0 @@ -# frozen_string_literal: true - -module RubyLLM - module Providers - module Bedrock - # Tools methods for the AWS Bedrock API implementation - module Tools - module_function - - def tool_for(tool) - { - type: 'function', - function: { - name: tool.name, - description: tool.description, - parameters: tool.parameters - } - } - end - - def parse_tool_calls(tool_calls, parse_arguments: true) - return {} unless tool_calls - - tool_calls.each_with_object({}) do |call, hash| - function_args = call['function']['arguments'] - hash[call['id']] = ToolCall.new( - id: call['id'], - name: call['function']['name'], - arguments: parse_arguments ? parse_arguments(function_args) : function_args - ) - end - end - - private - - def parse_arguments(arguments) - return {} unless arguments - - case arguments - when String - JSON.parse(arguments) - when Hash - arguments - else - {} - end - rescue JSON::ParserError - {} - end - end - end - end -end diff --git a/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations.yml b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations.yml deleted file mode 100644 index 6fcc57be0..000000000 --- a/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations.yml +++ /dev/null @@ -1,119 +0,0 @@ ---- -http_interactions: -- request: - method: post - uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke - body: - encoding: UTF-8 - string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"\n\nHuman: - Who was Ruby''s creator?"}],"temperature":0.7,"max_tokens":4096}' - headers: - User-Agent: - - Faraday v2.12.2 - Host: - - bedrock-runtime.us-west-2.amazonaws.com - X-Amz-Date: - - 20250325T173340Z - X-Amz-Security-Token: - - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i - X-Amz-Content-Sha256: - - c56a0aec87689b8f166eb4fb30443eed5b7a2264ff9c2e92f3bf062e913be0e0 - Authorization: - - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, - SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=7cfe5d1e8ff9b8a53e290de10e252bea3bf83e131f05f5c63160730ae25f1b67 - Content-Type: - - application/json - Accept: - - application/json - Accept-Encoding: - - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 - response: - status: - code: 200 - message: OK - headers: - Date: - - Tue, 25 Mar 2025 17:33:02 GMT - Content-Type: - - application/json - Content-Length: - - '635' - Connection: - - keep-alive - X-Amzn-Requestid: - - ec6af9e7-4d52-48ef-94cd-aa2015c467dd - X-Amzn-Bedrock-Invocation-Latency: - - '3915' - X-Amzn-Bedrock-Output-Token-Count: - - '87' - X-Amzn-Bedrock-Input-Token-Count: - - '17' - body: - encoding: UTF-8 - string: '{"id":"msg_bdrk_01QQmcf4XLa4Cd6kiPxcmJV7","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I - apologize, but I cannot find a specific context about who Ruby''s creator - is from our previous conversation. However, I can tell you that Ruby is a - programming language created by Yukihiro Matsumoto (often called \"Matz\") - from Japan. He first released Ruby in 1995, designing it to be a dynamic, - object-oriented programming language that prioritizes programmer happiness - and productivity."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":17,"output_tokens":87}}' - recorded_at: Tue, 25 Mar 2025 17:33:44 GMT -- request: - method: post - uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke - body: - encoding: UTF-8 - string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"\n\nHuman: - Who was Ruby''s creator?\n\nAssistant: I apologize, but I cannot find a specific - context about who Ruby''s creator is from our previous conversation. However, - I can tell you that Ruby is a programming language created by Yukihiro Matsumoto - (often called \"Matz\") from Japan. He first released Ruby in 1995, designing - it to be a dynamic, object-oriented programming language that prioritizes - programmer happiness and productivity.\n\nHuman: What year did he create Ruby?"}],"temperature":0.7,"max_tokens":4096}' - headers: - User-Agent: - - Faraday v2.12.2 - Host: - - bedrock-runtime.us-west-2.amazonaws.com - X-Amz-Date: - - 20250325T173344Z - X-Amz-Security-Token: - - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i - X-Amz-Content-Sha256: - - 2a7153f7766f359e70b001cf5f11256fc4021b7a87b1eebfeff6ee72863d5dff - Authorization: - - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, - SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=1110ac74ef6058078f7725f1cc7bad40aebb9613bc5b3e649f6b5d040c4c3f16 - Content-Type: - - application/json - Accept: - - application/json - Accept-Encoding: - - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 - response: - status: - code: 200 - message: OK - headers: - Date: - - Tue, 25 Mar 2025 17:33:03 GMT - Content-Type: - - application/json - Content-Length: - - '328' - Connection: - - keep-alive - X-Amzn-Requestid: - - 75ede0a1-054e-45de-813c-268940c554b5 - X-Amzn-Bedrock-Invocation-Latency: - - '743' - X-Amzn-Bedrock-Output-Token-Count: - - '25' - X-Amzn-Bedrock-Input-Token-Count: - - '114' - body: - encoding: UTF-8 - string: '{"id":"msg_bdrk_01CVgzsoXEFjZHb7q9GpQWT8","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"According - to my previous response, Yukihiro Matsumoto first released Ruby in 1995."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":114,"output_tokens":25}}' - recorded_at: Tue, 25 Mar 2025 17:33:45 GMT -recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations_with_bedrock.yml b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations_with_bedrock.yml new file mode 100644 index 000000000..01769fccc --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_multi-turn_conversations_with_bedrock.yml @@ -0,0 +1,120 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"Who + was Ruby''s creator?"}],"temperature":0.7,"max_tokens":4096}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222413Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 45bc07e92f3e4928655fb657829c1790fcb1594aafa5abfd051050018ced0ffe + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=2e0f1b029bf5d0ceab9c70d16e04ab18f92f86a7148e7771f81b35959fe9613d + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:23:35 GMT + Content-Type: + - application/json + Content-Length: + - '583' + Connection: + - keep-alive + X-Amzn-Requestid: + - 747f0c38-c89c-4041-8527-984920f9e9ae + X-Amzn-Bedrock-Invocation-Latency: + - '2790' + X-Amzn-Bedrock-Output-Token-Count: + - '77' + X-Amzn-Bedrock-Input-Token-Count: + - '13' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_01C5ZuowJFSpehKK5jvJ2tEA","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"Ruby + was created by Yukihiro Matsumoto (often called \"Matz\") in Japan. He first + released Ruby in 1995, designing it to be a programming language that prioritizes + human productivity and programmer happiness. Matz wanted to create a language + that was more natural and enjoyable to use compared to other programming languages + of that time."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":13,"output_tokens":77}}' + recorded_at: Tue, 25 Mar 2025 22:24:16 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"Who + was Ruby''s creator?"},{"role":"assistant","content":"Ruby was created by + Yukihiro Matsumoto (often called \"Matz\") in Japan. He first released Ruby + in 1995, designing it to be a programming language that prioritizes human + productivity and programmer happiness. Matz wanted to create a language that + was more natural and enjoyable to use compared to other programming languages + of that time."},{"role":"user","content":"What year did he create Ruby?"}],"temperature":0.7,"max_tokens":4096}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222416Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 474b86ca5c9de2f96dfc3faafd2c86535b93b25d30ab62cc053e3f1ba0b92df2 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=d86a5908714b7a1faf95628bcfea98883e77b494e20afdb21e6ffd6d0a9d11ad + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:23:36 GMT + Content-Type: + - application/json + Content-Length: + - '411' + Connection: + - keep-alive + X-Amzn-Requestid: + - 16de93f7-0a1c-433f-be10-77c8d1bdae48 + X-Amzn-Bedrock-Invocation-Latency: + - '1143' + X-Amzn-Bedrock-Output-Token-Count: + - '49' + X-Amzn-Bedrock-Input-Token-Count: + - '100' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_01QxrkK4Jn5mBeVvowN9pD7g","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"Yukihiro + Matsumoto first released Ruby in 1995. Specifically, he publicly released + Ruby version 0.95 in December of that year, making 1995 the year Ruby was + created."}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":100,"output_tokens":49}}' + recorded_at: Tue, 25 Mar 2025 22:24:17 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation.yml b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation.yml deleted file mode 100644 index b1ced5672..000000000 --- a/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation.yml +++ /dev/null @@ -1,55 +0,0 @@ ---- -http_interactions: -- request: - method: post - uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke - body: - encoding: UTF-8 - string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"\n\nHuman: - What''s 2 + 2?"}],"temperature":0.7,"max_tokens":4096}' - headers: - User-Agent: - - Faraday v2.12.2 - Host: - - bedrock-runtime.us-west-2.amazonaws.com - X-Amz-Date: - - 20250325T173339Z - X-Amz-Security-Token: - - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i - X-Amz-Content-Sha256: - - 433228d8fa6dfe7b1ac9ba49a6cb06882cf07f05124baee8a2f48d38b4d7e855 - Authorization: - - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, - SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=8aaf70e624d3df533de47499fde8d1285474f77fddf7fa1e6d71c4c738528a0e - Content-Type: - - application/json - Accept: - - application/json - Accept-Encoding: - - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 - response: - status: - code: 200 - message: OK - headers: - Date: - - Tue, 25 Mar 2025 17:32:58 GMT - Content-Type: - - application/json - Content-Length: - - '245' - Connection: - - keep-alive - X-Amzn-Requestid: - - 763d4d4c-c80b-4361-8203-1366e8046222 - X-Amzn-Bedrock-Invocation-Latency: - - '838' - X-Amzn-Bedrock-Output-Token-Count: - - '5' - X-Amzn-Bedrock-Input-Token-Count: - - '20' - body: - encoding: UTF-8 - string: '{"id":"msg_bdrk_01BfVD9JeTeEH5mcUYcBLkpp","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"4"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":20,"output_tokens":5}}' - recorded_at: Tue, 25 Mar 2025 17:33:40 GMT -recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation_with_bedrock.yml b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation_with_bedrock.yml new file mode 100644 index 000000000..8179ddd2b --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_have_a_basic_conversation_with_bedrock.yml @@ -0,0 +1,55 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + 2 + 2?"}],"temperature":0.7,"max_tokens":4096}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222412Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - b5957298a88788e861cf24c2faff82644662c726eed2de6599a41f3789caa703 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=2223548f016c41afc671c3964098931aeb2d9ca07b3a18c13402c5289f4a7aa9 + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:23:32 GMT + Content-Type: + - application/json + Content-Length: + - '245' + Connection: + - keep-alive + X-Amzn-Requestid: + - 186ac8b6-6234-45f8-8b4c-c893e78a949d + X-Amzn-Bedrock-Invocation-Latency: + - '946' + X-Amzn-Bedrock-Output-Token-Count: + - '5' + X-Amzn-Bedrock-Input-Token-Count: + - '16' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_018iG3YNksavaUeLbt48zRrB","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"4"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":16,"output_tokens":5}}' + recorded_at: Tue, 25 Mar 2025 22:24:13 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_in_multi-turn_conversations_with_bedrock.yml b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_in_multi-turn_conversations_with_bedrock.yml new file mode 100644 index 000000000..c8a31a137 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_in_multi-turn_conversations_with_bedrock.yml @@ -0,0 +1,245 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222238Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 175fe07b52825d214f2938f4dcfb11a1099924121ee86d46d6b027b10fa92fa4 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=34d98b1daf7e73fed2dfec76ff790ee8db4cdbe70bbba364e32721b52446233a + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:21:58 GMT + Content-Type: + - application/json + Content-Length: + - '460' + Connection: + - keep-alive + X-Amzn-Requestid: + - ca2ada88-c8d1-46ab-8a2d-eb1e6c3868dc + X-Amzn-Bedrock-Invocation-Latency: + - '1709' + X-Amzn-Bedrock-Output-Token-Count: + - '90' + X-Amzn-Bedrock-Input-Token-Count: + - '389' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_01AheF11cFX5N1bR8XeJD7qj","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''ll + help you check the current weather in Berlin using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_01RjqxekiaA5DWYzAHaxdL2m","name":"weather","input":{"latitude":"52.5200","longitude":"13.4050"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":389,"output_tokens":90}}' + recorded_at: Tue, 25 Mar 2025 22:22:40 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","content":[{"type":"text","text":"I''ll + help you check the current weather in Berlin using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_01RjqxekiaA5DWYzAHaxdL2m","name":"weather","input":{"latitude":"52.5200","longitude":"13.4050"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_01RjqxekiaA5DWYzAHaxdL2m","content":"Current + weather at 52.5200, 13.4050: 15°C, Wind: 10 km/h"}]}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222240Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 617b7969b05cf69b588756ce435959ced5b8a73c9c075762eb0a24b2c60a5253 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=543d20b349fd60ff4b8506070cdc500d472b804705630d8b20460cdbc235c9d0 + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:22:01 GMT + Content-Type: + - application/json + Content-Length: + - '558' + Connection: + - keep-alive + X-Amzn-Requestid: + - 424dc6e1-3c02-4310-8858-95b20fc10390 + X-Amzn-Bedrock-Invocation-Latency: + - '2392' + X-Amzn-Bedrock-Output-Token-Count: + - '79' + X-Amzn-Bedrock-Input-Token-Count: + - '518' + body: + encoding: ASCII-8BIT + string: !binary |- + eyJpZCI6Im1zZ19iZHJrXzAxM2JIc2lKVTg0UGh4NTg4SHp6YXRQZyIsInR5cGUiOiJtZXNzYWdlIiwicm9sZSI6ImFzc2lzdGFudCIsIm1vZGVsIjoiY2xhdWRlLTMtNS1oYWlrdS0yMDI0MTAyMiIsImNvbnRlbnQiOlt7InR5cGUiOiJ0ZXh0IiwidGV4dCI6IkJhc2VkIG9uIHRoZSB3ZWF0aGVyIGluZm9ybWF0aW9uLCBoZXJlJ3MgdGhlIGN1cnJlbnQgd2VhdGhlciBpbiBCZXJsaW46XG4tIFRlbXBlcmF0dXJlOiAxNcKwQyAoNTnCsEYpXG4tIFdpbmQgU3BlZWQ6IDEwIGttL2hcblxuSXQgc2VlbXMgbGlrZSBhIG1pbGQgZGF5IGluIEJlcmxpbiB3aXRoIGEgbW9kZXJhdGUgYnJlZXplLiBUaGUgdGVtcGVyYXR1cmUgaXMgY29tZm9ydGFibGUsIHBlcmZlY3QgZm9yIGxpZ2h0IG91dGRvb3IgYWN0aXZpdGllcy4gV291bGQgeW91IGxpa2UgdG8ga25vdyBhbnl0aGluZyBlbHNlIGFib3V0IHRoZSB3ZWF0aGVyPyJ9XSwic3RvcF9yZWFzb24iOiJlbmRfdHVybiIsInN0b3Bfc2VxdWVuY2UiOm51bGwsInVzYWdlIjp7ImlucHV0X3Rva2VucyI6NTE4LCJvdXRwdXRfdG9rZW5zIjo3OX19 + recorded_at: Tue, 25 Mar 2025 22:22:42 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","content":[{"type":"text","text":"I''ll + help you check the current weather in Berlin using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_01RjqxekiaA5DWYzAHaxdL2m","name":"weather","input":{"latitude":"52.5200","longitude":"13.4050"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_01RjqxekiaA5DWYzAHaxdL2m","content":"Current + weather at 52.5200, 13.4050: 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":"Based + on the weather information, here''s the current weather in Berlin:\n- Temperature: + 15°C (59°F)\n- Wind Speed: 10 km/h\n\nIt seems like a mild day in Berlin with + a moderate breeze. The temperature is comfortable, perfect for light outdoor + activities. Would you like to know anything else about the weather?"},{"role":"user","content":"What''s + the weather in Paris? (48.8575, 2.3514)"}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222242Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - c96f336256d9638968fb74b20338a6f8437c6065e81907ce6344384541430ce9 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=27ede3aea3e719588d13ac055d6b9bb1071a83bc0e6e7e2a8a64444966d2ee50 + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:22:02 GMT + Content-Type: + - application/json + Content-Length: + - '465' + Connection: + - keep-alive + X-Amzn-Requestid: + - 8db67aae-dfc5-4594-86fa-a1e83ddcb2e7 + X-Amzn-Bedrock-Invocation-Latency: + - '1338' + X-Amzn-Bedrock-Output-Token-Count: + - '89' + X-Amzn-Bedrock-Input-Token-Count: + - '619' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_0184kfx1R6kc44pc5K8FGVTX","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''ll + retrieve the current weather information for Paris using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_018WvqrpbvkYWcXZ8BbicqK2","name":"weather","input":{"latitude":"48.8575","longitude":"2.3514"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":619,"output_tokens":89}}' + recorded_at: Tue, 25 Mar 2025 22:22:44 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","content":[{"type":"text","text":"I''ll + help you check the current weather in Berlin using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_01RjqxekiaA5DWYzAHaxdL2m","name":"weather","input":{"latitude":"52.5200","longitude":"13.4050"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_01RjqxekiaA5DWYzAHaxdL2m","content":"Current + weather at 52.5200, 13.4050: 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":"Based + on the weather information, here''s the current weather in Berlin:\n- Temperature: + 15°C (59°F)\n- Wind Speed: 10 km/h\n\nIt seems like a mild day in Berlin with + a moderate breeze. The temperature is comfortable, perfect for light outdoor + activities. Would you like to know anything else about the weather?"},{"role":"user","content":"What''s + the weather in Paris? (48.8575, 2.3514)"},{"role":"assistant","content":[{"type":"text","text":"I''ll + retrieve the current weather information for Paris using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_018WvqrpbvkYWcXZ8BbicqK2","name":"weather","input":{"latitude":"48.8575","longitude":"2.3514"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_018WvqrpbvkYWcXZ8BbicqK2","content":"Current + weather at 48.8575, 2.3514: 15°C, Wind: 10 km/h"}]}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222244Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - c4e15659416061fb996a331cbf4318dbf3dfaf53a241eed16e620bbeb12c30ce + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=e577db0e44c13043ac927ea038d5e25515010a3bb50f2158b8f4b59d570cb32f + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:22:05 GMT + Content-Type: + - application/json + Content-Length: + - '602' + Connection: + - keep-alive + X-Amzn-Requestid: + - c52cad00-c97a-42e7-b601-e3c8e9c21c76 + X-Amzn-Bedrock-Invocation-Latency: + - '2293' + X-Amzn-Bedrock-Output-Token-Count: + - '93' + X-Amzn-Bedrock-Input-Token-Count: + - '747' + body: + encoding: ASCII-8BIT + string: !binary |- + eyJpZCI6Im1zZ19iZHJrXzAxOXFremVnaW50ZW1zNnBzNEJBNllXeiIsInR5cGUiOiJtZXNzYWdlIiwicm9sZSI6ImFzc2lzdGFudCIsIm1vZGVsIjoiY2xhdWRlLTMtNS1oYWlrdS0yMDI0MTAyMiIsImNvbnRlbnQiOlt7InR5cGUiOiJ0ZXh0IiwidGV4dCI6IkhlcmUncyB0aGUgY3VycmVudCB3ZWF0aGVyIGluIFBhcmlzOlxuLSBUZW1wZXJhdHVyZTogMTXCsEMgKDU5wrBGKVxuLSBXaW5kIFNwZWVkOiAxMCBrbS9oXG5cbkludGVyZXN0aW5nbHksIHRoZSB3ZWF0aGVyIGluIFBhcmlzIGlzIHF1aXRlIHNpbWlsYXIgdG8gQmVybGluIGF0IHRoZSBtb21lbnQsIHdpdGggdGhlIHNhbWUgdGVtcGVyYXR1cmUgYW5kIHdpbmQgc3BlZWQuIEl0J3MgYSBtaWxkIGRheSB0aGF0IHNob3VsZCBiZSBwbGVhc2FudCBmb3Igd2Fsa2luZyBhcm91bmQgYW5kIGV4cGxvcmluZyB0aGUgY2l0eS4gSXMgdGhlcmUgYW55dGhpbmcgZWxzZSB5b3UnZCBsaWtlIHRvIGtub3cgYWJvdXQgdGhlIHdlYXRoZXI/In1dLCJzdG9wX3JlYXNvbiI6ImVuZF90dXJuIiwic3RvcF9zZXF1ZW5jZSI6bnVsbCwidXNhZ2UiOnsiaW5wdXRfdG9rZW5zIjo3NDcsIm91dHB1dF90b2tlbnMiOjkzfX0= + recorded_at: Tue, 25 Mar 2025 22:22:46 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_with_bedrock.yml b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_with_bedrock.yml new file mode 100644 index 000000000..37a0c4049 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_with_bedrock.yml @@ -0,0 +1,117 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T214315Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 175fe07b52825d214f2938f4dcfb11a1099924121ee86d46d6b027b10fa92fa4 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=b4a8925c3fad30e1dc39636ad2378fa02662aa8bc1567778c04bed77b1ec510b + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 21:42:36 GMT + Content-Type: + - application/json + Content-Length: + - '465' + Connection: + - keep-alive + X-Amzn-Requestid: + - e3bd8814-37bc-4a6f-82ac-b2d062d34147 + X-Amzn-Bedrock-Invocation-Latency: + - '1800' + X-Amzn-Bedrock-Output-Token-Count: + - '91' + X-Amzn-Bedrock-Input-Token-Count: + - '389' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_01Ne8yKo4hhdGv3xkVzDN5fP","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"Let + me check the current weather in Berlin for you using the coordinates you provided."},{"type":"tool_use","id":"toolu_bdrk_0154fPJkniARCyGTTWHgN2CV","name":"weather","input":{"latitude":"52.5200","longitude":"13.4050"}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":389,"output_tokens":91}}' + recorded_at: Tue, 25 Mar 2025 21:43:17 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","content":[{"type":"text","text":"Let + me check the current weather in Berlin for you using the coordinates you provided."},{"type":"tool_use","id":"toolu_bdrk_0154fPJkniARCyGTTWHgN2CV","name":"weather","input":{"latitude":"52.5200","longitude":"13.4050"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_0154fPJkniARCyGTTWHgN2CV","content":"Current + weather at 52.5200, 13.4050: 15°C, Wind: 10 km/h"}]}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T214317Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - cff2e059af1693e25b684a93f8bc18a8e8615af89575672e62308129a2a0789a + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=4c8bebcd9cd21115855985fc879fcfca2108d3f65fb0e444ad23bbe06a67942c + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 21:42:38 GMT + Content-Type: + - application/json + Content-Length: + - '529' + Connection: + - keep-alive + X-Amzn-Requestid: + - 4cd943ef-539f-4698-9a0a-2e0afa0528dc + X-Amzn-Bedrock-Invocation-Latency: + - '2504' + X-Amzn-Bedrock-Output-Token-Count: + - '73' + X-Amzn-Bedrock-Input-Token-Count: + - '519' + body: + encoding: ASCII-8BIT + string: !binary |- + eyJpZCI6Im1zZ19iZHJrXzAxMmtyMTJYNUNuUzFScFFFWFAxNHdYRCIsInR5cGUiOiJtZXNzYWdlIiwicm9sZSI6ImFzc2lzdGFudCIsIm1vZGVsIjoiY2xhdWRlLTMtNS1oYWlrdS0yMDI0MTAyMiIsImNvbnRlbnQiOlt7InR5cGUiOiJ0ZXh0IiwidGV4dCI6IkJhc2VkIG9uIHRoZSB3ZWF0aGVyIGluZm9ybWF0aW9uLCBoZXJlJ3MgdGhlIGN1cnJlbnQgd2VhdGhlciBpbiBCZXJsaW46XG4tIFRlbXBlcmF0dXJlOiAxNcKwQyAoNTnCsEYpXG4tIFdpbmQgU3BlZWQ6IDEwIGttL2hcblxuSXQgc2VlbXMgbGlrZSBhIG1pbGQgZGF5IGluIEJlcmxpbiB3aXRoIGEgbW9kZXJhdGUgYnJlZXplLiBUaGUgdGVtcGVyYXR1cmUgaXMgY29tZm9ydGFibGUsIHBlcmZlY3QgZm9yIGxpZ2h0IG91dGRvb3IgYWN0aXZpdGllcyBvciB3YWxraW5nIGFyb3VuZCB0aGUgY2l0eS4ifV0sInN0b3BfcmVhc29uIjoiZW5kX3R1cm4iLCJzdG9wX3NlcXVlbmNlIjpudWxsLCJ1c2FnZSI6eyJpbnB1dF90b2tlbnMiOjUxOSwib3V0cHV0X3Rva2VucyI6NzN9fQ== + recorded_at: Tue, 25 Mar 2025 21:43:20 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_with_multi-turn_streaming_conversations_with_bedrock.yml b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_with_multi-turn_streaming_conversations_with_bedrock.yml new file mode 100644 index 000000000..3c950606a --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_with_multi-turn_streaming_conversations_with_bedrock.yml @@ -0,0 +1,365 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke-with-response-stream + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T220703Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 175fe07b52825d214f2938f4dcfb11a1099924121ee86d46d6b027b10fa92fa4 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=024e659098284d92279f67cd0b591e037a612f06c43f36626ce06748c1b1c8b7 + Content-Type: + - application/json + Accept: + - application/vnd.amazon.eventstream + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:06:22 GMT + Content-Type: + - application/vnd.amazon.eventstream + Transfer-Encoding: + - chunked + Connection: + - keep-alive + X-Amzn-Requestid: + - c620a31c-c1e9-461c-bda8-0fa5e2cdf972 + X-Amzn-Bedrock-Content-Type: + - application/json + body: + encoding: ASCII-8BIT + string: !binary |- + AAAB7gAAAEvYlO1qCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHRnlkQ0lzSW0xbGMzTmhaMlVpT25zaWFXUWlPaUp0YzJkZlltUnlhMTh3TVRkRFlYTm5SMWRDV1hVeFpFaFdWbmQwTkdGSFZVY2lMQ0owZVhCbElqb2liV1Z6YzJGblpTSXNJbkp2YkdVaU9pSmhjM05wYzNSaGJuUWlMQ0p0YjJSbGJDSTZJbU5zWVhWa1pTMHpMVFV0YUdGcGEzVXRNakF5TkRFd01qSWlMQ0pqYjI1MFpXNTBJanBiWFN3aWMzUnZjRjl5WldGemIyNGlPbTUxYkd3c0luTjBiM0JmYzJWeGRXVnVZMlVpT201MWJHd3NJblZ6WVdkbElqcDdJbWx1Y0hWMFgzUnZhMlZ1Y3lJNk16ZzVMQ0p2ZFhSd2RYUmZkRzlyWlc1eklqb3hmWDE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVoifXAr6J8AAAD8AAAASwno+i0LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOXpkR0Z5ZENJc0ltbHVaR1Y0SWpvd0xDSmpiMjUwWlc1MFgySnNiMk5ySWpwN0luUjVjR1VpT2lKMFpYaDBJaXdpZEdWNGRDSTZJaUo5ZlE9PSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0QifZaMhKQAAADlAAAAS2QYD94LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSk1aWFFpZlgwPSIsInAiOiJhYmNkZWZnIn095k2yAAABOwAAAEuZgR1DCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUlnYldVZ1kyaGxZMnNnZEdobElHTjFjbkpsYm5RZ2QyVmhkR2hsY2lCbWIzSWdRbVZ5YkdsdUlIVnphVzVuSW4xOSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLIn2WASsfAAABKgAAAEvEAaNxCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUlnZEdobElIQnliM1pwWkdWa0lHTnZiM0prYVc1aGRHVnpMaUo5ZlE9PSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUiJ944YSZQAAALcAAABLJsuTdQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5emRHOXdJaXdpYVc1a1pYZ2lPakI5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dSJ9wGABeAAAAWAAAABL1kLjmQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5emRHRnlkQ0lzSW1sdVpHVjRJam94TENKamIyNTBaVzUwWDJKc2IyTnJJanA3SW5SNWNHVWlPaUowYjI5c1gzVnpaU0lzSW1sa0lqb2lkRzl2YkhWZlltUnlhMTh3TVRWT1RWVmxhbVJYTmxCbVJEUlhXR3B2VWtWUlVsVWlMQ0p1WVcxbElqb2lkMlZoZEdobGNpSXNJbWx1Y0hWMElqcDdmWDE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1Qifa5smiEAAAD0AAAASzmYsewLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpveExDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWFXNXdkWFJmYW5OdmJsOWtaV3gwWVNJc0luQmhjblJwWVd4ZmFuTnZiaUk2SWlKOWZRPT0iLCJwIjoiYWJjZGVmIn3p9R6DAAABLQAAAEt2IX9hCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNkludGNJbXdpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQ1NiJ9ARzv0QAAAS4AAABLMYEFsQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJbUYwYVhSMVpHVWlmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzIn1oiWBuAAABJwAAAEs8kWfACzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklsd2lPaUFpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMCJ9izjqsAAAATMAAABLqfFWggs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJbHdpTlRJdU5USXdNRndpSW4xOSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQifQt8Q1sAAAEiAAAAS/Rx6LALOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpveExDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWFXNXdkWFJmYW5OdmJsOWtaV3gwWVNJc0luQmhjblJwWVd4ZmFuTnZiaUk2SWl3Z1hDSnNiMjVuYVhSMVpDSjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSiJ9FP2w9AAAAP0AAABLNIjTnQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJbVZjSWpvZ1hDSmNJbjBpZlgwPSIsInAiOiJhYmMifbJaATUAAACsAAAASzH7NeYLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOXpkRzl3SWl3aWFXNWtaWGdpT2pGOSIsInAiOiJhYmNkZWZnaGlqIn31LGSlAAABEgAAAEtVUFA2CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5a1pXeDBZU0lzSW1SbGJIUmhJanA3SW5OMGIzQmZjbVZoYzI5dUlqb2lkRzl2YkY5MWMyVWlMQ0p6ZEc5d1gzTmxjWFZsYm1ObElqcHVkV3hzZlN3aWRYTmhaMlVpT25zaWIzVjBjSFYwWDNSdmEyVnVjeUk2T0RoOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcCJ9Hf/T/wAAAVwAAABLspO2Hgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2liV1Z6YzJGblpWOXpkRzl3SWl3aVlXMWhlbTl1TFdKbFpISnZZMnN0YVc1MmIyTmhkR2x2YmsxbGRISnBZM01pT25zaWFXNXdkWFJVYjJ0bGJrTnZkVzUwSWpvek9Ea3NJbTkxZEhCMWRGUnZhMlZ1UTI5MWJuUWlPalk0TENKcGJuWnZZMkYwYVc5dVRHRjBaVzVqZVNJNk16azBNeXdpWm1seWMzUkNlWFJsVEdGMFpXNWplU0k2T0RNeWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSCJ9B57sAQ== + recorded_at: Tue, 25 Mar 2025 22:07:07 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke-with-response-stream + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","content":[{"type":"text","text":"Let + me check the current weather for Berlin using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","name":"weather","input":{"latitude":"52.5200","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","content":"Current + weather at 52.5200, : 15°C, Wind: 10 km/h"}]}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222158Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 39f2dab80df11c1de2f428b35b23d31293e8389166e7c31746c11a36b465a454 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=3a59f20fac243e8a0965af2d1c08394196061d19d61d9b8c8049cb21b457de2f + Content-Type: + - application/json + Accept: + - application/vnd.amazon.eventstream + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:21:16 GMT + Content-Type: + - application/vnd.amazon.eventstream + Transfer-Encoding: + - chunked + Connection: + - keep-alive + X-Amzn-Requestid: + - c66f54ea-f947-4352-a007-6088a5cd3978 + X-Amzn-Bedrock-Content-Type: + - application/json + body: + encoding: ASCII-8BIT + string: !binary |- + AAABvAAAAEuaR3HBCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHRnlkQ0lzSW0xbGMzTmhaMlVpT25zaWFXUWlPaUp0YzJkZlltUnlhMTh3TVV4UVVHVjNibU15ZDNGTk5EUXpXR0ZZUVVaQllVRWlMQ0owZVhCbElqb2liV1Z6YzJGblpTSXNJbkp2YkdVaU9pSmhjM05wYzNSaGJuUWlMQ0p0YjJSbGJDSTZJbU5zWVhWa1pTMHpMVFV0YUdGcGEzVXRNakF5TkRFd01qSWlMQ0pqYjI1MFpXNTBJanBiWFN3aWMzUnZjRjl5WldGemIyNGlPbTUxYkd3c0luTjBiM0JmYzJWeGRXVnVZMlVpT201MWJHd3NJblZ6WVdkbElqcDdJbWx1Y0hWMFgzUnZhMlZ1Y3lJNk5UQTJMQ0p2ZFhSd2RYUmZkRzlyWlc1eklqb3lmWDE5IiwicCI6ImFiIn3X7BsWAAAA8gAAAEu22ERMCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEdGeWRDSXNJbWx1WkdWNElqb3dMQ0pqYjI1MFpXNTBYMkpzYjJOcklqcDdJblI1Y0dVaU9pSjBaWGgwSWl3aWRHVjRkQ0k2SWlKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3QifZMekHEAAAECAAAASzWwx7QLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSmNibHh1U1NKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUYifdSSl4AAAAE2AAAAS2ER2fILOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSWdZWEJ2Ykc5bmFYcGxMQ0JpZFhRZ2FYUWdjMlZsYlhNZ2RHaGxjbVVpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaIn0eru+5AAABOQAAAEvjQU4jCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUlnYldsbmFIUWdhR0YyWlNCaVpXVnVJR0VnYzIxaGJHd2dhWE56ZFdVZ2QybDBhQ0IwYUdVZ1puVnVZM1JwYjI0aWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERSJ9tI0e6wAAAR4AAABLkKC9Nws6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lJZ1kyRnNiQzRnVEdWMElHMWxJSFJ5ZVNCaFoyRnBiaUIzYVhSb0lHSnZkR2dpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3gifRfBWvMAAAEeAAAAS5CgvTcLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSWdkR2hsSUd4aGRHbDBkV1JsSUdGdVpDQnNiMjVuYVhSMVpHVTZJbjE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGIn3DuoHjAAAA3wAAAEuPia/5CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEc5d0lpd2lhVzVrWlhnaU9qQjkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NTY3OCJ933gtaAAAAWkAAABL21KB6As6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5emRHRnlkQ0lzSW1sdVpHVjRJam94TENKamIyNTBaVzUwWDJKc2IyTnJJanA3SW5SNWNHVWlPaUowYjI5c1gzVnpaU0lzSW1sa0lqb2lkRzl2YkhWZlltUnlhMTh3TVRKbVRETmxabGhDY2tWaVozRjFRMDVsYzFkNlJtUWlMQ0p1WVcxbElqb2lkMlZoZEdobGNpSXNJbWx1Y0hWMElqcDdmWDE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIifTSquX4AAAEkAAAAS3sxHRALOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpveExDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWFXNXdkWFJmYW5OdmJsOWtaV3gwWVNJc0luQmhjblJwWVd4ZmFuTnZiaUk2SWlKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxIn065KDUAAABDwAAAEvNIAMFCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNkludGNJbXhoZEdsMGRXUmxJbjE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dSJ9IgFU+gAAASYAAABLAfFOcAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJbHdpT2lCY0lqVXlMalV5TURBaWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTiJ9BRcswQAAAPsAAABLu8gmPQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJbHdpSW4xOSIsInAiOiJhYmNkZWZnaGlqa2xtIn3Af37iAAABKgAAAEvEAaNxCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklpd2dYQ0pzYjI1bmFYUWlmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVViJ9jr0x3QAAAQsAAABLOKClxQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJblZrWlZ3aU9pSjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dSJ9v/X/nAAAASYAAABLAfFOcAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJaUJjSWx3aWZTSjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVViJ9nI5PDAAAAM4AAABL0gkRyws6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5emRHOXdJaXdpYVc1a1pYZ2lPakY5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSIn3w8dW3AAABBQAAAEuHkBukCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5a1pXeDBZU0lzSW1SbGJIUmhJanA3SW5OMGIzQmZjbVZoYzI5dUlqb2lkRzl2YkY5MWMyVWlMQ0p6ZEc5d1gzTmxjWFZsYm1ObElqcHVkV3hzZlN3aWRYTmhaMlVpT25zaWIzVjBjSFYwWDNSdmEyVnVjeUk2TVRBMmZYMD0iLCJwIjoiYWJjIn3rYdycAAABdwAAAEsEgqgLCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHOXdJaXdpWVcxaGVtOXVMV0psWkhKdlkyc3RhVzUyYjJOaGRHbHZiazFsZEhKcFkzTWlPbnNpYVc1d2RYUlViMnRsYmtOdmRXNTBJam8xTURZc0ltOTFkSEIxZEZSdmEyVnVRMjkxYm5RaU9qZ3lMQ0pwYm5adlkyRjBhVzl1VEdGMFpXNWplU0k2TWpjeE5Dd2labWx5YzNSQ2VYUmxUR0YwWlc1amVTSTZOalF4ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQ1Njc4In37k5td + recorded_at: Tue, 25 Mar 2025 22:22:00 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke-with-response-stream + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","content":[{"type":"text","text":"Let + me check the current weather for Berlin using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","name":"weather","input":{"latitude":"52.5200","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","content":"Current + weather at 52.5200, : 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":[{"type":"text","text":"\n\nI + apologize, but it seems there might have been a small issue with the function + call. Let me try again with both the latitude and longitude:"},{"type":"tool_use","id":"toolu_bdrk_012fL3efXBrEbgquCNesWzFd","name":"weather","input":{"latitude":"52.5200","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_012fL3efXBrEbgquCNesWzFd","content":"Current + weather at 52.5200, : 15°C, Wind: 10 km/h"}]}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222200Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 6f22a6d3fc17ab5c783c0edfcd330aaa63e8ef18ead4d0982665011e447e5224 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=cdeb0b08112c622175fa8480bd344781e6ef1f5be2e8c7ba02feb5adbcf177dd + Content-Type: + - application/json + Accept: + - application/vnd.amazon.eventstream + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:21:19 GMT + Content-Type: + - application/vnd.amazon.eventstream + Transfer-Encoding: + - chunked + Connection: + - keep-alive + X-Amzn-Requestid: + - e5182753-ee8f-4ad4-af75-406472fe4294 + X-Amzn-Bedrock-Content-Type: + - application/json + body: + encoding: ASCII-8BIT + string: !binary |- + AAAB3wAAAEtE1XxcCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHRnlkQ0lzSW0xbGMzTmhaMlVpT25zaWFXUWlPaUp0YzJkZlltUnlhMTh3TVRWWFJGUjNjRE14WVZVNVVXaDROa3haT1VkRWMzWWlMQ0owZVhCbElqb2liV1Z6YzJGblpTSXNJbkp2YkdVaU9pSmhjM05wYzNSaGJuUWlMQ0p0YjJSbGJDSTZJbU5zWVhWa1pTMHpMVFV0YUdGcGEzVXRNakF5TkRFd01qSWlMQ0pqYjI1MFpXNTBJanBiWFN3aWMzUnZjRjl5WldGemIyNGlPbTUxYkd3c0luTjBiM0JmYzJWeGRXVnVZMlVpT201MWJHd3NJblZ6WVdkbElqcDdJbWx1Y0hWMFgzUnZhMlZ1Y3lJNk5qUXhMQ0p2ZFhSd2RYUmZkRzlyWlc1eklqb3lmWDE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSksifVTHfJYAAAERAAAASxLwKuYLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOXpkR0Z5ZENJc0ltbHVaR1Y0SWpvd0xDSmpiMjUwWlc1MFgySnNiMk5ySWpwN0luUjVjR1VpT2lKMFpYaDBJaXdpZEdWNGRDSTZJaUo5ZlE9PSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFkifaHOKG0AAADsAAAAS2kIba8LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSmNibHh1U1NKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpaiJ9u30z/wAAAScAAABLPJFnwAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lJZ1lYQnZiRzluYVhwbElHWnZjaUIwYUdVZ2FXNWpiMjUyWlc1cFpXNWpaUzRnU1hRaWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkMifQ/gt2IAAAFPAAAAS5XTW0wLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSWdZWEJ3WldGeWN5QjBhR0YwSUhSb1pTQjNaV0YwYUdWeUlHWjFibU4wYVc5dUlHbHpJRzV2ZENCM2IzSnJhVzVuSUdOdmNuSmxZM1JzZVNCaGRDSjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGRyJ9M2w3hgAAASoAAABLxAGjcQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lJZ2RHaGxJRzF2YldWdWRDNGdTRzkzWlhabGNpd2dZbUZ6WldRZ2IyNGdkR2hsSUhCaGNuUnBZV3dpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3gifVfgpZsAAAEqAAAAS8QBo3ELOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSWdhVzVtYjNKdFlYUnBiMjRnY21WalpXbDJaV1FzSUhSb1pTQjBaVzF3WlhKaGRIVnlaU0JwYmlCQ1pYSnNhVzRnYVhNZ1lYSnZkVzRpZlgwPSIsInAiOiJhYmNkIn3cT7qKAAABCwAAAEs4oKXFCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUprSURFMXdyQkRJSGRwZEdnZ1lTQjNhVzRpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5In1tMgUMAAABJgAAAEsB8U5wCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUprSUhOd1pXVmtJRzltSURFd0lHdHRMMmd1SUNKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWIn0l8U6EAAABIQAAAEuz0ZJgCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUpjYmx4dVJtOXlJSFJvWlNCdGIzTjBJR0ZqWTNWeVlYUmxJR0Z1WkNCMWNDMTBieTBpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2dyJ9YBy0zAAAASYAAABLAfFOcAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lKa1lYUmxJSGRsWVhSb1pYSWdhVzVtYjNKdFlYUnBiMjRzSUVrZ2NtVmpiMjF0Wlc0aWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQiJ9dtqPDwAAASAAAABLjrG70As6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lKa0lHTm9aV05yYVc1bklHRWdiRzlqWVd3Z2QyVmhkR2hsY2lCelpYSjJhV05sSUc5eUlIZGxZbk5wZEdVdUlGZHZkV3dpZlgwPSIsInAiOiJhYiJ9jhGL/AAAAUQAAABL4gNqXQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lKa0lIbHZkU0JzYVd0bElHMWxJSFJ2SUdobGJIQWdlVzkxSUdacGJtUWdZU0J5Wld4cFlXSnNaU0IzWldGMGFHVnlJbjE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUCJ9bX2JYQAAASwAAABLS0FW0Qs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lJZ2MyOTFjbU5sSUdadmNpQkNaWEpzYVc0L0luMTkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NSJ9DzGoPAAAAKcAAABLRisE9ws6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5emRHOXdJaXdpYVc1a1pYZ2lPakI5IiwicCI6ImFiY2RlIn14AZkDAAABPQAAAEsWwejjCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5a1pXeDBZU0lzSW1SbGJIUmhJanA3SW5OMGIzQmZjbVZoYzI5dUlqb2laVzVrWDNSMWNtNGlMQ0p6ZEc5d1gzTmxjWFZsYm1ObElqcHVkV3hzZlN3aWRYTmhaMlVpT25zaWIzVjBjSFYwWDNSdmEyVnVjeUk2T1RoOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NTYifRNaFOAAAAF1AAAAS35C+2sLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pYldWemMyRm5aVjl6ZEc5d0lpd2lZVzFoZW05dUxXSmxaSEp2WTJzdGFXNTJiMk5oZEdsdmJrMWxkSEpwWTNNaU9uc2lhVzV3ZFhSVWIydGxia052ZFc1MElqbzJOREVzSW05MWRIQjFkRlJ2YTJWdVEyOTFiblFpT2prNExDSnBiblp2WTJGMGFXOXVUR0YwWlc1amVTSTZNekE1TVN3aVptbHljM1JDZVhSbFRHRjBaVzVqZVNJNk5USXdmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2In0G4VOT + recorded_at: Tue, 25 Mar 2025 22:22:04 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke-with-response-stream + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","content":[{"type":"text","text":"Let + me check the current weather for Berlin using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","name":"weather","input":{"latitude":"52.5200","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","content":"Current + weather at 52.5200, : 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":[{"type":"text","text":"\n\nI + apologize, but it seems there might have been a small issue with the function + call. Let me try again with both the latitude and longitude:"},{"type":"tool_use","id":"toolu_bdrk_012fL3efXBrEbgquCNesWzFd","name":"weather","input":{"latitude":"52.5200","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_012fL3efXBrEbgquCNesWzFd","content":"Current + weather at 52.5200, : 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":"\n\nI + apologize for the inconvenience. It appears that the weather function is not + working correctly at the moment. However, based on the partial information + received, the temperature in Berlin is around 15°C with a wind speed of 10 + km/h. \n\nFor the most accurate and up-to-date weather information, I recommend + checking a local weather service or website. Would you like me to help you + find a reliable weather source for Berlin?"},{"role":"user","content":"What''s + the weather in Paris? (48.8575, 2.3514)"}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222204Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 576502aa6ad10130bfcb9b581d4ef6dca8e199c1e11e4f10150a2a89e24d06ad + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=54ec86ef80dda18c124d1176efd1bf52d32cf4086264da7af276b5dd3d460533 + Content-Type: + - application/json + Accept: + - application/vnd.amazon.eventstream + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:21:22 GMT + Content-Type: + - application/vnd.amazon.eventstream + Transfer-Encoding: + - chunked + Connection: + - keep-alive + X-Amzn-Requestid: + - 31051375-0aed-43ae-901e-e9becd7567c6 + X-Amzn-Bedrock-Content-Type: + - application/json + body: + encoding: ASCII-8BIT + string: !binary |- + AAAByQAAAEurdR5+CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHRnlkQ0lzSW0xbGMzTmhaMlVpT25zaWFXUWlPaUp0YzJkZlltUnlhMTh3TVRKSVJIVnlZWEJZYm5RMldrdDFZVFYxT1ROaFpIRWlMQ0owZVhCbElqb2liV1Z6YzJGblpTSXNJbkp2YkdVaU9pSmhjM05wYzNSaGJuUWlMQ0p0YjJSbGJDSTZJbU5zWVhWa1pTMHpMVFV0YUdGcGEzVXRNakF5TkRFd01qSWlMQ0pqYjI1MFpXNTBJanBiWFN3aWMzUnZjRjl5WldGemIyNGlPbTUxYkd3c0luTjBiM0JmYzJWeGRXVnVZMlVpT201MWJHd3NJblZ6WVdkbElqcDdJbWx1Y0hWMFgzUnZhMlZ1Y3lJNk56WXhMQ0p2ZFhSd2RYUmZkRzlyWlc1eklqb3lmWDE5IiwicCI6ImFiY2RlZmdoaWprbG1ubyJ9w/PgCQAAAOAAAABLrPiArgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5emRHRnlkQ0lzSW1sdVpHVjRJam93TENKamIyNTBaVzUwWDJKc2IyTnJJanA3SW5SNWNHVWlPaUowWlhoMElpd2lkR1Y0ZENJNklpSjlmUT09IiwicCI6ImFiIn2SGprmAAABGwAAAEtYQDJHCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUpKSjJ4c0luMTkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxMjM0NTY3OCJ9w+GVFAAAAWEAAABL6yLKKQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lJZ1kyaGxZMnNnZEdobElHTjFjbkpsYm5RZ2QyVmhkR2hsY2lCbWIzSWdVR0Z5YVhNZ2RYTnBibWNnZEdobElIQnliM1pwWkdWa0lHTnZiM0prYVc1aGRHVnpMaUo5ZlE9PSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRIn2D3tCTAAAAyAAAAEtdSeRrCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEc5d0lpd2lhVzVrWlhnaU9qQjkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0wifYcXTLMAAAE4AAAAS94hZ5MLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOXpkR0Z5ZENJc0ltbHVaR1Y0SWpveExDSmpiMjUwWlc1MFgySnNiMk5ySWpwN0luUjVjR1VpT2lKMGIyOXNYM1Z6WlNJc0ltbGtJam9pZEc5dmJIVmZZbVJ5YTE4d01WSnRWM0ZaWnpOb00xcGpiVzlwWkROV2NYWjRXRElpTENKdVlXMWxJam9pZDJWaGRHaGxjaUlzSW1sdWNIVjBJanA3ZlgxOSIsInAiOiJhYmNkZWYifT9xji8AAAEIAAAAS38A3xULOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpveExDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWFXNXdkWFJmYW5OdmJsOWtaV3gwWVNJc0luQmhjblJwWVd4ZmFuTnZiaUk2SWlKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXoifdcQu9oAAAD9AAAASzSI050LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpveExDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWFXNXdkWFJmYW5OdmJsOWtaV3gwWVNJc0luQmhjblJwWVd4ZmFuTnZiaUk2SW50Y0lteGhkQ0o5ZlE9PSIsInAiOiJhYmNkZWZnIn0pr17dAAABKQAAAEuDodmhCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNkltbDBkV1JsWENJaWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZIn04WAFFAAABKQAAAEuDodmhCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklqb2dYQ0kwT0M0aWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZIn2eMvOnAAAA+QAAAEvBCHVdCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklqZzFOelVpZlgwPSIsInAiOiJhYmNkZWZnIn02xOytAAABIAAAAEuOsbvQCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklsd2lJbjE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYIn1o8KJHAAABFAAAAEvaEKWWCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklpd2dYQ0pzYnlKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNEIn1b6NSkAAABCwAAAEs4oKXFCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNkltNW5JbjE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDIn2CDdXjAAABHQAAAEvXAMfnCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNkltbDBkV1JsWENJNklGd2lYQ0o5SW4xOSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFIn08owdtAAAArwAAAEt2W082CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEc5d0lpd2lhVzVrWlhnaU9qRjkiLCJwIjoiYWJjZGVmZ2hpamtsbSJ93cUYDwAAARkAAABLIoBhJws6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2liV1Z6YzJGblpWOWtaV3gwWVNJc0ltUmxiSFJoSWpwN0luTjBiM0JmY21WaGMyOXVJam9pZEc5dmJGOTFjMlVpTENKemRHOXdYM05sY1hWbGJtTmxJanB1ZFd4c2ZTd2lkWE5oWjJVaU9uc2liM1YwY0hWMFgzUnZhMlZ1Y3lJNk9EaDlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3In3z0VQIAAABYgAAAEusgrD5CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHOXdJaXdpWVcxaGVtOXVMV0psWkhKdlkyc3RhVzUyYjJOaGRHbHZiazFsZEhKcFkzTWlPbnNpYVc1d2RYUlViMnRsYmtOdmRXNTBJam8zTmpFc0ltOTFkSEIxZEZSdmEyVnVRMjkxYm5RaU9qWTBMQ0pwYm5adlkyRjBhVzl1VEdGMFpXNWplU0k2TWpNME1Td2labWx5YzNSQ2VYUmxUR0YwWlc1amVTSTZOalk0ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OIn2FY7Bn + recorded_at: Tue, 25 Mar 2025 22:22:06 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke-with-response-stream + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","content":[{"type":"text","text":"Let + me check the current weather for Berlin using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","name":"weather","input":{"latitude":"52.5200","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","content":"Current + weather at 52.5200, : 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":[{"type":"text","text":"\n\nI + apologize, but it seems there might have been a small issue with the function + call. Let me try again with both the latitude and longitude:"},{"type":"tool_use","id":"toolu_bdrk_012fL3efXBrEbgquCNesWzFd","name":"weather","input":{"latitude":"52.5200","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_012fL3efXBrEbgquCNesWzFd","content":"Current + weather at 52.5200, : 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":"\n\nI + apologize for the inconvenience. It appears that the weather function is not + working correctly at the moment. However, based on the partial information + received, the temperature in Berlin is around 15°C with a wind speed of 10 + km/h. \n\nFor the most accurate and up-to-date weather information, I recommend + checking a local weather service or website. Would you like me to help you + find a reliable weather source for Berlin?"},{"role":"user","content":"What''s + the weather in Paris? (48.8575, 2.3514)"},{"role":"assistant","content":[{"type":"text","text":"I''ll + check the current weather for Paris using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_01RmWqYg3h3Zcmoid3VqvxX2","name":"weather","input":{"latitude":"48.8575","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_01RmWqYg3h3Zcmoid3VqvxX2","content":"Current + weather at 48.8575, : 15°C, Wind: 10 km/h"}]}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222206Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 55a01ca368111dc26dae6f75321c82b5c411e7d8083db5351d2999b156e68018 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=84ee5e6fcd96f9733319574f4bc5c7034e14f337558ece7b86c4055ef8ea3e9e + Content-Type: + - application/json + Accept: + - application/vnd.amazon.eventstream + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:21:25 GMT + Content-Type: + - application/vnd.amazon.eventstream + Transfer-Encoding: + - chunked + Connection: + - keep-alive + X-Amzn-Requestid: + - 8091ccc9-47ea-4ec6-8019-713df7fd6965 + X-Amzn-Bedrock-Content-Type: + - application/json + body: + encoding: ASCII-8BIT + string: !binary |- + AAAB7wAAAEvl9MTaCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHRnlkQ0lzSW0xbGMzTmhaMlVpT25zaWFXUWlPaUp0YzJkZlltUnlhMTh3TVRKbVFtWnJWMlpuV0RGaVNFcGtZV3RuWVVwWVUzUWlMQ0owZVhCbElqb2liV1Z6YzJGblpTSXNJbkp2YkdVaU9pSmhjM05wYzNSaGJuUWlMQ0p0YjJSbGJDSTZJbU5zWVhWa1pTMHpMVFV0YUdGcGEzVXRNakF5TkRFd01qSWlMQ0pqYjI1MFpXNTBJanBiWFN3aWMzUnZjRjl5WldGemIyNGlPbTUxYkd3c0luTjBiM0JmYzJWeGRXVnVZMlVpT201MWJHd3NJblZ6WVdkbElqcDdJbWx1Y0hWMFgzUnZhMlZ1Y3lJNk9EYzRMQ0p2ZFhSd2RYUmZkRzlyWlc1eklqb3pmWDE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowIn1UnSwpAAAA9QAAAEsE+JhcCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEdGeWRDSXNJbWx1WkdWNElqb3dMQ0pqYjI1MFpXNTBYMkpzYjJOcklqcDdJblI1Y0dVaU9pSjBaWGgwSWl3aWRHVjRkQ0k2SWlKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnciffZcoq0AAAEHAAAAS/1QSMQLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSmNibHh1U1NCaGNHOXNiMmNpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQyJ9zVTJAgAAAQoAAABLBcCMdQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lKcGVtVXNJR0oxZENCMGFHVnlaU0J6WldWdGN5SjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0In0lDx5XAAABPAAAAEsrocFTCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUlnZEc4Z1ltVWdZVzRnYjI1bmIybHVaeUJwYzNOMVpTQjNhWFJvSUhSb1pTQjNaV0YwYUdWeUluMTkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVCJ9fNg9IQAAAQ8AAABLzSADBQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lJZ1puVnVZM1JwYjI0dUlFeGxkQ0J0WlNCMGNua2dZV2RoYVc0Z2QybDBhQ0o5ZlE9PSIsInAiOiJhYmNkZWZnaGkifeqlzTMAAAEAAAAAS09wlNQLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSWdkR2hsSUdaMWJHd2dZMjl2Y21ScGJtRjBaWE02SW4xOSIsInAiOiJhYmNkZWZnaGlqIn1Vnw6BAAAA1wAAAEu/+eQ4CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEc5d0lpd2lhVzVrWlhnaU9qQjkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAifYRP1sIAAAFXAAAAS8VDhw8LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOXpkR0Z5ZENJc0ltbHVaR1Y0SWpveExDSmpiMjUwWlc1MFgySnNiMk5ySWpwN0luUjVjR1VpT2lKMGIyOXNYM1Z6WlNJc0ltbGtJam9pZEc5dmJIVmZZbVJ5YTE4d01VRnpNVkJaVjJneVdFczFRMmxYWVhFNE5HazBWMjRpTENKdVlXMWxJam9pZDJWaGRHaGxjaUlzSW1sdWNIVjBJanA3ZlgxOSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLIn1dkjI/AAAA/gAAAEtzKKlNCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklpSjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3AifcjktS0AAAEVAAAAS+dwjCYLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpveExDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWFXNXdkWFJmYW5OdmJsOWtaV3gwWVNJc0luQmhjblJwWVd4ZmFuTnZiaUk2SW50Y0lteGhkR2wwZFdSbEluMTkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBIn0l2I/3AAAA+wAAAEu7yCY9CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklsd2lPaUJjSWlKOWZRPT0iLCJwIjoiYWJjZGUifTqPPewAAAEQAAAASy+QA1YLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpveExDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWFXNXdkWFJmYW5OdmJsOWtaV3gwWVNJc0luQmhjblJwWVd4ZmFuTnZiaUk2SWpRNExqZ2lmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDRCJ9Y/78mAAAAPsAAABLu8gmPQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJalUzSW4xOSIsInAiOiJhYmNkZWZnaGlqa2xtIn2HhxepAAAA/QAAAEs0iNOdCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklqVmNJaUo5ZlE9PSIsInAiOiJhYmNkZWZnaGlqayJ95TO/JAAAATIAAABLlJF/Mgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJaXdnWENKc2IyNGlmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2NyJ9bmNbEQAAASIAAABL9HHosAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam94TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pYVc1d2RYUmZhbk52Ymw5a1pXeDBZU0lzSW5CaGNuUnBZV3hmYW5OdmJpSTZJbWRwZEhWa1pWd2lPaUJjSWpJaWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKIn2HjGX4AAABBQAAAEuHkBukCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3hMQ0prWld4MFlTSTZleUowZVhCbElqb2lhVzV3ZFhSZmFuTnZibDlrWld4MFlTSXNJbkJoY25ScFlXeGZhbk52YmlJNklpNHpOVEUwWENKOUluMTkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vIn1dVgocAAAApgAAAEt7Sy1HCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEc5d0lpd2lhVzVrWlhnaU9qRjkiLCJwIjoiYWJjZCJ9fFYeiwAAASoAAABLxAGjcQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2liV1Z6YzJGblpWOWtaV3gwWVNJc0ltUmxiSFJoSWpwN0luTjBiM0JmY21WaGMyOXVJam9pZEc5dmJGOTFjMlVpTENKemRHOXdYM05sY1hWbGJtTmxJanB1ZFd4c2ZTd2lkWE5oWjJVaU9uc2liM1YwY0hWMFgzUnZhMlZ1Y3lJNk1UQXlmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU4ifVrDDRIAAAFTAAAASzDDIc8LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pYldWemMyRm5aVjl6ZEc5d0lpd2lZVzFoZW05dUxXSmxaSEp2WTJzdGFXNTJiMk5oZEdsdmJrMWxkSEpwWTNNaU9uc2lhVzV3ZFhSVWIydGxia052ZFc1MElqbzROemdzSW05MWRIQjFkRlJ2YTJWdVEyOTFiblFpT2pjd0xDSnBiblp2WTJGMGFXOXVUR0YwWlc1amVTSTZNakF6TlN3aVptbHljM1JDZVhSbFRHRjBaVzVqZVNJNk1UYzFNbjE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHkifekoGwc= + recorded_at: Tue, 25 Mar 2025 22:22:08 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke-with-response-stream + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the weather in Berlin? (52.5200, 13.4050)"},{"role":"assistant","content":[{"type":"text","text":"Let + me check the current weather for Berlin using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","name":"weather","input":{"latitude":"52.5200","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_015NMUejdW6PfD4WXjoREQRU","content":"Current + weather at 52.5200, : 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":[{"type":"text","text":"\n\nI + apologize, but it seems there might have been a small issue with the function + call. Let me try again with both the latitude and longitude:"},{"type":"tool_use","id":"toolu_bdrk_012fL3efXBrEbgquCNesWzFd","name":"weather","input":{"latitude":"52.5200","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_012fL3efXBrEbgquCNesWzFd","content":"Current + weather at 52.5200, : 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":"\n\nI + apologize for the inconvenience. It appears that the weather function is not + working correctly at the moment. However, based on the partial information + received, the temperature in Berlin is around 15°C with a wind speed of 10 + km/h. \n\nFor the most accurate and up-to-date weather information, I recommend + checking a local weather service or website. Would you like me to help you + find a reliable weather source for Berlin?"},{"role":"user","content":"What''s + the weather in Paris? (48.8575, 2.3514)"},{"role":"assistant","content":[{"type":"text","text":"I''ll + check the current weather for Paris using the provided coordinates."},{"type":"tool_use","id":"toolu_bdrk_01RmWqYg3h3Zcmoid3VqvxX2","name":"weather","input":{"latitude":"48.8575","longitude":""}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_01RmWqYg3h3Zcmoid3VqvxX2","content":"Current + weather at 48.8575, : 15°C, Wind: 10 km/h"}]},{"role":"assistant","content":[{"type":"text","text":"\n\nI + apologize, but there seems to be an ongoing issue with the weather function. + Let me try again with the full coordinates:"},{"type":"tool_use","id":"toolu_bdrk_01As1PYWh2XK5CiWaq84i4Wn","name":"weather","input":{"latitude":"48.8575","longitude":"2.3514"}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_01As1PYWh2XK5CiWaq84i4Wn","content":"Current + weather at 48.8575, 2.3514: 15°C, Wind: 10 km/h"}]}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"weather","description":"Gets + current weather for a location","input_schema":{"type":"object","properties":{"latitude":{"type":"string","description":"Latitude + (e.g., 52.5200)"},"longitude":{"type":"string","description":"Longitude (e.g., + 13.4050)"}},"required":["latitude","longitude"]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222208Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 7792064bc9716dd94455bb5c1a8d5d650a98b3fb7bc797ef6601a345c57a22e6 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=2c9da80fc819e6b3df19a0ed5d746f794ef782a32be5cdfdc65a1463a819a30d + Content-Type: + - application/json + Accept: + - application/vnd.amazon.eventstream + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:21:27 GMT + Content-Type: + - application/vnd.amazon.eventstream + Transfer-Encoding: + - chunked + Connection: + - keep-alive + X-Amzn-Requestid: + - 742ce604-99db-4386-ab42-fd2969abe68d + X-Amzn-Bedrock-Content-Type: + - application/json + body: + encoding: ASCII-8BIT + string: !binary |- + AAAB2wAAAEuxVdqcCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHRnlkQ0lzSW0xbGMzTmhaMlVpT25zaWFXUWlPaUp0YzJkZlltUnlhMTh3TVZsUlFVdEtNbFJRVFcxcFZISXliVzFFTTBOdldGSWlMQ0owZVhCbElqb2liV1Z6YzJGblpTSXNJbkp2YkdVaU9pSmhjM05wYzNSaGJuUWlMQ0p0YjJSbGJDSTZJbU5zWVhWa1pTMHpMVFV0YUdGcGEzVXRNakF5TkRFd01qSWlMQ0pqYjI1MFpXNTBJanBiWFN3aWMzUnZjRjl5WldGemIyNGlPbTUxYkd3c0luTjBiM0JmYzJWeGRXVnVZMlVpT201MWJHd3NJblZ6WVdkbElqcDdJbWx1Y0hWMFgzUnZhMlZ1Y3lJNk1UQXhPU3dpYjNWMGNIVjBYM1J2YTJWdWN5STZNMzE5ZlE9PSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQyJ9RAzfuwAAARIAAABLVVBQNgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5emRHRnlkQ0lzSW1sdVpHVjRJam93TENKamIyNTBaVzUwWDJKc2IyTnJJanA3SW5SNWNHVWlPaUowWlhoMElpd2lkR1Y0ZENJNklpSjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVoifRCdkjgAAAElAAAAS0ZRNKALOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSmNibHh1U1NCaGNHOXNiMmNpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQ1NiJ9DDsUngAAATAAAABL7lEsUgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lKcGVtVWdabTl5SUhSb1pTQnBibU52Ym5OcGMzUmxiblFnY21WemRXeDBjeTRnVkdobElIZGxZWFJvWlhJZ1puVnVZM1JwYjI0aWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW4ifUIgGzsAAAFBAAAASyrj5S0LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSWdZWEJ3WldGeWN5QjBieUJpWlNCbGVIQmxjbWxsYm1OcGJtY2djMjl0WlNCMFpXTm9ibWxqWVd3Z1pHbG1abWxqZFd4MGFXVnpMaUo5ZlE9PSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekEifcWOxSMAAAExAAAAS9MxBeILOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSWdWR2hsSUhCaGNuUnBZV3dnYVc1bWIzSnRZWFJwYjI0Z2MzVm5aMlZ6ZEhNZ2RHaGhkQ0JRWVhKcGN5QnBjeUJqZFhKeVpXNTBiSGtpZlgwPSIsInAiOiJhYmNkZWZnaGlqayJ9DDFZuwAAAUMAAABLUCO2TQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lJZ1lYUWdNVFhDc0VNZ2QybDBhQ0JoSUhkcGJtUWdjM0JsWldRZ2IyWWdNVEFnSW4xOSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQ1Njc4In2sHCXeAAABQgAAAEttQ5/9CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUpyYlM5b0xpQmNibHh1U0c5M1pYWmxjaXdnWjJsMlpXNGdkR2hsSUhKbGNHVmhkR1VpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMyJ9KrkG6QAAAWMAAABLkeKZSQs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lKa0lHbGtaVzUwYVdOaGJDQnlaV0ZrYVc1bmN5Qm1iM0lnWkdsbVptVnlaVzUwSUd4dlkyRjBhVzl1Y3l3Z1NTQmpZVzV1YjNRZ1kyOXVabWtpZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaMDEyMzQifaKmNWAAAAF3AAAASwSCqAsLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSmtaVzUwYkhrZ1kyOXVabWx5YlNCMGFHVWdZV05qZFhKaFkza2diMllnZEdocGN5QnBibVp2Y20xaGRHbHZiaTRnUm05eUlIUm9aU0J0YjNOMElISmxiR2xoWW14bElHRnVJbjE5IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2NzgifcrUOHUAAAFGAAAAS5jDOT0LOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSmtJR04xY25KbGJuUWdkMlZoZEdobGNpQnBibVp2Y20xaGRHbHZiaUJtYjNJZ1VHRnlhWE1zSUVrZ2NtVmpiMjF0Wlc0aWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTiJ9PJAD0AAAARIAAABLVVBQNgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lKa0lHTm9aV05yYVc1bklHRWdiRzlqWVd3Z2QyVmhkR2hsY2lCelpYSjJhV05sSUc5eUlHRWlmWDA9IiwicCI6ImFiY2QifVJC0IsAAAElAAAAS0ZRNKALOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSWdjbVZ3ZFhSaFlteGxJSGRsWVhSb1pYSWdkMlZpYzJsMFpTNGdWMjkxYkdRZ2VXOTFJR3hwYTJVZ2JXVWdkRzhpZlgwPSIsInAiOiJhYmNkZWZnaGlqayJ9GpDWtgAAAR8AAABLrcCUhws6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lJZ2MzVm5aMlZ6ZENCemIyMWxJSE52ZFhKalpYTWdabTl5SUhWd0xYUnZMV1JoZEdVaWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1In0EKYPnAAABCwAAAEs4oKXFCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUlnZDJWaGRHaGxjaUJwYm1admNtMWhkR2x2Ymo4aWZYMD0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1In2rYEZYAAAAqgAAAEu+u8BGCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEc5d0lpd2lhVzVrWlhnaU9qQjkiLCJwIjoiYWJjZGVmZ2gifcwchfMAAAE0AAAASxvRipILOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pYldWemMyRm5aVjlrWld4MFlTSXNJbVJsYkhSaElqcDdJbk4wYjNCZmNtVmhjMjl1SWpvaVpXNWtYM1IxY200aUxDSnpkRzl3WDNObGNYVmxibU5sSWpwdWRXeHNmU3dpZFhOaFoyVWlPbnNpYjNWMGNIVjBYM1J2YTJWdWN5STZNVEUxZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWCJ9PotWFgAAAVwAAABLspO2Hgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2liV1Z6YzJGblpWOXpkRzl3SWl3aVlXMWhlbTl1TFdKbFpISnZZMnN0YVc1MmIyTmhkR2x2YmsxbGRISnBZM01pT25zaWFXNXdkWFJVYjJ0bGJrTnZkVzUwSWpveE1ERTVMQ0p2ZFhSd2RYUlViMnRsYmtOdmRXNTBJam94TVRVc0ltbHVkbTlqWVhScGIyNU1ZWFJsYm1ONUlqb3lOamN5TENKbWFYSnpkRUo1ZEdWTVlYUmxibU41SWpveE1UUTBmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDRCJ9bAhYkw== + recorded_at: Tue, 25 Mar 2025 22:22:11 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses.yml b/spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses.yml deleted file mode 100644 index 357acbf05..000000000 --- a/spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses.yml +++ /dev/null @@ -1,52 +0,0 @@ ---- -http_interactions: -- request: - method: post - uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke-with-response-stream - body: - encoding: UTF-8 - string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"Count - from 1 to 3"}],"temperature":0.7,"max_tokens":4096}' - headers: - User-Agent: - - Faraday v2.12.2 - Host: - - bedrock-runtime.us-west-2.amazonaws.com - X-Amz-Date: - - 20250325T181243Z - X-Amz-Security-Token: - - IQoJb3JpZ2luX2VjELH//////////wEaCXVzLXdlc3QtMiJGMEQCIDILea5LoQt0bejpnAvGlODePXWs+SVOVYKadYo/Rk9CAiBvj7jTOPExaq5kzi5J7Q2EI2OPjkGHwhnRvXZUSqD6LCqSAwgaEAYaDDIyMTg3MTkxNTQ2MyIMSNqSOazgocTuFlBsKu8C3Nbecj9HZFdBXZ7kaWtb3fByaNE9zYowsMzOb08Ugng8UL9qxzm5g0wT+NFvcJg4JvHNBlfxQyqhoqRAzSN8FAevZ2Pf59sReQbMAaKK0+CdIlH+begPkvTzwGvbj6CQhcRWkeD0UWRgLj1qpwJc2MhogI4CaSILeh/gkUC2fwtLaLK8KoXkrC+XWSvs/P+Qn5gF/YcwWYmlJjo1Y7zkaSRPD/V/SXrdEKCb7lHMkB9HgGSiV0kLg5y6KAcmBbG2HWd7S4qRu0Ko3lm3PIch5E4X7UDcxVLBtX0YErNR7vIRQvpbZ9itrjDqF1Wcckw26asVdC1UxbOSWrnAGqk8RFZrS17i4CP+XV+dQ3jD4/+ILYjKvXynqYA4TAwahL0104h7JCFELXmQOEwPmIPX4hmutyUEkkmfout1krQmzjE01ltNitPgVJzOI3On9YHKqBNp0aEgd6xC9frMseZ8Bb+d8B1Jx9oIBCVvH8RRtTDox4u/BjqnAVK1mPlee72ZUnkKg4jzL3LJL3OmDMBOEGSHmUoOTQ40feXeTlY01glfh7Cx3ExDxAUNIz96zwmhuVVSdwP9aGGsumlYsTJRz7wpDvgq8eGV+9JD9uG+55rtGsH8EzBS9Xw5bxJpBvmpIuRw6fKr7wBiCTTVyRR+Jl7JqGoQelm41v/mXIjzsO43zWUbwInRyFYzqxwmiPSHGwJJnn7eQriksmHxxh4i - X-Amz-Content-Sha256: - - 7444002409115a1a6689bf49e7a862dedd9b424399df485afe093d31dbd9e873 - Authorization: - - AWS4-HMAC-SHA256 Credential=ASIATHKEYYHDRJWE7TAB/20250325/us-west-2/bedrock/aws4_request, - SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=a7a876c520661dd43232ab57fadf3f9102ff2ae37722dbff323c719b48c48f65 - Content-Type: - - application/json - Accept: - - application/vnd.amazon.eventstream - Accept-Encoding: - - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 - response: - status: - code: 200 - message: OK - headers: - Date: - - Tue, 25 Mar 2025 18:12:02 GMT - Content-Type: - - application/vnd.amazon.eventstream - Transfer-Encoding: - - chunked - Connection: - - keep-alive - X-Amzn-Requestid: - - 07114d7d-0375-4bf3-a5f8-abcee919319e - X-Amzn-Bedrock-Content-Type: - - application/json - body: - encoding: ASCII-8BIT - string: !binary |- - AAABwwAAAEvhxQbfCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHRnlkQ0lzSW0xbGMzTmhaMlVpT25zaWFXUWlPaUp0YzJkZlltUnlhMTh3TVVOdVFUVmFSM1pxZFZCNlVFeDZTR0ZEVFhremVYTWlMQ0owZVhCbElqb2liV1Z6YzJGblpTSXNJbkp2YkdVaU9pSmhjM05wYzNSaGJuUWlMQ0p0YjJSbGJDSTZJbU5zWVhWa1pTMHpMVFV0YUdGcGEzVXRNakF5TkRFd01qSWlMQ0pqYjI1MFpXNTBJanBiWFN3aWMzUnZjRjl5WldGemIyNGlPbTUxYkd3c0luTjBiM0JmYzJWeGRXVnVZMlVpT201MWJHd3NJblZ6WVdkbElqcDdJbWx1Y0hWMFgzUnZhMlZ1Y3lJNk1UVXNJbTkxZEhCMWRGOTBiMnRsYm5NaU9qRjlmWDA9IiwicCI6ImFiY2RlZmdoaSJ9G3dfmQAAAOMAAABL61j6fgs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5emRHRnlkQ0lzSW1sdVpHVjRJam93TENKamIyNTBaVzUwWDJKc2IyTnJJanA3SW5SNWNHVWlPaUowWlhoMElpd2lkR1Y0ZENJNklpSjlmUT09IiwicCI6ImFiY2RlIn1qLPEUAAABAQAAAEtyEL1kCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUpJWlhKbEluMTkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSEkifVk6xqcAAAEAAAAAS09wlNQLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSW5jeUJqYjNWdWRHbHVaeUJtY205dElERWdkRzhnTXpvaWZYMD0iLCJwIjoiYWJjZGVmIn2rSl0OAAABIAAAAEuOsbvQCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUpjYmx4dU1WeHVNbHh1TXlKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWjAxIn3+SA/UAAAAvgAAAEsr2/EECzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEc5d0lpd2lhVzVrWlhnaU9qQjkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQiJ9gGjF+gAAAT0AAABLFsHo4ws6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2liV1Z6YzJGblpWOWtaV3gwWVNJc0ltUmxiSFJoSWpwN0luTjBiM0JmY21WaGMyOXVJam9pWlc1a1gzUjFjbTRpTENKemRHOXdYM05sY1hWbGJtTmxJanB1ZFd4c2ZTd2lkWE5oWjJVaU9uc2liM1YwY0hWMFgzUnZhMlZ1Y3lJNk1qQjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVowMTIzNDU2In1sT4BMAAABTQAAAEvvEwgsCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHOXdJaXdpWVcxaGVtOXVMV0psWkhKdlkyc3RhVzUyYjJOaGRHbHZiazFsZEhKcFkzTWlPbnNpYVc1d2RYUlViMnRsYmtOdmRXNTBJam94TlN3aWIzVjBjSFYwVkc5clpXNURiM1Z1ZENJNk1qQXNJbWx1ZG05allYUnBiMjVNWVhSbGJtTjVJam95TnpRNExDSm1hWEp6ZEVKNWRHVk1ZWFJsYm1ONUlqb3lNREV5ZlgwPSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzIn0wmND4 - recorded_at: Tue, 25 Mar 2025 18:12:46 GMT -recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses_with_bedrock.yml b/spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses_with_bedrock.yml new file mode 100644 index 000000000..66379ea6b --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_streaming_responses_claude-3-5-haiku_supports_streaming_responses_with_bedrock.yml @@ -0,0 +1,52 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke-with-response-stream + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"Count + from 1 to 3"}],"temperature":0.7,"max_tokens":4096}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250325T222417Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 7444002409115a1a6689bf49e7a862dedd9b424399df485afe093d31dbd9e873 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250325/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=a6817f7cca281c11cde92c54f945d0cf0f981b0323908e5f1a2926f8813603bb + Content-Type: + - application/json + Accept: + - application/vnd.amazon.eventstream + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Tue, 25 Mar 2025 22:23:36 GMT + Content-Type: + - application/vnd.amazon.eventstream + Transfer-Encoding: + - chunked + Connection: + - keep-alive + X-Amzn-Requestid: + - 7711d91d-bcd2-40d4-bbab-05d98ac0a501 + X-Amzn-Bedrock-Content-Type: + - application/json + body: + encoding: ASCII-8BIT + string: !binary |- + AAAB1AAAAEszBU1NCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaWJXVnpjMkZuWlY5emRHRnlkQ0lzSW0xbGMzTmhaMlVpT25zaWFXUWlPaUp0YzJkZlltUnlhMTh3TVZCME1WTlVOMDVvWVRJM1pUUnBTM1ZUT1UxNVRuUWlMQ0owZVhCbElqb2liV1Z6YzJGblpTSXNJbkp2YkdVaU9pSmhjM05wYzNSaGJuUWlMQ0p0YjJSbGJDSTZJbU5zWVhWa1pTMHpMVFV0YUdGcGEzVXRNakF5TkRFd01qSWlMQ0pqYjI1MFpXNTBJanBiWFN3aWMzUnZjRjl5WldGemIyNGlPbTUxYkd3c0luTjBiM0JmYzJWeGRXVnVZMlVpT201MWJHd3NJblZ6WVdkbElqcDdJbWx1Y0hWMFgzUnZhMlZ1Y3lJNk1UVXNJbTkxZEhCMWRGOTBiMnRsYm5NaU9qRjlmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6In3c/zoiAAAA/gAAAEtzKKlNCzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEdGeWRDSXNJbWx1WkdWNElqb3dMQ0pqYjI1MFpXNTBYMkpzYjJOcklqcDdJblI1Y0dVaU9pSjBaWGgwSWl3aWRHVjRkQ0k2SWlKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUYifZfw5TUAAAESAAAAS1VQUDYLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSklaWEpsSW4xOSIsInAiOiJhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ekFCQ0RFRkdISUpLTE1OT1BRUlNUVVZXWFlaIn1Pz3z6AAAA+wAAAEu7yCY9CzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTlrWld4MFlTSXNJbWx1WkdWNElqb3dMQ0prWld4MFlTSTZleUowZVhCbElqb2lkR1Y0ZEY5a1pXeDBZU0lzSW5SbGVIUWlPaUluY3lKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkMifdlQ658AAAEWAAAAS6DQ9vYLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSWdZMjkxYm5ScGJtY2lmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSktMTU5PUFFSU1RVViJ97gzgcgAAAOgAAABLnIjLbws6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lJZ1puSnZiU0F4SUNKOWZRPT0iLCJwIjoiYWIiffakF7gAAAETAAAAS2gweYYLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pWTI5dWRHVnVkRjlpYkc5amExOWtaV3gwWVNJc0ltbHVaR1Y0SWpvd0xDSmtaV3gwWVNJNmV5SjBlWEJsSWpvaWRHVjRkRjlrWld4MFlTSXNJblJsZUhRaU9pSjBieUF6T2lKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWVyJ9pZ+XhwAAAPAAAABLzBgXLAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lKY2JseHVNVnh1TWlKOWZRPT0iLCJwIjoiYWJjZGVmZ2hpaiJ9SijBugAAAPIAAABLtthETAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2lZMjl1ZEdWdWRGOWliRzlqYTE5a1pXeDBZU0lzSW1sdVpHVjRJam93TENKa1pXeDBZU0k2ZXlKMGVYQmxJam9pZEdWNGRGOWtaV3gwWVNJc0luUmxlSFFpT2lKY2JqTWlmWDA9IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0In259nQVAAAA1gAAAEuCmc2ICzpldmVudC10eXBlBwAFY2h1bmsNOmNvbnRlbnQtdHlwZQcAEGFwcGxpY2F0aW9uL2pzb24NOm1lc3NhZ2UtdHlwZQcABWV2ZW50eyJieXRlcyI6ImV5SjBlWEJsSWpvaVkyOXVkR1Z1ZEY5aWJHOWphMTl6ZEc5d0lpd2lhVzVrWlhnaU9qQjkiLCJwIjoiYWJjZGVmZ2hpamtsbW5vcHFyc3R1dnd4eXpBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWiJ95uzNJQAAAScAAABLPJFnwAs6ZXZlbnQtdHlwZQcABWNodW5rDTpjb250ZW50LXR5cGUHABBhcHBsaWNhdGlvbi9qc29uDTptZXNzYWdlLXR5cGUHAAVldmVudHsiYnl0ZXMiOiJleUowZVhCbElqb2liV1Z6YzJGblpWOWtaV3gwWVNJc0ltUmxiSFJoSWpwN0luTjBiM0JmY21WaGMyOXVJam9pWlc1a1gzUjFjbTRpTENKemRHOXdYM05sY1hWbGJtTmxJanB1ZFd4c2ZTd2lkWE5oWjJVaU9uc2liM1YwY0hWMFgzUnZhMlZ1Y3lJNk1qQjlmUT09IiwicCI6ImFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6QUJDREVGR0hJSksifVv+HrwAAAE/AAAAS2wBu4MLOmV2ZW50LXR5cGUHAAVjaHVuaw06Y29udGVudC10eXBlBwAQYXBwbGljYXRpb24vanNvbg06bWVzc2FnZS10eXBlBwAFZXZlbnR7ImJ5dGVzIjoiZXlKMGVYQmxJam9pYldWemMyRm5aVjl6ZEc5d0lpd2lZVzFoZW05dUxXSmxaSEp2WTJzdGFXNTJiMk5oZEdsdmJrMWxkSEpwWTNNaU9uc2lhVzV3ZFhSVWIydGxia052ZFc1MElqb3hOU3dpYjNWMGNIVjBWRzlyWlc1RGIzVnVkQ0k2TWpBc0ltbHVkbTlqWVhScGIyNU1ZWFJsYm1ONUlqb3hNVFExTENKbWFYSnpkRUo1ZEdWTVlYUmxibU41SWpvNE1qTjlmUT09IiwicCI6ImFiY2RlIn1l5ijz + recorded_at: Tue, 25 Mar 2025 22:24:19 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/ruby_llm/chat_spec.rb b/spec/ruby_llm/chat_spec.rb index 58c05810b..740e17fe5 100644 --- a/spec/ruby_llm/chat_spec.rb +++ b/spec/ruby_llm/chat_spec.rb @@ -12,9 +12,10 @@ ['gemini-2.0-flash', nil], ['deepseek-chat', nil], ['gpt-4o-mini', nil], - ['claude-3-5-haiku', 'bedrock'], + %w[claude-3-5-haiku bedrock] ].each do |model, provider| - it "#{model} can have a basic conversation" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations + provider_suffix = provider ? " with #{provider}" : '' + it "#{model} can have a basic conversation#{provider_suffix}" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations chat = RubyLLM.chat(model: model, provider: provider) response = chat.ask("What's 2 + 2?") @@ -24,7 +25,7 @@ expect(response.output_tokens).to be_positive end - it "#{model} can handle multi-turn conversations" do # rubocop:disable RSpec/MultipleExpectations + it "#{model} can handle multi-turn conversations#{provider_suffix}" do # rubocop:disable RSpec/MultipleExpectations chat = RubyLLM.chat(model: model, provider: provider) first = chat.ask("Who was Ruby's creator?") diff --git a/spec/ruby_llm/chat_streaming_spec.rb b/spec/ruby_llm/chat_streaming_spec.rb index 99e99c347..545cf6dac 100644 --- a/spec/ruby_llm/chat_streaming_spec.rb +++ b/spec/ruby_llm/chat_streaming_spec.rb @@ -8,13 +8,14 @@ describe 'streaming responses' do [ - # ['claude-3-5-haiku-20241022', nil], - # ['gemini-2.0-flash', nil], - # ['deepseek-chat', nil], - # ['gpt-4o-mini', nil], - ['claude-3-5-haiku', 'bedrock'], + ['claude-3-5-haiku-20241022', nil], + ['gemini-2.0-flash', nil], + ['deepseek-chat', nil], + ['gpt-4o-mini', nil], + %w[claude-3-5-haiku bedrock], ].each do |model, provider| - it "#{model} supports streaming responses" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations + provider_suffix = provider ? " with #{provider}" : '' + it "#{model} supports streaming responses#{provider_suffix}" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations chat = RubyLLM.chat(model: model, provider: provider) chunks = [] diff --git a/spec/ruby_llm/chat_tools_spec.rb b/spec/ruby_llm/chat_tools_spec.rb index 0de23b9bb..01f2c726c 100644 --- a/spec/ruby_llm/chat_tools_spec.rb +++ b/spec/ruby_llm/chat_tools_spec.rb @@ -18,12 +18,14 @@ def execute(latitude:, longitude:) describe 'function calling' do [ - 'claude-3-5-haiku-20241022', - 'gemini-2.0-flash', - 'gpt-4o-mini' - ].each do |model| - it "#{model} can use tools" do # rubocop:disable RSpec/MultipleExpectations - chat = RubyLLM.chat(model: model) + ['claude-3-5-haiku-20241022', nil], + ['gemini-2.0-flash', nil], + ['gpt-4o-mini', nil], + %w[claude-3-5-haiku bedrock], + ].each do |model, provider| + provider_suffix = provider ? " with #{provider}" : '' + it "#{model} can use tools#{provider_suffix}" do # rubocop:disable RSpec/MultipleExpectations + chat = RubyLLM.chat(model: model, provider: provider) .with_tool(Weather) response = chat.ask("What's the weather in Berlin? (52.5200, 13.4050)") @@ -31,8 +33,8 @@ def execute(latitude:, longitude:) expect(response.content).to include('10') end - it "#{model} can use tools in multi-turn conversations" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations - chat = RubyLLM.chat(model: model) + it "#{model} can use tools in multi-turn conversations#{provider_suffix}" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations + chat = RubyLLM.chat(model: model, provider: provider) .with_tool(Weather) response = chat.ask("What's the weather in Berlin? (52.5200, 13.4050)") @@ -44,8 +46,8 @@ def execute(latitude:, longitude:) expect(response.content).to include('10') end - it "#{model} can use tools with multi-turn streaming conversations" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations - chat = RubyLLM.chat(model: model) + it "#{model} can use tools with multi-turn streaming conversations#{provider_suffix}" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations + chat = RubyLLM.chat(model: model, provider: provider) .with_tool(Weather) chunks = [] From da861ccdff08282d5d347da12b5fe4ad8d32bfd7 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 15:30:53 -0700 Subject: [PATCH 38/85] Separate concerns in HTTP layer - take signing out of headers method --- lib/ruby_llm/providers/bedrock.rb | 35 +++++++++++++++++-------------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index c2791751b..3c1d2c234 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -25,34 +25,37 @@ def api_base end def post(url, payload) + signature = sign_request(url, payload, streaming: block_given?) + connection.post url, payload do |req| - req.headers.merge! headers(method: :post, - path: "#{connection.url_prefix}#{url}", - body: payload.to_json, - streaming: block_given?) + req.headers.merge! build_headers(signature.headers, streaming: block_given?) yield req if block_given? end end - def headers(method: :post, path: nil, body: nil, streaming: false) + def sign_request(url, payload, streaming: false) signer = Signing::Signer.new({ - access_key_id: aws_access_key_id, - secret_access_key: aws_secret_access_key, - session_token: aws_session_token, - region: aws_region, - service: 'bedrock' - }) + access_key_id: aws_access_key_id, + secret_access_key: aws_secret_access_key, + session_token: aws_session_token, + region: aws_region, + service: 'bedrock' + }) + request = { connection: connection, - http_method: method, - url: path || completion_url, - body: body || '' + http_method: :post, + url: url || completion_url, + body: payload&.to_json || '' } - signature = signer.sign_request(request) + signer.sign_request(request) + end + + def build_headers(signature_headers, streaming: false) accept_header = streaming ? 'application/vnd.amazon.eventstream' : 'application/json' - signature.headers.merge( + signature_headers.merge( 'Content-Type' => 'application/json', 'Accept' => accept_header ) From 79dac6a36c9fd8655fee530b610e8e1fdc1efd0a Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 15:34:28 -0700 Subject: [PATCH 39/85] Remove extra verbose methods --- lib/ruby_llm/providers/bedrock.rb | 26 +++++------------------- lib/ruby_llm/providers/bedrock/models.rb | 7 ++++++- 2 files changed, 11 insertions(+), 22 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 3c1d2c234..93f901f58 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -21,7 +21,7 @@ module Bedrock module_function def api_base - @api_base ||= "https://bedrock-runtime.#{aws_region}.amazonaws.com" + @api_base ||= "https://bedrock-runtime.#{RubyLLM.config.bedrock_region}.amazonaws.com" end def post(url, payload) @@ -35,10 +35,10 @@ def post(url, payload) def sign_request(url, payload, streaming: false) signer = Signing::Signer.new({ - access_key_id: aws_access_key_id, - secret_access_key: aws_secret_access_key, - session_token: aws_session_token, - region: aws_region, + access_key_id: RubyLLM.config.bedrock_api_key, + secret_access_key: RubyLLM.config.bedrock_secret_key, + session_token: RubyLLM.config.bedrock_session_token, + region: RubyLLM.config.bedrock_region, service: 'bedrock' }) @@ -69,22 +69,6 @@ def slug 'bedrock' end - def aws_region - RubyLLM.config.bedrock_region - end - - def aws_access_key_id - RubyLLM.config.bedrock_api_key - end - - def aws_secret_access_key - RubyLLM.config.bedrock_secret_key - end - - def aws_session_token - RubyLLM.config.bedrock_session_token - end - def configuration_requirements %i[bedrock_api_key bedrock_secret_key bedrock_region] end diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index 3412cadc4..ca0bd4226 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -5,9 +5,14 @@ module Providers module Bedrock # Models methods for the AWS Bedrock API implementation module Models + def initialize + super + @api_base = "https://bedrock.#{RubyLLM.config.bedrock_region}.amazonaws.com" + end + def list_models @connection = nil # reset connection since base url is different - @api_base = "https://bedrock.#{aws_region}.amazonaws.com" + @api_base = "https://bedrock.#{RubyLLM.config.bedrock_region}.amazonaws.com" response = connection.get(models_url) do |req| req.headers.merge! headers(method: :get, path: "#{connection.url_prefix}#{models_url}") From 9bc7f575a2a60e6ced254745ab366a9583b3fae3 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 15:40:00 -0700 Subject: [PATCH 40/85] Fix model refresh after previous refactor --- lib/ruby_llm/providers/bedrock/models.rb | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index ca0bd4226..056fcc31d 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -5,17 +5,13 @@ module Providers module Bedrock # Models methods for the AWS Bedrock API implementation module Models - def initialize - super - @api_base = "https://bedrock.#{RubyLLM.config.bedrock_region}.amazonaws.com" - end def list_models @connection = nil # reset connection since base url is different @api_base = "https://bedrock.#{RubyLLM.config.bedrock_region}.amazonaws.com" + signature = sign_request(models_url, nil, streaming: block_given?) response = connection.get(models_url) do |req| - req.headers.merge! headers(method: :get, - path: "#{connection.url_prefix}#{models_url}") + req.headers.merge! signature.headers end @connection = nil # reset connection since base url is different From e9021c374637f29fbda3f1f2bae439c86dd854d5 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:13:18 -0700 Subject: [PATCH 41/85] Fix signature by including full URL --- lib/ruby_llm/providers/bedrock.rb | 6 +++--- lib/ruby_llm/providers/bedrock/models.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 93f901f58..e7f01440a 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -25,15 +25,15 @@ def api_base end def post(url, payload) - signature = sign_request(url, payload, streaming: block_given?) - + signature = sign_request("#{connection.url_prefix}#{url}", payload) connection.post url, payload do |req| req.headers.merge! build_headers(signature.headers, streaming: block_given?) + yield req if block_given? end end - def sign_request(url, payload, streaming: false) + def sign_request(url, payload) signer = Signing::Signer.new({ access_key_id: RubyLLM.config.bedrock_api_key, secret_access_key: RubyLLM.config.bedrock_secret_key, diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index 056fcc31d..36e53cae7 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -9,7 +9,7 @@ module Models def list_models @connection = nil # reset connection since base url is different @api_base = "https://bedrock.#{RubyLLM.config.bedrock_region}.amazonaws.com" - signature = sign_request(models_url, nil, streaming: block_given?) + signature = sign_request(models_url, nil) response = connection.get(models_url) do |req| req.headers.merge! signature.headers end From 5ed222f55c6d3a4eb9fa1ccf68e350f319363e11 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:15:19 -0700 Subject: [PATCH 42/85] Refactor handle_stream for bedrock into smaller methods --- lib/ruby_llm/providers/bedrock/streaming.rb | 257 ++++++++++---------- 1 file changed, 131 insertions(+), 126 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index c572da8d2..c76c5b325 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -14,117 +14,11 @@ def stream_url end def handle_stream(&block) - proc do |chunk, _bytes, env| if env && env.status != 200 - # Accumulate error chunks - buffer = String.new - buffer << chunk - begin - error_data = JSON.parse(buffer) - error_response = env.merge(body: error_data) - ErrorMiddleware.parse_error(provider: self, response: error_response) - rescue JSON::ParserError - # Keep accumulating if we don't have complete JSON yet - RubyLLM.logger.debug "Accumulating error chunk: #{chunk}" - end + handle_error_response(chunk, env) else - begin - # Process each event stream message in the chunk - offset = 0 - while offset < chunk.bytesize - # Read the prelude (total length + headers length) - break if chunk.bytesize - offset < 12 # Need at least prelude size - - total_length = chunk[offset...offset + 4].unpack('N').first - headers_length = chunk[offset + 4...offset + 8].unpack('N').first - - # Validate lengths to ensure they're reasonable - if total_length.nil? || headers_length.nil? || - total_length <= 0 || total_length > 1_000_000 || # Sanity check for message size - headers_length <= 0 || headers_length > total_length - RubyLLM.logger.debug "Invalid lengths detected, trying next potential message" - # Try to find the next message prelude marker - next_prelude = find_next_prelude(chunk, offset + 4) - offset = next_prelude || chunk.bytesize - next - end - - # Verify we have the complete message - message_end = offset + total_length - break if chunk.bytesize < message_end - - # Extract headers and payload - headers_end = offset + 12 + headers_length - payload_end = message_end - 4 # Subtract 4 bytes for message CRC - - # Safety check for valid positions - if headers_end >= payload_end || headers_end >= chunk.bytesize || payload_end > chunk.bytesize - RubyLLM.logger.debug "Invalid positions detected, trying next potential message" - # Try to find the next message prelude marker - next_prelude = find_next_prelude(chunk, offset + 4) - offset = next_prelude || chunk.bytesize - next - end - - # Get payload - payload = chunk[headers_end...payload_end] - - # Safety check for payload - if payload.nil? || payload.empty? - RubyLLM.logger.debug "Empty or nil payload detected, skipping chunk" - offset = message_end - next - end - - # Find valid JSON in the payload - json_start = payload.index('{') - json_end = payload.rindex('}') - - if json_start.nil? || json_end.nil? || json_start >= json_end - RubyLLM.logger.debug "No valid JSON found in payload, skipping chunk" - offset = message_end - next - end - - # Extract just the JSON portion - json_payload = payload[json_start..json_end] - - begin - # Parse the JSON payload - json_data = JSON.parse(json_payload) - - # Handle Base64 encoded bytes - if json_data['bytes'] - decoded_bytes = Base64.strict_decode64(json_data['bytes']) - data = JSON.parse(decoded_bytes) - - block.call( - Chunk.new( - role: :assistant, - model_id: data.dig('message', 'model') || @model_id, - content: extract_streaming_content(data), - input_tokens: extract_input_tokens(data), - output_tokens: extract_output_tokens(data), - tool_calls: extract_tool_calls(data) - ) - ) - end - - rescue JSON::ParserError => e - RubyLLM.logger.debug "Failed to parse payload as JSON: #{e.message}" - RubyLLM.logger.debug "Attempted JSON payload: #{json_payload.inspect}" - rescue StandardError => e - RubyLLM.logger.debug "Error processing payload: #{e.message}" - end - - # Move to next message - offset = message_end - end - rescue StandardError => e - RubyLLM.logger.debug "Error processing chunk: #{e.message}" - RubyLLM.logger.debug "Chunk size: #{chunk.bytesize}" - end + process_chunk(chunk, &block) end end end @@ -136,29 +30,147 @@ def json_delta?(data) def extract_streaming_content(data) if data.is_a?(Hash) case data['type'] - when 'message_start' - # No content yet in message_start - '' when 'content_block_start' # Initial content block, might have some text data.dig('content_block', 'text').to_s when 'content_block_delta' # Incremental content updates data.dig('delta', 'text').to_s - when 'message_delta' - # Might contain updates to usage stats, but no new content - '' else - # Fall back to the existing extract_content method for other formats - extract_content(data) + '' end else - extract_content(data) + '' end end private + def handle_error_response(chunk, env) + buffer = String.new + buffer << chunk + begin + error_data = JSON.parse(buffer) + error_response = env.merge(body: error_data) + ErrorMiddleware.parse_error(provider: self, response: error_response) + rescue JSON::ParserError + # Keep accumulating if we don't have complete JSON yet + RubyLLM.logger.debug "Accumulating error chunk: #{chunk}" + end + end + + def process_chunk(chunk, &block) + offset = 0 + while offset < chunk.bytesize + offset = process_message(chunk, offset, &block) + end + rescue StandardError => e + RubyLLM.logger.debug "Error processing chunk: #{e.message}" + RubyLLM.logger.debug "Chunk size: #{chunk.bytesize}" + end + + def process_message(chunk, offset, &block) + return chunk.bytesize if !can_read_prelude?(chunk, offset) + + total_length, headers_length = read_prelude(chunk, offset) + return find_next_message(chunk, offset) if !valid_lengths?(total_length, headers_length) + + message_end = offset + total_length + return chunk.bytesize if chunk.bytesize < message_end + + headers_end, payload_end = calculate_positions(offset, total_length, headers_length) + return find_next_message(chunk, offset) if !valid_positions?(headers_end, payload_end, chunk.bytesize) + + payload = extract_payload(chunk, headers_end, payload_end) + return message_end if !valid_payload?(payload) + + process_payload(payload, &block) + message_end + end + + def can_read_prelude?(chunk, offset) + chunk.bytesize - offset >= 12 + end + + def read_prelude(chunk, offset) + total_length = chunk[offset...offset + 4].unpack('N').first + headers_length = chunk[offset + 4...offset + 8].unpack('N').first + [total_length, headers_length] + end + + def valid_lengths?(total_length, headers_length) + return false if total_length.nil? || headers_length.nil? + return false if total_length <= 0 || total_length > 1_000_000 + return false if headers_length <= 0 || headers_length > total_length + true + end + + def calculate_positions(offset, total_length, headers_length) + headers_end = offset + 12 + headers_length + payload_end = offset + total_length - 4 # Subtract 4 bytes for message CRC + [headers_end, payload_end] + end + + def valid_positions?(headers_end, payload_end, chunk_size) + return false if headers_end >= payload_end + return false if headers_end >= chunk_size + return false if payload_end > chunk_size + true + end + + def find_next_message(chunk, offset) + next_prelude = find_next_prelude(chunk, offset + 4) + next_prelude || chunk.bytesize + end + + def extract_payload(chunk, headers_end, payload_end) + chunk[headers_end...payload_end] + end + + def valid_payload?(payload) + return false if payload.nil? || payload.empty? + + json_start = payload.index('{') + json_end = payload.rindex('}') + + return false if json_start.nil? || json_end.nil? || json_start >= json_end + true + end + + def process_payload(payload, &block) + json_start = payload.index('{') + json_end = payload.rindex('}') + json_payload = payload[json_start..json_end] + + begin + json_data = JSON.parse(json_payload) + process_json_data(json_data, &block) + rescue JSON::ParserError => e + RubyLLM.logger.debug "Failed to parse payload as JSON: #{e.message}" + RubyLLM.logger.debug "Attempted JSON payload: #{json_payload.inspect}" + rescue StandardError => e + RubyLLM.logger.debug "Error processing payload: #{e.message}" + end + end + + def process_json_data(json_data, &block) + return unless json_data['bytes'] + + decoded_bytes = Base64.strict_decode64(json_data['bytes']) + data = JSON.parse(decoded_bytes) + + block.call( + Chunk.new( + role: :assistant, + model_id: data.dig('message', 'model') || @model_id, + content: extract_streaming_content(data), + input_tokens: extract_input_tokens(data), + output_tokens: extract_output_tokens(data), + tool_calls: extract_tool_calls(data) + ) + ) + end + def extract_input_tokens(data) data.dig('message', 'usage', 'input_tokens') end @@ -183,15 +195,8 @@ def extract_content(data) end end - def split_event_stream_chunk(chunk) - # Find the position of the first '{' character which indicates start of JSON - json_start = chunk.index('{') - return [nil, nil] unless json_start - - header = chunk[0...json_start].strip - payload = chunk[json_start..-1] - - [header, payload] + def extract_tool_calls(data) + data.dig('message', 'tool_calls') || data.dig('tool_calls') end def find_next_prelude(chunk, start_offset) From 1f5a34d8a367b3bf206bb1f3dbe1bd53cddf9f1b Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:16:25 -0700 Subject: [PATCH 43/85] rubocop -A --- lib/ruby_llm/providers/bedrock.rb | 12 ++--- lib/ruby_llm/providers/bedrock/models.rb | 1 - lib/ruby_llm/providers/bedrock/streaming.rb | 49 +++++++++++---------- spec/ruby_llm/chat_streaming_spec.rb | 2 +- spec/ruby_llm/chat_tools_spec.rb | 2 +- 5 files changed, 33 insertions(+), 33 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index e7f01440a..68f439e77 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -35,12 +35,12 @@ def post(url, payload) def sign_request(url, payload) signer = Signing::Signer.new({ - access_key_id: RubyLLM.config.bedrock_api_key, - secret_access_key: RubyLLM.config.bedrock_secret_key, - session_token: RubyLLM.config.bedrock_session_token, - region: RubyLLM.config.bedrock_region, - service: 'bedrock' - }) + access_key_id: RubyLLM.config.bedrock_api_key, + secret_access_key: RubyLLM.config.bedrock_secret_key, + session_token: RubyLLM.config.bedrock_session_token, + region: RubyLLM.config.bedrock_region, + service: 'bedrock' + }) request = { connection: connection, diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index 36e53cae7..cf4392eed 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -5,7 +5,6 @@ module Providers module Bedrock # Models methods for the AWS Bedrock API implementation module Models - def list_models @connection = nil # reset connection since base url is different @api_base = "https://bedrock.#{RubyLLM.config.bedrock_region}.amazonaws.com" diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index c76c5b325..2fb40ba54 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -59,32 +59,30 @@ def handle_error_response(chunk, env) end end - def process_chunk(chunk, &block) + def process_chunk(chunk, &) offset = 0 - while offset < chunk.bytesize - offset = process_message(chunk, offset, &block) - end + offset = process_message(chunk, offset, &) while offset < chunk.bytesize rescue StandardError => e RubyLLM.logger.debug "Error processing chunk: #{e.message}" RubyLLM.logger.debug "Chunk size: #{chunk.bytesize}" end - def process_message(chunk, offset, &block) - return chunk.bytesize if !can_read_prelude?(chunk, offset) + def process_message(chunk, offset, &) + return chunk.bytesize unless can_read_prelude?(chunk, offset) total_length, headers_length = read_prelude(chunk, offset) - return find_next_message(chunk, offset) if !valid_lengths?(total_length, headers_length) + return find_next_message(chunk, offset) unless valid_lengths?(total_length, headers_length) message_end = offset + total_length return chunk.bytesize if chunk.bytesize < message_end headers_end, payload_end = calculate_positions(offset, total_length, headers_length) - return find_next_message(chunk, offset) if !valid_positions?(headers_end, payload_end, chunk.bytesize) + return find_next_message(chunk, offset) unless valid_positions?(headers_end, payload_end, chunk.bytesize) payload = extract_payload(chunk, headers_end, payload_end) - return message_end if !valid_payload?(payload) + return message_end unless valid_payload?(payload) - process_payload(payload, &block) + process_payload(payload, &) message_end end @@ -93,8 +91,8 @@ def can_read_prelude?(chunk, offset) end def read_prelude(chunk, offset) - total_length = chunk[offset...offset + 4].unpack('N').first - headers_length = chunk[offset + 4...offset + 8].unpack('N').first + total_length = chunk[offset...offset + 4].unpack1('N') + headers_length = chunk[offset + 4...offset + 8].unpack1('N') [total_length, headers_length] end @@ -102,12 +100,13 @@ def valid_lengths?(total_length, headers_length) return false if total_length.nil? || headers_length.nil? return false if total_length <= 0 || total_length > 1_000_000 return false if headers_length <= 0 || headers_length > total_length + true end def calculate_positions(offset, total_length, headers_length) headers_end = offset + 12 + headers_length - payload_end = offset + total_length - 4 # Subtract 4 bytes for message CRC + payload_end = offset + total_length - 4 # Subtract 4 bytes for message CRC [headers_end, payload_end] end @@ -115,6 +114,7 @@ def valid_positions?(headers_end, payload_end, chunk_size) return false if headers_end >= payload_end return false if headers_end >= chunk_size return false if payload_end > chunk_size + true end @@ -129,22 +129,23 @@ def extract_payload(chunk, headers_end, payload_end) def valid_payload?(payload) return false if payload.nil? || payload.empty? - + json_start = payload.index('{') json_end = payload.rindex('}') - + return false if json_start.nil? || json_end.nil? || json_start >= json_end + true end - def process_payload(payload, &block) + def process_payload(payload, &) json_start = payload.index('{') json_end = payload.rindex('}') json_payload = payload[json_start..json_end] - + begin json_data = JSON.parse(json_payload) - process_json_data(json_data, &block) + process_json_data(json_data, &) rescue JSON::ParserError => e RubyLLM.logger.debug "Failed to parse payload as JSON: #{e.message}" RubyLLM.logger.debug "Attempted JSON payload: #{json_payload.inspect}" @@ -196,19 +197,19 @@ def extract_content(data) end def extract_tool_calls(data) - data.dig('message', 'tool_calls') || data.dig('tool_calls') + data.dig('message', 'tool_calls') || data['tool_calls'] end def find_next_prelude(chunk, start_offset) # Look for potential message prelude by scanning for reasonable length values (start_offset...(chunk.bytesize - 8)).each do |pos| - potential_total_length = chunk[pos...pos + 4].unpack('N').first - potential_headers_length = chunk[pos + 4...pos + 8].unpack('N').first - + potential_total_length = chunk[pos...pos + 4].unpack1('N') + potential_headers_length = chunk[pos + 4...pos + 8].unpack1('N') + # Check if these look like valid lengths if potential_total_length && potential_headers_length && - potential_total_length > 0 && potential_total_length < 1_000_000 && - potential_headers_length > 0 && potential_headers_length < potential_total_length + potential_total_length.positive? && potential_total_length < 1_000_000 && + potential_headers_length.positive? && potential_headers_length < potential_total_length return pos end end diff --git a/spec/ruby_llm/chat_streaming_spec.rb b/spec/ruby_llm/chat_streaming_spec.rb index 545cf6dac..ce6ae8f5f 100644 --- a/spec/ruby_llm/chat_streaming_spec.rb +++ b/spec/ruby_llm/chat_streaming_spec.rb @@ -12,7 +12,7 @@ ['gemini-2.0-flash', nil], ['deepseek-chat', nil], ['gpt-4o-mini', nil], - %w[claude-3-5-haiku bedrock], + %w[claude-3-5-haiku bedrock] ].each do |model, provider| provider_suffix = provider ? " with #{provider}" : '' it "#{model} supports streaming responses#{provider_suffix}" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations diff --git a/spec/ruby_llm/chat_tools_spec.rb b/spec/ruby_llm/chat_tools_spec.rb index 01f2c726c..b5580ffc4 100644 --- a/spec/ruby_llm/chat_tools_spec.rb +++ b/spec/ruby_llm/chat_tools_spec.rb @@ -21,7 +21,7 @@ def execute(latitude:, longitude:) ['claude-3-5-haiku-20241022', nil], ['gemini-2.0-flash', nil], ['gpt-4o-mini', nil], - %w[claude-3-5-haiku bedrock], + %w[claude-3-5-haiku bedrock] ].each do |model, provider| provider_suffix = provider ? " with #{provider}" : '' it "#{model} can use tools#{provider_suffix}" do # rubocop:disable RSpec/MultipleExpectations From 30d44dda170a9d8c19eb07e800df5a7cbad58884 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:19:35 -0700 Subject: [PATCH 44/85] Refactor extract_content and find_next_prelude into smaller methods --- lib/ruby_llm/providers/bedrock/streaming.rb | 100 ++++++++++++++------ 1 file changed, 71 insertions(+), 29 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index 2fb40ba54..f9a29ba77 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -44,6 +44,52 @@ def extract_streaming_content(data) end end + def extract_content(data) + return unless data.is_a?(Hash) + + content_extractors = [ + :extract_completion_content, + :extract_output_text_content, + :extract_array_content, + :extract_content_block_text + ] + + content_extractors.each do |extractor| + content = send(extractor, data) + return content if content + end + + nil + end + + def extract_completion_content(data) + data['completion'] if data.key?('completion') + end + + def extract_output_text_content(data) + data.dig('results', 0, 'outputText') + end + + def extract_array_content(data) + return unless data.key?('content') + + if data['content'].is_a?(Array) + data['content'].map { |item| item['text'] }.join + else + data['content'] + end + end + + def extract_content_block_text(data) + return unless data.key?('content_block') && data['content_block'].key?('text') + + data['content_block']['text'] + end + + def extract_tool_calls(data) + data.dig('message', 'tool_calls') || data['tool_calls'] + end + private def handle_error_response(chunk, env) @@ -180,40 +226,36 @@ def extract_output_tokens(data) data.dig('message', 'usage', 'output_tokens') || data.dig('usage', 'output_tokens') end - def extract_content(data) - case data - when Hash - if data.key?('completion') - data['completion'] - elsif data.dig('results', 0, 'outputText') - data.dig('results', 0, 'outputText') - elsif data.key?('content') - data['content'].is_a?(Array) ? data['content'].map { |item| item['text'] }.join : data['content'] - elsif data.key?('content_block') && data['content_block'].key?('text') - # Handle the newly decoded JSON structure - data['content_block']['text'] - end + def find_next_prelude(chunk, start_offset) + scan_range(chunk, start_offset).each do |pos| + return pos if valid_prelude_at_position?(chunk, pos) end + nil end - def extract_tool_calls(data) - data.dig('message', 'tool_calls') || data['tool_calls'] + def scan_range(chunk, start_offset) + (start_offset...(chunk.bytesize - 8)) end - def find_next_prelude(chunk, start_offset) - # Look for potential message prelude by scanning for reasonable length values - (start_offset...(chunk.bytesize - 8)).each do |pos| - potential_total_length = chunk[pos...pos + 4].unpack1('N') - potential_headers_length = chunk[pos + 4...pos + 8].unpack1('N') - - # Check if these look like valid lengths - if potential_total_length && potential_headers_length && - potential_total_length.positive? && potential_total_length < 1_000_000 && - potential_headers_length.positive? && potential_headers_length < potential_total_length - return pos - end - end - nil + def valid_prelude_at_position?(chunk, pos) + lengths = extract_potential_lengths(chunk, pos) + valid_prelude_lengths?(*lengths) + end + + def extract_potential_lengths(chunk, pos) + [ + chunk[pos...pos + 4].unpack1('N'), + chunk[pos + 4...pos + 8].unpack1('N') + ] + end + + def valid_prelude_lengths?(total_length, headers_length) + return false unless total_length && headers_length + return false unless total_length.positive? && headers_length.positive? + return false unless total_length < 1_000_000 + return false unless headers_length < total_length + + true end end end From 000fcbb53f63b0d0e880cc21ec5883362c5c82ad Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:24:20 -0700 Subject: [PATCH 45/85] Refactor process_payload and process_json_data --- lib/ruby_llm/providers/bedrock/streaming.rb | 86 ++++++++++++++------- 1 file changed, 59 insertions(+), 27 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index f9a29ba77..c72502f37 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -47,11 +47,11 @@ def extract_streaming_content(data) def extract_content(data) return unless data.is_a?(Hash) - content_extractors = [ - :extract_completion_content, - :extract_output_text_content, - :extract_array_content, - :extract_content_block_text + content_extractors = %i[ + extract_completion_content + extract_output_text_content + extract_array_content + extract_content_block_text ] content_extractors.each do |extractor| @@ -185,39 +185,71 @@ def valid_payload?(payload) end def process_payload(payload, &) + json_payload = extract_json_payload(payload) + parse_and_process_json(json_payload, &) + rescue JSON::ParserError => e + log_json_parse_error(e, json_payload) + rescue StandardError => e + log_general_error(e) + end + + def extract_json_payload(payload) json_start = payload.index('{') json_end = payload.rindex('}') - json_payload = payload[json_start..json_end] + payload[json_start..json_end] + end - begin - json_data = JSON.parse(json_payload) - process_json_data(json_data, &) - rescue JSON::ParserError => e - RubyLLM.logger.debug "Failed to parse payload as JSON: #{e.message}" - RubyLLM.logger.debug "Attempted JSON payload: #{json_payload.inspect}" - rescue StandardError => e - RubyLLM.logger.debug "Error processing payload: #{e.message}" - end + def parse_and_process_json(json_payload, &) + json_data = JSON.parse(json_payload) + process_json_data(json_data, &) + end + + def log_json_parse_error(error, json_payload) + RubyLLM.logger.debug "Failed to parse payload as JSON: #{error.message}" + RubyLLM.logger.debug "Attempted JSON payload: #{json_payload.inspect}" + end + + def log_general_error(error) + RubyLLM.logger.debug "Error processing payload: #{error.message}" end - def process_json_data(json_data, &block) + def process_json_data(json_data, &) return unless json_data['bytes'] + data = decode_and_parse_data(json_data) + create_and_yield_chunk(data, &) + end + + def decode_and_parse_data(json_data) decoded_bytes = Base64.strict_decode64(json_data['bytes']) - data = JSON.parse(decoded_bytes) - - block.call( - Chunk.new( - role: :assistant, - model_id: data.dig('message', 'model') || @model_id, - content: extract_streaming_content(data), - input_tokens: extract_input_tokens(data), - output_tokens: extract_output_tokens(data), - tool_calls: extract_tool_calls(data) - ) + JSON.parse(decoded_bytes) + end + + def create_and_yield_chunk(data, &block) + block.call(build_chunk(data)) + end + + def build_chunk(data) + Chunk.new( + **extract_chunk_attributes(data) ) end + def extract_chunk_attributes(data) + { + role: :assistant, + model_id: extract_model_id(data), + content: extract_streaming_content(data), + input_tokens: extract_input_tokens(data), + output_tokens: extract_output_tokens(data), + tool_calls: extract_tool_calls(data) + } + end + + def extract_model_id(data) + data.dig('message', 'model') || @model_id + end + def extract_input_tokens(data) data.dig('message', 'usage', 'input_tokens') end From 7737415604fc4d6f89c8f64988456321ff0784c6 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:26:03 -0700 Subject: [PATCH 46/85] Refactor sign_request into smaller methods --- lib/ruby_llm/providers/bedrock.rb | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 68f439e77..3c2c1dc2d 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -34,22 +34,28 @@ def post(url, payload) end def sign_request(url, payload) - signer = Signing::Signer.new({ - access_key_id: RubyLLM.config.bedrock_api_key, - secret_access_key: RubyLLM.config.bedrock_secret_key, - session_token: RubyLLM.config.bedrock_session_token, - region: RubyLLM.config.bedrock_region, - service: 'bedrock' - }) + signer = create_signer + request = build_request(url, payload) + signer.sign_request(request) + end + + def create_signer + Signing::Signer.new({ + access_key_id: RubyLLM.config.bedrock_api_key, + secret_access_key: RubyLLM.config.bedrock_secret_key, + session_token: RubyLLM.config.bedrock_session_token, + region: RubyLLM.config.bedrock_region, + service: 'bedrock' + }) + end - request = { + def build_request(url, payload) + { connection: connection, http_method: :post, url: url || completion_url, body: payload&.to_json || '' } - - signer.sign_request(request) end def build_headers(signature_headers, streaming: false) From 24fd8de5298e2a9f0ebe4841133d9e7415f8e1b9 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:28:00 -0700 Subject: [PATCH 47/85] Refactor some methods in capabilities.rb --- .../providers/bedrock/capabilities.rb | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/capabilities.rb b/lib/ruby_llm/providers/bedrock/capabilities.rb index d081d870d..c5fc16391 100644 --- a/lib/ruby_llm/providers/bedrock/capabilities.rb +++ b/lib/ruby_llm/providers/bedrock/capabilities.rb @@ -12,9 +12,7 @@ module Capabilities # @return [Integer] the context window size in tokens def context_window_for(model_id) case model_id - when /anthropic\.claude-3-opus/ then 200_000 - when /anthropic\.claude-3-sonnet/ then 200_000 - when /anthropic\.claude-3-haiku/ then 200_000 + when /anthropic\.claude-3-(opus|sonnet|haiku)/ then 200_000 when /anthropic\.claude-2/ then 100_000 else 4_096 end @@ -121,21 +119,23 @@ def supports_structured_output?(model_id) model_id.match?(/anthropic\.claude-3/) end + # Model family patterns for capability lookup + MODEL_FAMILIES = { + /anthropic\.claude-3-opus/ => :claude3_opus, + /anthropic\.claude-3-sonnet/ => :claude3_sonnet, + /anthropic\.claude-3-5-sonnet/ => :claude3_sonnet, + /anthropic\.claude-3-7-sonnet/ => :claude3_sonnet, + /anthropic\.claude-3-haiku/ => :claude3_haiku, + /anthropic\.claude-3-5-haiku/ => :claude3_5_haiku, + /anthropic\.claude-v2/ => :claude2, + /anthropic\.claude-instant/ => :claude_instant + }.freeze + # Determines the model family for pricing and capability lookup # @param model_id [String] the model identifier # @return [Symbol] the model family identifier def model_family(model_id) - case model_id - when /anthropic\.claude-3-opus/ then :claude3_opus - when /anthropic\.claude-3-sonnet/ then :claude3_sonnet - when /anthropic\.claude-3-5-sonnet/ then :claude3_sonnet - when /anthropic\.claude-3-7-sonnet/ then :claude3_sonnet - when /anthropic\.claude-3-haiku/ then :claude3_haiku - when /anthropic\.claude-3-5-haiku/ then :claude3_5_haiku - when /anthropic\.claude-v2/ then :claude2 - when /anthropic\.claude-instant/ then :claude_instant - else :other - end + MODEL_FAMILIES.find { |pattern, _family| model_id.match?(pattern) }&.last || :other end # Pricing information for Bedrock models (per million tokens) From 0bbddf3d30eb02952e3f98906ea19908a82f074c Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:31:12 -0700 Subject: [PATCH 48/85] Refactor parse_list_models_response --- lib/ruby_llm/providers/bedrock/models.rb | 77 +++++++++++++++--------- 1 file changed, 50 insertions(+), 27 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/models.rb b/lib/ruby_llm/providers/bedrock/models.rb index cf4392eed..9ad4cbf8b 100644 --- a/lib/ruby_llm/providers/bedrock/models.rb +++ b/lib/ruby_llm/providers/bedrock/models.rb @@ -25,34 +25,57 @@ def models_url def parse_list_models_response(response, slug, capabilities) data = response.body['modelSummaries'] || [] + data.filter { |model| model['modelId'].include?('claude') } + .map { |model| create_model_info(model, slug, capabilities) } + end - data = data.filter { |model| model['modelId'].include?('claude') } - data.map do |model| - model_id = model['modelId'] - ModelInfo.new( - id: model_id, - created_at: nil, - display_name: model['modelName'] || capabilities.format_display_name(model_id), - provider: slug, - context_window: capabilities.context_window_for(model_id), - max_tokens: capabilities.max_tokens_for(model_id), - type: capabilities.model_type(model_id), - family: capabilities.model_family(model_id).to_s, - supports_vision: capabilities.supports_vision?(model_id), - supports_functions: capabilities.supports_functions?(model_id), - supports_json_mode: capabilities.supports_json_mode?(model_id), - input_price_per_million: capabilities.input_price_for(model_id), - output_price_per_million: capabilities.output_price_for(model_id), - metadata: { - provider_name: model['providerName'], - customizations_supported: model['customizationsSupported'] || [], - inference_configurations: model['inferenceTypesSupported'] || [], - response_streaming_supported: model['responseStreamingSupported'] || false, - input_modalities: model['inputModalities'] || [], - output_modalities: model['outputModalities'] || [] - } - ) - end + def create_model_info(model, slug, capabilities) + model_id = model['modelId'] + ModelInfo.new( + **base_model_attributes(model_id, model, slug), + **capability_attributes(model_id, capabilities), + **pricing_attributes(model_id, capabilities), + metadata: build_metadata(model) + ) + end + + def base_model_attributes(model_id, model, slug) + { + id: model_id, + created_at: nil, + display_name: model['modelName'] || capabilities.format_display_name(model_id), + provider: slug + } + end + + def capability_attributes(model_id, capabilities) + { + context_window: capabilities.context_window_for(model_id), + max_tokens: capabilities.max_tokens_for(model_id), + type: capabilities.model_type(model_id), + family: capabilities.model_family(model_id).to_s, + supports_vision: capabilities.supports_vision?(model_id), + supports_functions: capabilities.supports_functions?(model_id), + supports_json_mode: capabilities.supports_json_mode?(model_id) + } + end + + def pricing_attributes(model_id, capabilities) + { + input_price_per_million: capabilities.input_price_for(model_id), + output_price_per_million: capabilities.output_price_for(model_id) + } + end + + def build_metadata(model) + { + provider_name: model['providerName'], + customizations_supported: model['customizationsSupported'] || [], + inference_configurations: model['inferenceTypesSupported'] || [], + response_streaming_supported: model['responseStreamingSupported'] || false, + input_modalities: model['inputModalities'] || [], + output_modalities: model['outputModalities'] || [] + } end end end From 9d6d31cb3e0e0da765b5b0fae6a0eb3dd5263728 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:31:36 -0700 Subject: [PATCH 49/85] Ignore unused method that is passed from library --- lib/ruby_llm/providers/bedrock/chat.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index e61dc2bcc..9e821cc90 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -11,7 +11,7 @@ def completion_url "model/#{@model_id}/invoke" end - def render_payload(messages, tools:, temperature:, model:, stream: false) + def render_payload(messages, tools:, temperature:, model:, stream: false) # rubocop:disable Lint/UnusedMethodArgument # Hold model_id in instance variable for use in completion_url and stream_url @model_id = model case model From 9346762af733974701f538554eb9f00d82197fa9 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:34:57 -0700 Subject: [PATCH 50/85] Refactor process_message --- lib/ruby_llm/providers/bedrock/streaming.rb | 23 +++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index c72502f37..3a045b58a 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -116,20 +116,31 @@ def process_chunk(chunk, &) def process_message(chunk, offset, &) return chunk.bytesize unless can_read_prelude?(chunk, offset) + message_info = extract_message_info(chunk, offset) + return find_next_message(chunk, offset) unless message_info + + process_valid_message(chunk, offset, message_info, &) + end + + def extract_message_info(chunk, offset) total_length, headers_length = read_prelude(chunk, offset) - return find_next_message(chunk, offset) unless valid_lengths?(total_length, headers_length) + return unless valid_lengths?(total_length, headers_length) message_end = offset + total_length - return chunk.bytesize if chunk.bytesize < message_end + return unless chunk.bytesize >= message_end headers_end, payload_end = calculate_positions(offset, total_length, headers_length) - return find_next_message(chunk, offset) unless valid_positions?(headers_end, payload_end, chunk.bytesize) + return unless valid_positions?(headers_end, payload_end, chunk.bytesize) + + { total_length:, headers_length:, headers_end:, payload_end: } + end - payload = extract_payload(chunk, headers_end, payload_end) - return message_end unless valid_payload?(payload) + def process_valid_message(chunk, offset, message_info, &) + payload = extract_payload(chunk, message_info[:headers_end], message_info[:payload_end]) + return find_next_message(chunk, offset) unless valid_payload?(payload) process_payload(payload, &) - message_end + offset + message_info[:total_length] end def can_read_prelude?(chunk, offset) From 64e80e37d6f50cdc82d85d7e8b97538416846081 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:36:19 -0700 Subject: [PATCH 51/85] Refactor sha256_hexdigest --- lib/ruby_llm/providers/bedrock/signing.rb | 41 ++++++++++++++++------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index e1b3846b0..f39bba2ee 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -533,23 +533,40 @@ def host(uri) # @param [File, Tempfile, IO#read, String] value # @return [String] def sha256_hexdigest(value) - if (value.is_a?(File) || value.is_a?(Tempfile)) && !value.path.nil? && File.exist?(value.path) - OpenSSL::Digest::SHA256.file(value).hexdigest + if file_like?(value) + digest_file(value) elsif value.respond_to?(:read) - sha256 = OpenSSL::Digest.new('SHA256') - loop do - chunk = value.read(1024 * 1024) # 1MB - break unless chunk - - sha256.update(chunk) - end - value.rewind - sha256.hexdigest + digest_io(value) else - OpenSSL::Digest::SHA256.hexdigest(value) + digest_string(value) end end + def file_like?(value) + (value.is_a?(File) || value.is_a?(Tempfile)) && !value.path.nil? && File.exist?(value.path) + end + + def digest_file(value) + OpenSSL::Digest::SHA256.file(value).hexdigest + end + + def digest_io(value) + sha256 = OpenSSL::Digest.new('SHA256') + update_digest_from_io(sha256, value) + value.rewind + sha256.hexdigest + end + + def update_digest_from_io(digest, io) + while (chunk = io.read(1024 * 1024)) # 1MB + digest.update(chunk) + end + end + + def digest_string(value) + OpenSSL::Digest::SHA256.hexdigest(value) + end + def hmac(key, value) OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), key, value) end From 88573cf0cf5b5ebf1e1a8bb71e9dfa866a030527 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 16:37:42 -0700 Subject: [PATCH 52/85] Refactor normalized_querystring --- lib/ruby_llm/providers/bedrock/signing.rb | 61 +++++++++++++++-------- 1 file changed, 39 insertions(+), 22 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index f39bba2ee..c0f1da330 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -467,32 +467,49 @@ def path(url) end def normalized_querystring(querystring) + params = normalize_params(querystring) + sort_params(params).map(&:first).join('&') + end + + def normalize_params(querystring) params = querystring.split('&') - params = params.map { |p| p.match(/=/) ? p : "#{p}=" } - # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html - # Sort the parameter names by character code point in ascending order. - # Parameters with duplicate names should be sorted by value. - # - # Default sort <=> in JRuby will swap members - # occasionally when <=> is 0 (considered still sorted), but this - # causes our normalized query string to not match the sent querystring. - # When names match, we then sort by their values. When values also - # match then we sort by their original order + params.map { |p| ensure_param_has_equals(p) } + end + + def ensure_param_has_equals(param) + param.match(/=/) ? param : "#{param}=" + end + + # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html + # Sort the parameter names by character code point in ascending order. + # Parameters with duplicate names should be sorted by value. + # When names match, sort by values. When values also match, + # preserve original order to maintain stable sorting. + def sort_params(params) params.each.with_index.sort do |a, b| - a, a_offset = a - b, b_offset = b - a_name, a_value = a.split('=') - b_name, b_value = b.split('=') - if a_name == b_name - if a_value == b_value - a_offset <=> b_offset - else - a_value <=> b_value - end + compare_params(a, b) + end + end + + def compare_params(param_a, param_b) + a, a_offset = param_a + b, b_offset = param_b + a_name, a_value = a.split('=') + b_name, b_value = b.split('=') + + compare_param_components(a_name, a_value, b_name, b_value, a_offset, b_offset) + end + + def compare_param_components(a_name, a_value, b_name, b_value, a_offset, b_offset) + if a_name == b_name + if a_value == b_value + a_offset <=> b_offset else - a_name <=> b_name + a_value <=> b_value end - end.map(&:first).join('&') + else + a_name <=> b_name + end end def signed_headers(headers) From 7e0a3dfe4bf6d24b83cc52eba35ed39615c4347d Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 17:08:18 -0700 Subject: [PATCH 53/85] Refactor sign_request --- lib/ruby_llm/providers/bedrock/signing.rb | 142 +++++++++++++++------- 1 file changed, 98 insertions(+), 44 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index c0f1da330..656b71484 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -334,73 +334,127 @@ def initialize(options = {}) # a `#headers` method. The headers must be applied to your request. # def sign_request(request) - creds, = fetch_credentials + creds = fetch_credentials.first + request_components = extract_request_components(request) + sigv4_headers = build_sigv4_headers(request_components, creds) + signature = compute_signature(request_components, creds, sigv4_headers) + build_signature_response(request_components, sigv4_headers, signature) + end + + private + + def extract_request_components(request) http_method = extract_http_method(request) url = extract_url(request) Signer.normalize_path(url) if @normalize_path headers = downcase_headers(request[:headers]) + datetime = extract_datetime(headers) + content_sha256 = extract_content_sha256(headers, request[:body]) - datetime = headers['x-amz-date'] - datetime ||= Time.now.utc.strftime('%Y%m%dT%H%M%SZ') - date = datetime[0, 8] + { + http_method: http_method, + url: url, + headers: headers, + datetime: datetime, + date: datetime[0, 8], + content_sha256: content_sha256 + } + end - content_sha256 = headers['x-amz-content-sha256'] - content_sha256 ||= sha256_hexdigest(request[:body] || '') + def extract_datetime(headers) + headers['x-amz-date'] || Time.now.utc.strftime('%Y%m%dT%H%M%SZ') + end - sigv4_headers = {} - sigv4_headers['host'] = headers['host'] || host(url) - sigv4_headers['x-amz-date'] = datetime - if creds.session_token && !@omit_session_token - if @signing_algorithm == :'sigv4-s3express' - sigv4_headers['x-amz-s3session-token'] = creds.session_token - else - sigv4_headers['x-amz-security-token'] = creds.session_token - end - end + def extract_content_sha256(headers, body) + headers['x-amz-content-sha256'] || sha256_hexdigest(body || '') + end - sigv4_headers['x-amz-content-sha256'] ||= content_sha256 if @apply_checksum_header + def build_sigv4_headers(components, creds) + headers = { + 'host' => components[:headers]['host'] || host(components[:url]), + 'x-amz-date' => components[:datetime] + } - sigv4_headers['x-amz-region-set'] = @region if @signing_algorithm == :sigv4a && @region && !@region.empty? - headers = headers.merge(sigv4_headers) # merge so we do not modify given headers hash + add_session_token_header(headers, creds) + add_content_sha256_header(headers, components[:content_sha256]) + add_region_header(headers) - algorithm = sts_algorithm + headers + end - # compute signature parts - creq = canonical_request(http_method, url, headers, content_sha256) - sts = string_to_sign(datetime, creq, algorithm) + def add_session_token_header(headers, creds) + return unless creds.session_token && !@omit_session_token - sig = - if @signing_algorithm == :sigv4a - asymmetric_signature(creds, sts) - else - signature(creds.secret_access_key, date, sts) - end + if @signing_algorithm == :'sigv4-s3express' + headers['x-amz-s3session-token'] = creds.session_token + else + headers['x-amz-security-token'] = creds.session_token + end + end + def add_content_sha256_header(headers, content_sha256) + headers['x-amz-content-sha256'] = content_sha256 if @apply_checksum_header + end + + def add_region_header(headers) + headers['x-amz-region-set'] = @region if @signing_algorithm == :sigv4a && @region && !@region.empty? + end + + def compute_signature(components, creds, sigv4_headers) algorithm = sts_algorithm + headers = components[:headers].merge(sigv4_headers) - # apply signature - sigv4_headers['authorization'] = [ - "#{algorithm} Credential=#{credential(creds, date)}", - "SignedHeaders=#{signed_headers(headers)}", - "Signature=#{sig}" + creq = canonical_request( + components[:http_method], + components[:url], + headers, + components[:content_sha256] + ) + sts = string_to_sign(components[:datetime], creq, algorithm) + sig = generate_signature(creds, components[:date], sts) + + { + algorithm: algorithm, + credential: credential(creds, components[:date]), + signed_headers: signed_headers(headers), + signature: sig, + canonical_request: creq, + string_to_sign: sts + } + end + + def generate_signature(creds, date, string_to_sign) + if @signing_algorithm == :sigv4a + asymmetric_signature(creds, string_to_sign) + else + signature(creds.secret_access_key, date, string_to_sign) + end + end + + def build_signature_response(components, sigv4_headers, signature) + authorization = [ + "#{signature[:algorithm]} Credential=#{signature[:credential]}", + "SignedHeaders=#{signature[:signed_headers]}", + "Signature=#{signature[:signature]}" ].join(', ') - # skip signing the session token, but include it in the headers - sigv4_headers['x-amz-security-token'] = creds.session_token if creds.session_token && @omit_session_token + headers = sigv4_headers.merge('authorization' => authorization) + + # Add session token if omitted from signing + if @omit_session_token && components[:creds]&.session_token + headers['x-amz-security-token'] = components[:creds].session_token + end - # Returning the signature components. Signature.new( - headers: sigv4_headers, - string_to_sign: sts, - canonical_request: creq, - content_sha256: content_sha256, - signature: sig + headers: headers, + string_to_sign: signature[:string_to_sign], + canonical_request: signature[:canonical_request], + content_sha256: components[:content_sha256], + signature: signature[:signature] ) end - private - def sts_algorithm @signing_algorithm == :sigv4a ? 'AWS4-ECDSA-P256-SHA256' : 'AWS4-HMAC-SHA256' end From 73052dbbdb7bb11bdbf37ad96a153468c6ec80ee Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 17:12:21 -0700 Subject: [PATCH 54/85] Refactor build_signature_response and compare_param_components --- lib/ruby_llm/providers/bedrock/signing.rb | 55 +++++++++++++++-------- 1 file changed, 36 insertions(+), 19 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 656b71484..5c4c523e7 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -186,6 +186,8 @@ def set? end end + ParamComponent = Struct.new(:name, :value, :offset) + # Handles AWS request signing using SigV4 or SigV4a class Signer # @overload initialize(service:, region:, access_key_id:, secret_access_key:, session_token:nil, **options) @@ -433,18 +435,7 @@ def generate_signature(creds, date, string_to_sign) end def build_signature_response(components, sigv4_headers, signature) - authorization = [ - "#{signature[:algorithm]} Credential=#{signature[:credential]}", - "SignedHeaders=#{signature[:signed_headers]}", - "Signature=#{signature[:signature]}" - ].join(', ') - - headers = sigv4_headers.merge('authorization' => authorization) - - # Add session token if omitted from signing - if @omit_session_token && components[:creds]&.session_token - headers['x-amz-security-token'] = components[:creds].session_token - end + headers = build_headers(sigv4_headers, signature, components) Signature.new( headers: headers, @@ -455,6 +446,29 @@ def build_signature_response(components, sigv4_headers, signature) ) end + def build_headers(sigv4_headers, signature, components) + headers = sigv4_headers.merge( + 'authorization' => build_authorization_header(signature) + ) + + add_omitted_session_token(headers, components[:creds]) if @omit_session_token + headers + end + + def build_authorization_header(signature) + [ + "#{signature[:algorithm]} Credential=#{signature[:credential]}", + "SignedHeaders=#{signature[:signed_headers]}", + "Signature=#{signature[:signature]}" + ].join(', ') + end + + def add_omitted_session_token(headers, creds) + return unless creds&.session_token + + headers['x-amz-security-token'] = creds.session_token + end + def sts_algorithm @signing_algorithm == :sigv4a ? 'AWS4-ECDSA-P256-SHA256' : 'AWS4-HMAC-SHA256' end @@ -551,18 +565,21 @@ def compare_params(param_a, param_b) a_name, a_value = a.split('=') b_name, b_value = b.split('=') - compare_param_components(a_name, a_value, b_name, b_value, a_offset, b_offset) + compare_param_components( + ParamComponent.new(a_name, a_value, a_offset), + ParamComponent.new(b_name, b_value, b_offset) + ) end - def compare_param_components(a_name, a_value, b_name, b_value, a_offset, b_offset) - if a_name == b_name - if a_value == b_value - a_offset <=> b_offset + def compare_param_components(a, b) + if a.name == b.name + if a.value == b.value + a.offset <=> b.offset else - a_value <=> b_value + a.value <=> b.value end else - a_name <=> b_name + a.name <=> b.name end end From 52ac83df3c63529a128b783f7bbb5fc0b3a14a26 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 17:13:34 -0700 Subject: [PATCH 55/85] Name some parameters --- lib/ruby_llm/providers/bedrock/signing.rb | 26 +++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 5c4c523e7..8d19c99bf 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -559,27 +559,27 @@ def sort_params(params) end end - def compare_params(param_a, param_b) - a, a_offset = param_a - b, b_offset = param_b - a_name, a_value = a.split('=') - b_name, b_value = b.split('=') + def compare_params(param_pair1, param_pair2) + param1, offset1 = param_pair1 + param2, offset2 = param_pair2 + name1, value1 = param1.split('=') + name2, value2 = param2.split('=') compare_param_components( - ParamComponent.new(a_name, a_value, a_offset), - ParamComponent.new(b_name, b_value, b_offset) + ParamComponent.new(name1, value1, offset1), + ParamComponent.new(name2, value2, offset2) ) end - def compare_param_components(a, b) - if a.name == b.name - if a.value == b.value - a.offset <=> b.offset + def compare_param_components(component1, component2) + if component1.name == component2.name + if component1.value == component2.value + component1.offset <=> component2.offset else - a.value <=> b.value + component1.value <=> component2.value end else - a.name <=> b.name + component1.name <=> component2.name end end From 226b24ad9f7f376ded277172ce3c05f9e7635278 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 17:42:46 -0700 Subject: [PATCH 56/85] Split bedrock streaming into separate smaller modules --- lib/ruby_llm/providers/bedrock/streaming.rb | 320 ++---------------- .../providers/bedrock/streaming/base.rb | 60 ++++ .../bedrock/streaming/content_extraction.rb | 116 +++++++ .../bedrock/streaming/message_processing.rb | 79 +++++ .../bedrock/streaming/payload_processing.rb | 90 +++++ .../bedrock/streaming/prelude_handling.rb | 96 ++++++ 6 files changed, 466 insertions(+), 295 deletions(-) create mode 100644 lib/ruby_llm/providers/bedrock/streaming/base.rb create mode 100644 lib/ruby_llm/providers/bedrock/streaming/content_extraction.rb create mode 100644 lib/ruby_llm/providers/bedrock/streaming/message_processing.rb create mode 100644 lib/ruby_llm/providers/bedrock/streaming/payload_processing.rb create mode 100644 lib/ruby_llm/providers/bedrock/streaming/prelude_handling.rb diff --git a/lib/ruby_llm/providers/bedrock/streaming.rb b/lib/ruby_llm/providers/bedrock/streaming.rb index 3a045b58a..3dad651d8 100644 --- a/lib/ruby_llm/providers/bedrock/streaming.rb +++ b/lib/ruby_llm/providers/bedrock/streaming.rb @@ -1,305 +1,35 @@ # frozen_string_literal: true -require 'base64' +require_relative 'streaming/base' +require_relative 'streaming/content_extraction' +require_relative 'streaming/message_processing' +require_relative 'streaming/payload_processing' +require_relative 'streaming/prelude_handling' module RubyLLM module Providers module Bedrock - # Streaming methods for the AWS Bedrock API implementation + # Streaming implementation for the AWS Bedrock API. + # This module provides functionality for handling streaming responses from AWS Bedrock, + # including message processing, content extraction, and error handling. + # + # The implementation is split into several focused modules: + # - Base: Core streaming functionality and module coordination + # - ContentExtraction: Extracting content from response data + # - MessageProcessing: Processing streaming message chunks + # - PayloadProcessing: Handling JSON payloads and chunk creation + # - PreludeHandling: Managing message preludes and headers + # + # @example Using the streaming module + # class BedrockClient + # include RubyLLM::Providers::Bedrock::Streaming + # + # def stream_response(&block) + # handle_stream(&block) + # end + # end module Streaming - module_function - - def stream_url - "model/#{@model_id}/invoke-with-response-stream" - end - - def handle_stream(&block) - proc do |chunk, _bytes, env| - if env && env.status != 200 - handle_error_response(chunk, env) - else - process_chunk(chunk, &block) - end - end - end - - def json_delta?(data) - data['type'] == 'content_block_delta' && data.dig('delta', 'type') == 'input_json_delta' - end - - def extract_streaming_content(data) - if data.is_a?(Hash) - case data['type'] - when 'content_block_start' - # Initial content block, might have some text - data.dig('content_block', 'text').to_s - when 'content_block_delta' - # Incremental content updates - data.dig('delta', 'text').to_s - else - '' - end - else - '' - end - end - - def extract_content(data) - return unless data.is_a?(Hash) - - content_extractors = %i[ - extract_completion_content - extract_output_text_content - extract_array_content - extract_content_block_text - ] - - content_extractors.each do |extractor| - content = send(extractor, data) - return content if content - end - - nil - end - - def extract_completion_content(data) - data['completion'] if data.key?('completion') - end - - def extract_output_text_content(data) - data.dig('results', 0, 'outputText') - end - - def extract_array_content(data) - return unless data.key?('content') - - if data['content'].is_a?(Array) - data['content'].map { |item| item['text'] }.join - else - data['content'] - end - end - - def extract_content_block_text(data) - return unless data.key?('content_block') && data['content_block'].key?('text') - - data['content_block']['text'] - end - - def extract_tool_calls(data) - data.dig('message', 'tool_calls') || data['tool_calls'] - end - - private - - def handle_error_response(chunk, env) - buffer = String.new - buffer << chunk - begin - error_data = JSON.parse(buffer) - error_response = env.merge(body: error_data) - ErrorMiddleware.parse_error(provider: self, response: error_response) - rescue JSON::ParserError - # Keep accumulating if we don't have complete JSON yet - RubyLLM.logger.debug "Accumulating error chunk: #{chunk}" - end - end - - def process_chunk(chunk, &) - offset = 0 - offset = process_message(chunk, offset, &) while offset < chunk.bytesize - rescue StandardError => e - RubyLLM.logger.debug "Error processing chunk: #{e.message}" - RubyLLM.logger.debug "Chunk size: #{chunk.bytesize}" - end - - def process_message(chunk, offset, &) - return chunk.bytesize unless can_read_prelude?(chunk, offset) - - message_info = extract_message_info(chunk, offset) - return find_next_message(chunk, offset) unless message_info - - process_valid_message(chunk, offset, message_info, &) - end - - def extract_message_info(chunk, offset) - total_length, headers_length = read_prelude(chunk, offset) - return unless valid_lengths?(total_length, headers_length) - - message_end = offset + total_length - return unless chunk.bytesize >= message_end - - headers_end, payload_end = calculate_positions(offset, total_length, headers_length) - return unless valid_positions?(headers_end, payload_end, chunk.bytesize) - - { total_length:, headers_length:, headers_end:, payload_end: } - end - - def process_valid_message(chunk, offset, message_info, &) - payload = extract_payload(chunk, message_info[:headers_end], message_info[:payload_end]) - return find_next_message(chunk, offset) unless valid_payload?(payload) - - process_payload(payload, &) - offset + message_info[:total_length] - end - - def can_read_prelude?(chunk, offset) - chunk.bytesize - offset >= 12 - end - - def read_prelude(chunk, offset) - total_length = chunk[offset...offset + 4].unpack1('N') - headers_length = chunk[offset + 4...offset + 8].unpack1('N') - [total_length, headers_length] - end - - def valid_lengths?(total_length, headers_length) - return false if total_length.nil? || headers_length.nil? - return false if total_length <= 0 || total_length > 1_000_000 - return false if headers_length <= 0 || headers_length > total_length - - true - end - - def calculate_positions(offset, total_length, headers_length) - headers_end = offset + 12 + headers_length - payload_end = offset + total_length - 4 # Subtract 4 bytes for message CRC - [headers_end, payload_end] - end - - def valid_positions?(headers_end, payload_end, chunk_size) - return false if headers_end >= payload_end - return false if headers_end >= chunk_size - return false if payload_end > chunk_size - - true - end - - def find_next_message(chunk, offset) - next_prelude = find_next_prelude(chunk, offset + 4) - next_prelude || chunk.bytesize - end - - def extract_payload(chunk, headers_end, payload_end) - chunk[headers_end...payload_end] - end - - def valid_payload?(payload) - return false if payload.nil? || payload.empty? - - json_start = payload.index('{') - json_end = payload.rindex('}') - - return false if json_start.nil? || json_end.nil? || json_start >= json_end - - true - end - - def process_payload(payload, &) - json_payload = extract_json_payload(payload) - parse_and_process_json(json_payload, &) - rescue JSON::ParserError => e - log_json_parse_error(e, json_payload) - rescue StandardError => e - log_general_error(e) - end - - def extract_json_payload(payload) - json_start = payload.index('{') - json_end = payload.rindex('}') - payload[json_start..json_end] - end - - def parse_and_process_json(json_payload, &) - json_data = JSON.parse(json_payload) - process_json_data(json_data, &) - end - - def log_json_parse_error(error, json_payload) - RubyLLM.logger.debug "Failed to parse payload as JSON: #{error.message}" - RubyLLM.logger.debug "Attempted JSON payload: #{json_payload.inspect}" - end - - def log_general_error(error) - RubyLLM.logger.debug "Error processing payload: #{error.message}" - end - - def process_json_data(json_data, &) - return unless json_data['bytes'] - - data = decode_and_parse_data(json_data) - create_and_yield_chunk(data, &) - end - - def decode_and_parse_data(json_data) - decoded_bytes = Base64.strict_decode64(json_data['bytes']) - JSON.parse(decoded_bytes) - end - - def create_and_yield_chunk(data, &block) - block.call(build_chunk(data)) - end - - def build_chunk(data) - Chunk.new( - **extract_chunk_attributes(data) - ) - end - - def extract_chunk_attributes(data) - { - role: :assistant, - model_id: extract_model_id(data), - content: extract_streaming_content(data), - input_tokens: extract_input_tokens(data), - output_tokens: extract_output_tokens(data), - tool_calls: extract_tool_calls(data) - } - end - - def extract_model_id(data) - data.dig('message', 'model') || @model_id - end - - def extract_input_tokens(data) - data.dig('message', 'usage', 'input_tokens') - end - - def extract_output_tokens(data) - data.dig('message', 'usage', 'output_tokens') || data.dig('usage', 'output_tokens') - end - - def find_next_prelude(chunk, start_offset) - scan_range(chunk, start_offset).each do |pos| - return pos if valid_prelude_at_position?(chunk, pos) - end - nil - end - - def scan_range(chunk, start_offset) - (start_offset...(chunk.bytesize - 8)) - end - - def valid_prelude_at_position?(chunk, pos) - lengths = extract_potential_lengths(chunk, pos) - valid_prelude_lengths?(*lengths) - end - - def extract_potential_lengths(chunk, pos) - [ - chunk[pos...pos + 4].unpack1('N'), - chunk[pos + 4...pos + 8].unpack1('N') - ] - end - - def valid_prelude_lengths?(total_length, headers_length) - return false unless total_length && headers_length - return false unless total_length.positive? && headers_length.positive? - return false unless total_length < 1_000_000 - return false unless headers_length < total_length - - true - end + include Base end end end diff --git a/lib/ruby_llm/providers/bedrock/streaming/base.rb b/lib/ruby_llm/providers/bedrock/streaming/base.rb new file mode 100644 index 000000000..62fe66fb1 --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/streaming/base.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + module Bedrock + module Streaming + # Base module for AWS Bedrock streaming functionality. + # Serves as the core module that includes all other streaming-related modules + # and provides fundamental streaming operations. + # + # Responsibilities: + # - Stream URL management + # - Stream handling and error processing + # - Coordinating the functionality of other streaming modules + # + # @example + # module MyStreamingImplementation + # include RubyLLM::Providers::Bedrock::Streaming::Base + # end + module Base + def self.included(base) + base.include ContentExtraction + base.include MessageProcessing + base.include PayloadProcessing + base.include PreludeHandling + end + + def stream_url + "model/#{@model_id}/invoke-with-response-stream" + end + + def handle_stream(&block) + proc do |chunk, _bytes, env| + if env && env.status != 200 + handle_error_response(chunk, env) + else + process_chunk(chunk, &block) + end + end + end + + private + + def handle_error_response(chunk, env) + buffer = String.new + buffer << chunk + begin + error_data = JSON.parse(buffer) + error_response = env.merge(body: error_data) + ErrorMiddleware.parse_error(provider: self, response: error_response) + rescue JSON::ParserError + # Keep accumulating if we don't have complete JSON yet + RubyLLM.logger.debug "Accumulating error chunk: #{chunk}" + end + end + end + end + end + end +end diff --git a/lib/ruby_llm/providers/bedrock/streaming/content_extraction.rb b/lib/ruby_llm/providers/bedrock/streaming/content_extraction.rb new file mode 100644 index 000000000..c9043e648 --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/streaming/content_extraction.rb @@ -0,0 +1,116 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + module Bedrock + module Streaming + # Module for handling content extraction from AWS Bedrock streaming responses. + # Provides methods to extract and process various types of content from the response data. + # + # Responsibilities: + # - Extracting content from different response formats + # - Processing JSON deltas and content blocks + # - Extracting metadata (tokens, model IDs, tool calls) + # - Handling different content structures (arrays, blocks, completions) + # + # @example Content extraction from a response + # content = extract_content(response_data) + # streaming_content = extract_streaming_content(delta_data) + # tool_calls = extract_tool_calls(message_data) + module ContentExtraction + def json_delta?(data) + data['type'] == 'content_block_delta' && data.dig('delta', 'type') == 'input_json_delta' + end + + def extract_streaming_content(data) + return '' unless data.is_a?(Hash) + + extract_content_by_type(data) + end + + def extract_content(data) + return unless data.is_a?(Hash) + + try_content_extractors(data) + end + + def extract_completion_content(data) + data['completion'] if data.key?('completion') + end + + def extract_output_text_content(data) + data.dig('results', 0, 'outputText') + end + + def extract_array_content(data) + return unless data.key?('content') + + content = data['content'] + content.is_a?(Array) ? join_array_content(content) : content + end + + def extract_content_block_text(data) + return unless data.key?('content_block') && data['content_block'].key?('text') + + data['content_block']['text'] + end + + def extract_tool_calls(data) + data.dig('message', 'tool_calls') || data['tool_calls'] + end + + def extract_model_id(data) + data.dig('message', 'model') || @model_id + end + + def extract_input_tokens(data) + data.dig('message', 'usage', 'input_tokens') + end + + def extract_output_tokens(data) + data.dig('message', 'usage', 'output_tokens') || data.dig('usage', 'output_tokens') + end + + private + + def extract_content_by_type(data) + case data['type'] + when 'content_block_start' then extract_block_start_content(data) + when 'content_block_delta' then extract_delta_content(data) + else '' + end + end + + def extract_block_start_content(data) + data.dig('content_block', 'text').to_s + end + + def extract_delta_content(data) + data.dig('delta', 'text').to_s + end + + def try_content_extractors(data) + content_extractors.each do |extractor| + content = send(extractor, data) + return content if content + end + nil + end + + def content_extractors + %i[ + extract_completion_content + extract_output_text_content + extract_array_content + extract_content_block_text + ] + end + + def join_array_content(content_array) + content_array.map { |item| item['text'] }.join + end + end + end + end + end +end diff --git a/lib/ruby_llm/providers/bedrock/streaming/message_processing.rb b/lib/ruby_llm/providers/bedrock/streaming/message_processing.rb new file mode 100644 index 000000000..a43864eb1 --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/streaming/message_processing.rb @@ -0,0 +1,79 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + module Bedrock + module Streaming + # Module for processing streaming messages from AWS Bedrock. + # Handles the core message processing logic, including validation and chunking. + # + # Responsibilities: + # - Processing incoming message chunks + # - Validating message structure and content + # - Managing message offsets and boundaries + # - Error handling during message processing + # + # @example Processing a message chunk + # offset = process_message(chunk, current_offset) do |processed_chunk| + # handle_processed_chunk(processed_chunk) + # end + module MessageProcessing + def process_chunk(chunk, &) + offset = 0 + offset = process_message(chunk, offset, &) while offset < chunk.bytesize + rescue StandardError => e + RubyLLM.logger.debug "Error processing chunk: #{e.message}" + RubyLLM.logger.debug "Chunk size: #{chunk.bytesize}" + end + + def process_message(chunk, offset, &) + return chunk.bytesize unless can_read_prelude?(chunk, offset) + + message_info = extract_message_info(chunk, offset) + return find_next_message(chunk, offset) unless message_info + + process_valid_message(chunk, offset, message_info, &) + end + + def process_valid_message(chunk, offset, message_info, &) + payload = extract_payload(chunk, message_info[:headers_end], message_info[:payload_end]) + return find_next_message(chunk, offset) unless valid_payload?(payload) + + process_payload(payload, &) + offset + message_info[:total_length] + end + + private + + def extract_message_info(chunk, offset) + total_length, headers_length = read_prelude(chunk, offset) + return unless valid_lengths?(total_length, headers_length) + + message_end = offset + total_length + return unless chunk.bytesize >= message_end + + headers_end, payload_end = calculate_positions(offset, total_length, headers_length) + return unless valid_positions?(headers_end, payload_end, chunk.bytesize) + + { total_length:, headers_length:, headers_end:, payload_end: } + end + + def extract_payload(chunk, headers_end, payload_end) + chunk[headers_end...payload_end] + end + + def valid_payload?(payload) + return false if payload.nil? || payload.empty? + + json_start = payload.index('{') + json_end = payload.rindex('}') + + return false if json_start.nil? || json_end.nil? || json_start >= json_end + + true + end + end + end + end + end +end diff --git a/lib/ruby_llm/providers/bedrock/streaming/payload_processing.rb b/lib/ruby_llm/providers/bedrock/streaming/payload_processing.rb new file mode 100644 index 000000000..2b7153286 --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/streaming/payload_processing.rb @@ -0,0 +1,90 @@ +# frozen_string_literal: true + +require 'base64' + +module RubyLLM + module Providers + module Bedrock + module Streaming + # Module for processing payloads from AWS Bedrock streaming responses. + # Handles JSON payload extraction, decoding, and chunk creation. + # + # Responsibilities: + # - Extracting and validating JSON payloads + # - Decoding Base64-encoded response data + # - Creating response chunks from processed data + # - Error handling for JSON parsing and processing + # + # @example Processing a payload + # process_payload(raw_payload) do |chunk| + # yield_chunk_to_client(chunk) + # end + module PayloadProcessing + def process_payload(payload, &) + json_payload = extract_json_payload(payload) + parse_and_process_json(json_payload, &) + rescue JSON::ParserError => e + log_json_parse_error(e, json_payload) + rescue StandardError => e + log_general_error(e) + end + + private + + def extract_json_payload(payload) + json_start = payload.index('{') + json_end = payload.rindex('}') + payload[json_start..json_end] + end + + def parse_and_process_json(json_payload, &) + json_data = JSON.parse(json_payload) + process_json_data(json_data, &) + end + + def process_json_data(json_data, &) + return unless json_data['bytes'] + + data = decode_and_parse_data(json_data) + create_and_yield_chunk(data, &) + end + + def decode_and_parse_data(json_data) + decoded_bytes = Base64.strict_decode64(json_data['bytes']) + JSON.parse(decoded_bytes) + end + + def create_and_yield_chunk(data, &block) + block.call(build_chunk(data)) + end + + def build_chunk(data) + Chunk.new( + **extract_chunk_attributes(data) + ) + end + + def extract_chunk_attributes(data) + { + role: :assistant, + model_id: extract_model_id(data), + content: extract_streaming_content(data), + input_tokens: extract_input_tokens(data), + output_tokens: extract_output_tokens(data), + tool_calls: extract_tool_calls(data) + } + end + + def log_json_parse_error(error, json_payload) + RubyLLM.logger.debug "Failed to parse payload as JSON: #{error.message}" + RubyLLM.logger.debug "Attempted JSON payload: #{json_payload.inspect}" + end + + def log_general_error(error) + RubyLLM.logger.debug "Error processing payload: #{error.message}" + end + end + end + end + end +end diff --git a/lib/ruby_llm/providers/bedrock/streaming/prelude_handling.rb b/lib/ruby_llm/providers/bedrock/streaming/prelude_handling.rb new file mode 100644 index 000000000..db06ccd2d --- /dev/null +++ b/lib/ruby_llm/providers/bedrock/streaming/prelude_handling.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +module RubyLLM + module Providers + module Bedrock + module Streaming + # Module for handling message preludes in AWS Bedrock streaming responses. + # Manages the parsing and validation of message headers and prelude data. + # + # Responsibilities: + # - Reading and validating message preludes + # - Calculating message positions and boundaries + # - Finding and validating prelude positions in chunks + # - Ensuring message integrity through length validation + # + # @example Reading a prelude + # if can_read_prelude?(chunk, offset) + # total_length, headers_length = read_prelude(chunk, offset) + # process_message_with_lengths(total_length, headers_length) + # end + module PreludeHandling + def can_read_prelude?(chunk, offset) + chunk.bytesize - offset >= 12 + end + + def read_prelude(chunk, offset) + total_length = chunk[offset...offset + 4].unpack1('N') + headers_length = chunk[offset + 4...offset + 8].unpack1('N') + [total_length, headers_length] + end + + def valid_lengths?(total_length, headers_length) + return false if total_length.nil? || headers_length.nil? + return false if total_length <= 0 || total_length > 1_000_000 + return false if headers_length <= 0 || headers_length > total_length + + true + end + + def calculate_positions(offset, total_length, headers_length) + headers_end = offset + 12 + headers_length + payload_end = offset + total_length - 4 # Subtract 4 bytes for message CRC + [headers_end, payload_end] + end + + def valid_positions?(headers_end, payload_end, chunk_size) + return false if headers_end >= payload_end + return false if headers_end >= chunk_size + return false if payload_end > chunk_size + + true + end + + def find_next_message(chunk, offset) + next_prelude = find_next_prelude(chunk, offset + 4) + next_prelude || chunk.bytesize + end + + def find_next_prelude(chunk, start_offset) + scan_range(chunk, start_offset).each do |pos| + return pos if valid_prelude_at_position?(chunk, pos) + end + nil + end + + private + + def scan_range(chunk, start_offset) + (start_offset...(chunk.bytesize - 8)) + end + + def valid_prelude_at_position?(chunk, pos) + lengths = extract_potential_lengths(chunk, pos) + valid_prelude_lengths?(*lengths) + end + + def extract_potential_lengths(chunk, pos) + [ + chunk[pos...pos + 4].unpack1('N'), + chunk[pos + 4...pos + 8].unpack1('N') + ] + end + + def valid_prelude_lengths?(total_length, headers_length) + return false unless total_length && headers_length + return false unless total_length.positive? && headers_length.positive? + return false unless total_length < 1_000_000 + return false unless headers_length < total_length + + true + end + end + end + end + end +end From a80149f9ce4c20a63f0ccbe6168a5182cbbce930 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 17:45:45 -0700 Subject: [PATCH 57/85] Refactor some methods in signing --- lib/ruby_llm/providers/bedrock/signing.rb | 42 ++++++++++++++++++----- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 8d19c99bf..9d757dc9a 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -252,10 +252,7 @@ def initialize(options = {}) @service = extract_service(options) @region = extract_region(options) @credentials_provider = extract_credentials_provider(options) - @unsigned_headers = Set.new(options.fetch(:unsigned_headers, []).map(&:downcase)) - @unsigned_headers << 'authorization' - @unsigned_headers << 'x-amzn-trace-id' - @unsigned_headers << 'expect' + @unsigned_headers = initialize_unsigned_headers(options) @uri_escape_path = options.fetch(:uri_escape_path, true) @apply_checksum_header = options.fetch(:apply_checksum_header, true) @signing_algorithm = options.fetch(:signing_algorithm, :sigv4) @@ -263,6 +260,11 @@ def initialize(options = {}) @omit_session_token = options.fetch(:omit_session_token, false) end + def initialize_unsigned_headers(options) + headers = Set.new(options.fetch(:unsigned_headers, []).map(&:downcase)) + headers.merge(%w[authorization x-amzn-trace-id expect]) + end + # @return [String] attr_reader :service @@ -351,8 +353,13 @@ def extract_request_components(request) url = extract_url(request) Signer.normalize_path(url) if @normalize_path headers = downcase_headers(request[:headers]) + + build_request_components(http_method, url, headers, request[:body]) + end + + def build_request_components(http_method, url, headers, body) datetime = extract_datetime(headers) - content_sha256 = extract_content_sha256(headers, request[:body]) + content_sha256 = extract_content_sha256(headers, body) { http_method: http_method, @@ -407,6 +414,14 @@ def compute_signature(components, creds, sigv4_headers) algorithm = sts_algorithm headers = components[:headers].merge(sigv4_headers) + signature_components = build_signature_components( + components, creds, headers, algorithm + ) + + build_signature_result(signature_components, headers, creds, components[:date]) + end + + def build_signature_components(components, creds, headers, algorithm) creq = canonical_request( components[:http_method], components[:url], @@ -416,13 +431,22 @@ def compute_signature(components, creds, sigv4_headers) sts = string_to_sign(components[:datetime], creq, algorithm) sig = generate_signature(creds, components[:date], sts) + { + creq: creq, + sts: sts, + sig: sig + } + end + + def build_signature_result(components, headers, creds, date) + algorithm = sts_algorithm { algorithm: algorithm, - credential: credential(creds, components[:date]), + credential: credential(creds, date), signed_headers: signed_headers(headers), - signature: sig, - canonical_request: creq, - string_to_sign: sts + signature: components[:sig], + canonical_request: components[:creq], + string_to_sign: components[:sts] } end From f6099d3960ee9211ba1bd1e19d7392605f5c0ad5 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 17:58:38 -0700 Subject: [PATCH 58/85] Refactor build_signature_components --- lib/ruby_llm/providers/bedrock/signing.rb | 26 ++++++++++++++++------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 9d757dc9a..bf999dd30 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -422,20 +422,30 @@ def compute_signature(components, creds, sigv4_headers) end def build_signature_components(components, creds, headers, algorithm) - creq = canonical_request( + creq = build_canonical_request(components, headers) + sts = build_string_to_sign(components, algorithm, creq) + { + creq: creq, + sts: sts, + sig: generate_signature(creds, components[:date], sts) + } + end + + def build_canonical_request(components, headers) + canonical_request( components[:http_method], components[:url], headers, components[:content_sha256] ) - sts = string_to_sign(components[:datetime], creq, algorithm) - sig = generate_signature(creds, components[:date], sts) + end - { - creq: creq, - sts: sts, - sig: sig - } + def build_string_to_sign(components, algorithm, creq) + string_to_sign( + components[:datetime], + creq, + algorithm + ) end def build_signature_result(components, headers, creds, date) From 5d8184c0abd487aceb44d7218bd50aa7039ed1b7 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 18:18:33 -0700 Subject: [PATCH 59/85] Split Signer into smaller classes --- lib/ruby_llm/providers/bedrock/signing.rb | 955 ++++++++++++---------- 1 file changed, 509 insertions(+), 446 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index bf999dd30..8af934b08 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -188,381 +188,143 @@ def set? ParamComponent = Struct.new(:name, :value, :offset) - # Handles AWS request signing using SigV4 or SigV4a - class Signer - # @overload initialize(service:, region:, access_key_id:, secret_access_key:, session_token:nil, **options) - # @param [String] :service The service signing name, e.g. 's3'. - # @param [String] :region The region name, e.g. 'us-east-1'. When signing - # with sigv4a, this should be a comma separated list of regions. - # @param [String] :access_key_id - # @param [String] :secret_access_key - # @param [String] :session_token (nil) - # - # @overload initialize(service:, region:, credentials:, **options) - # @param [String] :service The service signing name, e.g. 's3'. - # @param [String] :region The region name, e.g. 'us-east-1'. When signing - # with sigv4a, this should be a comma separated list of regions. - # @param [Credentials] :credentials Any object that responds to the following - # methods: - # - # * `#access_key_id` => String - # * `#secret_access_key` => String - # * `#session_token` => String, nil - # * `#set?` => Boolean - # - # @overload initialize(service:, region:, credentials_provider:, **options) - # @param [String] :service The service signing name, e.g. 's3'. - # @param [String] :region The region name, e.g. 'us-east-1'. When signing - # with sigv4a, this should be a comma separated list of regions. - # @param [#credentials] :credentials_provider An object that responds - # to `#credentials`, returning an object that responds to the following - # methods: - # - # * `#access_key_id` => String - # * `#secret_access_key` => String - # * `#session_token` => String, nil - # * `#set?` => Boolean - # - # @option options [Array] :unsigned_headers ([]) A list of - # headers that should not be signed. This is useful when a proxy - # modifies headers, such as 'User-Agent', invalidating a signature. - # - # @option options [Boolean] :uri_escape_path (true) When `true`, - # the request URI path is uri-escaped as part of computing the canonical - # request string. This is required for every service, except Amazon S3, - # as of late 2016. - # - # @option options [Boolean] :apply_checksum_header (true) When `true`, - # the computed content checksum is returned in the hash of signature - # headers. This is required for AWS Glacier, and optional for - # every other AWS service as of late 2016. - # - # @option options [Symbol] :signing_algorithm (:sigv4) The - # algorithm to use for signing. - # - # @option options [Boolean] :omit_session_token (false) - # (Supported only when `aws-crt` is available) If `true`, - # then security token is added to the final signing result, - # but is treated as "unsigned" and does not contribute - # to the authorization signature. - # - # @option options [Boolean] :normalize_path (true) When `true`, the - # uri paths will be normalized when building the canonical request. - def initialize(options = {}) - @service = extract_service(options) - @region = extract_region(options) - @credentials_provider = extract_credentials_provider(options) - @unsigned_headers = initialize_unsigned_headers(options) - @uri_escape_path = options.fetch(:uri_escape_path, true) - @apply_checksum_header = options.fetch(:apply_checksum_header, true) - @signing_algorithm = options.fetch(:signing_algorithm, :sigv4) - @normalize_path = options.fetch(:normalize_path, true) - @omit_session_token = options.fetch(:omit_session_token, false) - end - - def initialize_unsigned_headers(options) - headers = Set.new(options.fetch(:unsigned_headers, []).map(&:downcase)) - headers.merge(%w[authorization x-amzn-trace-id expect]) - end - - # @return [String] - attr_reader :service - - # @return [String] - attr_reader :region - - # @return [#credentials] Returns an object that responds to - # `#credentials`, returning an object that responds to the following - # methods: - # - # * `#access_key_id` => String - # * `#secret_access_key` => String - # * `#session_token` => String, nil - # * `#set?` => Boolean - # - attr_reader :credentials_provider - - # @return [Set] Returns a set of header names that should not be signed. - # All header names have been downcased. - attr_reader :unsigned_headers - - # @return [Boolean] When `true` the `x-amz-content-sha256` header will be signed and - # returned in the signature headers. - attr_reader :apply_checksum_header - - # Computes a version 4 signature signature. Returns the resultant - # signature as a hash of headers to apply to your HTTP request. The given - # request is not modified. - # - # signature = signer.sign_request( - # http_method: 'PUT', - # url: 'https://domain.com', - # headers: { - # 'Abc' => 'xyz', - # }, - # body: 'body' # String or IO object - # ) - # - # # Apply the following hash of headers to your HTTP request - # signature.headers['host'] - # signature.headers['x-amz-date'] - # signature.headers['x-amz-security-token'] - # signature.headers['x-amz-content-sha256'] - # signature.headers['authorization'] - # - # In addition to computing the signature headers, the canonicalized - # request, string to sign and content sha256 checksum are also available. - # These values are useful for debugging signature errors returned by AWS. - # - # signature.canonical_request #=> "..." - # signature.string_to_sign #=> "..." - # signature.content_sha256 #=> "..." - # - # @param [Hash] request - # - # @option request [required, String] :http_method One of - # 'GET', 'HEAD', 'PUT', 'POST', 'PATCH', or 'DELETE' - # - # @option request [required, String, URI::HTTP, URI::HTTPS] :url - # The request URI. Must be a valid HTTP or HTTPS URI. - # - # @option request [optional, Hash] :headers ({}) A hash of headers - # to sign. If the 'X-Amz-Content-Sha256' header is set, the `:body` - # is optional and will not be read. - # - # @option request [optional, String, IO] :body ('') The HTTP request body. - # A sha256 checksum is computed of the body unless the - # 'X-Amz-Content-Sha256' header is set. - # - # @return [Signature] Return an instance of {Signature} that has - # a `#headers` method. The headers must be applied to your request. - # - def sign_request(request) - creds = fetch_credentials.first - request_components = extract_request_components(request) - sigv4_headers = build_sigv4_headers(request_components, creds) - signature = compute_signature(request_components, creds, sigv4_headers) - - build_signature_response(request_components, sigv4_headers, signature) - end - - private - - def extract_request_components(request) - http_method = extract_http_method(request) - url = extract_url(request) - Signer.normalize_path(url) if @normalize_path - headers = downcase_headers(request[:headers]) - - build_request_components(http_method, url, headers, request[:body]) - end - - def build_request_components(http_method, url, headers, body) - datetime = extract_datetime(headers) - content_sha256 = extract_content_sha256(headers, body) - - { - http_method: http_method, - url: url, - headers: headers, - datetime: datetime, - date: datetime[0, 8], - content_sha256: content_sha256 - } - end - - def extract_datetime(headers) - headers['x-amz-date'] || Time.now.utc.strftime('%Y%m%dT%H%M%SZ') - end - - def extract_content_sha256(headers, body) - headers['x-amz-content-sha256'] || sha256_hexdigest(body || '') - end - - def build_sigv4_headers(components, creds) - headers = { - 'host' => components[:headers]['host'] || host(components[:url]), - 'x-amz-date' => components[:datetime] - } - - add_session_token_header(headers, creds) - add_content_sha256_header(headers, components[:content_sha256]) - add_region_header(headers) + # Utility methods for URI manipulation and hashing + module UriUtils + module_function - headers + def uri_escape_path(path) + path.gsub(%r{[^/]+}) { |part| uri_escape(part) } end - def add_session_token_header(headers, creds) - return unless creds.session_token && !@omit_session_token - - if @signing_algorithm == :'sigv4-s3express' - headers['x-amz-s3session-token'] = creds.session_token + def uri_escape(string) + if string.nil? + nil else - headers['x-amz-security-token'] = creds.session_token + CGI.escape(string.encode('UTF-8')).gsub('+', '%20').gsub('%7E', '~') end end - def add_content_sha256_header(headers, content_sha256) - headers['x-amz-content-sha256'] = content_sha256 if @apply_checksum_header + def normalize_path(uri) + normalized_path = Pathname.new(uri.path).cleanpath.to_s + # Pathname is probably not correct to use. Empty paths will + # resolve to "." and should be disregarded + normalized_path = '' if normalized_path == '.' + # Ensure trailing slashes are correctly preserved + normalized_path << '/' if uri.path.end_with?('/') && !normalized_path.end_with?('/') + uri.path = normalized_path end - def add_region_header(headers) - headers['x-amz-region-set'] = @region if @signing_algorithm == :sigv4a && @region && !@region.empty? + def host(uri) + # Handles known and unknown URI schemes; default_port nil when unknown. + if uri.default_port == uri.port + uri.host + else + "#{uri.host}:#{uri.port}" + end end + end - def compute_signature(components, creds, sigv4_headers) - algorithm = sts_algorithm - headers = components[:headers].merge(sigv4_headers) - - signature_components = build_signature_components( - components, creds, headers, algorithm - ) - - build_signature_result(signature_components, headers, creds, components[:date]) - end + # Cryptographic hash and digest utilities + module CryptoUtils + module_function - def build_signature_components(components, creds, headers, algorithm) - creq = build_canonical_request(components, headers) - sts = build_string_to_sign(components, algorithm, creq) - { - creq: creq, - sts: sts, - sig: generate_signature(creds, components[:date], sts) - } + # @param [File, Tempfile, IO#read, String] value + # @return [String] + def sha256_hexdigest(value) + if file_like?(value) + digest_file(value) + elsif value.respond_to?(:read) + digest_io(value) + else + digest_string(value) + end end - def build_canonical_request(components, headers) - canonical_request( - components[:http_method], - components[:url], - headers, - components[:content_sha256] - ) + def file_like?(value) + (value.is_a?(File) || value.is_a?(Tempfile)) && !value.path.nil? && File.exist?(value.path) end - def build_string_to_sign(components, algorithm, creq) - string_to_sign( - components[:datetime], - creq, - algorithm - ) + def digest_file(value) + OpenSSL::Digest::SHA256.file(value).hexdigest end - def build_signature_result(components, headers, creds, date) - algorithm = sts_algorithm - { - algorithm: algorithm, - credential: credential(creds, date), - signed_headers: signed_headers(headers), - signature: components[:sig], - canonical_request: components[:creq], - string_to_sign: components[:sts] - } + def digest_io(value) + sha256 = OpenSSL::Digest.new('SHA256') + update_digest_from_io(sha256, value) + value.rewind + sha256.hexdigest end - def generate_signature(creds, date, string_to_sign) - if @signing_algorithm == :sigv4a - asymmetric_signature(creds, string_to_sign) - else - signature(creds.secret_access_key, date, string_to_sign) + def update_digest_from_io(digest, io) + while (chunk = io.read(1024 * 1024)) # 1MB + digest.update(chunk) end end - def build_signature_response(components, sigv4_headers, signature) - headers = build_headers(sigv4_headers, signature, components) - - Signature.new( - headers: headers, - string_to_sign: signature[:string_to_sign], - canonical_request: signature[:canonical_request], - content_sha256: components[:content_sha256], - signature: signature[:signature] - ) + def digest_string(value) + OpenSSL::Digest::SHA256.hexdigest(value) end - def build_headers(sigv4_headers, signature, components) - headers = sigv4_headers.merge( - 'authorization' => build_authorization_header(signature) - ) - - add_omitted_session_token(headers, components[:creds]) if @omit_session_token - headers + def hmac(key, value) + OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), key, value) end - def build_authorization_header(signature) - [ - "#{signature[:algorithm]} Credential=#{signature[:credential]}", - "SignedHeaders=#{signature[:signed_headers]}", - "Signature=#{signature[:signature]}" - ].join(', ') + def hexhmac(key, value) + OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha256'), key, value) end + end - def add_omitted_session_token(headers, creds) - return unless creds&.session_token - - headers['x-amz-security-token'] = creds.session_token - end + # Configuration for canonical request creation + class CanonicalRequestConfig + attr_reader :uri_escape_path, :unsigned_headers - def sts_algorithm - @signing_algorithm == :sigv4a ? 'AWS4-ECDSA-P256-SHA256' : 'AWS4-HMAC-SHA256' + def initialize(options = {}) + @uri_escape_path = options[:uri_escape_path] || true + @unsigned_headers = options[:unsigned_headers] || Set.new end + end - def canonical_request(http_method, url, headers, content_sha256) - [ - http_method, - path(url), - normalized_querystring(url.query || ''), - "#{canonical_headers(headers)}\n", - signed_headers(headers), - content_sha256 - ].join("\n") + # Handles canonical requests for AWS signature + class CanonicalRequest + # Builds a canonical request for AWS signature + # @param [Hash] params Parameters for the canonical request + def initialize(params = {}) + @http_method = params[:http_method] + @url = params[:url] + @headers = params[:headers] + @content_sha256 = params[:content_sha256] + @config = params[:config] || CanonicalRequestConfig.new end - def string_to_sign(datetime, canonical_request, algorithm) + def to_s [ - algorithm, - datetime, - credential_scope(datetime[0, 8]), - sha256_hexdigest(canonical_request) + @http_method, + path, + normalized_querystring(@url.query || ''), + "#{canonical_headers}\n", + signed_headers, + @content_sha256 ].join("\n") end - def credential_scope(date) - [ - date, - (@region unless @signing_algorithm == :sigv4a), - @service, - 'aws4_request' - ].compact.join('/') - end - - def credential(credentials, date) - "#{credentials.access_key_id}/#{credential_scope(date)}" - end - - def signature(secret_access_key, date, string_to_sign) - k_date = hmac("AWS4#{secret_access_key}", date) - k_region = hmac(k_date, @region) - k_service = hmac(k_region, @service) - k_credentials = hmac(k_service, 'aws4_request') - hexhmac(k_credentials, string_to_sign) + # Returns the list of signed headers for authorization + def signed_headers + @headers.inject([]) do |signed_headers, (header, _)| + if @config.unsigned_headers.include?(header) + signed_headers + else + signed_headers << header + end + end.sort.join(';') end - def asymmetric_signature(creds, string_to_sign) - ec, = Aws::Sigv4::AsymmetricCredentials.derive_asymmetric_key( - creds.access_key_id, creds.secret_access_key - ) - sts_digest = OpenSSL::Digest::SHA256.digest(string_to_sign) - s = ec.dsa_sign_asn1(sts_digest) - - Digest.hexencode(s) - end + private - def path(url) - path = url.path + def path + path = @url.path path = '/' if path == '' - if @uri_escape_path - uri_escape_path(path) + if @config.uri_escape_path + UriUtils.uri_escape_path(path) else path end @@ -582,11 +344,6 @@ def ensure_param_has_equals(param) param.match(/=/) ? param : "#{param}=" end - # From: https://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html - # Sort the parameter names by character code point in ascending order. - # Parameters with duplicate names should be sorted by value. - # When names match, sort by values. When values also match, - # preserve original order to maintain stable sorting. def sort_params(params) params.each.with_index.sort do |a, b| compare_params(a, b) @@ -617,19 +374,9 @@ def compare_param_components(component1, component2) end end - def signed_headers(headers) - headers.inject([]) do |signed_headers, (header, _)| - if @unsigned_headers.include?(header) - signed_headers - else - signed_headers << header - end - end.sort.join(';') - end - - def canonical_headers(headers) - headers = headers.inject([]) do |hdrs, (k, v)| - if @unsigned_headers.include?(k) + def canonical_headers + headers = @headers.inject([]) do |hdrs, (k, v)| + if @config.unsigned_headers.include?(k) hdrs else hdrs << [k, v] @@ -642,87 +389,96 @@ def canonical_headers(headers) def canonical_header_value(value) value.gsub(/\s+/, ' ').strip end + end - def host(uri) - # Handles known and unknown URI schemes; default_port nil when unknown. - if uri.default_port == uri.port - uri.host - else - "#{uri.host}:#{uri.port}" - end + # Handles signature computation + class SignatureComputation + def initialize(service, region, signing_algorithm) + @service = service + @region = region + @signing_algorithm = signing_algorithm end - # @param [File, Tempfile, IO#read, String] value - # @return [String] - def sha256_hexdigest(value) - if file_like?(value) - digest_file(value) - elsif value.respond_to?(:read) - digest_io(value) - else - digest_string(value) - end + def string_to_sign(datetime, canonical_request, algorithm) + [ + algorithm, + datetime, + credential_scope(datetime[0, 8]), + CryptoUtils.sha256_hexdigest(canonical_request) + ].join("\n") end - def file_like?(value) - (value.is_a?(File) || value.is_a?(Tempfile)) && !value.path.nil? && File.exist?(value.path) + def credential_scope(date) + [ + date, + (@region unless @signing_algorithm == :sigv4a), + @service, + 'aws4_request' + ].compact.join('/') end - def digest_file(value) - OpenSSL::Digest::SHA256.file(value).hexdigest + def credential(credentials, date) + "#{credentials.access_key_id}/#{credential_scope(date)}" end - def digest_io(value) - sha256 = OpenSSL::Digest.new('SHA256') - update_digest_from_io(sha256, value) - value.rewind - sha256.hexdigest + def signature(secret_access_key, date, string_to_sign) + k_date = CryptoUtils.hmac("AWS4#{secret_access_key}", date) + k_region = CryptoUtils.hmac(k_date, @region) + k_service = CryptoUtils.hmac(k_region, @service) + k_credentials = CryptoUtils.hmac(k_service, 'aws4_request') + CryptoUtils.hexhmac(k_credentials, string_to_sign) end - def update_digest_from_io(digest, io) - while (chunk = io.read(1024 * 1024)) # 1MB - digest.update(chunk) - end - end + def asymmetric_signature(creds, string_to_sign) + ec, = Aws::Sigv4::AsymmetricCredentials.derive_asymmetric_key( + creds.access_key_id, creds.secret_access_key + ) + sts_digest = OpenSSL::Digest::SHA256.digest(string_to_sign) + s = ec.dsa_sign_asn1(sts_digest) - def digest_string(value) - OpenSSL::Digest::SHA256.hexdigest(value) + Digest.hexencode(s) end + end - def hmac(key, value) - OpenSSL::HMAC.digest(OpenSSL::Digest.new('sha256'), key, value) - end + # Extracts and validates request components + class RequestExtractor + # Extract and process request components + # @param [Hash] request The request to process + # @param [Hash] options Options for extraction + # @return [Hash] Processed request components + def self.extract_components(request, options = {}) + normalize_path = options.fetch(:normalize_path, true) - def hexhmac(key, value) - OpenSSL::HMAC.hexdigest(OpenSSL::Digest.new('sha256'), key, value) - end + # Extract base components + http_method, url, headers = extract_base_components(request) + UriUtils.normalize_path(url) if normalize_path - def extract_service(options) - if options[:service] - options[:service] - else - msg = 'missing required option :service' - raise ArgumentError, msg - end - end + # Process headers and compute content SHA256 + datetime = headers['x-amz-date'] || Time.now.utc.strftime('%Y%m%dT%H%M%SZ') + content_sha256 = extract_content_sha256(headers, request[:body]) - def extract_region(options) - raise Errors::MissingRegionError unless options[:region] + { + http_method: http_method, + url: url, + headers: headers, + datetime: datetime, + date: datetime[0, 8], + content_sha256: content_sha256 + } + end - options[:region] + def self.extract_base_components(request) + http_method = extract_http_method(request) + url = extract_url(request) + headers = downcase_headers(request[:headers]) + [http_method, url, headers] end - def extract_credentials_provider(options) - if options[:credentials_provider] - options[:credentials_provider] - elsif options.key?(:credentials) || options.key?(:access_key_id) - StaticCredentialsProvider.new(options) - else - raise Errors::MissingCredentialsError - end + def self.extract_content_sha256(headers, body) + headers['x-amz-content-sha256'] || CryptoUtils.sha256_hexdigest(body || '') end - def extract_http_method(request) + def self.extract_http_method(request) if request[:http_method] request[:http_method].upcase else @@ -731,7 +487,7 @@ def extract_http_method(request) end end - def extract_url(request) + def self.extract_url(request) if request[:url] URI.parse(request[:url].to_s) else @@ -740,26 +496,90 @@ def extract_url(request) end end - def downcase_headers(headers) + def self.downcase_headers(headers) (headers || {}).to_hash.transform_keys(&:downcase) end + end - def extract_expires_in(options) - case options[:expires_in] - when nil then 900 - when Integer then options[:expires_in] + # Handles generating headers for AWS request signing + class HeaderBuilder + def initialize(options = {}) + @signing_algorithm = options[:signing_algorithm] + @apply_checksum_header = options[:apply_checksum_header] + @omit_session_token = options[:omit_session_token] + @region = options[:region] + end + + # Build headers for a signed request + # @param [Hash] components Request components + # @param [Credentials] creds AWS credentials + # @return [Hash] Generated headers + def build_sigv4_headers(components, creds) + headers = { + 'host' => components[:headers]['host'] || UriUtils.host(components[:url]), + 'x-amz-date' => components[:datetime] + } + + add_session_token_header(headers, creds) + add_content_sha256_header(headers, components[:content_sha256]) + add_region_header(headers) + + headers + end + + # Build authorization headers for a signature + # @param [Hash] sigv4_headers Headers for the signature + # @param [Hash] signature The computed signature + # @param [Hash] components Request components + # @return [Hash] Headers with authorization + def build_headers(sigv4_headers, signature, components) + headers = sigv4_headers.merge( + 'authorization' => build_authorization_header(signature) + ) + + add_omitted_session_token(headers, components[:creds]) if @omit_session_token + headers + end + + def build_authorization_header(signature) + [ + "#{signature[:algorithm]} Credential=#{signature[:credential]}", + "SignedHeaders=#{signature[:signed_headers]}", + "Signature=#{signature[:signature]}" + ].join(', ') + end + + private + + def add_session_token_header(headers, creds) + return unless creds.session_token && !@omit_session_token + + if @signing_algorithm == :'sigv4-s3express' + headers['x-amz-s3session-token'] = creds.session_token else - msg = 'expected :expires_in to be a number of seconds' - raise ArgumentError, msg + headers['x-amz-security-token'] = creds.session_token end end - def uri_escape(string) - self.class.uri_escape(string) + def add_content_sha256_header(headers, content_sha256) + headers['x-amz-content-sha256'] = content_sha256 if @apply_checksum_header + end + + def add_region_header(headers) + headers['x-amz-region-set'] = @region if @signing_algorithm == :sigv4a && @region && !@region.empty? + end + + def add_omitted_session_token(headers, creds) + return unless creds&.session_token + + headers['x-amz-security-token'] = creds.session_token end + end - def uri_escape_path(string) - self.class.uri_escape_path(string) + # Credential management and fetching + class CredentialManager + def initialize(credentials_provider) + @credentials_provider = credentials_provider end def fetch_credentials @@ -774,10 +594,6 @@ def fetch_credentials end end - # Returns true if credentials are set (not nil or empty) - # Credentials may not implement the Credentials interface - # and may just be credential like Client response objects - # (eg those returned by sts#assume_role) def credentials_set?(credentials) !credentials.access_key_id.nil? && !credentials.access_key_id.empty? && @@ -791,8 +607,8 @@ def presigned_url_expiration(options, expiration, datetime) expiration_seconds = (expiration - datetime).to_i # In the static stability case, credentials may expire in the past - # but still be valid. For those cases, use the user configured - # expires_in and ingore expiration. + # but still be valid. For those cases, use the user configured + # expires_in and ignore expiration. if expiration_seconds <= 0 expires_in else @@ -800,36 +616,283 @@ def presigned_url_expiration(options, expiration, datetime) end end + private + + def extract_expires_in(options) + case options[:expires_in] + when nil then 900 + when Integer then options[:expires_in] + else + msg = 'expected :expires_in to be a number of seconds' + raise ArgumentError, msg + end + end + end + + # Result builder for signature computation + class SignatureResultBuilder + def initialize(signature_computation) + @signature_computation = signature_computation + end + + def build_result(request_data) + { + algorithm: request_data[:algorithm], + credential: @signature_computation.credential( + request_data[:credentials], + request_data[:date] + ), + signed_headers: request_data[:canonical_request].signed_headers, + signature: request_data[:signature], + canonical_request: request_data[:creq], + string_to_sign: request_data[:sts] + } + end + end + + # Core functionality for computing signatures + class SignatureGenerator + def initialize(options = {}) + @signing_algorithm = options[:signing_algorithm] || :sigv4 + @uri_escape_path = options[:uri_escape_path] || true + @unsigned_headers = options[:unsigned_headers] || Set.new + @service = options[:service] + @region = options[:region] + + @signature_computation = SignatureComputation.new(@service, @region, @signing_algorithm) + @result_builder = SignatureResultBuilder.new(@signature_computation) + end + + def sts_algorithm + @signing_algorithm == :sigv4a ? 'AWS4-ECDSA-P256-SHA256' : 'AWS4-HMAC-SHA256' + end + + def compute_signature(components, creds, sigv4_headers) + algorithm = sts_algorithm + headers = components[:headers].merge(sigv4_headers) + + # Create request configuration and canonical request + canonical_request = create_canonical_request(components, headers) + + # Generate string to sign and signature + creq = canonical_request.to_s + sts = @signature_computation.string_to_sign( + components[:datetime], + creq, + algorithm + ) + + sig = generate_signature(creds, components[:date], sts) + + # Build the final result + @result_builder.build_result( + algorithm: algorithm, + credentials: creds, + date: components[:date], + signature: sig, + creq: creq, + sts: sts, + canonical_request: canonical_request + ) + end + + private + + def create_canonical_request(components, headers) + canon_req_config = CanonicalRequestConfig.new( + uri_escape_path: @uri_escape_path, + unsigned_headers: @unsigned_headers + ) + + CanonicalRequest.new( + http_method: components[:http_method], + url: components[:url], + headers: headers, + content_sha256: components[:content_sha256], + config: canon_req_config + ) + end + + def generate_signature(creds, date, string_to_sign) + if @signing_algorithm == :sigv4a + @signature_computation.asymmetric_signature(creds, string_to_sign) + else + @signature_computation.signature(creds.secret_access_key, date, string_to_sign) + end + end + end + + # Utility for extracting options and config + class SignerOptionExtractor + def self.extract_service(options) + if options[:service] + options[:service] + else + msg = 'missing required option :service' + raise ArgumentError, msg + end + end + + def self.extract_region(options) + raise Errors::MissingRegionError unless options[:region] + + options[:region] + end + + def self.extract_credentials_provider(options) + if options[:credentials_provider] + options[:credentials_provider] + elsif options.key?(:credentials) || options.key?(:access_key_id) + StaticCredentialsProvider.new(options) + else + raise Errors::MissingCredentialsError + end + end + + def self.initialize_unsigned_headers(options) + headers = Set.new(options.fetch(:unsigned_headers, []).map(&:downcase)) + headers.merge(%w[authorization x-amzn-trace-id expect]) + end + end + + # Handles initialization of Signer dependencies + class SignerInitializer + def self.create_components(options = {}) + service = SignerOptionExtractor.extract_service(options) + region = SignerOptionExtractor.extract_region(options) + credentials_provider = SignerOptionExtractor.extract_credentials_provider(options) + unsigned_headers = SignerOptionExtractor.initialize_unsigned_headers(options) + + uri_escape_path = options.fetch(:uri_escape_path, true) + apply_checksum_header = options.fetch(:apply_checksum_header, true) + signing_algorithm = options.fetch(:signing_algorithm, :sigv4) + normalize_path = options.fetch(:normalize_path, true) + omit_session_token = options.fetch(:omit_session_token, false) + + # Create generator + signature_generator = SignatureGenerator.new( + signing_algorithm: signing_algorithm, + uri_escape_path: uri_escape_path, + unsigned_headers: unsigned_headers, + service: service, + region: region + ) + + # Create header builder + header_builder = HeaderBuilder.new( + signing_algorithm: signing_algorithm, + apply_checksum_header: apply_checksum_header, + omit_session_token: omit_session_token, + region: region + ) + + # Create credential manager + credential_manager = CredentialManager.new(credentials_provider) + + { + service: service, + region: region, + credentials_provider: credentials_provider, + unsigned_headers: unsigned_headers, + uri_escape_path: uri_escape_path, + apply_checksum_header: apply_checksum_header, + signing_algorithm: signing_algorithm, + normalize_path: normalize_path, + omit_session_token: omit_session_token, + signature_generator: signature_generator, + header_builder: header_builder, + credential_manager: credential_manager + } + end + end + + # Handles AWS request signing using SigV4 or SigV4a + class Signer + # Initialize a new signer with the provided options + # @param [Hash] options Configuration options for the signer + def initialize(options = {}) + components = SignerInitializer.create_components(options) + + @service = components[:service] + @region = components[:region] + @credentials_provider = components[:credentials_provider] + @unsigned_headers = components[:unsigned_headers] + @uri_escape_path = components[:uri_escape_path] + @apply_checksum_header = components[:apply_checksum_header] + @signing_algorithm = components[:signing_algorithm] + @normalize_path = components[:normalize_path] + @omit_session_token = components[:omit_session_token] + + @signature_generator = components[:signature_generator] + @header_builder = components[:header_builder] + @credential_manager = components[:credential_manager] + end + + # @return [String] + attr_reader :service + + # @return [String] + attr_reader :region + + # @return [#credentials] + attr_reader :credentials_provider + + # @return [Set] + attr_reader :unsigned_headers + + # @return [Boolean] + attr_reader :apply_checksum_header + + # Sign an AWS request with SigV4 or SigV4a + # @param [Hash] request The request to sign + # @return [Signature] The signature with headers to apply + def sign_request(request) + creds = @credential_manager.fetch_credentials.first + request_components = RequestExtractor.extract_components( + request, + normalize_path: @normalize_path + ) + + # Generate headers and compute signature + sigv4_headers = @header_builder.build_sigv4_headers(request_components, creds) + signature = @signature_generator.compute_signature( + request_components, + creds, + sigv4_headers + ) + + build_signature_response(request_components, sigv4_headers, signature) + end + + private + + def build_signature_response(components, sigv4_headers, signature) + headers = @header_builder.build_headers(sigv4_headers, signature, components) + + Signature.new( + headers: headers, + string_to_sign: signature[:string_to_sign], + canonical_request: signature[:canonical_request], + content_sha256: components[:content_sha256], + signature: signature[:signature] + ) + end + class << self - # Kept for backwards compatability - # Always return false since we are not using crt signing functionality def use_crt? false end - # @api private def uri_escape_path(path) - path.gsub(%r{[^/]+}) { |part| uri_escape(part) } + UriUtils.uri_escape_path(path) end - # @api private def uri_escape(string) - if string.nil? - nil - else - CGI.escape(string.encode('UTF-8')).gsub('+', '%20').gsub('%7E', '~') - end + UriUtils.uri_escape(string) end - # @api private def normalize_path(uri) - normalized_path = Pathname.new(uri.path).cleanpath.to_s - # Pathname is probably not correct to use. Empty paths will - # resolve to "." and should be disregarded - normalized_path = '' if normalized_path == '.' - # Ensure trailing slashes are correctly preserved - normalized_path << '/' if uri.path.end_with?('/') && !normalized_path.end_with?('/') - uri.path = normalized_path + UriUtils.normalize_path(uri) end end end From 611a32c6308555fad7b00d5b023388c72a9600f3 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 21:12:45 -0700 Subject: [PATCH 60/85] Refactor extract_components --- lib/ruby_llm/providers/bedrock/signing.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 8af934b08..a15967a23 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -457,6 +457,10 @@ def self.extract_components(request, options = {}) datetime = headers['x-amz-date'] || Time.now.utc.strftime('%Y%m%dT%H%M%SZ') content_sha256 = extract_content_sha256(headers, request[:body]) + build_component_hash(http_method, url, headers, datetime, content_sha256) + end + + def self.build_component_hash(http_method, url, headers, datetime, content_sha256) { http_method: http_method, url: url, From 1a2825d7fceb99d8241cd38149e5da0865c1494e Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 21:14:48 -0700 Subject: [PATCH 61/85] Refactor build_result --- lib/ruby_llm/providers/bedrock/signing.rb | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index a15967a23..7e98babbd 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -640,18 +640,28 @@ def initialize(signature_computation) end def build_result(request_data) + result_hash(request_data) + end + + private + + def result_hash(request_data) { algorithm: request_data[:algorithm], - credential: @signature_computation.credential( - request_data[:credentials], - request_data[:date] - ), + credential: credential_from_request(request_data), signed_headers: request_data[:canonical_request].signed_headers, signature: request_data[:signature], canonical_request: request_data[:creq], string_to_sign: request_data[:sts] } end + + def credential_from_request(request_data) + @signature_computation.credential( + request_data[:credentials], + request_data[:date] + ) + end end # Core functionality for computing signatures From 8809f390efa91925c1e02edd70b4b58aedb7bc82 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 21:21:59 -0700 Subject: [PATCH 62/85] Refactor signature generation methods --- lib/ruby_llm/providers/bedrock/signing.rb | 29 +++++++++++++++-------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 7e98babbd..5dcdf1b69 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -685,33 +685,42 @@ def compute_signature(components, creds, sigv4_headers) algorithm = sts_algorithm headers = components[:headers].merge(sigv4_headers) - # Create request configuration and canonical request + # Process request and generate signature canonical_request = create_canonical_request(components, headers) + sig = compute_signature_from_request(canonical_request, components, creds, algorithm) - # Generate string to sign and signature + # Build and return the final result + build_signature_result(components, creds, canonical_request, sig, algorithm) + end + + private + + def compute_signature_from_request(canonical_request, components, creds, algorithm) creq = canonical_request.to_s - sts = @signature_computation.string_to_sign( + sts = generate_string_to_sign(components, creq, algorithm) + generate_signature(creds, components[:date], sts) + end + + def generate_string_to_sign(components, creq, algorithm) + @signature_computation.string_to_sign( components[:datetime], creq, algorithm ) + end - sig = generate_signature(creds, components[:date], sts) - - # Build the final result + def build_signature_result(components, creds, canonical_request, sig, algorithm) @result_builder.build_result( algorithm: algorithm, credentials: creds, date: components[:date], signature: sig, - creq: creq, - sts: sts, + creq: canonical_request.to_s, + sts: generate_string_to_sign(components, canonical_request.to_s, algorithm), canonical_request: canonical_request ) end - private - def create_canonical_request(components, headers) canon_req_config = CanonicalRequestConfig.new( uri_escape_path: @uri_escape_path, From d05817f0b76bf8cdefcfd55e07b8832cdca34f2e Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 21:23:23 -0700 Subject: [PATCH 63/85] Refactor create_canonical_request --- lib/ruby_llm/providers/bedrock/signing.rb | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 5dcdf1b69..dc6514e6c 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -722,10 +722,7 @@ def build_signature_result(components, creds, canonical_request, sig, algorithm) end def create_canonical_request(components, headers) - canon_req_config = CanonicalRequestConfig.new( - uri_escape_path: @uri_escape_path, - unsigned_headers: @unsigned_headers - ) + canon_req_config = create_canonical_request_config CanonicalRequest.new( http_method: components[:http_method], @@ -736,6 +733,13 @@ def create_canonical_request(components, headers) ) end + def create_canonical_request_config + CanonicalRequestConfig.new( + uri_escape_path: @uri_escape_path, + unsigned_headers: @unsigned_headers + ) + end + def generate_signature(creds, date, string_to_sign) if @signing_algorithm == :sigv4a @signature_computation.asymmetric_signature(creds, string_to_sign) From 533b9dca4f58226ad74ed5184071ab8e7b8c3170 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 21:24:52 -0700 Subject: [PATCH 64/85] Refactor create_components --- lib/ruby_llm/providers/bedrock/signing.rb | 85 ++++++++++++++--------- 1 file changed, 51 insertions(+), 34 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index dc6514e6c..a308f0b72 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -785,51 +785,68 @@ def self.initialize_unsigned_headers(options) # Handles initialization of Signer dependencies class SignerInitializer def self.create_components(options = {}) - service = SignerOptionExtractor.extract_service(options) - region = SignerOptionExtractor.extract_region(options) - credentials_provider = SignerOptionExtractor.extract_credentials_provider(options) + service, region, credentials_provider = extract_core_components(options) unsigned_headers = SignerOptionExtractor.initialize_unsigned_headers(options) - uri_escape_path = options.fetch(:uri_escape_path, true) - apply_checksum_header = options.fetch(:apply_checksum_header, true) - signing_algorithm = options.fetch(:signing_algorithm, :sigv4) - normalize_path = options.fetch(:normalize_path, true) - omit_session_token = options.fetch(:omit_session_token, false) + config_options = extract_config_options(options) - # Create generator - signature_generator = SignatureGenerator.new( - signing_algorithm: signing_algorithm, - uri_escape_path: uri_escape_path, - unsigned_headers: unsigned_headers, + components = { service: service, - region: region - ) + region: region, + credentials_provider: credentials_provider, + unsigned_headers: unsigned_headers + }.merge(config_options) - # Create header builder - header_builder = HeaderBuilder.new( - signing_algorithm: signing_algorithm, - apply_checksum_header: apply_checksum_header, - omit_session_token: omit_session_token, - region: region - ) + create_service_components(components) + end - # Create credential manager - credential_manager = CredentialManager.new(credentials_provider) + def self.extract_core_components(options) + service = SignerOptionExtractor.extract_service(options) + region = SignerOptionExtractor.extract_region(options) + credentials_provider = SignerOptionExtractor.extract_credentials_provider(options) + [service, region, credentials_provider] + end + + def self.extract_config_options(options) { - service: service, - region: region, - credentials_provider: credentials_provider, - unsigned_headers: unsigned_headers, - uri_escape_path: uri_escape_path, - apply_checksum_header: apply_checksum_header, - signing_algorithm: signing_algorithm, - normalize_path: normalize_path, - omit_session_token: omit_session_token, + uri_escape_path: options.fetch(:uri_escape_path, true), + apply_checksum_header: options.fetch(:apply_checksum_header, true), + signing_algorithm: options.fetch(:signing_algorithm, :sigv4), + normalize_path: options.fetch(:normalize_path, true), + omit_session_token: options.fetch(:omit_session_token, false) + } + end + + def self.create_service_components(components) + signature_generator = create_signature_generator(components) + header_builder = create_header_builder(components) + credential_manager = CredentialManager.new(components[:credentials_provider]) + + components.merge( signature_generator: signature_generator, header_builder: header_builder, credential_manager: credential_manager - } + ) + end + + def self.create_signature_generator(components) + SignatureGenerator.new( + signing_algorithm: components[:signing_algorithm], + uri_escape_path: components[:uri_escape_path], + unsigned_headers: components[:unsigned_headers], + service: components[:service], + region: components[:region] + ) + end + + def self.create_header_builder(components) + HeaderBuilder.new( + signing_algorithm: components[:signing_algorithm], + apply_checksum_header: components[:apply_checksum_header], + omit_session_token: components[:omit_session_token], + region: components[:region] + ) end end From 7d82e525376b17a5d5ada0ca2609beab3ea0b0fb Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 21:26:00 -0700 Subject: [PATCH 65/85] Refactor Signer init --- lib/ruby_llm/providers/bedrock/signing.rb | 34 +++++++++++++---------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index a308f0b72..8bf3af588 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -856,20 +856,8 @@ class Signer # @param [Hash] options Configuration options for the signer def initialize(options = {}) components = SignerInitializer.create_components(options) - - @service = components[:service] - @region = components[:region] - @credentials_provider = components[:credentials_provider] - @unsigned_headers = components[:unsigned_headers] - @uri_escape_path = components[:uri_escape_path] - @apply_checksum_header = components[:apply_checksum_header] - @signing_algorithm = components[:signing_algorithm] - @normalize_path = components[:normalize_path] - @omit_session_token = components[:omit_session_token] - - @signature_generator = components[:signature_generator] - @header_builder = components[:header_builder] - @credential_manager = components[:credential_manager] + setup_configuration(components) + setup_service_components(components) end # @return [String] @@ -910,6 +898,24 @@ def sign_request(request) private + def setup_configuration(components) + @service = components[:service] + @region = components[:region] + @credentials_provider = components[:credentials_provider] + @unsigned_headers = components[:unsigned_headers] + @uri_escape_path = components[:uri_escape_path] + @apply_checksum_header = components[:apply_checksum_header] + @signing_algorithm = components[:signing_algorithm] + @normalize_path = components[:normalize_path] + @omit_session_token = components[:omit_session_token] + end + + def setup_service_components(components) + @signature_generator = components[:signature_generator] + @header_builder = components[:header_builder] + @credential_manager = components[:credential_manager] + end + def build_signature_response(components, sigv4_headers, signature) headers = @header_builder.build_headers(sigv4_headers, signature, components) From eb20d95efa8fa779a3e7eda8f6656c0946ef9227 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 21:27:07 -0700 Subject: [PATCH 66/85] Refactor sign_request --- lib/ruby_llm/providers/bedrock/signing.rb | 35 ++++++++++++++--------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 8bf3af588..4fce4048d 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -880,19 +880,9 @@ def initialize(options = {}) # @return [Signature] The signature with headers to apply def sign_request(request) creds = @credential_manager.fetch_credentials.first - request_components = RequestExtractor.extract_components( - request, - normalize_path: @normalize_path - ) - - # Generate headers and compute signature - sigv4_headers = @header_builder.build_sigv4_headers(request_components, creds) - signature = @signature_generator.compute_signature( - request_components, - creds, - sigv4_headers - ) - + request_components = extract_request_components(request) + sigv4_headers = build_sigv4_headers(request_components, creds) + signature = compute_signature(request_components, creds, sigv4_headers) build_signature_response(request_components, sigv4_headers, signature) end @@ -916,6 +906,25 @@ def setup_service_components(components) @credential_manager = components[:credential_manager] end + def extract_request_components(request) + RequestExtractor.extract_components( + request, + normalize_path: @normalize_path + ) + end + + def build_sigv4_headers(request_components, creds) + @header_builder.build_sigv4_headers(request_components, creds) + end + + def compute_signature(request_components, creds, sigv4_headers) + @signature_generator.compute_signature( + request_components, + creds, + sigv4_headers + ) + end + def build_signature_response(components, sigv4_headers, signature) headers = @header_builder.build_headers(sigv4_headers, signature, components) From 468df041d8704ede6f7df703af7d707c88f8a66f Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 21:48:16 -0700 Subject: [PATCH 67/85] Include Apache license and notice from AWS SDK for Ruby --- THIRD_PARTY_LICENSES/APACHE-2.0.txt | 202 ++++++++++++++++++++++ THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt | 8 + lib/ruby_llm/providers/bedrock/signing.rb | 4 + 3 files changed, 214 insertions(+) create mode 100644 THIRD_PARTY_LICENSES/APACHE-2.0.txt create mode 100644 THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt diff --git a/THIRD_PARTY_LICENSES/APACHE-2.0.txt b/THIRD_PARTY_LICENSES/APACHE-2.0.txt new file mode 100644 index 000000000..7a4a3ea24 --- /dev/null +++ b/THIRD_PARTY_LICENSES/APACHE-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt b/THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt new file mode 100644 index 000000000..76189a3bf --- /dev/null +++ b/THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt @@ -0,0 +1,8 @@ +This project includes code derived from AWS SDK for Ruby, licensed under the Apache License 2.0. +The original work can be found at https://github.com/aws/aws-sdk-ruby +Modifications were made here by RubyLLM. + +The following notice is from the original project and is included per the Apache 2.0 license: + +AWS SDK for Ruby +Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. \ No newline at end of file diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 4fce4048d..3b428ac82 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -1,3 +1,7 @@ +# Portions of this file are derived from AWS SDK for Ruby (Apache License 2.0) +# Modifications made by RubyLLM in 2025 +# See THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt for details + # frozen_string_literal: true require 'openssl' From 296d1ae3b797c51c3f29ea8a90f3d96dc2345472 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 22:03:56 -0700 Subject: [PATCH 68/85] Update some capabilities --- lib/ruby_llm/providers/bedrock/capabilities.rb | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/capabilities.rb b/lib/ruby_llm/providers/bedrock/capabilities.rb index c5fc16391..3c9f4e185 100644 --- a/lib/ruby_llm/providers/bedrock/capabilities.rb +++ b/lib/ruby_llm/providers/bedrock/capabilities.rb @@ -12,9 +12,8 @@ module Capabilities # @return [Integer] the context window size in tokens def context_window_for(model_id) case model_id - when /anthropic\.claude-3-(opus|sonnet|haiku)/ then 200_000 when /anthropic\.claude-2/ then 100_000 - else 4_096 + else 200_000 end end @@ -22,7 +21,7 @@ def context_window_for(model_id) # @param model_id [String] the model identifier # @return [Integer] the maximum output tokens def max_tokens_for(_model_id) - 4096 + 4_096 end # Returns the input price per million tokens for the given model ID From 01e90763afb6ff6f3f126306bab1946402f6105d Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 25 Mar 2025 22:09:47 -0700 Subject: [PATCH 69/85] Update more capabilities --- .../providers/bedrock/capabilities.rb | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/capabilities.rb b/lib/ruby_llm/providers/bedrock/capabilities.rb index 3c9f4e185..c9f91de43 100644 --- a/lib/ruby_llm/providers/bedrock/capabilities.rb +++ b/lib/ruby_llm/providers/bedrock/capabilities.rb @@ -56,31 +56,21 @@ def supports_streaming?(model_id) # @param model_id [String] the model identifier # @return [Boolean] true if the model supports images def supports_images?(model_id) - case model_id - when /anthropic\.claude-3/ - true - else - false - end + model_id.match?(/anthropic\.claude/) end # Determines if the model supports vision capabilities # @param model_id [String] the model identifier # @return [Boolean] true if the model supports vision def supports_vision?(model_id) - case model_id - when /anthropic\.claude-3/ - true - else - false - end + model_id.match?(/anthropic\.claude/) end # Determines if the model supports function calling # @param model_id [String] the model identifier # @return [Boolean] true if the model supports functions def supports_functions?(model_id) - model_id.match?(/anthropic\.claude-3/) + model_id.match?(/anthropic\.claude/) end # Determines if the model supports audio input/output @@ -94,7 +84,7 @@ def supports_audio?(_model_id) # @param model_id [String] the model identifier # @return [Boolean] true if the model supports JSON mode def supports_json_mode?(model_id) - model_id.match?(/anthropic\.claude-3/) + model_id.match?(/anthropic\.claude/) end # Formats the model ID into a human-readable display name @@ -115,7 +105,7 @@ def model_type(_model_id) # @param model_id [String] the model identifier # @return [Boolean] true if the model supports structured output def supports_structured_output?(model_id) - model_id.match?(/anthropic\.claude-3/) + model_id.match?(/anthropic\.claude/) end # Model family patterns for capability lookup From 69d16284e5e7ee7d539ed0cc226303ee9a2372ed Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 07:48:49 -0700 Subject: [PATCH 70/85] Fix specs based on cassettes with us-west-2 in URL - Make default region us-west-2 throughout --- README.md | 2 +- bin/console | 2 +- docs/guides/getting-started.md | 3 +-- docs/guides/rails.md | 2 +- docs/installation.md | 2 +- lib/tasks/models.rake | 2 +- spec/spec_helper.rb | 2 +- 7 files changed, 7 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 6225f735a..714cce3e9 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end ``` diff --git a/bin/console b/bin/console index f3c2b322a..393074602 100755 --- a/bin/console +++ b/bin/console @@ -14,7 +14,7 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index 4428f908b..815853b1d 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -32,9 +32,8 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) - end ``` diff --git a/docs/guides/rails.md b/docs/guides/rails.md index 8cf783355..cd53e4532 100644 --- a/docs/guides/rails.md +++ b/docs/guides/rails.md @@ -98,7 +98,7 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end ``` diff --git a/docs/installation.md b/docs/installation.md index 510977c22..a5f2a0656 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -57,7 +57,7 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) # Optional: Set default models diff --git a/lib/tasks/models.rake b/lib/tasks/models.rake index c43a225e2..1b0d02da5 100644 --- a/lib/tasks/models.rake +++ b/lib/tasks/models.rake @@ -74,7 +74,7 @@ namespace :models do # rubocop:disable Metrics/BlockLength config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-east-1') + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2cd1d8fb5..a4fdfedeb 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -92,7 +92,7 @@ config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', 'test') config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', 'test') config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', 'test') - config.bedrock_region = ENV.fetch('AWS_REGION', 'test') + config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', 'test') config.max_retries = 50 end From 96f63b67f4c6a3ba16d6e9074ca082af745c4508 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 08:06:02 -0700 Subject: [PATCH 71/85] Merge from main --- .github/workflows/cicd.yml | 12 +- docs/guides/index.md | 5 +- docs/guides/models.md | 109 +++++++-- lib/ruby_llm/models.json | 61 +++-- lib/ruby_llm/models.rb | 22 +- lib/ruby_llm/providers/gemini/tools.rb | 24 +- ...41022_can_use_tools_without_parameters.yml | 168 +++++++++++++ ..._tools_without_parameters_with_bedrock.yml | 123 ++++++++++ ...flash_can_use_tools_without_parameters.yml | 174 +++++++++++++ ...-mini_can_use_tools_without_parameters.yml | 231 ++++++++++++++++++ spec/ruby_llm/chat_tools_spec.rb | 14 ++ spec/ruby_llm/models_spec.rb | 16 ++ 12 files changed, 882 insertions(+), 77 deletions(-) create mode 100644 spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku-20241022_can_use_tools_without_parameters.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_without_parameters_with_bedrock.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_function_calling_gemini-2_0-flash_can_use_tools_without_parameters.yml create mode 100644 spec/fixtures/vcr_cassettes/chat_function_calling_gpt-4o-mini_can_use_tools_without_parameters.yml diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 6d8ee5ae1..4ff12c4b7 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - ruby-version: ['3.1'] + ruby-version: ['3.1', '3.2', '3.3', '3.4'] steps: - uses: actions/checkout@v4 @@ -49,13 +49,6 @@ jobs: run: bundle exec rubocop - name: Run tests - env: - # For PRs, we use VCR cassettes - # For main branch, we use real API keys for verification - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} - GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }} - DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }} run: bundle exec rspec - name: Upload coverage to Codecov @@ -71,9 +64,6 @@ jobs: needs: test if: github.event_name == 'push' && github.ref == 'refs/heads/main' runs-on: ubuntu-latest - permissions: - contents: read - packages: write steps: - uses: actions/checkout@v4 diff --git a/docs/guides/index.md b/docs/guides/index.md index c5152ac6b..8988a8089 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -36,10 +36,13 @@ Explore how to create vector embeddings for semantic search and other applicatio ### [Error Handling]({% link guides/error-handling.md %}) Master the techniques for robust error handling in AI applications. +### [Working with Models]({% link guides/models.md %}) +Learn how to discover, select, and work with different AI models across providers. + ## Getting Help If you can't find what you're looking for in these guides, consider: 1. Checking the [API Documentation]() for detailed information about specific classes and methods -2. Looking at the [GitHub repository](https://github.com/yourusername/ruby_llm) for examples and the latest updates +2. Looking at the [GitHub repository](https://github.com/crmne/ruby_llm) for examples and the latest updates 3. Filing an issue on GitHub if you find a bug or have a feature request \ No newline at end of file diff --git a/docs/guides/models.md b/docs/guides/models.md index 621cf9370..a1a83282b 100644 --- a/docs/guides/models.md +++ b/docs/guides/models.md @@ -10,6 +10,86 @@ permalink: /guides/models RubyLLM provides a clean interface for discovering and working with AI models from multiple providers. This guide explains how to find, filter, and select the right model for your needs. +## Finding Models + +### Basic Model Selection + +The simplest way to use a model is to specify it when creating a chat: + +```ruby +# Use the default model +chat = RubyLLM.chat + +# Specify a model +chat = RubyLLM.chat(model: 'gpt-4o-mini') + +# Change models mid-conversation +chat.with_model('claude-3-5-sonnet') +``` + +### Model Resolution + +{: .warning-title } +> Coming in v1.1.0 +> +> Provider-Specific Match and Alias Resolution will be available in the next release. + +When you specify a model, RubyLLM follows these steps to find it: + +1. **Exact Match**: First tries to find an exact match for the model ID + ```ruby + # Uses the actual gemini-2.0-flash model + chat = RubyLLM.chat(model: 'gemini-2.0-flash') + ``` + +2. **Provider-Specific Match**: If a provider is specified, looks for an exact match in that provider + ```ruby + # Looks for gemini-2.0-flash in Gemini + chat = RubyLLM.chat(model: 'gemini-2.0-flash', provider: 'gemini') + ``` + +3. **Alias Resolution**: Only if no exact match is found, checks for aliases + ```ruby + # No exact match for 'claude-3', uses alias + chat = RubyLLM.chat(model: 'claude-3') + ``` + +The same model ID can exist both as a concrete model and as an alias, particularly when the same model is available through different providers: + +```ruby +# Use native Claude 3.5 +chat = RubyLLM.chat(model: 'claude-3-5-sonnet') + +# Use Claude 3.5 through Bedrock +chat = RubyLLM.chat(model: 'claude-3-5-sonnet', provider: 'bedrock') +``` + +### Model Aliases + +{: .warning-title } +> Coming in v1.1.0 +> +> Alias Resolution will be available in the next release. + +RubyLLM provides convenient aliases for popular models, so you don't have to remember specific version numbers: + +```ruby +# These are equivalent +chat = RubyLLM.chat(model: 'claude-3-5-sonnet') +chat = RubyLLM.chat(model: 'claude-3-5-sonnet-20241022') + +# These are also equivalent +chat = RubyLLM.chat(model: 'gpt-4o') +chat = RubyLLM.chat(model: 'gpt-4o-2024-11-20') +``` + +If you want to ensure you're always getting a specific version, use the full model ID: + +```ruby +# Always gets this exact version +chat = RubyLLM.chat(model: 'claude-3-sonnet-20240229') +``` + ## Exploring Available Models RubyLLM automatically discovers available models from all configured providers: @@ -63,32 +143,6 @@ google_models = RubyLLM.models.by_provider('gemini') deepseek_models = RubyLLM.models.by_provider('deepseek') ``` -## Using Model Aliases - -{: .warning-title } -> Coming in v1.1.0 -> -> This feature is available in the upcoming version but not in the latest release. - -RubyLLM provides convenient aliases for popular models, so you don't have to remember specific version numbers: - -```ruby -# These are equivalent -chat = RubyLLM.chat(model: 'claude-3-5-sonnet') -chat = RubyLLM.chat(model: 'claude-3-5-sonnet-20241022') - -# These are also equivalent -chat = RubyLLM.chat(model: 'gpt-4o') -chat = RubyLLM.chat(model: 'gpt-4o-2024-11-20') -``` - -You can also specify a different provider to use with a model: - -```ruby -# Use a specific model via a different provider -chat = RubyLLM.chat(model: 'claude-3-5-sonnet', provider: 'bedrock') -``` - ## Chaining Filters You can chain multiple filters to find exactly what you need: @@ -176,4 +230,5 @@ When selecting models for your application: 1. **Consider context windows** - Larger context windows support longer conversations but may cost more 2. **Balance cost vs. quality** - More capable models cost more but may give better results 3. **Check capabilities** - Make sure the model supports features you need (vision, functions, etc.) -4. **Use appropriate model types** - Use embedding models for vector operations, chat models for conversations \ No newline at end of file +4. **Use appropriate model types** - Use embedding models for vector operations, chat models for conversations +5. **Version control** - Use exact model IDs in production for consistency, aliases for development \ No newline at end of file diff --git a/lib/ruby_llm/models.json b/lib/ruby_llm/models.json index f47f7b60a..5151aa1f2 100644 --- a/lib/ruby_llm/models.json +++ b/lib/ruby_llm/models.json @@ -1874,8 +1874,8 @@ "created_at": null, "display_name": "Gemini 2.0 Pro Experimental", "provider": "gemini", - "context_window": 2097152, - "max_tokens": 8192, + "context_window": 1048576, + "max_tokens": 65536, "type": "chat", "family": "other", "supports_vision": true, @@ -1884,10 +1884,10 @@ "input_price_per_million": 0.075, "output_price_per_million": 0.3, "metadata": { - "version": "2.0", - "description": "Experimental release (February 5th, 2025) of Gemini 2.0 Pro", - "input_token_limit": 2097152, - "output_token_limit": 8192, + "version": "2.5-exp-03-25", + "description": "Experimental release (March 25th, 2025) of Gemini 2.5 Pro", + "input_token_limit": 1048576, + "output_token_limit": 65536, "supported_generation_methods": [ "generateContent", "countTokens" @@ -1899,8 +1899,8 @@ "created_at": null, "display_name": "Gemini 2.0 Pro Experimental 02-05", "provider": "gemini", - "context_window": 2097152, - "max_tokens": 8192, + "context_window": 1048576, + "max_tokens": 65536, "type": "chat", "family": "other", "supports_vision": true, @@ -1909,10 +1909,35 @@ "input_price_per_million": 0.075, "output_price_per_million": 0.3, "metadata": { - "version": "2.0", - "description": "Experimental release (February 5th, 2025) of Gemini 2.0 Pro", - "input_token_limit": 2097152, - "output_token_limit": 8192, + "version": "2.5-exp-03-25", + "description": "Experimental release (March 25th, 2025) of Gemini 2.5 Pro", + "input_token_limit": 1048576, + "output_token_limit": 65536, + "supported_generation_methods": [ + "generateContent", + "countTokens" + ] + } + }, + { + "id": "gemini-2.5-pro-exp-03-25", + "created_at": null, + "display_name": "Gemini 2.5 Pro Experimental 03-25", + "provider": "gemini", + "context_window": 1048576, + "max_tokens": 65536, + "type": "chat", + "family": "other", + "supports_vision": true, + "supports_functions": true, + "supports_json_mode": true, + "input_price_per_million": 0.075, + "output_price_per_million": 0.3, + "metadata": { + "version": "2.5-exp-03-25", + "description": "Experimental release (March 25th, 2025) of Gemini 2.5 Pro", + "input_token_limit": 1048576, + "output_token_limit": 65536, "supported_generation_methods": [ "generateContent", "countTokens" @@ -1972,8 +1997,8 @@ "created_at": null, "display_name": "Gemini Experimental 1206", "provider": "gemini", - "context_window": 2097152, - "max_tokens": 8192, + "context_window": 1048576, + "max_tokens": 65536, "type": "chat", "family": "other", "supports_vision": false, @@ -1982,10 +2007,10 @@ "input_price_per_million": 0.075, "output_price_per_million": 0.3, "metadata": { - "version": "2.0", - "description": "Experimental release (February 5th, 2025) of Gemini 2.0 Pro", - "input_token_limit": 2097152, - "output_token_limit": 8192, + "version": "2.5-exp-03-25", + "description": "Experimental release (March 25th, 2025) of Gemini 2.5 Pro", + "input_token_limit": 1048576, + "output_token_limit": 65536, "supported_generation_methods": [ "generateContent", "countTokens" diff --git a/lib/ruby_llm/models.rb b/lib/ruby_llm/models.rb index 41778a1f7..85d6486b1 100644 --- a/lib/ruby_llm/models.rb +++ b/lib/ruby_llm/models.rb @@ -86,12 +86,11 @@ def each(&) # Find a specific model by ID def find(model_id, provider = nil) - return find_with_provider(model_id, provider) if provider - - # Find native model - all.find { |m| m.id == model_id } || - all.find { |m| m.id == Aliases.resolve(model_id) } || - raise(ModelNotFoundError, "Unknown model: #{model_id}") + if provider + find_with_provider(model_id, provider) + else + find_without_provider(model_id) + end end # Filter to only chat models @@ -132,9 +131,16 @@ def refresh! private def find_with_provider(model_id, provider) - provider_id = Aliases.resolve(model_id, provider) - all.find { |m| m.id == provider_id && m.provider == provider.to_s } || + resolved_id = Aliases.resolve(model_id, provider) + all.find { |m| m.id == model_id && m.provider == provider.to_s } || + all.find { |m| m.id == resolved_id && m.provider == provider.to_s } || raise(ModelNotFoundError, "Unknown model: #{model_id} for provider: #{provider}") end + + def find_without_provider(model_id) + all.find { |m| m.id == model_id } || + all.find { |m| m.id == Aliases.resolve(model_id) } || + raise(ModelNotFoundError, "Unknown model: #{model_id}") + end end end diff --git a/lib/ruby_llm/providers/gemini/tools.rb b/lib/ruby_llm/providers/gemini/tools.rb index 6ed53e17c..8cb85b1c3 100644 --- a/lib/ruby_llm/providers/gemini/tools.rb +++ b/lib/ruby_llm/providers/gemini/tools.rb @@ -54,22 +54,22 @@ def function_declaration_for(tool) { name: tool.name, description: tool.description, - parameters: { - type: 'OBJECT', - properties: format_parameters(tool.parameters), - required: tool.parameters.select { |_, p| p.required }.keys.map(&:to_s) - } - } + parameters: tool.parameters.any? ? format_parameters(tool.parameters) : nil + }.compact end # Format tool parameters for Gemini API def format_parameters(parameters) - parameters.transform_values do |param| - { - type: param_type_for_gemini(param.type), - description: param.description - }.compact - end + { + type: 'OBJECT', + properties: parameters.transform_values do |param| + { + type: param_type_for_gemini(param.type), + description: param.description + }.compact + end, + required: parameters.select { |_, p| p.required }.keys.map(&:to_s) + } end # Convert RubyLLM param types to Gemini API types diff --git a/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku-20241022_can_use_tools_without_parameters.yml b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku-20241022_can_use_tools_without_parameters.yml new file mode 100644 index 000000000..625509fae --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku-20241022_can_use_tools_without_parameters.yml @@ -0,0 +1,168 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-3-5-haiku-20241022","messages":[{"role":"user","content":"What''s + the best language to learn?"}],"temperature":0.7,"stream":false,"max_tokens":8192,"tools":[{"name":"best_language_to_learn","description":"Gets + the best language to learn","input_schema":{"type":"object","properties":{},"required":[]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 26 Mar 2025 12:36:24 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Requests-Limit: + - '50' + Anthropic-Ratelimit-Requests-Remaining: + - '49' + Anthropic-Ratelimit-Requests-Reset: + - '2025-03-26T12:36:22Z' + Anthropic-Ratelimit-Input-Tokens-Limit: + - '50000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '50000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-03-26T12:36:23Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '10000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '10000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-03-26T12:36:23Z' + Anthropic-Ratelimit-Tokens-Limit: + - '60000' + Anthropic-Ratelimit-Tokens-Remaining: + - '60000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-03-26T12:36:23Z' + Request-Id: + - "" + Anthropic-Organization-Id: + - 382da897-d586-4e0c-bd1c-a9407bbe3b7a + Via: + - 1.1 google + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"id":"msg_016N5q71zpXwLnoYPDYjpi92","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''ll + help you find out the best language to learn by using the available tool."},{"type":"tool_use","id":"toolu_01HiFoEeZiQJ6CFwCNzT6Nwu","name":"best_language_to_learn","input":{}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":327,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":58}}' + recorded_at: Wed, 26 Mar 2025 12:36:24 GMT +- request: + method: post + uri: https://api.anthropic.com/v1/messages + body: + encoding: UTF-8 + string: '{"model":"claude-3-5-haiku-20241022","messages":[{"role":"user","content":"What''s + the best language to learn?"},{"role":"assistant","content":[{"type":"text","text":"I''ll + help you find out the best language to learn by using the available tool."},{"type":"tool_use","id":"toolu_01HiFoEeZiQJ6CFwCNzT6Nwu","name":"best_language_to_learn","input":{}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_01HiFoEeZiQJ6CFwCNzT6Nwu","content":"Ruby"}]}],"temperature":0.7,"stream":false,"max_tokens":8192,"tools":[{"name":"best_language_to_learn","description":"Gets + the best language to learn","input_schema":{"type":"object","properties":{},"required":[]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + X-Api-Key: + - "" + Anthropic-Version: + - '2023-06-01' + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 26 Mar 2025 12:36:30 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Anthropic-Ratelimit-Requests-Limit: + - '50' + Anthropic-Ratelimit-Requests-Remaining: + - '49' + Anthropic-Ratelimit-Requests-Reset: + - '2025-03-26T12:36:25Z' + Anthropic-Ratelimit-Input-Tokens-Limit: + - '50000' + Anthropic-Ratelimit-Input-Tokens-Remaining: + - '50000' + Anthropic-Ratelimit-Input-Tokens-Reset: + - '2025-03-26T12:36:27Z' + Anthropic-Ratelimit-Output-Tokens-Limit: + - '10000' + Anthropic-Ratelimit-Output-Tokens-Remaining: + - '10000' + Anthropic-Ratelimit-Output-Tokens-Reset: + - '2025-03-26T12:36:30Z' + Anthropic-Ratelimit-Tokens-Limit: + - '60000' + Anthropic-Ratelimit-Tokens-Remaining: + - '60000' + Anthropic-Ratelimit-Tokens-Reset: + - '2025-03-26T12:36:27Z' + Request-Id: + - "" + Anthropic-Organization-Id: + - 382da897-d586-4e0c-bd1c-a9407bbe3b7a + Via: + - 1.1 google + Cf-Cache-Status: + - DYNAMIC + X-Robots-Tag: + - none + Server: + - cloudflare + Cf-Ray: + - "" + body: + encoding: ASCII-8BIT + string: '{"id":"msg_01VXzvCFsc6ggwCM6qbGXKTK","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"According + to the tool, Ruby is recommended as the best language to learn. Ruby is a + dynamic, object-oriented programming language known for its simplicity and + readability. It''s particularly popular for web development, especially with + the Ruby on Rails framework. \n\nSome reasons why Ruby might be a great language + to learn:\n1. Easy to read and write, with a clean and elegant syntax\n2. + Great for web development and scripting\n3. Strong community support\n4. Used + in many startups and tech companies\n5. Good for beginners due to its intuitive + nature\n\nHowever, the \"best\" language can depend on your personal goals, + such as:\n- Web development\n- Data science\n- Mobile app development\n- Game + development\n- Career opportunities\n\nWould you like to know more about Ruby + or discuss how it might fit your specific learning objectives?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":397,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":182}}' + recorded_at: Wed, 26 Mar 2025 12:36:30 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_without_parameters_with_bedrock.yml b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_without_parameters_with_bedrock.yml new file mode 100644 index 000000000..ab01ebf11 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_function_calling_claude-3-5-haiku_can_use_tools_without_parameters_with_bedrock.yml @@ -0,0 +1,123 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the best language to learn?"}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"best_language_to_learn","description":"Gets + the best language to learn","input_schema":{"type":"object","properties":{},"required":[]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250326T150506Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - 5470b3b9e786d362a84d10e00f2ca7de6f684b36347b98361f45534636f69c1a + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250326/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=263824a7661419acc292499e27d1b7bbcc2603c05cb7d1a0ed4fdf98d90c1e00 + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 26 Mar 2025 15:05:07 GMT + Content-Type: + - application/json + Content-Length: + - '416' + Connection: + - keep-alive + X-Amzn-Requestid: + - 802ee619-9882-4513-875a-4a256afd98bb + X-Amzn-Bedrock-Invocation-Latency: + - '1115' + X-Amzn-Bedrock-Output-Token-Count: + - '56' + X-Amzn-Bedrock-Input-Token-Count: + - '327' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_018qctpm5xnJFAhg8iGsyh9j","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"I''ll + help you find out by using the best language to learn tool."},{"type":"tool_use","id":"toolu_bdrk_01TF3yhqhTcW4bJP3sM7xUs9","name":"best_language_to_learn","input":{}}],"stop_reason":"tool_use","stop_sequence":null,"usage":{"input_tokens":327,"output_tokens":56}}' + recorded_at: Wed, 26 Mar 2025 15:05:07 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What''s + the best language to learn?"},{"role":"assistant","content":[{"type":"text","text":"I''ll + help you find out by using the best language to learn tool."},{"type":"tool_use","id":"toolu_bdrk_01TF3yhqhTcW4bJP3sM7xUs9","name":"best_language_to_learn","input":{}}]},{"role":"user","content":[{"type":"tool_result","tool_use_id":"toolu_bdrk_01TF3yhqhTcW4bJP3sM7xUs9","content":"Ruby"}]}],"temperature":0.7,"max_tokens":4096,"tools":[{"name":"best_language_to_learn","description":"Gets + the best language to learn","input_schema":{"type":"object","properties":{},"required":[]}}]}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250326T150507Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - a738efc8beb12807ced753e348efc0b06c5046235adeb7968b61259e31610ad7 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250326/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=8774c9b3e047112b6d546d2e74241e502a9aced903220c5cc6afc0e34008a24e + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 26 Mar 2025 15:05:13 GMT + Content-Type: + - application/json + Content-Length: + - '1127' + Connection: + - keep-alive + X-Amzn-Requestid: + - 41a8cb1f-0730-45b1-aa83-5f5c4bf52355 + X-Amzn-Bedrock-Invocation-Latency: + - '5256' + X-Amzn-Bedrock-Output-Token-Count: + - '189' + X-Amzn-Bedrock-Input-Token-Count: + - '395' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_01EYFqfYHqwkAhndtoyQ47z6","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"According + to the tool, Ruby is currently considered the best language to learn! Ruby + is a dynamic, object-oriented programming language known for its simplicity + and readability. It''s particularly popular for web development, especially + with the Ruby on Rails framework. \n\nSome reasons why Ruby might be a great + language to learn include:\n1. Easy to read and write syntax\n2. Strong community + support\n3. Versatile for web development\n4. Good for beginners due to its + intuitive nature\n5. Used in many startups and tech companies\n\nOf course, + the \"best\" language can depend on your specific goals, whether you''re interested + in web development, data science, mobile apps, or other programming areas. + But Ruby is certainly an excellent choice for many aspiring programmers!\n\nWould + you like to know more about Ruby or discuss why it might be a good language + for you to learn?"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":395,"output_tokens":189}}' + recorded_at: Wed, 26 Mar 2025 15:05:13 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_function_calling_gemini-2_0-flash_can_use_tools_without_parameters.yml b/spec/fixtures/vcr_cassettes/chat_function_calling_gemini-2_0-flash_can_use_tools_without_parameters.yml new file mode 100644 index 000000000..51236722c --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_function_calling_gemini-2_0-flash_can_use_tools_without_parameters.yml @@ -0,0 +1,174 @@ +--- +http_interactions: +- request: + method: post + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent + body: + encoding: UTF-8 + string: '{"contents":[{"role":"user","parts":[{"text":"What''s the best language + to learn?"}]}],"generationConfig":{"temperature":0.7},"tools":[{"functionDeclarations":[{"name":"best_language_to_learn","description":"Gets + the best language to learn"}]}]}' + headers: + User-Agent: + - Faraday v2.12.2 + X-Goog-Api-Key: + - "" + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=UTF-8 + Vary: + - Origin + - Referer + - X-Origin + Date: + - Wed, 26 Mar 2025 12:36:31 GMT + Server: + - scaffolding on HTTPServer2 + X-Xss-Protection: + - '0' + X-Frame-Options: + - SAMEORIGIN + X-Content-Type-Options: + - nosniff + Server-Timing: + - gfet4t7; dur=505 + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Transfer-Encoding: + - chunked + body: + encoding: ASCII-8BIT + string: | + { + "candidates": [ + { + "content": { + "parts": [ + { + "functionCall": { + "name": "best_language_to_learn", + "args": {} + } + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.00011707750880824668 + } + ], + "usageMetadata": { + "promptTokenCount": 22, + "candidatesTokenCount": 7, + "totalTokenCount": 29, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 22 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 7 + } + ] + }, + "modelVersion": "gemini-2.0-flash" + } + recorded_at: Wed, 26 Mar 2025 12:36:31 GMT +- request: + method: post + uri: https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent + body: + encoding: UTF-8 + string: '{"contents":[{"role":"user","parts":[{"text":"What''s the best language + to learn?"}]},{"role":"model","parts":[{"functionCall":{"name":"best_language_to_learn","args":{}}}]},{"role":"user","parts":[{"functionResponse":{"name":"59a92e0a-8267-4a36-b42f-6c8f41da2add","response":{"name":"59a92e0a-8267-4a36-b42f-6c8f41da2add","content":"Ruby"}}}]}],"generationConfig":{"temperature":0.7},"tools":[{"functionDeclarations":[{"name":"best_language_to_learn","description":"Gets + the best language to learn"}]}]}' + headers: + User-Agent: + - Faraday v2.12.2 + X-Goog-Api-Key: + - "" + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Content-Type: + - application/json; charset=UTF-8 + Vary: + - Origin + - Referer + - X-Origin + Date: + - Wed, 26 Mar 2025 12:36:31 GMT + Server: + - scaffolding on HTTPServer2 + X-Xss-Protection: + - '0' + X-Frame-Options: + - SAMEORIGIN + X-Content-Type-Options: + - nosniff + Server-Timing: + - gfet4t7; dur=472 + Alt-Svc: + - h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 + Transfer-Encoding: + - chunked + body: + encoding: ASCII-8BIT + string: | + { + "candidates": [ + { + "content": { + "parts": [ + { + "text": "I think Ruby is the best language to learn.\n" + } + ], + "role": "model" + }, + "finishReason": "STOP", + "avgLogprobs": -0.11496557972647926 + } + ], + "usageMetadata": { + "promptTokenCount": 98, + "candidatesTokenCount": 11, + "totalTokenCount": 109, + "promptTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 98 + } + ], + "candidatesTokensDetails": [ + { + "modality": "TEXT", + "tokenCount": 11 + } + ] + }, + "modelVersion": "gemini-2.0-flash" + } + recorded_at: Wed, 26 Mar 2025 12:36:31 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/fixtures/vcr_cassettes/chat_function_calling_gpt-4o-mini_can_use_tools_without_parameters.yml b/spec/fixtures/vcr_cassettes/chat_function_calling_gpt-4o-mini_can_use_tools_without_parameters.yml new file mode 100644 index 000000000..57e087283 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_function_calling_gpt-4o-mini_can_use_tools_without_parameters.yml @@ -0,0 +1,231 @@ +--- +http_interactions: +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"gpt-4o-mini","messages":[{"role":"user","content":"What''s + the best language to learn?"}],"temperature":0.7,"stream":false,"tools":[{"type":"function","function":{"name":"best_language_to_learn","description":"Gets + the best language to learn","parameters":{"type":"object","properties":{},"required":[]}}}],"tool_choice":"auto"}' + headers: + User-Agent: + - Faraday v2.12.2 + Authorization: + - Bearer + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 26 Mar 2025 12:36:32 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - "" + Openai-Processing-Ms: + - '835' + Openai-Version: + - '2020-10-01' + X-Ratelimit-Limit-Requests: + - '10000' + X-Ratelimit-Limit-Tokens: + - '200000' + X-Ratelimit-Remaining-Requests: + - '9999' + X-Ratelimit-Remaining-Tokens: + - '199988' + X-Ratelimit-Reset-Requests: + - 8.64s + X-Ratelimit-Reset-Tokens: + - 3ms + X-Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Cf-Cache-Status: + - DYNAMIC + Set-Cookie: + - "" + - "" + X-Content-Type-Options: + - nosniff + Server: + - cloudflare + Cf-Ray: + - "" + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: | + { + "id": "chatcmpl-BFKHgIGAwLUS0wTdClrDXbEzwGlhE", + "object": "chat.completion", + "created": 1742992592, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": null, + "tool_calls": [ + { + "id": "call_rSFV6B5evXpOXNsNQfv379HV", + "type": "function", + "function": { + "name": "best_language_to_learn", + "arguments": "{}" + } + } + ], + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "tool_calls" + } + ], + "usage": { + "prompt_tokens": 48, + "completion_tokens": 14, + "total_tokens": 62, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_27322b4e16" + } + recorded_at: Wed, 26 Mar 2025 12:36:32 GMT +- request: + method: post + uri: https://api.openai.com/v1/chat/completions + body: + encoding: UTF-8 + string: '{"model":"gpt-4o-mini","messages":[{"role":"user","content":"What''s + the best language to learn?"},{"role":"assistant","tool_calls":[{"id":"call_rSFV6B5evXpOXNsNQfv379HV","type":"function","function":{"name":"best_language_to_learn","arguments":"{}"}}]},{"role":"tool","content":"Ruby","tool_call_id":"call_rSFV6B5evXpOXNsNQfv379HV"}],"temperature":0.7,"stream":false,"tools":[{"type":"function","function":{"name":"best_language_to_learn","description":"Gets + the best language to learn","parameters":{"type":"object","properties":{},"required":[]}}}],"tool_choice":"auto"}' + headers: + User-Agent: + - Faraday v2.12.2 + Authorization: + - Bearer + Content-Type: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + Accept: + - "*/*" + response: + status: + code: 200 + message: OK + headers: + Date: + - Wed, 26 Mar 2025 12:36:33 GMT + Content-Type: + - application/json + Transfer-Encoding: + - chunked + Connection: + - keep-alive + Access-Control-Expose-Headers: + - X-Request-ID + Openai-Organization: + - "" + Openai-Processing-Ms: + - '261' + Openai-Version: + - '2020-10-01' + X-Ratelimit-Limit-Requests: + - '10000' + X-Ratelimit-Limit-Tokens: + - '200000' + X-Ratelimit-Remaining-Requests: + - '9998' + X-Ratelimit-Remaining-Tokens: + - '199986' + X-Ratelimit-Reset-Requests: + - 16.249s + X-Ratelimit-Reset-Tokens: + - 4ms + X-Request-Id: + - "" + Strict-Transport-Security: + - max-age=31536000; includeSubDomains; preload + Cf-Cache-Status: + - DYNAMIC + Set-Cookie: + - "" + - "" + X-Content-Type-Options: + - nosniff + Server: + - cloudflare + Cf-Ray: + - "" + Alt-Svc: + - h3=":443"; ma=86400 + body: + encoding: ASCII-8BIT + string: | + { + "id": "chatcmpl-BFKHhaUl3iiFyJeMb0nD071b7EYPZ", + "object": "chat.completion", + "created": 1742992593, + "model": "gpt-4o-mini-2024-07-18", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "The best language to learn is Ruby.", + "refusal": null, + "annotations": [] + }, + "logprobs": null, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 73, + "completion_tokens": 10, + "total_tokens": 83, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 0, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": "fp_bbfba58e46" + } + recorded_at: Wed, 26 Mar 2025 12:36:33 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/ruby_llm/chat_tools_spec.rb b/spec/ruby_llm/chat_tools_spec.rb index b5580ffc4..855c610ab 100644 --- a/spec/ruby_llm/chat_tools_spec.rb +++ b/spec/ruby_llm/chat_tools_spec.rb @@ -16,6 +16,14 @@ def execute(latitude:, longitude:) end end + class BestLanguageToLearn < RubyLLM::Tool # rubocop:disable Lint/ConstantDefinitionInBlock,RSpec/LeakyConstantDeclaration + description 'Gets the best language to learn' + + def execute + 'Ruby' + end + end + describe 'function calling' do [ ['claude-3-5-haiku-20241022', nil], @@ -46,6 +54,12 @@ def execute(latitude:, longitude:) expect(response.content).to include('10') end + it "#{model} can use tools without parameters#{provider_suffix}" do + chat = RubyLLM.chat(model: model, provider: provider).with_tool(BestLanguageToLearn) + response = chat.ask("What's the best language to learn?") + expect(response.content).to include('Ruby') + end + it "#{model} can use tools with multi-turn streaming conversations#{provider_suffix}" do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations chat = RubyLLM.chat(model: model, provider: provider) .with_tool(Weather) diff --git a/spec/ruby_llm/models_spec.rb b/spec/ruby_llm/models_spec.rb index a1ac753c2..ea608692d 100644 --- a/spec/ruby_llm/models_spec.rb +++ b/spec/ruby_llm/models_spec.rb @@ -62,6 +62,22 @@ end end + describe '#find' do + it 'prioritizes exact matches over aliases' do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations + # This test covers the case from the issue + chat_model = RubyLLM.chat(model: 'gemini-2.0-flash') + expect(chat_model.model.id).to eq('gemini-2.0-flash') + + # Even with provider specified, exact match wins + chat_model = RubyLLM.chat(model: 'gemini-2.0-flash', provider: 'gemini') + expect(chat_model.model.id).to eq('gemini-2.0-flash') + + # Only use alias when exact match isn't found + chat_model = RubyLLM.chat(model: 'claude-3') + expect(chat_model.model.id).to eq('claude-3-sonnet-20240229') + end + end + describe '#refresh!' do it 'updates models and returns a chainable Models instance' do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations # Use a temporary file to avoid modifying actual models.json From 39854abdab14670ca8e4afbf2f958d180aea38cc Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 10:29:02 -0700 Subject: [PATCH 72/85] Remove unused methods in content_extraction --- .../bedrock/streaming/content_extraction.rb | 53 ------------------- 1 file changed, 53 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/streaming/content_extraction.rb b/lib/ruby_llm/providers/bedrock/streaming/content_extraction.rb index c9043e648..29dc8166e 100644 --- a/lib/ruby_llm/providers/bedrock/streaming/content_extraction.rb +++ b/lib/ruby_llm/providers/bedrock/streaming/content_extraction.rb @@ -12,11 +12,6 @@ module Streaming # - Processing JSON deltas and content blocks # - Extracting metadata (tokens, model IDs, tool calls) # - Handling different content structures (arrays, blocks, completions) - # - # @example Content extraction from a response - # content = extract_content(response_data) - # streaming_content = extract_streaming_content(delta_data) - # tool_calls = extract_tool_calls(message_data) module ContentExtraction def json_delta?(data) data['type'] == 'content_block_delta' && data.dig('delta', 'type') == 'input_json_delta' @@ -28,33 +23,6 @@ def extract_streaming_content(data) extract_content_by_type(data) end - def extract_content(data) - return unless data.is_a?(Hash) - - try_content_extractors(data) - end - - def extract_completion_content(data) - data['completion'] if data.key?('completion') - end - - def extract_output_text_content(data) - data.dig('results', 0, 'outputText') - end - - def extract_array_content(data) - return unless data.key?('content') - - content = data['content'] - content.is_a?(Array) ? join_array_content(content) : content - end - - def extract_content_block_text(data) - return unless data.key?('content_block') && data['content_block'].key?('text') - - data['content_block']['text'] - end - def extract_tool_calls(data) data.dig('message', 'tool_calls') || data['tool_calls'] end @@ -88,27 +56,6 @@ def extract_block_start_content(data) def extract_delta_content(data) data.dig('delta', 'text').to_s end - - def try_content_extractors(data) - content_extractors.each do |extractor| - content = send(extractor, data) - return content if content - end - nil - end - - def content_extractors - %i[ - extract_completion_content - extract_output_text_content - extract_array_content - extract_content_block_text - ] - end - - def join_array_content(content_array) - content_array.map { |item| item['text'] }.join - end end end end From 204a9c35c6e802cc1527574c67bdd387d8a38719 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 10:38:31 -0700 Subject: [PATCH 73/85] Remove unused methods in signing --- lib/ruby_llm/providers/bedrock/signing.rb | 37 +---------------------- 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 3b428ac82..14fd12293 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -17,15 +17,11 @@ module Providers module Bedrock module Signing # Utility class for creating AWS signature version 4 signature. This class - # provides two methods for generating signatures: + # provides a method for generating signatures: # # * {#sign_request} - Computes a signature of the given request, returning # the hash of headers that should be applied to the request. # - # * {#presign_url} - Computes a presigned request with an expiration. - # By default, the body of this request is not signed and the request - # expires in 15 minutes. - # # ## Configuration # # To use the signer, you need to specify the service, region, and credentials. @@ -608,33 +604,6 @@ def credentials_set?(credentials) !credentials.secret_access_key.nil? && !credentials.secret_access_key.empty? end - - def presigned_url_expiration(options, expiration, datetime) - expires_in = extract_expires_in(options) - return expires_in unless expiration - - expiration_seconds = (expiration - datetime).to_i - # In the static stability case, credentials may expire in the past - # but still be valid. For those cases, use the user configured - # expires_in and ignore expiration. - if expiration_seconds <= 0 - expires_in - else - [expires_in, expiration_seconds].min - end - end - - private - - def extract_expires_in(options) - case options[:expires_in] - when nil then 900 - when Integer then options[:expires_in] - else - msg = 'expected :expires_in to be a number of seconds' - raise ArgumentError, msg - end - end end # Result builder for signature computation @@ -942,10 +911,6 @@ def build_signature_response(components, sigv4_headers, signature) end class << self - def use_crt? - false - end - def uri_escape_path(path) UriUtils.uri_escape_path(path) end From 47ae38272a8cd595d9d66387356d59450f3506a2 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 11:01:38 -0700 Subject: [PATCH 74/85] Remove support for other kinds of signatures - Just supporting sigv4 for now --- lib/ruby_llm/providers/bedrock/signing.rb | 36 ++++------------------- 1 file changed, 6 insertions(+), 30 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 14fd12293..2d50aedef 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -411,10 +411,10 @@ def string_to_sign(datetime, canonical_request, algorithm) def credential_scope(date) [ date, - (@region unless @signing_algorithm == :sigv4a), + @region, @service, 'aws4_request' - ].compact.join('/') + ].join('/') end def credential(credentials, date) @@ -428,16 +428,6 @@ def signature(secret_access_key, date, string_to_sign) k_credentials = CryptoUtils.hmac(k_service, 'aws4_request') CryptoUtils.hexhmac(k_credentials, string_to_sign) end - - def asymmetric_signature(creds, string_to_sign) - ec, = Aws::Sigv4::AsymmetricCredentials.derive_asymmetric_key( - creds.access_key_id, creds.secret_access_key - ) - sts_digest = OpenSSL::Digest::SHA256.digest(string_to_sign) - s = ec.dsa_sign_asn1(sts_digest) - - Digest.hexencode(s) - end end # Extracts and validates request components @@ -526,7 +516,6 @@ def build_sigv4_headers(components, creds) add_session_token_header(headers, creds) add_content_sha256_header(headers, components[:content_sha256]) - add_region_header(headers) headers end @@ -558,21 +547,13 @@ def build_authorization_header(signature) def add_session_token_header(headers, creds) return unless creds.session_token && !@omit_session_token - if @signing_algorithm == :'sigv4-s3express' - headers['x-amz-s3session-token'] = creds.session_token - else - headers['x-amz-security-token'] = creds.session_token - end + headers['x-amz-security-token'] = creds.session_token end def add_content_sha256_header(headers, content_sha256) headers['x-amz-content-sha256'] = content_sha256 if @apply_checksum_header end - def add_region_header(headers) - headers['x-amz-region-set'] = @region if @signing_algorithm == :sigv4a && @region && !@region.empty? - end - def add_omitted_session_token(headers, creds) return unless creds&.session_token @@ -640,7 +621,7 @@ def credential_from_request(request_data) # Core functionality for computing signatures class SignatureGenerator def initialize(options = {}) - @signing_algorithm = options[:signing_algorithm] || :sigv4 + @signing_algorithm = :sigv4 # Always use sigv4 @uri_escape_path = options[:uri_escape_path] || true @unsigned_headers = options[:unsigned_headers] || Set.new @service = options[:service] @@ -651,7 +632,7 @@ def initialize(options = {}) end def sts_algorithm - @signing_algorithm == :sigv4a ? 'AWS4-ECDSA-P256-SHA256' : 'AWS4-HMAC-SHA256' + 'AWS4-HMAC-SHA256' # Always use HMAC-SHA256 end def compute_signature(components, creds, sigv4_headers) @@ -714,11 +695,7 @@ def create_canonical_request_config end def generate_signature(creds, date, string_to_sign) - if @signing_algorithm == :sigv4a - @signature_computation.asymmetric_signature(creds, string_to_sign) - else - @signature_computation.signature(creds.secret_access_key, date, string_to_sign) - end + @signature_computation.signature(creds.secret_access_key, date, string_to_sign) end end @@ -785,7 +762,6 @@ def self.extract_config_options(options) { uri_escape_path: options.fetch(:uri_escape_path, true), apply_checksum_header: options.fetch(:apply_checksum_header, true), - signing_algorithm: options.fetch(:signing_algorithm, :sigv4), normalize_path: options.fetch(:normalize_path, true), omit_session_token: options.fetch(:omit_session_token, false) } From 62662c82d5f4e35a5f153a849b318d0adb1ccaf2 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 11:04:56 -0700 Subject: [PATCH 75/85] Remove query string normalization code --- lib/ruby_llm/providers/bedrock/signing.rb | 46 +---------------------- 1 file changed, 1 insertion(+), 45 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index 2d50aedef..b68c33fd8 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -300,7 +300,7 @@ def to_s [ @http_method, path, - normalized_querystring(@url.query || ''), + '', # Empty string instead of normalized_querystring since we don't use query params "#{canonical_headers}\n", signed_headers, @content_sha256 @@ -330,50 +330,6 @@ def path end end - def normalized_querystring(querystring) - params = normalize_params(querystring) - sort_params(params).map(&:first).join('&') - end - - def normalize_params(querystring) - params = querystring.split('&') - params.map { |p| ensure_param_has_equals(p) } - end - - def ensure_param_has_equals(param) - param.match(/=/) ? param : "#{param}=" - end - - def sort_params(params) - params.each.with_index.sort do |a, b| - compare_params(a, b) - end - end - - def compare_params(param_pair1, param_pair2) - param1, offset1 = param_pair1 - param2, offset2 = param_pair2 - name1, value1 = param1.split('=') - name2, value2 = param2.split('=') - - compare_param_components( - ParamComponent.new(name1, value1, offset1), - ParamComponent.new(name2, value2, offset2) - ) - end - - def compare_param_components(component1, component2) - if component1.name == component2.name - if component1.value == component2.value - component1.offset <=> component2.offset - else - component1.value <=> component2.value - end - else - component1.name <=> component2.name - end - end - def canonical_headers headers = @headers.inject([]) do |hdrs, (k, v)| if @config.unsigned_headers.include?(k) From a2775d62c57ef184c6ac87515dbf7ba9b75ad474 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 11:07:53 -0700 Subject: [PATCH 76/85] Remove support for hashing non-strings --- lib/ruby_llm/providers/bedrock/signing.rb | 33 +---------------------- 1 file changed, 1 insertion(+), 32 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/signing.rb b/lib/ruby_llm/providers/bedrock/signing.rb index b68c33fd8..0dd7392ef 100644 --- a/lib/ruby_llm/providers/bedrock/signing.rb +++ b/lib/ruby_llm/providers/bedrock/signing.rb @@ -228,40 +228,9 @@ def host(uri) module CryptoUtils module_function - # @param [File, Tempfile, IO#read, String] value + # @param [String] value # @return [String] def sha256_hexdigest(value) - if file_like?(value) - digest_file(value) - elsif value.respond_to?(:read) - digest_io(value) - else - digest_string(value) - end - end - - def file_like?(value) - (value.is_a?(File) || value.is_a?(Tempfile)) && !value.path.nil? && File.exist?(value.path) - end - - def digest_file(value) - OpenSSL::Digest::SHA256.file(value).hexdigest - end - - def digest_io(value) - sha256 = OpenSSL::Digest.new('SHA256') - update_digest_from_io(sha256, value) - value.rewind - sha256.hexdigest - end - - def update_digest_from_io(digest, io) - while (chunk = io.read(1024 * 1024)) # 1MB - digest.update(chunk) - end - end - - def digest_string(value) OpenSSL::Digest::SHA256.hexdigest(value) end From f14d27680497d2d88fa2e0925a69c96e7d94df05 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 11:27:08 -0700 Subject: [PATCH 77/85] Remove unneeded handle_error_response method --- lib/ruby_llm/providers/bedrock/streaming/base.rb | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/streaming/base.rb b/lib/ruby_llm/providers/bedrock/streaming/base.rb index 62fe66fb1..3b1f9bda8 100644 --- a/lib/ruby_llm/providers/bedrock/streaming/base.rb +++ b/lib/ruby_llm/providers/bedrock/streaming/base.rb @@ -30,9 +30,10 @@ def stream_url end def handle_stream(&block) + buffer = String.new proc do |chunk, _bytes, env| if env && env.status != 200 - handle_error_response(chunk, env) + handle_failed_response(chunk, buffer, env) else process_chunk(chunk, &block) end @@ -40,19 +41,6 @@ def handle_stream(&block) end private - - def handle_error_response(chunk, env) - buffer = String.new - buffer << chunk - begin - error_data = JSON.parse(buffer) - error_response = env.merge(body: error_data) - ErrorMiddleware.parse_error(provider: self, response: error_response) - rescue JSON::ParserError - # Keep accumulating if we don't have complete JSON yet - RubyLLM.logger.debug "Accumulating error chunk: #{chunk}" - end - end end end end From 9b11f493a7cda543789e9f6f601dd2a37fef80fc Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 11:46:57 -0700 Subject: [PATCH 78/85] Refactor some length checks --- .../bedrock/streaming/prelude_handling.rb | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/lib/ruby_llm/providers/bedrock/streaming/prelude_handling.rb b/lib/ruby_llm/providers/bedrock/streaming/prelude_handling.rb index db06ccd2d..920d94702 100644 --- a/lib/ruby_llm/providers/bedrock/streaming/prelude_handling.rb +++ b/lib/ruby_llm/providers/bedrock/streaming/prelude_handling.rb @@ -30,11 +30,7 @@ def read_prelude(chunk, offset) end def valid_lengths?(total_length, headers_length) - return false if total_length.nil? || headers_length.nil? - return false if total_length <= 0 || total_length > 1_000_000 - return false if headers_length <= 0 || headers_length > total_length - - true + validate_length_constraints(total_length, headers_length) end def calculate_positions(offset, total_length, headers_length) @@ -71,7 +67,7 @@ def scan_range(chunk, start_offset) def valid_prelude_at_position?(chunk, pos) lengths = extract_potential_lengths(chunk, pos) - valid_prelude_lengths?(*lengths) + validate_length_constraints(*lengths) end def extract_potential_lengths(chunk, pos) @@ -81,11 +77,10 @@ def extract_potential_lengths(chunk, pos) ] end - def valid_prelude_lengths?(total_length, headers_length) - return false unless total_length && headers_length - return false unless total_length.positive? && headers_length.positive? - return false unless total_length < 1_000_000 - return false unless headers_length < total_length + def validate_length_constraints(total_length, headers_length) + return false if total_length.nil? || headers_length.nil? + return false if total_length <= 0 || total_length > 1_000_000 + return false if headers_length <= 0 || headers_length >= total_length true end From d18e019eb0003304b7fa84fdef167d1bd859a913 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 11:51:03 -0700 Subject: [PATCH 79/85] Add spec to cover prelude handling sad paths --- .../streaming/prelude_handling_spec.rb | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 spec/ruby_llm/providers/bedrock/streaming/prelude_handling_spec.rb diff --git a/spec/ruby_llm/providers/bedrock/streaming/prelude_handling_spec.rb b/spec/ruby_llm/providers/bedrock/streaming/prelude_handling_spec.rb new file mode 100644 index 000000000..1d0f2d646 --- /dev/null +++ b/spec/ruby_llm/providers/bedrock/streaming/prelude_handling_spec.rb @@ -0,0 +1,191 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe RubyLLM::Providers::Bedrock::Streaming::PreludeHandling do + let(:dummy_class) { Class.new { include RubyLLM::Providers::Bedrock::Streaming::PreludeHandling } } + let(:instance) { dummy_class.new } + + describe '#can_read_prelude?' do + subject { instance.can_read_prelude?(chunk, offset) } + + context 'when chunk has enough bytes after offset' do + let(:chunk) { "x" * 20 } + let(:offset) { 5 } + it { is_expected.to be true } + end + + context 'when chunk does not have enough bytes after offset' do + let(:chunk) { "x" * 15 } + let(:offset) { 5 } + it { is_expected.to be false } + end + end + + describe '#read_prelude' do + subject { instance.read_prelude(chunk, offset) } + + context 'with valid binary data' do + let(:offset) { 0 } + let(:total_length) { 100 } + let(:headers_length) { 50 } + let(:chunk) do + [ + total_length, # 4 bytes for total_length + headers_length # 4 bytes for headers_length + ].pack('NN') + "x" * 92 # Padding to make total 100 bytes + end + + it 'correctly unpacks the lengths' do + expect(subject).to eq([total_length, headers_length]) + end + end + + context 'with offset' do + let(:offset) { 4 } + let(:total_length) { 100 } + let(:headers_length) { 50 } + let(:chunk) do + "pad!" + # 4 bytes padding before offset + [ + total_length, + headers_length + ].pack('NN') + "x" * 92 + end + + it 'correctly unpacks the lengths from the offset' do + expect(subject).to eq([total_length, headers_length]) + end + end + end + + describe '#find_next_message' do + subject { instance.find_next_message(chunk, offset) } + + context 'when next prelude exists' do + let(:offset) { 0 } + let(:chunk) do + first_message = [100, 50].pack('NN') + "x" * 92 + second_message = [80, 40].pack('NN') + "x" * 72 + first_message + second_message + end + + it 'returns the position of the next message' do + expect(subject).to eq(100) + end + end + + context 'when no next prelude exists' do + let(:offset) { 0 } + let(:chunk) do + [100, 50].pack('NN') + "x" * 92 # Single message + end + + it 'returns the chunk size' do + expect(subject).to eq(chunk.bytesize) + end + end + end + + describe '#find_next_prelude' do + subject { instance.find_next_prelude(chunk, start_offset) } + + context 'when valid prelude exists' do + let(:start_offset) { 100 } + let(:chunk) do + first_message = "x" * 100 # Padding + second_message = [80, 40].pack('NN') + "x" * 72 + first_message + second_message + end + + it 'returns the position of the valid prelude' do + expect(subject).to eq(100) + end + end + + context 'when no valid prelude exists' do + let(:start_offset) { 0 } + let(:chunk) { "x" * 100 } # Invalid data + + it 'returns nil' do + expect(subject).to be_nil + end + end + end + + describe '#valid_lengths?' do + subject { instance.valid_lengths?(total_length, headers_length) } + + context 'with nil values' do + let(:total_length) { nil } + let(:headers_length) { nil } + it { is_expected.to be false } + end + + context 'with invalid values' do + context 'when total_length is zero' do + let(:total_length) { 0 } + let(:headers_length) { 10 } + it { is_expected.to be false } + end + + context 'when total_length exceeds maximum' do + let(:total_length) { 1_000_001 } + let(:headers_length) { 10 } + it { is_expected.to be false } + end + + context 'when headers_length is zero' do + let(:total_length) { 100 } + let(:headers_length) { 0 } + it { is_expected.to be false } + end + + context 'when headers_length exceeds total_length' do + let(:total_length) { 100 } + let(:headers_length) { 101 } + it { is_expected.to be false } + end + end + + context 'with valid values' do + let(:total_length) { 100 } + let(:headers_length) { 50 } + it { is_expected.to be true } + end + end + + describe '#valid_positions?' do + subject { instance.valid_positions?(headers_end, payload_end, chunk_size) } + + context 'with invalid positions' do + context 'when headers_end >= payload_end' do + let(:headers_end) { 100 } + let(:payload_end) { 100 } + let(:chunk_size) { 200 } + it { is_expected.to be false } + end + + context 'when headers_end >= chunk_size' do + let(:headers_end) { 200 } + let(:payload_end) { 300 } + let(:chunk_size) { 200 } + it { is_expected.to be false } + end + + context 'when payload_end > chunk_size' do + let(:headers_end) { 100 } + let(:payload_end) { 300 } + let(:chunk_size) { 200 } + it { is_expected.to be false } + end + end + + context 'with valid positions' do + let(:headers_end) { 100 } + let(:payload_end) { 150 } + let(:chunk_size) { 200 } + it { is_expected.to be true } + end + end +end \ No newline at end of file From 08e7f24e72e0fde6defcdd6b2e3856d6c90beff8 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Wed, 26 Mar 2025 12:14:44 -0700 Subject: [PATCH 80/85] Remove spec that was added for coverage and violated rubocop a bunch --- .../providers/bedrock/streaming/base.rb | 2 - .../streaming/prelude_handling_spec.rb | 191 ------------------ 2 files changed, 193 deletions(-) delete mode 100644 spec/ruby_llm/providers/bedrock/streaming/prelude_handling_spec.rb diff --git a/lib/ruby_llm/providers/bedrock/streaming/base.rb b/lib/ruby_llm/providers/bedrock/streaming/base.rb index 3b1f9bda8..bbf5e406b 100644 --- a/lib/ruby_llm/providers/bedrock/streaming/base.rb +++ b/lib/ruby_llm/providers/bedrock/streaming/base.rb @@ -39,8 +39,6 @@ def handle_stream(&block) end end end - - private end end end diff --git a/spec/ruby_llm/providers/bedrock/streaming/prelude_handling_spec.rb b/spec/ruby_llm/providers/bedrock/streaming/prelude_handling_spec.rb deleted file mode 100644 index 1d0f2d646..000000000 --- a/spec/ruby_llm/providers/bedrock/streaming/prelude_handling_spec.rb +++ /dev/null @@ -1,191 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe RubyLLM::Providers::Bedrock::Streaming::PreludeHandling do - let(:dummy_class) { Class.new { include RubyLLM::Providers::Bedrock::Streaming::PreludeHandling } } - let(:instance) { dummy_class.new } - - describe '#can_read_prelude?' do - subject { instance.can_read_prelude?(chunk, offset) } - - context 'when chunk has enough bytes after offset' do - let(:chunk) { "x" * 20 } - let(:offset) { 5 } - it { is_expected.to be true } - end - - context 'when chunk does not have enough bytes after offset' do - let(:chunk) { "x" * 15 } - let(:offset) { 5 } - it { is_expected.to be false } - end - end - - describe '#read_prelude' do - subject { instance.read_prelude(chunk, offset) } - - context 'with valid binary data' do - let(:offset) { 0 } - let(:total_length) { 100 } - let(:headers_length) { 50 } - let(:chunk) do - [ - total_length, # 4 bytes for total_length - headers_length # 4 bytes for headers_length - ].pack('NN') + "x" * 92 # Padding to make total 100 bytes - end - - it 'correctly unpacks the lengths' do - expect(subject).to eq([total_length, headers_length]) - end - end - - context 'with offset' do - let(:offset) { 4 } - let(:total_length) { 100 } - let(:headers_length) { 50 } - let(:chunk) do - "pad!" + # 4 bytes padding before offset - [ - total_length, - headers_length - ].pack('NN') + "x" * 92 - end - - it 'correctly unpacks the lengths from the offset' do - expect(subject).to eq([total_length, headers_length]) - end - end - end - - describe '#find_next_message' do - subject { instance.find_next_message(chunk, offset) } - - context 'when next prelude exists' do - let(:offset) { 0 } - let(:chunk) do - first_message = [100, 50].pack('NN') + "x" * 92 - second_message = [80, 40].pack('NN') + "x" * 72 - first_message + second_message - end - - it 'returns the position of the next message' do - expect(subject).to eq(100) - end - end - - context 'when no next prelude exists' do - let(:offset) { 0 } - let(:chunk) do - [100, 50].pack('NN') + "x" * 92 # Single message - end - - it 'returns the chunk size' do - expect(subject).to eq(chunk.bytesize) - end - end - end - - describe '#find_next_prelude' do - subject { instance.find_next_prelude(chunk, start_offset) } - - context 'when valid prelude exists' do - let(:start_offset) { 100 } - let(:chunk) do - first_message = "x" * 100 # Padding - second_message = [80, 40].pack('NN') + "x" * 72 - first_message + second_message - end - - it 'returns the position of the valid prelude' do - expect(subject).to eq(100) - end - end - - context 'when no valid prelude exists' do - let(:start_offset) { 0 } - let(:chunk) { "x" * 100 } # Invalid data - - it 'returns nil' do - expect(subject).to be_nil - end - end - end - - describe '#valid_lengths?' do - subject { instance.valid_lengths?(total_length, headers_length) } - - context 'with nil values' do - let(:total_length) { nil } - let(:headers_length) { nil } - it { is_expected.to be false } - end - - context 'with invalid values' do - context 'when total_length is zero' do - let(:total_length) { 0 } - let(:headers_length) { 10 } - it { is_expected.to be false } - end - - context 'when total_length exceeds maximum' do - let(:total_length) { 1_000_001 } - let(:headers_length) { 10 } - it { is_expected.to be false } - end - - context 'when headers_length is zero' do - let(:total_length) { 100 } - let(:headers_length) { 0 } - it { is_expected.to be false } - end - - context 'when headers_length exceeds total_length' do - let(:total_length) { 100 } - let(:headers_length) { 101 } - it { is_expected.to be false } - end - end - - context 'with valid values' do - let(:total_length) { 100 } - let(:headers_length) { 50 } - it { is_expected.to be true } - end - end - - describe '#valid_positions?' do - subject { instance.valid_positions?(headers_end, payload_end, chunk_size) } - - context 'with invalid positions' do - context 'when headers_end >= payload_end' do - let(:headers_end) { 100 } - let(:payload_end) { 100 } - let(:chunk_size) { 200 } - it { is_expected.to be false } - end - - context 'when headers_end >= chunk_size' do - let(:headers_end) { 200 } - let(:payload_end) { 300 } - let(:chunk_size) { 200 } - it { is_expected.to be false } - end - - context 'when payload_end > chunk_size' do - let(:headers_end) { 100 } - let(:payload_end) { 300 } - let(:chunk_size) { 200 } - it { is_expected.to be false } - end - end - - context 'with valid positions' do - let(:headers_end) { 100 } - let(:payload_end) { 150 } - let(:chunk_size) { 200 } - it { is_expected.to be true } - end - end -end \ No newline at end of file From 801b42dc29645585fc2f758b00d2c42d22b1a9be Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Mon, 31 Mar 2025 13:20:33 -0700 Subject: [PATCH 81/85] Use JSON.generate that does not encode non-ASCII characters - Needed for correct hash in Bedrock signing process --- lib/ruby_llm/providers/bedrock.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ruby_llm/providers/bedrock.rb b/lib/ruby_llm/providers/bedrock.rb index 3c2c1dc2d..2f94bab9a 100644 --- a/lib/ruby_llm/providers/bedrock.rb +++ b/lib/ruby_llm/providers/bedrock.rb @@ -54,7 +54,7 @@ def build_request(url, payload) connection: connection, http_method: :post, url: url || completion_url, - body: payload&.to_json || '' + body: JSON.generate(payload, ascii_only: false) || '' } end From 14df4ec97ec6c62bfcfa75435b8821ad1948fd9a Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Mon, 31 Mar 2025 14:22:27 -0700 Subject: [PATCH 82/85] Add support for system messages --- lib/ruby_llm/providers/bedrock/chat.rb | 29 ++++- ...an_handle_system_messages_with_bedrock.yml | 121 ++++++++++++++++++ spec/ruby_llm/chat_spec.rb | 19 +++ 3 files changed, 165 insertions(+), 4 deletions(-) create mode 100644 spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_system_messages_with_bedrock.yml diff --git a/lib/ruby_llm/providers/bedrock/chat.rb b/lib/ruby_llm/providers/bedrock/chat.rb index 9e821cc90..f7786c125 100644 --- a/lib/ruby_llm/providers/bedrock/chat.rb +++ b/lib/ruby_llm/providers/bedrock/chat.rb @@ -49,14 +49,34 @@ def build_message(data, content, tool_use) end def build_claude_request(messages, tools:, temperature:, model:) + system_messages, chat_messages = separate_messages(messages) + system_content = build_system_content(system_messages) + + build_base_payload(chat_messages, temperature, model).tap do |payload| + add_optional_fields(payload, system_content:, tools:) + end + end + + def separate_messages(messages) + messages.partition { |msg| msg.role == :system } + end + + def build_system_content(system_messages) + system_messages.map { |msg| format_message(msg)[:content] }.join("\n\n") + end + + def build_base_payload(chat_messages, temperature, model) { anthropic_version: 'bedrock-2023-05-31', - messages: messages.map { |msg| format_message(msg) }, + messages: chat_messages.map { |msg| format_message(msg) }, temperature: temperature, max_tokens: max_tokens_for(model) - }.tap do |payload| - payload[:tools] = tools.values.map { |t| function_for(t) } if tools.any? - end + } + end + + def add_optional_fields(payload, system_content:, tools:) + payload[:tools] = tools.values.map { |t| function_for(t) } if tools.any? + payload[:system] = system_content unless system_content.empty? end def format_message(msg) @@ -79,6 +99,7 @@ def format_basic_message(msg) def convert_role(role) case role when :tool, :user then 'user' + when :system then 'system' else 'assistant' end end diff --git a/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_system_messages_with_bedrock.yml b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_system_messages_with_bedrock.yml new file mode 100644 index 000000000..ac118ae79 --- /dev/null +++ b/spec/fixtures/vcr_cassettes/chat_basic_chat_functionality_claude-3-5-haiku_can_handle_system_messages_with_bedrock.yml @@ -0,0 +1,121 @@ +--- +http_interactions: +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What + is 15 + 27?"}],"temperature":0.7,"max_tokens":4096,"system":"You are a helpful + math tutor who always shows your work."}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250331T211814Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - fbe47914c1da712b79e70a557d3f0b86f91d283f7c2e07643feff6580105f319 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250331/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=d8e79c8ca7d9a7a14c0bb83d654a729530d80426c1d55860f839174e123ee24f + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 31 Mar 2025 21:17:35 GMT + Content-Type: + - application/json + Content-Length: + - '618' + Connection: + - keep-alive + X-Amzn-Requestid: + - 5e07f27e-93dd-4c55-a1b0-67c6a3884627 + X-Amzn-Bedrock-Invocation-Latency: + - '2573' + X-Amzn-Bedrock-Output-Token-Count: + - '123' + X-Amzn-Bedrock-Input-Token-Count: + - '29' + body: + encoding: UTF-8 + string: '{"id":"msg_bdrk_01BwYUhrKeru4SJ2Djr6e3V2","type":"message","role":"assistant","model":"claude-3-5-haiku-20241022","content":[{"type":"text","text":"Let + me solve this step by step:\n\n 15\n+ 27\n-----\n 42\n\nI''ll break down + the addition:\n1. First, add the ones column: 5 + 7 = 12\n2. Write down the + 2 in the ones place and carry the 1 to the tens column\n3. In the tens column, + add 1 (carried over) + 1 + 2 = 4\n4. The final answer is 42\n\nSo, 15 + 27 + = 42"}],"stop_reason":"end_turn","stop_sequence":null,"usage":{"input_tokens":29,"cache_creation_input_tokens":0,"cache_read_input_tokens":0,"output_tokens":123}}' + recorded_at: Mon, 31 Mar 2025 21:18:17 GMT +- request: + method: post + uri: https://bedrock-runtime.us-west-2.amazonaws.com/model/anthropic.claude-3-5-haiku-20241022-v1:0/invoke + body: + encoding: UTF-8 + string: '{"anthropic_version":"bedrock-2023-05-31","messages":[{"role":"user","content":"What + is 15 + 27?"},{"role":"assistant","content":"Let me solve this step by step:\n\n 15\n+ + 27\n-----\n 42\n\nI''ll break down the addition:\n1. First, add the ones + column: 5 + 7 = 12\n2. Write down the 2 in the ones place and carry the 1 + to the tens column\n3. In the tens column, add 1 (carried over) + 1 + 2 = + 4\n4. The final answer is 42\n\nSo, 15 + 27 = 42"},{"role":"user","content":"What + is 25 * 4?"}],"temperature":0.7,"max_tokens":4096,"system":"You are a helpful + math tutor who always shows your work.\n\nAlways include a fun fact about + numbers in your response."}' + headers: + User-Agent: + - Faraday v2.12.2 + Host: + - bedrock-runtime.us-west-2.amazonaws.com + X-Amz-Date: + - 20250331T211817Z + X-Amz-Security-Token: + - "" + X-Amz-Content-Sha256: + - a55ae236e70cacb5532c4a120209c0280ff759a285b5a875caafb5a942a086a4 + Authorization: + - AWS4-HMAC-SHA256 Credential=/20250331/us-west-2/bedrock/aws4_request, + SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-security-token, Signature=7ab569ddd72b52f7f4b253113a05cb0e00d5f111eb53846bc3effd93ce958a6f + Content-Type: + - application/json + Accept: + - application/json + Accept-Encoding: + - gzip;q=1.0,deflate;q=0.6,identity;q=0.3 + response: + status: + code: 200 + message: OK + headers: + Date: + - Mon, 31 Mar 2025 21:17:38 GMT + Content-Type: + - application/json + Content-Length: + - '773' + Connection: + - keep-alive + X-Amzn-Requestid: + - 5ca42b45-0c6c-474b-9f37-2b76a8dcc89c + X-Amzn-Bedrock-Invocation-Latency: + - '3047' + X-Amzn-Bedrock-Output-Token-Count: + - '159' + X-Amzn-Bedrock-Input-Token-Count: + - '176' + body: + encoding: ASCII-8BIT + string: !binary |- + eyJpZCI6Im1zZ19iZHJrXzAxTXpxcnNkcmF5anJZSnRDcW9RNDRleiIsInR5cGUiOiJtZXNzYWdlIiwicm9sZSI6ImFzc2lzdGFudCIsIm1vZGVsIjoiY2xhdWRlLTMtNS1oYWlrdS0yMDI0MTAyMiIsImNvbnRlbnQiOlt7InR5cGUiOiJ0ZXh0IiwidGV4dCI6IkxldCBtZSBzb2x2ZSB0aGlzIHN0ZXAgYnkgc3RlcDpcblxuICAgIDI1XG7DlyAgICA0XG4tLS0tLS0tXG4gICAxMDBcblxuSGVyZSdzIHRoZSBtdWx0aXBsaWNhdGlvbiBicmVha2Rvd246XG4xLiBNdWx0aXBseSA1IMOXIDQgPSAyMCwgd3JpdGUgZG93biAwLCBjYXJyeSB0aGUgMlxuMi4gTXVsdGlwbHkgMiDDlyA0ID0gOCwgdGhlbiBhZGQgdGhlIGNhcnJpZWQgMlxuMy4gU28gOCArIDIgPSAxMFxuNC4gVGhlIGZpbmFsIGFuc3dlciBpcyAxMDBcblxuU28sIDI1IMOXIDQgPSAxMDBcblxuRnVuIE51bWJlciBGYWN0OiBEaWQgeW91IGtub3cgdGhhdCAxMDAgaXMgY29uc2lkZXJlZCBhIFwicGVyZmVjdCBzcXVhcmVcIiBiZWNhdXNlIGl0J3MgdGhlIHJlc3VsdCBvZiAxMCDDlyAxMD8gUGVyZmVjdCBzcXVhcmVzIGhhdmUgd2hvbGUgbnVtYmVyIHJvb3RzIHRoYXQgbXVsdGlwbHkgZXZlbmx5IGludG8gdGhlbXNlbHZlcyEifV0sInN0b3BfcmVhc29uIjoiZW5kX3R1cm4iLCJzdG9wX3NlcXVlbmNlIjpudWxsLCJ1c2FnZSI6eyJpbnB1dF90b2tlbnMiOjE3NiwiY2FjaGVfY3JlYXRpb25faW5wdXRfdG9rZW5zIjowLCJjYWNoZV9yZWFkX2lucHV0X3Rva2VucyI6MCwib3V0cHV0X3Rva2VucyI6MTU5fX0= + recorded_at: Mon, 31 Mar 2025 21:18:20 GMT +recorded_with: VCR 6.3.1 diff --git a/spec/ruby_llm/chat_spec.rb b/spec/ruby_llm/chat_spec.rb index 740e17fe5..cd5583aba 100644 --- a/spec/ruby_llm/chat_spec.rb +++ b/spec/ruby_llm/chat_spec.rb @@ -35,5 +35,24 @@ expect(followup.content).to include('199') end end + + it 'claude-3-5-haiku can handle system messages with bedrock' do # rubocop:disable RSpec/ExampleLength,RSpec/MultipleExpectations + chat = RubyLLM.chat(model: 'claude-3-5-haiku', provider: 'bedrock') + + # Add a system message + chat.add_message(role: :system, content: 'You are a helpful math tutor who always shows your work.') + + response = chat.ask('What is 15 + 27?') + expect(response.content).to include('42') + expect(response.content).to match(/step|work|process/i) # Should show work as instructed + + # Add another system message + chat.add_message(role: :system, content: 'Always include a fun fact about numbers in your response.') + + response = chat.ask('What is 25 * 4?') + expect(response.content).to include('100') + expect(response.content).to match(/step|work|process/i) # Should still show work + expect(response.content).to match(/fact|interesting|did you know/i) # Should include a fun fact + end end end From 7510ba73f748f25c78ca36005d8dea91211ad6c1 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 1 Apr 2025 09:04:33 -0700 Subject: [PATCH 83/85] Remove duplicate error class --- lib/ruby_llm/error.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/ruby_llm/error.rb b/lib/ruby_llm/error.rb index 9f2e10e68..a0c752bf7 100644 --- a/lib/ruby_llm/error.rb +++ b/lib/ruby_llm/error.rb @@ -32,7 +32,6 @@ class OverloadedError < Error; end class PaymentRequiredError < Error; end class RateLimitError < Error; end class ServerError < Error; end - class ForbiddenError < Error; end class ServiceUnavailableError < Error; end class UnauthorizedError < Error; end From dc8152f957e6e8f45f26783c00a025768a25cdf4 Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 1 Apr 2025 09:10:29 -0700 Subject: [PATCH 84/85] Get the legal stuff right --- .../APACHE-2.0.txt | 0 NOTICE | 31 +++++++++++++++++++ README.md | 2 +- THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt | 8 ----- 4 files changed, 32 insertions(+), 9 deletions(-) rename {THIRD_PARTY_LICENSES => LICENSES}/APACHE-2.0.txt (100%) create mode 100644 NOTICE delete mode 100644 THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt diff --git a/THIRD_PARTY_LICENSES/APACHE-2.0.txt b/LICENSES/APACHE-2.0.txt similarity index 100% rename from THIRD_PARTY_LICENSES/APACHE-2.0.txt rename to LICENSES/APACHE-2.0.txt diff --git a/NOTICE b/NOTICE new file mode 100644 index 000000000..a2662b740 --- /dev/null +++ b/NOTICE @@ -0,0 +1,31 @@ +RubyLLM Project +Copyright 2025 Carmine Paolino. All Rights Reserved. +Licensed under the MIT License. + +This project includes software developed by third parties, licensed under +terms compatible with the MIT License. The required notices and license +texts are included below and/or in the LICENSES/ directory. + +-------------------------------------------------------------------------- + +This project incorporates code from aws-sdk-ruby. +The original NOTICE file for aws-sdk-ruby contains the following: + + --- BEGIN aws-sdk-ruby NOTICE --- + AWS SDK for Ruby + Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + --- END aws-sdk-ruby NOTICE --- + +The aws-sdk-ruby components used in this project are licensed under the +Apache License, Version 2.0 (the "License"). You may obtain a copy of +the License at: + + LICENSES/APACHE-2.0.txt + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +-------------------------------------------------------------------------- diff --git a/README.md b/README.md index 714cce3e9..1b9cb15cd 100644 --- a/README.md +++ b/README.md @@ -105,7 +105,7 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') + config.bedrock_region = ENV.fetch('AWS_REGION', nil) config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end ``` diff --git a/THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt b/THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt deleted file mode 100644 index 76189a3bf..000000000 --- a/THIRD_PARTY_LICENSES/AWS-SDK-NOTICE.txt +++ /dev/null @@ -1,8 +0,0 @@ -This project includes code derived from AWS SDK for Ruby, licensed under the Apache License 2.0. -The original work can be found at https://github.com/aws/aws-sdk-ruby -Modifications were made here by RubyLLM. - -The following notice is from the original project and is included per the Apache 2.0 license: - -AWS SDK for Ruby -Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. \ No newline at end of file From aee80e37f6141b39d9734a59f7300bf3efce8dbc Mon Sep 17 00:00:00 2001 From: Paul Shippy Date: Tue, 1 Apr 2025 09:11:29 -0700 Subject: [PATCH 85/85] Remove default region for Bedrock --- bin/console | 2 +- docs/guides/getting-started.md | 2 +- docs/guides/rails.md | 2 +- docs/installation.md | 2 +- lib/tasks/model_updater.rb | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/console b/bin/console index 393074602..b5e871689 100755 --- a/bin/console +++ b/bin/console @@ -14,7 +14,7 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') + config.bedrock_region = ENV.fetch('AWS_REGION', nil) config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end diff --git a/docs/guides/getting-started.md b/docs/guides/getting-started.md index 815853b1d..07cb5e460 100644 --- a/docs/guides/getting-started.md +++ b/docs/guides/getting-started.md @@ -32,7 +32,7 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') + config.bedrock_region = ENV.fetch('AWS_REGION', nil) config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end ``` diff --git a/docs/guides/rails.md b/docs/guides/rails.md index cd53e4532..4eb8108d3 100644 --- a/docs/guides/rails.md +++ b/docs/guides/rails.md @@ -98,7 +98,7 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') + config.bedrock_region = ENV.fetch('AWS_REGION', nil) config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end ``` diff --git a/docs/installation.md b/docs/installation.md index a5f2a0656..7fc021c7c 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -57,7 +57,7 @@ RubyLLM.configure do |config| config.deepseek_api_key = ENV.fetch('DEEPSEEK_API_KEY', nil) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') + config.bedrock_region = ENV.fetch('AWS_REGION', nil) config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) # Optional: Set default models diff --git a/lib/tasks/model_updater.rb b/lib/tasks/model_updater.rb index 681f360a2..7685a2191 100644 --- a/lib/tasks/model_updater.rb +++ b/lib/tasks/model_updater.rb @@ -27,7 +27,7 @@ def configure_from_env def configure_bedrock(config) config.bedrock_api_key = ENV.fetch('AWS_ACCESS_KEY_ID', nil) config.bedrock_secret_key = ENV.fetch('AWS_SECRET_ACCESS_KEY', nil) - config.bedrock_region = ENV.fetch('AWS_REGION', 'us-west-2') + config.bedrock_region = ENV.fetch('AWS_REGION', nil) config.bedrock_session_token = ENV.fetch('AWS_SESSION_TOKEN', nil) end