From 9268e4201bcba2f7fb704e626e8148df6c91d740 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Tue, 27 Jan 2026 22:03:37 +0000 Subject: [PATCH] Add e2e Ruby SDK test for Responses API with Anthropic This commit adds: - New test folder tests/responses_api_tests_proxy/ with Ruby SDK tests - config.yaml with anthropic/* and openai/* wildcards - Ruby spec file testing the Responses API with Anthropic model - Validates output_tokens_details is properly set (not null) for Ruby SDK compatibility - Tests basic, streaming, and tool call scenarios - New CircleCI job e2e_ruby_sdk_responses_api to run the tests This ensures the fix for output_tokens_details (reasoning_tokens default to 0) is working correctly with the Ruby OpenAI SDK. Co-authored-by: ishaan --- .circleci/config.yml | 107 ++++++++++ .../ruby_tests/Gemfile | 4 + .../spec/responses_api_anthropic_spec.rb | 194 ++++++++++++++++++ .../ruby_tests/spec/spec_helper.rb | 21 ++ 4 files changed, 326 insertions(+) create mode 100644 tests/responses_api_tests_proxy/ruby_tests/Gemfile create mode 100644 tests/responses_api_tests_proxy/ruby_tests/spec/responses_api_anthropic_spec.rb create mode 100644 tests/responses_api_tests_proxy/ruby_tests/spec/spec_helper.rb diff --git a/.circleci/config.yml b/.circleci/config.yml index 77f3c2b9a7d..caddef256ce 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2562,6 +2562,104 @@ jobs: python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5 no_output_timeout: 120m + # Store test results + - store_test_results: + path: test-results + e2e_ruby_sdk_responses_api: + machine: + image: ubuntu-2204:2023.10.1 + resource_class: xlarge + working_directory: ~/project + steps: + - checkout + - setup_google_dns + - run: + name: Install Docker CLI (In case it's not already installed) + command: | + curl -fsSL https://get.docker.com | sh + sudo usermod -aG docker $USER + docker version + - run: + name: Install dockerize + command: | + wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz + sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz + rm dockerize-linux-amd64-v0.6.1.tar.gz + - run: + name: Start PostgreSQL Database + command: | + docker run -d \ + --name postgres-db \ + -e POSTGRES_USER=postgres \ + -e POSTGRES_PASSWORD=postgres \ + -e POSTGRES_DB=circle_test \ + -p 5432:5432 \ + postgres:14 + - run: + name: Wait for PostgreSQL to be ready + command: dockerize -wait tcp://localhost:5432 -timeout 1m + - attach_workspace: + at: ~/project + - run: + name: Load Docker Database Image + command: | + gunzip -c litellm-docker-database.tar.gz | docker load + docker images | grep litellm-docker-database + - run: + name: Run Docker container + command: | + docker run -d \ + -p 4000:4000 \ + -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ + -e LITELLM_MASTER_KEY="sk-1234" \ + -e OPENAI_API_KEY=$OPENAI_API_KEY \ + -e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \ + -e LITELLM_LICENSE=$LITELLM_LICENSE \ + --add-host host.docker.internal:host-gateway \ + --name my-app \ + -v $(pwd)/tests/responses_api_tests_proxy/config.yaml:/app/config.yaml \ + litellm-docker-database:ci \ + --config /app/config.yaml \ + --port 4000 \ + --detailed_debug \ + - run: + name: Start outputting logs + command: docker logs -f my-app + background: true + - run: + name: Wait for app to be ready + command: dockerize -wait http://localhost:4000 -timeout 5m + - run: + name: Install Ruby and Bundler + command: | + # Import GPG keys first + gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB || { + curl -sSL https://rvm.io/mpapis.asc | gpg --import - + curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - + } + + # Install Ruby version manager (RVM) + curl -sSL https://get.rvm.io | bash -s stable + + # Source RVM from the correct location + source $HOME/.rvm/scripts/rvm + + # Install Ruby 3.2.2 + rvm install 3.2.2 + rvm use 3.2.2 --default + + # Install latest Bundler + gem install bundler + + - run: + name: Run Ruby SDK Responses API tests + command: | + source $HOME/.rvm/scripts/rvm + cd tests/responses_api_tests_proxy/ruby_tests + bundle install + bundle exec rspec --format documentation + no_output_timeout: 30m + # Store test results - store_test_results: path: test-results @@ -4005,6 +4103,14 @@ workflows: only: - main - /litellm_.*/ + - e2e_ruby_sdk_responses_api: + requires: + - build_docker_database_image + filters: + branches: + only: + - main + - /litellm_.*/ - proxy_logging_guardrails_model_info_tests: requires: - build_docker_database_image @@ -4242,6 +4348,7 @@ workflows: - local_testing_part2 - build_and_test - e2e_openai_endpoints + - e2e_ruby_sdk_responses_api - test_bad_database_url - llm_translation_testing - mcp_testing diff --git a/tests/responses_api_tests_proxy/ruby_tests/Gemfile b/tests/responses_api_tests_proxy/ruby_tests/Gemfile new file mode 100644 index 00000000000..bedab086bd7 --- /dev/null +++ b/tests/responses_api_tests_proxy/ruby_tests/Gemfile @@ -0,0 +1,4 @@ +source 'https://rubygems.org' + +gem 'rspec' +gem 'ruby-openai', '~> 7.4' diff --git a/tests/responses_api_tests_proxy/ruby_tests/spec/responses_api_anthropic_spec.rb b/tests/responses_api_tests_proxy/ruby_tests/spec/responses_api_anthropic_spec.rb new file mode 100644 index 00000000000..24e5b3480f6 --- /dev/null +++ b/tests/responses_api_tests_proxy/ruby_tests/spec/responses_api_anthropic_spec.rb @@ -0,0 +1,194 @@ +require 'openai' +require 'rspec' + +RSpec.describe 'OpenAI Responses API with Anthropic via LiteLLM' do + let(:client) do + OpenAI::Client.new( + access_token: "sk-1234", + uri_base: "http://0.0.0.0:4000" + ) + end + + describe 'basic responses API' do + it 'should create a basic response with anthropic model' do + puts "\n=== Testing basic Responses API with Anthropic ===" + + response = client.responses.create( + parameters: { + model: "anthropic/claude-sonnet-4-5-20250929", + input: "Say hello in one word", + max_output_tokens: 50 + } + ) + + puts "Response: #{response.inspect}" + + # Validate response structure + expect(response).to include('id') + expect(response).to include('output') + expect(response).to include('status') + expect(response['status']).to eq('completed') + + # Validate usage is present + expect(response).to include('usage') + usage = response['usage'] + expect(usage).to include('input_tokens') + expect(usage).to include('output_tokens') + expect(usage).to include('total_tokens') + expect(usage['input_tokens']).to be > 0 + expect(usage['output_tokens']).to be > 0 + + # CRITICAL: Validate output_tokens_details for Ruby SDK compatibility + # This is the key fix being tested - the Ruby SDK requires output_tokens_details + # to be an object with reasoning_tokens, NOT null + expect(usage).to include('output_tokens_details') + output_tokens_details = usage['output_tokens_details'] + expect(output_tokens_details).not_to be_nil, + "output_tokens_details should not be nil - Ruby SDK requires it to be an object" + expect(output_tokens_details).to include('reasoning_tokens') + expect(output_tokens_details['reasoning_tokens']).to be_a(Integer), + "reasoning_tokens should be an Integer (can be 0 for non-reasoning models)" + + puts "✓ output_tokens_details validated: reasoning_tokens=#{output_tokens_details['reasoning_tokens']}" + puts "=== Basic Responses API test passed ===" + end + + it 'should create a streaming response with anthropic model' do + puts "\n=== Testing streaming Responses API with Anthropic ===" + + collected_content = "" + response_completed_event = nil + + client.responses.create( + parameters: { + model: "anthropic/claude-sonnet-4-5-20250929", + input: "Say hello in one word", + max_output_tokens: 50, + stream: proc do |chunk, _bytesize| + puts "Received chunk type: #{chunk['type']}" if chunk['type'] + + if chunk['type'] == 'response.output_text.delta' && chunk['delta'] + collected_content += chunk['delta'] + elsif chunk['type'] == 'response.completed' && chunk['response'] + response_completed_event = chunk + end + end + } + ) + + puts "Collected content: #{collected_content}" + + # Validate we received the completed event + expect(response_completed_event).not_to be_nil, + "Expected to receive response.completed event" + + response = response_completed_event['response'] + expect(response).not_to be_nil + + # For streaming, validate the final response + expect(response).to include('id') + expect(response).to include('status') + + # Validate usage in streaming response + usage = response['usage'] + expect(usage).not_to be_nil, "Usage should be present in streaming response" + expect(usage).to include('input_tokens') + expect(usage).to include('output_tokens') + + # CRITICAL: Validate output_tokens_details for Ruby SDK compatibility in streaming + expect(usage).to include('output_tokens_details') + output_tokens_details = usage['output_tokens_details'] + expect(output_tokens_details).not_to be_nil, + "output_tokens_details should not be nil in streaming response - Ruby SDK requires it to be an object" + expect(output_tokens_details).to include('reasoning_tokens') + expect(output_tokens_details['reasoning_tokens']).to be_a(Integer), + "reasoning_tokens should be an Integer in streaming response" + + puts "✓ Streaming output_tokens_details validated: reasoning_tokens=#{output_tokens_details['reasoning_tokens']}" + puts "=== Streaming Responses API test passed ===" + end + + it 'should handle tool calls with anthropic model' do + puts "\n=== Testing Responses API with tool calls using Anthropic ===" + + tools = [ + { + type: "function", + name: "get_weather", + description: "Get current weather for a location", + parameters: { + type: "object", + properties: { + location: { + type: "string", + description: "City name" + } + }, + required: ["location"] + } + } + ] + + response = client.responses.create( + parameters: { + model: "anthropic/claude-sonnet-4-5-20250929", + input: "What's the weather in Paris?", + tools: tools, + max_output_tokens: 200 + } + ) + + puts "Tool call response: #{response.inspect}" + + # Validate response structure + expect(response).to include('id') + expect(response).to include('output') + + # Check for tool calls in output + output = response['output'] + expect(output).to be_an(Array) + expect(output.length).to be > 0 + + # Validate usage with output_tokens_details + usage = response['usage'] + expect(usage).not_to be_nil + expect(usage).to include('output_tokens_details') + output_tokens_details = usage['output_tokens_details'] + expect(output_tokens_details).not_to be_nil, + "output_tokens_details should not be nil for tool call responses" + + puts "✓ Tool call response validated with output_tokens_details" + puts "=== Tool calls test passed ===" + end + end + + describe 'OpenAI model comparison' do + it 'should work the same way with OpenAI model' do + puts "\n=== Testing Responses API with OpenAI model for comparison ===" + + response = client.responses.create( + parameters: { + model: "openai/gpt-4o-mini", + input: "Say hello in one word", + max_output_tokens: 50 + } + ) + + puts "OpenAI Response: #{response.inspect}" + + # Validate response structure matches Anthropic + expect(response).to include('id') + expect(response).to include('output') + expect(response).to include('usage') + + usage = response['usage'] + expect(usage).to include('output_tokens_details') + output_tokens_details = usage['output_tokens_details'] + expect(output_tokens_details).not_to be_nil, + "OpenAI model should also have output_tokens_details" + + puts "✓ OpenAI model response validated" + puts "=== OpenAI model test passed ===" + end + end +end diff --git a/tests/responses_api_tests_proxy/ruby_tests/spec/spec_helper.rb b/tests/responses_api_tests_proxy/ruby_tests/spec/spec_helper.rb new file mode 100644 index 00000000000..e7d59f41e47 --- /dev/null +++ b/tests/responses_api_tests_proxy/ruby_tests/spec/spec_helper.rb @@ -0,0 +1,21 @@ +RSpec.configure do |config| + config.expect_with :rspec do |expectations| + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + config.mock_with :rspec do |mocks| + mocks.verify_partial_doubles = true + end + + config.shared_context_metadata_behavior = :apply_to_host_groups + config.filter_run_when_matching :focus + config.example_status_persistence_file_path = "spec/examples.txt" + config.disable_monkey_patching! + + if config.files_to_run.one? + config.default_formatter = "doc" + end + + config.order = :random + Kernel.srand config.seed +end