scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-08-28 05:25:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 9

Add e2e Ruby SDK test for Responses API with Anthropic

Browse source 
This commit adds:
- New test folder tests/responses_api_tests_proxy/ with Ruby SDK tests
- config.yaml with anthropic/* and openai/* wildcards
- Ruby spec file testing the Responses API with Anthropic model
- Validates output_tokens_details is properly set (not null) for Ruby SDK compatibility
- Tests basic, streaming, and tool call scenarios
- New CircleCI job e2e_ruby_sdk_responses_api to run the tests

This ensures the fix for output_tokens_details (reasoning_tokens default to 0)
is working correctly with the Ruby OpenAI SDK.

Co-authored-by: ishaan <ishaan@berri.ai>
This commit is contained in:
Cursor Agent 2026-01-27 22:03:37 +00:00
parent 23f77accc7
commit 9268e4201b
4 changed files with 326 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

107
.circleci/config.yml
View file

@ -2562,6 +2562,104 @@ jobs:
python -m pytest -s -vv tests/openai_endpoints_tests --junitxml=test-results/junit.xml --durations=5
no_output_timeout: 120m
# Store test results
- store_test_results:
path: test-results
e2e_ruby_sdk_responses_api:
machine:
image: ubuntu-2204:2023.10.1
resource_class: xlarge
working_directory: ~/project
steps:
- checkout
- setup_google_dns
- run:
name: Install Docker CLI (In case it's not already installed)
command: |
curl -fsSL https://get.docker.com | sh
sudo usermod -aG docker $USER
docker version
- run:
name: Install dockerize
command: |
wget https://github.com/jwilder/dockerize/releases/download/v0.6.1/dockerize-linux-amd64-v0.6.1.tar.gz
sudo tar -C /usr/local/bin -xzvf dockerize-linux-amd64-v0.6.1.tar.gz
rm dockerize-linux-amd64-v0.6.1.tar.gz
- run:
name: Start PostgreSQL Database
command: |
docker run -d \
--name postgres-db \
-e POSTGRES_USER=postgres \
-e POSTGRES_PASSWORD=postgres \
-e POSTGRES_DB=circle_test \
-p 5432:5432 \
postgres:14
- run:
name: Wait for PostgreSQL to be ready
command: dockerize -wait tcp://localhost:5432 -timeout 1m
- attach_workspace:
at: ~/project
- run:
name: Load Docker Database Image
command: |
gunzip -c litellm-docker-database.tar.gz | docker load
docker images | grep litellm-docker-database
- run:
name: Run Docker container
command: |
docker run -d \
-p 4000:4000 \
-e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \
-e LITELLM_MASTER_KEY="sk-1234" \
-e OPENAI_API_KEY=$OPENAI_API_KEY \
-e ANTHROPIC_API_KEY=$ANTHROPIC_API_KEY \
-e LITELLM_LICENSE=$LITELLM_LICENSE \
--add-host host.docker.internal:host-gateway \
--name my-app \
-v $(pwd)/tests/responses_api_tests_proxy/config.yaml:/app/config.yaml \
litellm-docker-database:ci \
--config /app/config.yaml \
--port 4000 \
--detailed_debug \
- run:
name: Start outputting logs
command: docker logs -f my-app
background: true
- run:
name: Wait for app to be ready
command: dockerize -wait http://localhost:4000 -timeout 5m
- run:
name: Install Ruby and Bundler
command: |
# Import GPG keys first
gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB || {
curl -sSL https://rvm.io/mpapis.asc | gpg --import -
curl -sSL https://rvm.io/pkuczynski.asc | gpg --import -
}
# Install Ruby version manager (RVM)
curl -sSL https://get.rvm.io | bash -s stable
# Source RVM from the correct location
source $HOME/.rvm/scripts/rvm
# Install Ruby 3.2.2
rvm install 3.2.2
rvm use 3.2.2 --default
# Install latest Bundler
gem install bundler
- run:
name: Run Ruby SDK Responses API tests
command: |
source $HOME/.rvm/scripts/rvm
cd tests/responses_api_tests_proxy/ruby_tests
bundle install
bundle exec rspec --format documentation
no_output_timeout: 30m
# Store test results
- store_test_results:
path: test-results
@ -4005,6 +4103,14 @@ workflows:
only:
- main
- /litellm_.*/
- e2e_ruby_sdk_responses_api:
requires:
- build_docker_database_image
filters:
branches:
only:
- main
- /litellm_.*/
- proxy_logging_guardrails_model_info_tests:
requires:
- build_docker_database_image
@ -4242,6 +4348,7 @@ workflows:
- local_testing_part2
- build_and_test
- e2e_openai_endpoints
- e2e_ruby_sdk_responses_api
- test_bad_database_url
- llm_translation_testing
- mcp_testing

4
tests/responses_api_tests_proxy/ruby_tests/Gemfile Normal file
View file

@ -0,0 +1,4 @@
source 'https://rubygems.org'
gem 'rspec'
gem 'ruby-openai', '~> 7.4'

194
tests/responses_api_tests_proxy/ruby_tests/spec/responses_api_anthropic_spec.rb Normal file
View file

@ -0,0 +1,194 @@
require 'openai'
require 'rspec'
RSpec.describe 'OpenAI Responses API with Anthropic via LiteLLM' do
let(:client) do
OpenAI::Client.new(
access_token: "sk-1234",
uri_base: "http://0.0.0.0:4000"
)
end
describe 'basic responses API' do
it 'should create a basic response with anthropic model' do
puts "\n=== Testing basic Responses API with Anthropic ==="
response = client.responses.create(
parameters: {
model: "anthropic/claude-sonnet-4-5-20250929",
input: "Say hello in one word",
max_output_tokens: 50
}
)
puts "Response: #{response.inspect}"
# Validate response structure
expect(response).to include('id')
expect(response).to include('output')
expect(response).to include('status')
expect(response['status']).to eq('completed')
# Validate usage is present
expect(response).to include('usage')
usage = response['usage']
expect(usage).to include('input_tokens')
expect(usage).to include('output_tokens')
expect(usage).to include('total_tokens')
expect(usage['input_tokens']).to be > 0
expect(usage['output_tokens']).to be > 0
# CRITICAL: Validate output_tokens_details for Ruby SDK compatibility
# This is the key fix being tested - the Ruby SDK requires output_tokens_details
# to be an object with reasoning_tokens, NOT null
expect(usage).to include('output_tokens_details')
output_tokens_details = usage['output_tokens_details']
expect(output_tokens_details).not_to be_nil,
"output_tokens_details should not be nil - Ruby SDK requires it to be an object"
expect(output_tokens_details).to include('reasoning_tokens')
expect(output_tokens_details['reasoning_tokens']).to be_a(Integer),
"reasoning_tokens should be an Integer (can be 0 for non-reasoning models)"
puts "✓ output_tokens_details validated: reasoning_tokens=#{output_tokens_details['reasoning_tokens']}"
puts "=== Basic Responses API test passed ==="
end
it 'should create a streaming response with anthropic model' do
puts "\n=== Testing streaming Responses API with Anthropic ==="
collected_content = ""
response_completed_event = nil
client.responses.create(
parameters: {
model: "anthropic/claude-sonnet-4-5-20250929",
input: "Say hello in one word",
max_output_tokens: 50,
stream: proc do |chunk, _bytesize|
puts "Received chunk type: #{chunk['type']}" if chunk['type']
if chunk['type'] == 'response.output_text.delta' && chunk['delta']
collected_content += chunk['delta']
elsif chunk['type'] == 'response.completed' && chunk['response']
response_completed_event = chunk
end
end
}
)
puts "Collected content: #{collected_content}"
# Validate we received the completed event
expect(response_completed_event).not_to be_nil,
"Expected to receive response.completed event"
response = response_completed_event['response']
expect(response).not_to be_nil
# For streaming, validate the final response
expect(response).to include('id')
expect(response).to include('status')
# Validate usage in streaming response
usage = response['usage']
expect(usage).not_to be_nil, "Usage should be present in streaming response"
expect(usage).to include('input_tokens')
expect(usage).to include('output_tokens')
# CRITICAL: Validate output_tokens_details for Ruby SDK compatibility in streaming
expect(usage).to include('output_tokens_details')
output_tokens_details = usage['output_tokens_details']
expect(output_tokens_details).not_to be_nil,
"output_tokens_details should not be nil in streaming response - Ruby SDK requires it to be an object"
expect(output_tokens_details).to include('reasoning_tokens')
expect(output_tokens_details['reasoning_tokens']).to be_a(Integer),
"reasoning_tokens should be an Integer in streaming response"
puts "✓ Streaming output_tokens_details validated: reasoning_tokens=#{output_tokens_details['reasoning_tokens']}"
puts "=== Streaming Responses API test passed ==="
end
it 'should handle tool calls with anthropic model' do
puts "\n=== Testing Responses API with tool calls using Anthropic ==="
tools = [
{
type: "function",
name: "get_weather",
description: "Get current weather for a location",
parameters: {
type: "object",
properties: {
location: {
type: "string",
description: "City name"
}
},
required: ["location"]
}
}
]
response = client.responses.create(
parameters: {
model: "anthropic/claude-sonnet-4-5-20250929",
input: "What's the weather in Paris?",
tools: tools,
max_output_tokens: 200
}
)
puts "Tool call response: #{response.inspect}"
# Validate response structure
expect(response).to include('id')
expect(response).to include('output')
# Check for tool calls in output
output = response['output']
expect(output).to be_an(Array)
expect(output.length).to be > 0
# Validate usage with output_tokens_details
usage = response['usage']
expect(usage).not_to be_nil
expect(usage).to include('output_tokens_details')
output_tokens_details = usage['output_tokens_details']
expect(output_tokens_details).not_to be_nil,
"output_tokens_details should not be nil for tool call responses"
puts "✓ Tool call response validated with output_tokens_details"
puts "=== Tool calls test passed ==="
end
end
describe 'OpenAI model comparison' do
it 'should work the same way with OpenAI model' do
puts "\n=== Testing Responses API with OpenAI model for comparison ==="
response = client.responses.create(
parameters: {
model: "openai/gpt-4o-mini",
input: "Say hello in one word",
max_output_tokens: 50
}
)
puts "OpenAI Response: #{response.inspect}"
# Validate response structure matches Anthropic
expect(response).to include('id')
expect(response).to include('output')
expect(response).to include('usage')
usage = response['usage']
expect(usage).to include('output_tokens_details')
output_tokens_details = usage['output_tokens_details']
expect(output_tokens_details).not_to be_nil,
"OpenAI model should also have output_tokens_details"
puts "✓ OpenAI model response validated"
puts "=== OpenAI model test passed ==="
end
end
end

21
tests/responses_api_tests_proxy/ruby_tests/spec/spec_helper.rb Normal file
View file

@ -0,0 +1,21 @@
RSpec.configure do |config|
config.expect_with :rspec do |expectations|
expectations.include_chain_clauses_in_custom_matcher_descriptions = true
end
config.mock_with :rspec do |mocks|
mocks.verify_partial_doubles = true
end
config.shared_context_metadata_behavior = :apply_to_host_groups
config.filter_run_when_matching :focus
config.example_status_persistence_file_path = "spec/examples.txt"
config.disable_monkey_patching!
if config.files_to_run.one?
config.default_formatter = "doc"
end
config.order = :random
Kernel.srand config.seed
end