mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
test: refresh the suites that drifted from langfuse and OpenAI's retired Assistants API
Two unrelated causes, both leaving staging red with tests that no longer describe anything true. #38264 gave LangFuseLogger a langfuse_environment argument and started carrying it in the credentials dict. The handler test's fake logger did not accept the new keyword, so constructing it raised TypeError, and four cases in test_langfuse_unit_tests rebuilt the cache key by hand from three fields and missed on the four-field key production now writes. Caching itself was never broken: the handler sets and gets with the same dict. The fake now takes the argument and asserts it is forwarded, and the cache assertion issues a second identical request and expects the same logger back, which is the behaviour that matters and cannot rot the next time a credential field is added. OpenAI has retired the Assistants API. /v1/assistants and /v1/threads both answer 404 with a valid key, where every live route answers 401, so nothing calling them can pass again. test_custom_logger_passthrough covered generic passthrough logging and only used assistants because it is a route with no provider-specific handler; it moves to /v1/moderations, which is still unclaimed by _is_supported_openai_endpoint, so the same generic branch is exercised. The two tests there asserted the same thing against different dead routes, so they collapse into one. The Ruby suite existed solely to drive assistants, threads, messages and runs, so it goes along with the RVM and bundler steps that were installed only to run it, and the two dead OpenAI assistants cases leave test_openai_assistants_passthrough. The Azure assistants case in that file stays. Azure runs its own lifecycle and I could not reach the CI deployment to check whether that API is still there.
This commit is contained in:
Yuneng Jiang
parent
02c1c45b53
commit
feffb62266
8 changed files with 21 additions and 327 deletions
|
|
@ -2421,45 +2421,6 @@ jobs:
|
|||
- wait_for_service:
|
||||
url: http://localhost:4000
|
||||
timeout: "300"
|
||||
# Add Ruby installation and testing before the existing Node.js and Python tests
|
||||
- run:
|
||||
name: Install Ruby and Bundler
|
||||
command: |
|
||||
# Clone RVM at pinned tag and verify the commit SHA matches the
|
||||
# published tag before running its install script.
|
||||
RVM_VERSION="1.29.12"
|
||||
RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81"
|
||||
git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm
|
||||
RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)"
|
||||
if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then
|
||||
echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Import RVM signing keys (used by `rvm install` to verify Ruby tarballs)
|
||||
gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
|
||||
|
||||
# Install RVM from the verified checkout. The install script
|
||||
# sources `scripts/functions/installer` using paths relative to
|
||||
# its own working directory, so it must be run from /tmp/rvm.
|
||||
(cd /tmp/rvm && ./install --path "$HOME/.rvm")
|
||||
source "$HOME/.rvm/scripts/rvm"
|
||||
|
||||
# Install Ruby 3.2.2 (RVM verifies the tarball PGP signature)
|
||||
rvm install 3.2.2
|
||||
rvm use 3.2.2 --default
|
||||
|
||||
# Install latest Bundler
|
||||
gem install bundler
|
||||
|
||||
- run:
|
||||
name: Run Ruby tests
|
||||
command: |
|
||||
source $HOME/.rvm/scripts/rvm
|
||||
cd tests/pass_through_tests/ruby_passthrough_tests
|
||||
bundle install
|
||||
bundle exec rspec
|
||||
no_output_timeout: 30m
|
||||
# Install Node.js directly from nodejs.org with SHA256 verification,
|
||||
# instead of piping NodeSource's setup_24.x apt-repo installer into
|
||||
# sudo bash (which runs a mutable upstream script unattended).
|
||||
|
|
|
|||
|
|
@ -94,11 +94,13 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch):
|
|||
langfuse_public_key=None,
|
||||
langfuse_secret=None,
|
||||
langfuse_host=None,
|
||||
langfuse_environment=None,
|
||||
allow_env_credentials=True,
|
||||
):
|
||||
captured["langfuse_public_key"] = langfuse_public_key
|
||||
captured["langfuse_secret"] = langfuse_secret
|
||||
captured["langfuse_host"] = langfuse_host
|
||||
captured["langfuse_environment"] = langfuse_environment
|
||||
captured["allow_env_credentials"] = allow_env_credentials
|
||||
|
||||
class FakeDynamicLoggingCache:
|
||||
|
|
@ -117,6 +119,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch):
|
|||
"langfuse_public_key": "dynamic-public",
|
||||
"langfuse_secret_key": "dynamic-secret",
|
||||
"langfuse_host": "https://langfuse.example",
|
||||
"langfuse_environment": "dynamic-environment",
|
||||
},
|
||||
in_memory_dynamic_logger_cache=FakeDynamicLoggingCache(),
|
||||
)
|
||||
|
|
@ -124,6 +127,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch):
|
|||
assert captured["langfuse_public_key"] == "dynamic-public"
|
||||
assert captured["langfuse_secret"] == "dynamic-secret"
|
||||
assert captured["langfuse_host"] == "https://langfuse.example"
|
||||
assert captured["langfuse_environment"] == "dynamic-environment"
|
||||
assert captured["allow_env_credentials"] is False
|
||||
assert captured["cached_service_name"] == "langfuse"
|
||||
assert captured["cached_logging_obj"] is logger
|
||||
|
|
|
|||
|
|
@ -123,16 +123,12 @@ def test_get_langfuse_logger_for_request_with_dynamic_params(
|
|||
assert result.secret_key == "test_secret"
|
||||
assert result.langfuse_host == "https://test.langfuse.com"
|
||||
|
||||
# Check if the logger is cached
|
||||
cached_logger = dynamic_logging_cache.get_cache(
|
||||
credentials={
|
||||
"langfuse_public_key": "test_public_key",
|
||||
"langfuse_secret": "test_secret",
|
||||
"langfuse_host": "https://test.langfuse.com",
|
||||
},
|
||||
service_name="langfuse",
|
||||
logger_for_identical_repeat_request = LangFuseHandler.get_langfuse_logger_for_request(
|
||||
standard_callback_dynamic_params=standard_params,
|
||||
in_memory_dynamic_logger_cache=dynamic_logging_cache,
|
||||
globalLangfuseLogger=globalLangfuseLogger,
|
||||
)
|
||||
assert cached_logger is result
|
||||
assert logger_for_identical_repeat_request is result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("globalLangfuseLogger", [None, global_langfuse_logger])
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
source 'https://rubygems.org'
|
||||
|
||||
gem 'rspec'
|
||||
gem 'ruby-openai'
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
base64 (0.2.0)
|
||||
diff-lcs (1.6.0)
|
||||
event_stream_parser (1.0.0)
|
||||
faraday (2.8.1)
|
||||
base64
|
||||
faraday-net_http (>= 2.0, < 3.1)
|
||||
ruby2_keywords (>= 0.0.4)
|
||||
faraday-multipart (1.1.0)
|
||||
multipart-post (~> 2.0)
|
||||
faraday-net_http (3.0.2)
|
||||
multipart-post (2.4.1)
|
||||
rspec (3.13.0)
|
||||
rspec-core (~> 3.13.0)
|
||||
rspec-expectations (~> 3.13.0)
|
||||
rspec-mocks (~> 3.13.0)
|
||||
rspec-core (3.13.3)
|
||||
rspec-support (~> 3.13.0)
|
||||
rspec-expectations (3.13.3)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.13.0)
|
||||
rspec-mocks (3.13.2)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.13.0)
|
||||
rspec-support (3.13.2)
|
||||
ruby-openai (7.4.0)
|
||||
event_stream_parser (>= 0.3.0, < 2.0.0)
|
||||
faraday (>= 1)
|
||||
faraday-multipart (>= 1)
|
||||
ruby2_keywords (0.0.5)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
|
||||
DEPENDENCIES
|
||||
rspec
|
||||
ruby-openai
|
||||
|
||||
BUNDLED WITH
|
||||
2.6.5
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
require 'openai'
|
||||
require 'rspec'
|
||||
|
||||
RSpec.describe 'OpenAI Assistants Passthrough' do
|
||||
let(:client) do
|
||||
OpenAI::Client.new(
|
||||
access_token: "sk-1234",
|
||||
uri_base: "http://0.0.0.0:4000/openai",
|
||||
request_timeout: 600
|
||||
)
|
||||
end
|
||||
|
||||
|
||||
it 'performs basic assistant operations' do
|
||||
assistant = client.assistants.create(
|
||||
parameters: {
|
||||
name: "Math Tutor",
|
||||
instructions: "You are a personal math tutor. Write and run code to answer math questions.",
|
||||
tools: [{ type: "code_interpreter" }],
|
||||
model: "gpt-4o"
|
||||
}
|
||||
)
|
||||
expect(assistant).to include('id')
|
||||
expect(assistant['name']).to eq("Math Tutor")
|
||||
|
||||
assistants_list = client.assistants.list
|
||||
expect(assistants_list['data']).to be_an(Array)
|
||||
expect(assistants_list['data']).to include(include('id' => assistant['id']))
|
||||
|
||||
retrieved_assistant = client.assistants.retrieve(id: assistant['id'])
|
||||
expect(retrieved_assistant).to eq(assistant)
|
||||
|
||||
deleted_assistant = client.assistants.delete(id: assistant['id'])
|
||||
expect(deleted_assistant['deleted']).to be true
|
||||
expect(deleted_assistant['id']).to eq(assistant['id'])
|
||||
end
|
||||
|
||||
it 'performs streaming assistant operations' do
|
||||
puts "\n=== Starting Streaming Assistant Test ==="
|
||||
|
||||
assistant = client.assistants.create(
|
||||
parameters: {
|
||||
name: "Math Tutor",
|
||||
instructions: "You are a personal math tutor. Write and run code to answer math questions.",
|
||||
tools: [{ type: "code_interpreter" }],
|
||||
model: "gpt-4o"
|
||||
}
|
||||
)
|
||||
puts "Created assistant: #{assistant['id']}"
|
||||
expect(assistant).to include('id')
|
||||
|
||||
thread = client.threads.create
|
||||
puts "Created thread: #{thread['id']}"
|
||||
expect(thread).to include('id')
|
||||
|
||||
message = client.messages.create(
|
||||
thread_id: thread['id'],
|
||||
parameters: {
|
||||
role: "user",
|
||||
content: "I need to solve the equation `3x + 11 = 14`. Can you help me?"
|
||||
}
|
||||
)
|
||||
puts "Created message: #{message['id']}"
|
||||
puts "User question: #{message['content']}"
|
||||
expect(message).to include('id')
|
||||
expect(message['role']).to eq('user')
|
||||
|
||||
puts "\nStarting streaming response:"
|
||||
puts "------------------------"
|
||||
run = client.runs.create(
|
||||
thread_id: thread['id'],
|
||||
parameters: {
|
||||
assistant_id: assistant['id'],
|
||||
max_prompt_tokens: 256,
|
||||
max_completion_tokens: 16,
|
||||
stream: proc do |chunk, _bytesize|
|
||||
puts "Received chunk: #{chunk.inspect}" # Debug: Print raw chunk
|
||||
if chunk["object"] == "thread.message.delta"
|
||||
content = chunk.dig("delta", "content")
|
||||
puts "Content: #{content.inspect}" # Debug: Print content structure
|
||||
if content && content[0] && content[0]["text"]
|
||||
print content[0]["text"]["value"]
|
||||
$stdout.flush # Ensure output is printed immediately
|
||||
end
|
||||
end
|
||||
end
|
||||
}
|
||||
)
|
||||
puts "\n------------------------"
|
||||
puts "Run completed: #{run['id']}"
|
||||
expect(run).not_to be_nil
|
||||
ensure
|
||||
client.assistants.delete(id: assistant['id']) if assistant && assistant['id']
|
||||
client.threads.delete(id: thread['id']) if thread && thread['id']
|
||||
end
|
||||
end
|
||||
|
|
@ -1,7 +1,4 @@
|
|||
import pytest
|
||||
import openai
|
||||
import aiohttp
|
||||
import asyncio
|
||||
import tempfile
|
||||
from typing_extensions import override
|
||||
from openai import AssistantEventHandler
|
||||
|
|
@ -30,22 +27,6 @@ def test_pass_through_file_operations():
|
|||
print("file deleted", delete_file)
|
||||
|
||||
|
||||
def test_openai_assistants_e2e_operations():
|
||||
assistant = client.beta.assistants.create(
|
||||
name="Math Tutor",
|
||||
instructions="You are a personal math tutor. Write and run code to answer math questions.",
|
||||
tools=[{"type": "code_interpreter"}],
|
||||
model="gpt-4o",
|
||||
)
|
||||
print("assistant created", assistant)
|
||||
|
||||
get_assistant = client.beta.assistants.retrieve(assistant.id)
|
||||
print(get_assistant)
|
||||
|
||||
delete_assistant = client.beta.assistants.delete(assistant.id)
|
||||
print(delete_assistant)
|
||||
|
||||
|
||||
class EventHandler(AssistantEventHandler):
|
||||
@override
|
||||
def on_text_created(self, text) -> None:
|
||||
|
|
@ -69,39 +50,6 @@ class EventHandler(AssistantEventHandler):
|
|||
print(f"\n{output.logs}", flush=True)
|
||||
|
||||
|
||||
def test_openai_assistants_e2e_operations_stream():
|
||||
|
||||
assistant = client.beta.assistants.create(
|
||||
name="Math Tutor",
|
||||
instructions="You are a personal math tutor. Write and run code to answer math questions.",
|
||||
tools=[{"type": "code_interpreter"}],
|
||||
model="gpt-4o",
|
||||
)
|
||||
print("assistant created", assistant)
|
||||
|
||||
thread = client.beta.threads.create()
|
||||
print("thread created", thread)
|
||||
|
||||
message = client.beta.threads.messages.create(
|
||||
thread_id=thread.id,
|
||||
role="user",
|
||||
content="I need to solve the equation `3x + 11 = 14`. Can you help me?",
|
||||
)
|
||||
print("message created", message)
|
||||
|
||||
# Then, we use the `stream` SDK helper
|
||||
# with the `EventHandler` class to create the Run
|
||||
# and stream the response.
|
||||
|
||||
with client.beta.threads.runs.stream(
|
||||
thread_id=thread.id,
|
||||
assistant_id=assistant.id,
|
||||
instructions="Please address the user as Jane Doe. The user has a premium account.",
|
||||
event_handler=EventHandler(),
|
||||
) as stream:
|
||||
stream.until_done()
|
||||
|
||||
|
||||
def test_azure_openai_assistants_e2e_operations_stream():
|
||||
from openai import AzureOpenAI
|
||||
|
||||
|
|
|
|||
|
|
@ -1,7 +1,5 @@
|
|||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock, Mock, patch, MagicMock
|
||||
from typing import Optional
|
||||
from fastapi import Request
|
||||
import pytest
|
||||
|
|
@ -32,16 +30,16 @@ class TestCustomLogger(CustomLogger):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assistants_passthrough_logging():
|
||||
async def test_untracked_openai_route_passthrough_logging():
|
||||
"""Keep this on a route `_is_supported_openai_endpoint` does not claim, or the
|
||||
OpenAI-specific handler takes over and the generic payload stops being exercised."""
|
||||
test_custom_logger = TestCustomLogger()
|
||||
litellm._async_success_callback = [test_custom_logger]
|
||||
|
||||
TARGET_URL = "https://api.openai.com/v1/assistants"
|
||||
TARGET_URL = "https://api.openai.com/v1/moderations"
|
||||
REQUEST_BODY = {
|
||||
"instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
|
||||
"name": "Math Tutor",
|
||||
"tools": [{"type": "code_interpreter"}],
|
||||
"model": "gpt-4.1-mini",
|
||||
"model": "omni-moderation-latest",
|
||||
"input": "I want to bake a cake for my friend's birthday.",
|
||||
}
|
||||
TARGET_METHOD = "POST"
|
||||
|
||||
|
|
@ -50,7 +48,7 @@ async def test_assistants_passthrough_logging():
|
|||
scope={
|
||||
"type": "http",
|
||||
"method": TARGET_METHOD,
|
||||
"path": "/v1/assistants",
|
||||
"path": "/v1/moderations",
|
||||
"query_string": b"",
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
|
|
@ -58,7 +56,6 @@ async def test_assistants_passthrough_logging():
|
|||
b"authorization",
|
||||
f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(),
|
||||
),
|
||||
(b"openai-beta", b"assistants=v2"),
|
||||
],
|
||||
},
|
||||
),
|
||||
|
|
@ -66,7 +63,6 @@ async def test_assistants_passthrough_logging():
|
|||
custom_headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}",
|
||||
"OpenAI-Beta": "assistants=v2",
|
||||
},
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="test",
|
||||
|
|
@ -83,6 +79,8 @@ async def test_assistants_passthrough_logging():
|
|||
print("result status code", result.status_code)
|
||||
print("result content", result.body)
|
||||
|
||||
assert result.status_code == 200
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
assert test_custom_logger.logged_kwargs is not None
|
||||
|
|
@ -92,79 +90,8 @@ async def test_assistants_passthrough_logging():
|
|||
assert passthrough_logging_payload is not None
|
||||
assert passthrough_logging_payload["url"] == TARGET_URL
|
||||
assert passthrough_logging_payload["request_body"] == REQUEST_BODY
|
||||
|
||||
# assert that the response body content matches the response body content
|
||||
client_facing_response_body = json.loads(result.body)
|
||||
assert passthrough_logging_payload["response_body"] == client_facing_response_body
|
||||
|
||||
# assert that the request method is correct
|
||||
assert passthrough_logging_payload["request_method"] == TARGET_METHOD
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_threads_passthrough_logging():
|
||||
test_custom_logger = TestCustomLogger()
|
||||
litellm._async_success_callback = [test_custom_logger]
|
||||
|
||||
TARGET_URL = "https://api.openai.com/v1/threads"
|
||||
REQUEST_BODY = {}
|
||||
TARGET_METHOD = "POST"
|
||||
|
||||
result = await pass_through_request(
|
||||
request=Request(
|
||||
scope={
|
||||
"type": "http",
|
||||
"method": TARGET_METHOD,
|
||||
"path": "/v1/threads",
|
||||
"query_string": b"",
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(
|
||||
b"authorization",
|
||||
f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(),
|
||||
),
|
||||
(b"openai-beta", b"assistants=v2"),
|
||||
],
|
||||
},
|
||||
),
|
||||
target=TARGET_URL,
|
||||
custom_headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}",
|
||||
"OpenAI-Beta": "assistants=v2",
|
||||
},
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="test",
|
||||
user_id="test",
|
||||
team_id="test",
|
||||
end_user_id="test",
|
||||
),
|
||||
custom_body=REQUEST_BODY,
|
||||
forward_headers=False,
|
||||
merge_query_params=False,
|
||||
)
|
||||
|
||||
print("got result", result)
|
||||
print("result status code", result.status_code)
|
||||
print("result content", result.body)
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
assert test_custom_logger.logged_kwargs is not None
|
||||
passthrough_logging_payload = test_custom_logger.logged_kwargs[
|
||||
"passthrough_logging_payload"
|
||||
]
|
||||
assert passthrough_logging_payload is not None
|
||||
|
||||
# Fix for TypedDict access errors
|
||||
assert passthrough_logging_payload.get("url") == TARGET_URL
|
||||
assert passthrough_logging_payload.get("request_body") == REQUEST_BODY
|
||||
|
||||
# Fix for json.loads error with potential memoryview
|
||||
response_body = result.body
|
||||
client_facing_response_body = json.loads(response_body)
|
||||
|
||||
assert (
|
||||
passthrough_logging_payload.get("response_body") == client_facing_response_body
|
||||
)
|
||||
assert passthrough_logging_payload.get("request_method") == TARGET_METHOD
|
||||
client_facing_response_body = json.loads(result.body)
|
||||
assert client_facing_response_body["results"]
|
||||
assert passthrough_logging_payload["response_body"] == client_facing_response_body
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue