mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
Merge pull request #38637 from BerriAI/litellm_/red-tests-review-fbb718
test: refresh the suites that drifted from langfuse and OpenAI's retired Assistants API
This commit is contained in:
yuneng-jiang
GitHub
commit
a4a0386717
13 changed files with 165 additions and 494 deletions
|
|
@ -2421,45 +2421,6 @@ jobs:
|
|||
- wait_for_service:
|
||||
url: http://localhost:4000
|
||||
timeout: "300"
|
||||
# Add Ruby installation and testing before the existing Node.js and Python tests
|
||||
- run:
|
||||
name: Install Ruby and Bundler
|
||||
command: |
|
||||
# Clone RVM at pinned tag and verify the commit SHA matches the
|
||||
# published tag before running its install script.
|
||||
RVM_VERSION="1.29.12"
|
||||
RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81"
|
||||
git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm
|
||||
RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)"
|
||||
if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then
|
||||
echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Import RVM signing keys (used by `rvm install` to verify Ruby tarballs)
|
||||
gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB
|
||||
|
||||
# Install RVM from the verified checkout. The install script
|
||||
# sources `scripts/functions/installer` using paths relative to
|
||||
# its own working directory, so it must be run from /tmp/rvm.
|
||||
(cd /tmp/rvm && ./install --path "$HOME/.rvm")
|
||||
source "$HOME/.rvm/scripts/rvm"
|
||||
|
||||
# Install Ruby 3.2.2 (RVM verifies the tarball PGP signature)
|
||||
rvm install 3.2.2
|
||||
rvm use 3.2.2 --default
|
||||
|
||||
# Install latest Bundler
|
||||
gem install bundler
|
||||
|
||||
- run:
|
||||
name: Run Ruby tests
|
||||
command: |
|
||||
source $HOME/.rvm/scripts/rvm
|
||||
cd tests/pass_through_tests/ruby_passthrough_tests
|
||||
bundle install
|
||||
bundle exec rspec
|
||||
no_output_timeout: 30m
|
||||
# Install Node.js directly from nodejs.org with SHA256 verification,
|
||||
# instead of piping NodeSource's setup_24.x apt-repo installer into
|
||||
# sudo bash (which runs a mutable upstream script unattended).
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"TQ001": {
|
||||
"limit": 744
|
||||
"limit": 736
|
||||
},
|
||||
"TQ002": {
|
||||
"limit": 742
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
"limit": 469
|
||||
},
|
||||
"TQ005": {
|
||||
"limit": 2405
|
||||
"limit": 2399
|
||||
},
|
||||
"TQ006": {
|
||||
"limit": 34
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import { expect, test, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { expect, test, type Locator, type Page as PlaywrightPage } from "@playwright/test";
|
||||
import { ADMIN_STORAGE_PATH } from "../../constants";
|
||||
import { navigateToPage } from "../../helpers/navigation";
|
||||
import { Page } from "../../fixtures/pages";
|
||||
|
|
@ -17,43 +17,49 @@ async function openTemplateSelect(page: PlaywrightPage) {
|
|||
return trigger;
|
||||
}
|
||||
|
||||
function pollPixelsBelowTrigger(trigger: Locator, popup: Locator) {
|
||||
return expect.poll(async () => {
|
||||
const triggerBox = await trigger.boundingBox();
|
||||
const popupBox = await popup.boundingBox();
|
||||
if (!triggerBox || !popupBox) return null;
|
||||
return popupBox.y - (triggerBox.y + triggerBox.height);
|
||||
});
|
||||
}
|
||||
|
||||
function pollPopupOverlapsTrigger(trigger: Locator, popup: Locator) {
|
||||
return expect.poll(async () => {
|
||||
const triggerBox = await trigger.boundingBox();
|
||||
const popupBox = await popup.boundingBox();
|
||||
if (!triggerBox || !popupBox) return null;
|
||||
return popupBox.y < triggerBox.y + triggerBox.height && popupBox.y + popupBox.height > triggerBox.y;
|
||||
});
|
||||
}
|
||||
|
||||
test.describe("Auto Router template select anchoring", () => {
|
||||
test.use({ storageState: ADMIN_STORAGE_PATH });
|
||||
|
||||
test("opens the options below the trigger rather than over it", async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1280, height: 900 });
|
||||
const trigger = await openTemplateSelect(page);
|
||||
const triggerBox = await trigger.boundingBox();
|
||||
|
||||
await trigger.click();
|
||||
const popup = page.locator('[data-slot="select-content"]');
|
||||
await expect(popup).toBeVisible();
|
||||
const popupBox = await popup.boundingBox();
|
||||
|
||||
expect(triggerBox).not.toBeNull();
|
||||
expect(popupBox).not.toBeNull();
|
||||
|
||||
// Item-aligned mode reports "none" and puts the active item over the trigger.
|
||||
await expect(popup).toHaveAttribute("data-side", "bottom");
|
||||
expect(popupBox!.y).toBeGreaterThanOrEqual(triggerBox!.y + triggerBox!.height);
|
||||
await pollPixelsBelowTrigger(trigger, popup).toBeGreaterThanOrEqual(0);
|
||||
});
|
||||
|
||||
test("flips above the trigger instead of covering it when there is no room below", async ({ page }) => {
|
||||
await page.setViewportSize({ width: 1280, height: 560 });
|
||||
const trigger = await openTemplateSelect(page);
|
||||
await trigger.scrollIntoViewIfNeeded();
|
||||
const triggerBox = await trigger.boundingBox();
|
||||
|
||||
await trigger.click();
|
||||
const popup = page.locator('[data-slot="select-content"]');
|
||||
await expect(popup).toBeVisible();
|
||||
const popupBox = await popup.boundingBox();
|
||||
|
||||
expect(triggerBox).not.toBeNull();
|
||||
expect(popupBox).not.toBeNull();
|
||||
|
||||
const overlaps =
|
||||
popupBox!.y < triggerBox!.y + triggerBox!.height && popupBox!.y + popupBox!.height > triggerBox!.y;
|
||||
expect(overlaps).toBe(false);
|
||||
await pollPopupOverlapsTrigger(trigger, popup).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -19,32 +19,32 @@ from litellm import (
|
|||
# litellm.set_verbose=True
|
||||
|
||||
|
||||
TOLERATED_UPSTREAM_FAILURES = (Timeout, litellm.InternalServerError)
|
||||
|
||||
|
||||
def test_batch_completions():
|
||||
messages = [[{"role": "user", "content": "write a short poem"}] for _ in range(3)]
|
||||
model = "gpt-3.5-turbo"
|
||||
litellm.set_verbose = True
|
||||
try:
|
||||
result = batch_completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.2,
|
||||
request_timeout=1,
|
||||
)
|
||||
print(result)
|
||||
print(len(result))
|
||||
assert len(result) == 3
|
||||
|
||||
for response in result:
|
||||
assert response.choices[0].message.content is not None
|
||||
except Timeout as e:
|
||||
print(f"IN TIMEOUT")
|
||||
pass
|
||||
except litellm.InternalServerError as e:
|
||||
print(f"IN INTERNAL SERVER ERROR")
|
||||
pass
|
||||
except Exception as e:
|
||||
pytest.fail(f"An error occurred: {e}")
|
||||
result = batch_completion(
|
||||
model=model,
|
||||
messages=messages,
|
||||
max_tokens=10,
|
||||
temperature=0.2,
|
||||
request_timeout=1,
|
||||
)
|
||||
print(result)
|
||||
|
||||
assert len(result) == 3
|
||||
|
||||
for response in result:
|
||||
if isinstance(response, TOLERATED_UPSTREAM_FAILURES):
|
||||
continue
|
||||
assert not isinstance(
|
||||
response, Exception
|
||||
), f"batch_completion returned {type(response).__name__}: {response}"
|
||||
assert response.choices[0].message.content is not None
|
||||
|
||||
|
||||
# test_batch_completions()
|
||||
|
|
|
|||
|
|
@ -16,6 +16,8 @@ verbose_logger.setLevel(logging.DEBUG)
|
|||
litellm.set_verbose = True
|
||||
import time
|
||||
|
||||
INTERVAL_TOO_LONG_TO_FIRE_DURING_THIS_TEST = 3600
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opik_logging_http_request():
|
||||
|
|
@ -23,70 +25,60 @@ async def test_opik_logging_http_request():
|
|||
- Test that HTTP requests are made to Opik
|
||||
- Traces and spans are batched correctly
|
||||
"""
|
||||
try:
|
||||
from litellm.integrations.opik.opik import OpikLogger
|
||||
from litellm.integrations.opik.opik import OpikLogger
|
||||
|
||||
os.environ["OPIK_URL_OVERRIDE"] = "https://fake.comet.com/opik/api"
|
||||
os.environ["OPIK_API_KEY"] = "anything"
|
||||
os.environ["OPIK_WORKSPACE"] = "anything"
|
||||
os.environ["OPIK_URL_OVERRIDE"] = "https://fake.comet.com/opik/api"
|
||||
os.environ["OPIK_API_KEY"] = "anything"
|
||||
os.environ["OPIK_WORKSPACE"] = "anything"
|
||||
|
||||
# Initialize OpikLogger
|
||||
test_opik_logger = OpikLogger()
|
||||
test_opik_logger = OpikLogger()
|
||||
test_opik_logger.flush_interval = INTERVAL_TOO_LONG_TO_FIRE_DURING_THIS_TEST
|
||||
test_opik_logger.batch_size = 12
|
||||
|
||||
litellm.callbacks = [test_opik_logger]
|
||||
test_opik_logger.batch_size = 12
|
||||
litellm.set_verbose = True
|
||||
litellm.callbacks = [test_opik_logger]
|
||||
|
||||
# Create a mock for the async_client's post method
|
||||
mock_post = AsyncMock()
|
||||
mock_post.return_value.status_code = 202
|
||||
mock_post.return_value.text = "Accepted"
|
||||
test_opik_logger.async_httpx_client.post = mock_post
|
||||
mock_post = AsyncMock(return_value=Mock(status_code=202, text="Accepted"))
|
||||
test_opik_logger.async_httpx_client.post = mock_post
|
||||
|
||||
# Make multiple calls to ensure we don't hit the batch size
|
||||
for _ in range(5):
|
||||
response = await litellm.acompletion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Test message"}],
|
||||
max_tokens=10,
|
||||
temperature=0.2,
|
||||
mock_response="This is a mock response",
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
def opik_batch_calls():
|
||||
return [
|
||||
call
|
||||
for call in mock_post.call_args_list
|
||||
if "/traces/batch" in str(call) or "/spans/batch" in str(call)
|
||||
]
|
||||
|
||||
# Check batching of events and that the queue contains 5 trace events and 5 span events
|
||||
assert (
|
||||
mock_post.called == False
|
||||
), "HTTP request was made but events should have been batched"
|
||||
assert len(test_opik_logger.log_queue) == 10
|
||||
for _ in range(5):
|
||||
await litellm.acompletion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Test message"}],
|
||||
max_tokens=10,
|
||||
temperature=0.2,
|
||||
mock_response="This is a mock response",
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Now make calls to exceed the batch size
|
||||
for _ in range(3):
|
||||
response = await litellm.acompletion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Test message"}],
|
||||
max_tokens=10,
|
||||
temperature=0.2,
|
||||
mock_response="This is a mock response",
|
||||
)
|
||||
assert opik_batch_calls() == [], "events below batch_size must stay queued"
|
||||
assert len(test_opik_logger.log_queue) == 10
|
||||
|
||||
# Wait a short time for any asynchronous operations to complete
|
||||
await asyncio.sleep(1)
|
||||
for _ in range(3):
|
||||
await litellm.acompletion(
|
||||
model="gpt-3.5-turbo",
|
||||
messages=[{"role": "user", "content": "Test message"}],
|
||||
max_tokens=10,
|
||||
temperature=0.2,
|
||||
mock_response="This is a mock response",
|
||||
)
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# Check that the queue was flushed after exceeding batch size
|
||||
assert len(test_opik_logger.log_queue) < test_opik_logger.batch_size
|
||||
assert opik_batch_calls(), "crossing batch_size must flush the queue"
|
||||
events_left_over_after_the_size_triggered_flush = len(test_opik_logger.log_queue)
|
||||
assert 0 < events_left_over_after_the_size_triggered_flush < test_opik_logger.batch_size
|
||||
|
||||
# Check that the data has been sent when it goes above the flush interval
|
||||
await asyncio.sleep(test_opik_logger.flush_interval)
|
||||
assert len(test_opik_logger.log_queue) == 0
|
||||
calls_before_periodic_flush = len(opik_batch_calls())
|
||||
await test_opik_logger.flush_queue()
|
||||
|
||||
# Clean up
|
||||
for cb in litellm.callbacks:
|
||||
if isinstance(cb, OpikLogger):
|
||||
await cb.async_httpx_client.client.aclose()
|
||||
|
||||
except Exception as e:
|
||||
pytest.fail(f"Error occurred: {e}")
|
||||
assert len(opik_batch_calls()) > calls_before_periodic_flush
|
||||
assert len(test_opik_logger.log_queue) == 0
|
||||
|
||||
|
||||
def test_sync_opik_logging_http_request():
|
||||
|
|
|
|||
|
|
@ -94,11 +94,13 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch):
|
|||
langfuse_public_key=None,
|
||||
langfuse_secret=None,
|
||||
langfuse_host=None,
|
||||
langfuse_environment=None,
|
||||
allow_env_credentials=True,
|
||||
):
|
||||
captured["langfuse_public_key"] = langfuse_public_key
|
||||
captured["langfuse_secret"] = langfuse_secret
|
||||
captured["langfuse_host"] = langfuse_host
|
||||
captured["langfuse_environment"] = langfuse_environment
|
||||
captured["allow_env_credentials"] = allow_env_credentials
|
||||
|
||||
class FakeDynamicLoggingCache:
|
||||
|
|
@ -117,6 +119,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch):
|
|||
"langfuse_public_key": "dynamic-public",
|
||||
"langfuse_secret_key": "dynamic-secret",
|
||||
"langfuse_host": "https://langfuse.example",
|
||||
"langfuse_environment": "dynamic-environment",
|
||||
},
|
||||
in_memory_dynamic_logger_cache=FakeDynamicLoggingCache(),
|
||||
)
|
||||
|
|
@ -124,6 +127,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch):
|
|||
assert captured["langfuse_public_key"] == "dynamic-public"
|
||||
assert captured["langfuse_secret"] == "dynamic-secret"
|
||||
assert captured["langfuse_host"] == "https://langfuse.example"
|
||||
assert captured["langfuse_environment"] == "dynamic-environment"
|
||||
assert captured["allow_env_credentials"] is False
|
||||
assert captured["cached_service_name"] == "langfuse"
|
||||
assert captured["cached_logging_obj"] is logger
|
||||
|
|
|
|||
|
|
@ -123,16 +123,12 @@ def test_get_langfuse_logger_for_request_with_dynamic_params(
|
|||
assert result.secret_key == "test_secret"
|
||||
assert result.langfuse_host == "https://test.langfuse.com"
|
||||
|
||||
# Check if the logger is cached
|
||||
cached_logger = dynamic_logging_cache.get_cache(
|
||||
credentials={
|
||||
"langfuse_public_key": "test_public_key",
|
||||
"langfuse_secret": "test_secret",
|
||||
"langfuse_host": "https://test.langfuse.com",
|
||||
},
|
||||
service_name="langfuse",
|
||||
logger_for_identical_repeat_request = LangFuseHandler.get_langfuse_logger_for_request(
|
||||
standard_callback_dynamic_params=standard_params,
|
||||
in_memory_dynamic_logger_cache=dynamic_logging_cache,
|
||||
globalLangfuseLogger=globalLangfuseLogger,
|
||||
)
|
||||
assert cached_logger is result
|
||||
assert logger_for_identical_repeat_request is result
|
||||
|
||||
|
||||
@pytest.mark.parametrize("globalLangfuseLogger", [None, global_langfuse_logger])
|
||||
|
|
|
|||
|
|
@ -1,4 +0,0 @@
|
|||
source 'https://rubygems.org'
|
||||
|
||||
gem 'rspec'
|
||||
gem 'ruby-openai'
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
base64 (0.2.0)
|
||||
diff-lcs (1.6.0)
|
||||
event_stream_parser (1.0.0)
|
||||
faraday (2.8.1)
|
||||
base64
|
||||
faraday-net_http (>= 2.0, < 3.1)
|
||||
ruby2_keywords (>= 0.0.4)
|
||||
faraday-multipart (1.1.0)
|
||||
multipart-post (~> 2.0)
|
||||
faraday-net_http (3.0.2)
|
||||
multipart-post (2.4.1)
|
||||
rspec (3.13.0)
|
||||
rspec-core (~> 3.13.0)
|
||||
rspec-expectations (~> 3.13.0)
|
||||
rspec-mocks (~> 3.13.0)
|
||||
rspec-core (3.13.3)
|
||||
rspec-support (~> 3.13.0)
|
||||
rspec-expectations (3.13.3)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.13.0)
|
||||
rspec-mocks (3.13.2)
|
||||
diff-lcs (>= 1.2.0, < 2.0)
|
||||
rspec-support (~> 3.13.0)
|
||||
rspec-support (3.13.2)
|
||||
ruby-openai (7.4.0)
|
||||
event_stream_parser (>= 0.3.0, < 2.0.0)
|
||||
faraday (>= 1)
|
||||
faraday-multipart (>= 1)
|
||||
ruby2_keywords (0.0.5)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
|
||||
DEPENDENCIES
|
||||
rspec
|
||||
ruby-openai
|
||||
|
||||
BUNDLED WITH
|
||||
2.6.5
|
||||
|
|
@ -1,96 +0,0 @@
|
|||
require 'openai'
|
||||
require 'rspec'
|
||||
|
||||
RSpec.describe 'OpenAI Assistants Passthrough' do
|
||||
let(:client) do
|
||||
OpenAI::Client.new(
|
||||
access_token: "sk-1234",
|
||||
uri_base: "http://0.0.0.0:4000/openai",
|
||||
request_timeout: 600
|
||||
)
|
||||
end
|
||||
|
||||
|
||||
it 'performs basic assistant operations' do
|
||||
assistant = client.assistants.create(
|
||||
parameters: {
|
||||
name: "Math Tutor",
|
||||
instructions: "You are a personal math tutor. Write and run code to answer math questions.",
|
||||
tools: [{ type: "code_interpreter" }],
|
||||
model: "gpt-4o"
|
||||
}
|
||||
)
|
||||
expect(assistant).to include('id')
|
||||
expect(assistant['name']).to eq("Math Tutor")
|
||||
|
||||
assistants_list = client.assistants.list
|
||||
expect(assistants_list['data']).to be_an(Array)
|
||||
expect(assistants_list['data']).to include(include('id' => assistant['id']))
|
||||
|
||||
retrieved_assistant = client.assistants.retrieve(id: assistant['id'])
|
||||
expect(retrieved_assistant).to eq(assistant)
|
||||
|
||||
deleted_assistant = client.assistants.delete(id: assistant['id'])
|
||||
expect(deleted_assistant['deleted']).to be true
|
||||
expect(deleted_assistant['id']).to eq(assistant['id'])
|
||||
end
|
||||
|
||||
it 'performs streaming assistant operations' do
|
||||
puts "\n=== Starting Streaming Assistant Test ==="
|
||||
|
||||
assistant = client.assistants.create(
|
||||
parameters: {
|
||||
name: "Math Tutor",
|
||||
instructions: "You are a personal math tutor. Write and run code to answer math questions.",
|
||||
tools: [{ type: "code_interpreter" }],
|
||||
model: "gpt-4o"
|
||||
}
|
||||
)
|
||||
puts "Created assistant: #{assistant['id']}"
|
||||
expect(assistant).to include('id')
|
||||
|
||||
thread = client.threads.create
|
||||
puts "Created thread: #{thread['id']}"
|
||||
expect(thread).to include('id')
|
||||
|
||||
message = client.messages.create(
|
||||
thread_id: thread['id'],
|
||||
parameters: {
|
||||
role: "user",
|
||||
content: "I need to solve the equation `3x + 11 = 14`. Can you help me?"
|
||||
}
|
||||
)
|
||||
puts "Created message: #{message['id']}"
|
||||
puts "User question: #{message['content']}"
|
||||
expect(message).to include('id')
|
||||
expect(message['role']).to eq('user')
|
||||
|
||||
puts "\nStarting streaming response:"
|
||||
puts "------------------------"
|
||||
run = client.runs.create(
|
||||
thread_id: thread['id'],
|
||||
parameters: {
|
||||
assistant_id: assistant['id'],
|
||||
max_prompt_tokens: 256,
|
||||
max_completion_tokens: 16,
|
||||
stream: proc do |chunk, _bytesize|
|
||||
puts "Received chunk: #{chunk.inspect}" # Debug: Print raw chunk
|
||||
if chunk["object"] == "thread.message.delta"
|
||||
content = chunk.dig("delta", "content")
|
||||
puts "Content: #{content.inspect}" # Debug: Print content structure
|
||||
if content && content[0] && content[0]["text"]
|
||||
print content[0]["text"]["value"]
|
||||
$stdout.flush # Ensure output is printed immediately
|
||||
end
|
||||
end
|
||||
end
|
||||
}
|
||||
)
|
||||
puts "\n------------------------"
|
||||
puts "Run completed: #{run['id']}"
|
||||
expect(run).not_to be_nil
|
||||
ensure
|
||||
client.assistants.delete(id: assistant['id']) if assistant && assistant['id']
|
||||
client.threads.delete(id: thread['id']) if thread && thread['id']
|
||||
end
|
||||
end
|
||||
|
|
@ -1,141 +1,22 @@
|
|||
import pytest
|
||||
import openai
|
||||
import aiohttp
|
||||
import asyncio
|
||||
import tempfile
|
||||
from typing_extensions import override
|
||||
from openai import AssistantEventHandler
|
||||
|
||||
|
||||
client = openai.OpenAI(base_url="http://0.0.0.0:4000/openai", api_key="sk-1234")
|
||||
|
||||
|
||||
def test_pass_through_file_operations():
|
||||
# Create a temporary file
|
||||
with tempfile.NamedTemporaryFile(
|
||||
mode="w+", suffix=".txt", delete=False
|
||||
) as temp_file:
|
||||
temp_file.write("This is a test file for the OpenAI Assistants API.")
|
||||
temp_file.flush()
|
||||
|
||||
# create a file
|
||||
file = client.files.create(
|
||||
file=open(temp_file.name, "rb"),
|
||||
purpose="assistants",
|
||||
)
|
||||
print("file created", file)
|
||||
|
||||
# delete the file
|
||||
delete_file = client.files.delete(file.id)
|
||||
print("file deleted", delete_file)
|
||||
|
||||
|
||||
def test_openai_assistants_e2e_operations():
|
||||
assistant = client.beta.assistants.create(
|
||||
name="Math Tutor",
|
||||
instructions="You are a personal math tutor. Write and run code to answer math questions.",
|
||||
tools=[{"type": "code_interpreter"}],
|
||||
model="gpt-4o",
|
||||
)
|
||||
print("assistant created", assistant)
|
||||
|
||||
get_assistant = client.beta.assistants.retrieve(assistant.id)
|
||||
print(get_assistant)
|
||||
|
||||
delete_assistant = client.beta.assistants.delete(assistant.id)
|
||||
print(delete_assistant)
|
||||
|
||||
|
||||
class EventHandler(AssistantEventHandler):
|
||||
@override
|
||||
def on_text_created(self, text) -> None:
|
||||
print(f"\nassistant > ", end="", flush=True)
|
||||
|
||||
@override
|
||||
def on_text_delta(self, delta, snapshot):
|
||||
print(delta.value, end="", flush=True)
|
||||
|
||||
def on_tool_call_created(self, tool_call):
|
||||
print(f"\nassistant > {tool_call.type}\n", flush=True)
|
||||
|
||||
def on_tool_call_delta(self, delta, snapshot):
|
||||
if delta.type == "code_interpreter":
|
||||
if delta.code_interpreter.input:
|
||||
print(delta.code_interpreter.input, end="", flush=True)
|
||||
if delta.code_interpreter.outputs:
|
||||
print(f"\n\noutput >", flush=True)
|
||||
for output in delta.code_interpreter.outputs:
|
||||
if output.type == "logs":
|
||||
print(f"\n{output.logs}", flush=True)
|
||||
|
||||
|
||||
def test_openai_assistants_e2e_operations_stream():
|
||||
|
||||
assistant = client.beta.assistants.create(
|
||||
name="Math Tutor",
|
||||
instructions="You are a personal math tutor. Write and run code to answer math questions.",
|
||||
tools=[{"type": "code_interpreter"}],
|
||||
model="gpt-4o",
|
||||
)
|
||||
print("assistant created", assistant)
|
||||
|
||||
thread = client.beta.threads.create()
|
||||
print("thread created", thread)
|
||||
|
||||
message = client.beta.threads.messages.create(
|
||||
thread_id=thread.id,
|
||||
role="user",
|
||||
content="I need to solve the equation `3x + 11 = 14`. Can you help me?",
|
||||
)
|
||||
print("message created", message)
|
||||
|
||||
# Then, we use the `stream` SDK helper
|
||||
# with the `EventHandler` class to create the Run
|
||||
# and stream the response.
|
||||
|
||||
with client.beta.threads.runs.stream(
|
||||
thread_id=thread.id,
|
||||
assistant_id=assistant.id,
|
||||
instructions="Please address the user as Jane Doe. The user has a premium account.",
|
||||
event_handler=EventHandler(),
|
||||
) as stream:
|
||||
stream.until_done()
|
||||
|
||||
|
||||
def test_azure_openai_assistants_e2e_operations_stream():
|
||||
from openai import AzureOpenAI
|
||||
|
||||
client = AzureOpenAI(
|
||||
base_url="http://0.0.0.0:4000/azure-config-passthrough/openai",
|
||||
api_key="sk-1234",
|
||||
api_version="2025-01-01-preview",
|
||||
)
|
||||
assistant = client.beta.assistants.create(
|
||||
name="Math Tutor",
|
||||
instructions="You are a personal math tutor. Write and run code to answer math questions.",
|
||||
tools=[{"type": "code_interpreter"}],
|
||||
model="gpt-4o",
|
||||
)
|
||||
print("assistant created", assistant)
|
||||
|
||||
thread = client.beta.threads.create()
|
||||
print("thread created", thread)
|
||||
|
||||
message = client.beta.threads.messages.create(
|
||||
thread_id=thread.id,
|
||||
role="user",
|
||||
content="I need to solve the equation `3x + 11 = 14`. Can you help me?",
|
||||
)
|
||||
print("message created", message)
|
||||
|
||||
# Then, we use the `stream` SDK helper
|
||||
# with the `EventHandler` class to create the Run
|
||||
# and stream the response.
|
||||
|
||||
with client.beta.threads.runs.stream(
|
||||
thread_id=thread.id,
|
||||
assistant_id=assistant.id,
|
||||
instructions="Please address the user as Jane Doe. The user has a premium account.",
|
||||
event_handler=EventHandler(),
|
||||
) as stream:
|
||||
stream.until_done()
|
||||
|
|
|
|||
|
|
@ -1,7 +1,6 @@
|
|||
import json
|
||||
import os
|
||||
from datetime import datetime
|
||||
from unittest.mock import AsyncMock, Mock, patch, MagicMock
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Optional
|
||||
from fastapi import Request
|
||||
import pytest
|
||||
|
|
@ -31,17 +30,62 @@ class TestCustomLogger(CustomLogger):
|
|||
self.logged_kwargs = kwargs
|
||||
|
||||
|
||||
UPSTREAM_RESPONSE_BODY = {
|
||||
"id": "modr-abc123",
|
||||
"model": "omni-moderation-latest",
|
||||
"results": [
|
||||
{
|
||||
"flagged": False,
|
||||
"categories": {"violence": False},
|
||||
"category_scores": {"violence": 1.2e-06},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def upstream():
|
||||
received: dict = {}
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def do_POST(self):
|
||||
body = self.rfile.read(int(self.headers.get("content-length", 0) or 0))
|
||||
received["path"] = self.path
|
||||
received["body"] = json.loads(body or b"{}")
|
||||
payload = json.dumps(UPSTREAM_RESPONSE_BODY).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("content-type", "application/json")
|
||||
self.send_header("content-length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
try:
|
||||
yield f"http://127.0.0.1:{server.server_port}", received
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_assistants_passthrough_logging():
|
||||
async def test_passthrough_logging_payload_for_a_route_no_provider_handler_claims(
|
||||
upstream,
|
||||
):
|
||||
base_url, upstream_received = upstream
|
||||
|
||||
test_custom_logger = TestCustomLogger()
|
||||
litellm._async_success_callback = [test_custom_logger]
|
||||
|
||||
TARGET_URL = "https://api.openai.com/v1/assistants"
|
||||
TARGET_URL = f"{base_url}/v1/moderations"
|
||||
REQUEST_BODY = {
|
||||
"instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.",
|
||||
"name": "Math Tutor",
|
||||
"tools": [{"type": "code_interpreter"}],
|
||||
"model": "gpt-4.1-mini",
|
||||
"model": "omni-moderation-latest",
|
||||
"input": "I want to bake a cake for my friend's birthday.",
|
||||
}
|
||||
TARGET_METHOD = "POST"
|
||||
|
||||
|
|
@ -50,23 +94,18 @@ async def test_assistants_passthrough_logging():
|
|||
scope={
|
||||
"type": "http",
|
||||
"method": TARGET_METHOD,
|
||||
"path": "/v1/assistants",
|
||||
"path": "/v1/moderations",
|
||||
"query_string": b"",
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(
|
||||
b"authorization",
|
||||
f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(),
|
||||
),
|
||||
(b"openai-beta", b"assistants=v2"),
|
||||
(b"authorization", b"Bearer sk-test-passthrough"),
|
||||
],
|
||||
},
|
||||
),
|
||||
target=TARGET_URL,
|
||||
custom_headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}",
|
||||
"OpenAI-Beta": "assistants=v2",
|
||||
"Authorization": "Bearer sk-test-passthrough",
|
||||
},
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="test",
|
||||
|
|
@ -83,6 +122,10 @@ async def test_assistants_passthrough_logging():
|
|||
print("result status code", result.status_code)
|
||||
print("result content", result.body)
|
||||
|
||||
assert upstream_received.get("path") == "/v1/moderations"
|
||||
assert upstream_received.get("body") == REQUEST_BODY
|
||||
assert result.status_code == 200
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
assert test_custom_logger.logged_kwargs is not None
|
||||
|
|
@ -92,79 +135,8 @@ async def test_assistants_passthrough_logging():
|
|||
assert passthrough_logging_payload is not None
|
||||
assert passthrough_logging_payload["url"] == TARGET_URL
|
||||
assert passthrough_logging_payload["request_body"] == REQUEST_BODY
|
||||
|
||||
# assert that the response body content matches the response body content
|
||||
client_facing_response_body = json.loads(result.body)
|
||||
assert passthrough_logging_payload["response_body"] == client_facing_response_body
|
||||
|
||||
# assert that the request method is correct
|
||||
assert passthrough_logging_payload["request_method"] == TARGET_METHOD
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_threads_passthrough_logging():
|
||||
test_custom_logger = TestCustomLogger()
|
||||
litellm._async_success_callback = [test_custom_logger]
|
||||
|
||||
TARGET_URL = "https://api.openai.com/v1/threads"
|
||||
REQUEST_BODY = {}
|
||||
TARGET_METHOD = "POST"
|
||||
|
||||
result = await pass_through_request(
|
||||
request=Request(
|
||||
scope={
|
||||
"type": "http",
|
||||
"method": TARGET_METHOD,
|
||||
"path": "/v1/threads",
|
||||
"query_string": b"",
|
||||
"headers": [
|
||||
(b"content-type", b"application/json"),
|
||||
(
|
||||
b"authorization",
|
||||
f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(),
|
||||
),
|
||||
(b"openai-beta", b"assistants=v2"),
|
||||
],
|
||||
},
|
||||
),
|
||||
target=TARGET_URL,
|
||||
custom_headers={
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}",
|
||||
"OpenAI-Beta": "assistants=v2",
|
||||
},
|
||||
user_api_key_dict=UserAPIKeyAuth(
|
||||
api_key="test",
|
||||
user_id="test",
|
||||
team_id="test",
|
||||
end_user_id="test",
|
||||
),
|
||||
custom_body=REQUEST_BODY,
|
||||
forward_headers=False,
|
||||
merge_query_params=False,
|
||||
)
|
||||
|
||||
print("got result", result)
|
||||
print("result status code", result.status_code)
|
||||
print("result content", result.body)
|
||||
|
||||
await asyncio.sleep(1)
|
||||
|
||||
assert test_custom_logger.logged_kwargs is not None
|
||||
passthrough_logging_payload = test_custom_logger.logged_kwargs[
|
||||
"passthrough_logging_payload"
|
||||
]
|
||||
assert passthrough_logging_payload is not None
|
||||
|
||||
# Fix for TypedDict access errors
|
||||
assert passthrough_logging_payload.get("url") == TARGET_URL
|
||||
assert passthrough_logging_payload.get("request_body") == REQUEST_BODY
|
||||
|
||||
# Fix for json.loads error with potential memoryview
|
||||
response_body = result.body
|
||||
client_facing_response_body = json.loads(response_body)
|
||||
|
||||
assert (
|
||||
passthrough_logging_payload.get("response_body") == client_facing_response_body
|
||||
)
|
||||
assert passthrough_logging_payload.get("request_method") == TARGET_METHOD
|
||||
client_facing_response_body = json.loads(result.body)
|
||||
assert client_facing_response_body == UPSTREAM_RESPONSE_BODY
|
||||
assert passthrough_logging_payload["response_body"] == client_facing_response_body
|
||||
|
|
|
|||
|
|
@ -23,9 +23,10 @@ from litellm.proxy.management_helpers.access_group_team_sync import (
|
|||
sync_team_access_group_membership,
|
||||
)
|
||||
|
||||
TEAM = "ags-team-a"
|
||||
OTHER_TEAM = "ags-team-b"
|
||||
GROUPS = ("ags-group-1", "ags-group-2", "ags-group-3")
|
||||
_XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "master")
|
||||
TEAM = f"ags-team-a-{_XDIST_WORKER}"
|
||||
OTHER_TEAM = f"ags-team-b-{_XDIST_WORKER}"
|
||||
GROUPS = tuple(f"ags-group-{n}-{_XDIST_WORKER}" for n in (1, 2, 3))
|
||||
_DELETE_SEEDED = 'DELETE FROM "LiteLLM_AccessGroupTable" WHERE access_group_id = ANY($1::TEXT[])'
|
||||
_DELETE_TEAMS = 'DELETE FROM "LiteLLM_TeamTable" WHERE team_id = ANY($1::TEXT[])'
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue