diff --git a/.circleci/config.yml b/.circleci/config.yml index 4615a6a5a7e..55fa9410845 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2421,45 +2421,6 @@ jobs: - wait_for_service: url: http://localhost:4000 timeout: "300" - # Add Ruby installation and testing before the existing Node.js and Python tests - - run: - name: Install Ruby and Bundler - command: | - # Clone RVM at pinned tag and verify the commit SHA matches the - # published tag before running its install script. - RVM_VERSION="1.29.12" - RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81" - git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm - RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)" - if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then - echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2 - exit 1 - fi - - # Import RVM signing keys (used by `rvm install` to verify Ruby tarballs) - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB - - # Install RVM from the verified checkout. The install script - # sources `scripts/functions/installer` using paths relative to - # its own working directory, so it must be run from /tmp/rvm. - (cd /tmp/rvm && ./install --path "$HOME/.rvm") - source "$HOME/.rvm/scripts/rvm" - - # Install Ruby 3.2.2 (RVM verifies the tarball PGP signature) - rvm install 3.2.2 - rvm use 3.2.2 --default - - # Install latest Bundler - gem install bundler - - - run: - name: Run Ruby tests - command: | - source $HOME/.rvm/scripts/rvm - cd tests/pass_through_tests/ruby_passthrough_tests - bundle install - bundle exec rspec - no_output_timeout: 30m # Install Node.js directly from nodejs.org with SHA256 verification, # instead of piping NodeSource's setup_24.x apt-repo installer into # sudo bash (which runs a mutable upstream script unattended). diff --git a/test-quality-budget.json b/test-quality-budget.json index 4a7bc7edff2..db96156e4d9 100644 --- a/test-quality-budget.json +++ b/test-quality-budget.json @@ -1,6 +1,6 @@ { "TQ001": { - "limit": 744 + "limit": 736 }, "TQ002": { "limit": 742 @@ -12,7 +12,7 @@ "limit": 469 }, "TQ005": { - "limit": 2405 + "limit": 2399 }, "TQ006": { "limit": 34 diff --git a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts index 98bd1b84f11..1d080ec82b8 100644 --- a/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts +++ b/tests/e2e/ui/tests/modelsPage/autoRouterTemplateSelect.spec.ts @@ -1,4 +1,4 @@ -import { expect, test, type Page as PlaywrightPage } from "@playwright/test"; +import { expect, test, type Locator, type Page as PlaywrightPage } from "@playwright/test"; import { ADMIN_STORAGE_PATH } from "../../constants"; import { navigateToPage } from "../../helpers/navigation"; import { Page } from "../../fixtures/pages"; @@ -17,43 +17,49 @@ async function openTemplateSelect(page: PlaywrightPage) { return trigger; } +function pollPixelsBelowTrigger(trigger: Locator, popup: Locator) { + return expect.poll(async () => { + const triggerBox = await trigger.boundingBox(); + const popupBox = await popup.boundingBox(); + if (!triggerBox || !popupBox) return null; + return popupBox.y - (triggerBox.y + triggerBox.height); + }); +} + +function pollPopupOverlapsTrigger(trigger: Locator, popup: Locator) { + return expect.poll(async () => { + const triggerBox = await trigger.boundingBox(); + const popupBox = await popup.boundingBox(); + if (!triggerBox || !popupBox) return null; + return popupBox.y < triggerBox.y + triggerBox.height && popupBox.y + popupBox.height > triggerBox.y; + }); +} + test.describe("Auto Router template select anchoring", () => { test.use({ storageState: ADMIN_STORAGE_PATH }); test("opens the options below the trigger rather than over it", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 900 }); const trigger = await openTemplateSelect(page); - const triggerBox = await trigger.boundingBox(); await trigger.click(); const popup = page.locator('[data-slot="select-content"]'); await expect(popup).toBeVisible(); - const popupBox = await popup.boundingBox(); - - expect(triggerBox).not.toBeNull(); - expect(popupBox).not.toBeNull(); // Item-aligned mode reports "none" and puts the active item over the trigger. await expect(popup).toHaveAttribute("data-side", "bottom"); - expect(popupBox!.y).toBeGreaterThanOrEqual(triggerBox!.y + triggerBox!.height); + await pollPixelsBelowTrigger(trigger, popup).toBeGreaterThanOrEqual(0); }); test("flips above the trigger instead of covering it when there is no room below", async ({ page }) => { await page.setViewportSize({ width: 1280, height: 560 }); const trigger = await openTemplateSelect(page); await trigger.scrollIntoViewIfNeeded(); - const triggerBox = await trigger.boundingBox(); await trigger.click(); const popup = page.locator('[data-slot="select-content"]'); await expect(popup).toBeVisible(); - const popupBox = await popup.boundingBox(); - expect(triggerBox).not.toBeNull(); - expect(popupBox).not.toBeNull(); - - const overlaps = - popupBox!.y < triggerBox!.y + triggerBox!.height && popupBox!.y + popupBox!.height > triggerBox!.y; - expect(overlaps).toBe(false); + await pollPopupOverlapsTrigger(trigger, popup).toBe(false); }); }); diff --git a/tests/local_testing/test_batch_completions.py b/tests/local_testing/test_batch_completions.py index d3296988e8c..0f9b628823a 100644 --- a/tests/local_testing/test_batch_completions.py +++ b/tests/local_testing/test_batch_completions.py @@ -19,32 +19,32 @@ from litellm import ( # litellm.set_verbose=True +TOLERATED_UPSTREAM_FAILURES = (Timeout, litellm.InternalServerError) + + def test_batch_completions(): messages = [[{"role": "user", "content": "write a short poem"}] for _ in range(3)] model = "gpt-3.5-turbo" litellm.set_verbose = True - try: - result = batch_completion( - model=model, - messages=messages, - max_tokens=10, - temperature=0.2, - request_timeout=1, - ) - print(result) - print(len(result)) - assert len(result) == 3 - for response in result: - assert response.choices[0].message.content is not None - except Timeout as e: - print(f"IN TIMEOUT") - pass - except litellm.InternalServerError as e: - print(f"IN INTERNAL SERVER ERROR") - pass - except Exception as e: - pytest.fail(f"An error occurred: {e}") + result = batch_completion( + model=model, + messages=messages, + max_tokens=10, + temperature=0.2, + request_timeout=1, + ) + print(result) + + assert len(result) == 3 + + for response in result: + if isinstance(response, TOLERATED_UPSTREAM_FAILURES): + continue + assert not isinstance( + response, Exception + ), f"batch_completion returned {type(response).__name__}: {response}" + assert response.choices[0].message.content is not None # test_batch_completions() diff --git a/tests/local_testing/test_opik.py b/tests/local_testing/test_opik.py index 8be4b796360..2f6b15e1f27 100644 --- a/tests/local_testing/test_opik.py +++ b/tests/local_testing/test_opik.py @@ -16,6 +16,8 @@ verbose_logger.setLevel(logging.DEBUG) litellm.set_verbose = True import time +INTERVAL_TOO_LONG_TO_FIRE_DURING_THIS_TEST = 3600 + @pytest.mark.asyncio async def test_opik_logging_http_request(): @@ -23,70 +25,60 @@ async def test_opik_logging_http_request(): - Test that HTTP requests are made to Opik - Traces and spans are batched correctly """ - try: - from litellm.integrations.opik.opik import OpikLogger + from litellm.integrations.opik.opik import OpikLogger - os.environ["OPIK_URL_OVERRIDE"] = "https://fake.comet.com/opik/api" - os.environ["OPIK_API_KEY"] = "anything" - os.environ["OPIK_WORKSPACE"] = "anything" + os.environ["OPIK_URL_OVERRIDE"] = "https://fake.comet.com/opik/api" + os.environ["OPIK_API_KEY"] = "anything" + os.environ["OPIK_WORKSPACE"] = "anything" - # Initialize OpikLogger - test_opik_logger = OpikLogger() + test_opik_logger = OpikLogger() + test_opik_logger.flush_interval = INTERVAL_TOO_LONG_TO_FIRE_DURING_THIS_TEST + test_opik_logger.batch_size = 12 - litellm.callbacks = [test_opik_logger] - test_opik_logger.batch_size = 12 - litellm.set_verbose = True + litellm.callbacks = [test_opik_logger] - # Create a mock for the async_client's post method - mock_post = AsyncMock() - mock_post.return_value.status_code = 202 - mock_post.return_value.text = "Accepted" - test_opik_logger.async_httpx_client.post = mock_post + mock_post = AsyncMock(return_value=Mock(status_code=202, text="Accepted")) + test_opik_logger.async_httpx_client.post = mock_post - # Make multiple calls to ensure we don't hit the batch size - for _ in range(5): - response = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Test message"}], - max_tokens=10, - temperature=0.2, - mock_response="This is a mock response", - ) - await asyncio.sleep(1) + def opik_batch_calls(): + return [ + call + for call in mock_post.call_args_list + if "/traces/batch" in str(call) or "/spans/batch" in str(call) + ] - # Check batching of events and that the queue contains 5 trace events and 5 span events - assert ( - mock_post.called == False - ), "HTTP request was made but events should have been batched" - assert len(test_opik_logger.log_queue) == 10 + for _ in range(5): + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Test message"}], + max_tokens=10, + temperature=0.2, + mock_response="This is a mock response", + ) + await asyncio.sleep(1) - # Now make calls to exceed the batch size - for _ in range(3): - response = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=[{"role": "user", "content": "Test message"}], - max_tokens=10, - temperature=0.2, - mock_response="This is a mock response", - ) + assert opik_batch_calls() == [], "events below batch_size must stay queued" + assert len(test_opik_logger.log_queue) == 10 - # Wait a short time for any asynchronous operations to complete - await asyncio.sleep(1) + for _ in range(3): + await litellm.acompletion( + model="gpt-3.5-turbo", + messages=[{"role": "user", "content": "Test message"}], + max_tokens=10, + temperature=0.2, + mock_response="This is a mock response", + ) + await asyncio.sleep(1) - # Check that the queue was flushed after exceeding batch size - assert len(test_opik_logger.log_queue) < test_opik_logger.batch_size + assert opik_batch_calls(), "crossing batch_size must flush the queue" + events_left_over_after_the_size_triggered_flush = len(test_opik_logger.log_queue) + assert 0 < events_left_over_after_the_size_triggered_flush < test_opik_logger.batch_size - # Check that the data has been sent when it goes above the flush interval - await asyncio.sleep(test_opik_logger.flush_interval) - assert len(test_opik_logger.log_queue) == 0 + calls_before_periodic_flush = len(opik_batch_calls()) + await test_opik_logger.flush_queue() - # Clean up - for cb in litellm.callbacks: - if isinstance(cb, OpikLogger): - await cb.async_httpx_client.client.aclose() - - except Exception as e: - pytest.fail(f"Error occurred: {e}") + assert len(opik_batch_calls()) > calls_before_periodic_flush + assert len(test_opik_logger.log_queue) == 0 def test_sync_opik_logging_http_request(): diff --git a/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py b/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py index 1b198623381..2346a5ee047 100644 --- a/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py +++ b/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py @@ -94,11 +94,13 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch): langfuse_public_key=None, langfuse_secret=None, langfuse_host=None, + langfuse_environment=None, allow_env_credentials=True, ): captured["langfuse_public_key"] = langfuse_public_key captured["langfuse_secret"] = langfuse_secret captured["langfuse_host"] = langfuse_host + captured["langfuse_environment"] = langfuse_environment captured["allow_env_credentials"] = allow_env_credentials class FakeDynamicLoggingCache: @@ -117,6 +119,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch): "langfuse_public_key": "dynamic-public", "langfuse_secret_key": "dynamic-secret", "langfuse_host": "https://langfuse.example", + "langfuse_environment": "dynamic-environment", }, in_memory_dynamic_logger_cache=FakeDynamicLoggingCache(), ) @@ -124,6 +127,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch): assert captured["langfuse_public_key"] == "dynamic-public" assert captured["langfuse_secret"] == "dynamic-secret" assert captured["langfuse_host"] == "https://langfuse.example" + assert captured["langfuse_environment"] == "dynamic-environment" assert captured["allow_env_credentials"] is False assert captured["cached_service_name"] == "langfuse" assert captured["cached_logging_obj"] is logger diff --git a/tests/logging_callback_tests/test_langfuse_unit_tests.py b/tests/logging_callback_tests/test_langfuse_unit_tests.py index 1c25b169243..405b6e9e48e 100644 --- a/tests/logging_callback_tests/test_langfuse_unit_tests.py +++ b/tests/logging_callback_tests/test_langfuse_unit_tests.py @@ -123,16 +123,12 @@ def test_get_langfuse_logger_for_request_with_dynamic_params( assert result.secret_key == "test_secret" assert result.langfuse_host == "https://test.langfuse.com" - # Check if the logger is cached - cached_logger = dynamic_logging_cache.get_cache( - credentials={ - "langfuse_public_key": "test_public_key", - "langfuse_secret": "test_secret", - "langfuse_host": "https://test.langfuse.com", - }, - service_name="langfuse", + logger_for_identical_repeat_request = LangFuseHandler.get_langfuse_logger_for_request( + standard_callback_dynamic_params=standard_params, + in_memory_dynamic_logger_cache=dynamic_logging_cache, + globalLangfuseLogger=globalLangfuseLogger, ) - assert cached_logger is result + assert logger_for_identical_repeat_request is result @pytest.mark.parametrize("globalLangfuseLogger", [None, global_langfuse_logger]) diff --git a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile b/tests/pass_through_tests/ruby_passthrough_tests/Gemfile deleted file mode 100644 index 56860496b2b..00000000000 --- a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile +++ /dev/null @@ -1,4 +0,0 @@ -source 'https://rubygems.org' - -gem 'rspec' -gem 'ruby-openai' \ No newline at end of file diff --git a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock b/tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock deleted file mode 100644 index 2072798ccfc..00000000000 --- a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock +++ /dev/null @@ -1,42 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - base64 (0.2.0) - diff-lcs (1.6.0) - event_stream_parser (1.0.0) - faraday (2.8.1) - base64 - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) - faraday-multipart (1.1.0) - multipart-post (~> 2.0) - faraday-net_http (3.0.2) - multipart-post (2.4.1) - rspec (3.13.0) - rspec-core (~> 3.13.0) - rspec-expectations (~> 3.13.0) - rspec-mocks (~> 3.13.0) - rspec-core (3.13.3) - rspec-support (~> 3.13.0) - rspec-expectations (3.13.3) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-mocks (3.13.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-support (3.13.2) - ruby-openai (7.4.0) - event_stream_parser (>= 0.3.0, < 2.0.0) - faraday (>= 1) - faraday-multipart (>= 1) - ruby2_keywords (0.0.5) - -PLATFORMS - ruby - -DEPENDENCIES - rspec - ruby-openai - -BUNDLED WITH - 2.6.5 diff --git a/tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb b/tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb deleted file mode 100644 index 5a4dc0395f8..00000000000 --- a/tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb +++ /dev/null @@ -1,96 +0,0 @@ -require 'openai' -require 'rspec' - -RSpec.describe 'OpenAI Assistants Passthrough' do - let(:client) do - OpenAI::Client.new( - access_token: "sk-1234", - uri_base: "http://0.0.0.0:4000/openai", - request_timeout: 600 - ) - end - - - it 'performs basic assistant operations' do - assistant = client.assistants.create( - parameters: { - name: "Math Tutor", - instructions: "You are a personal math tutor. Write and run code to answer math questions.", - tools: [{ type: "code_interpreter" }], - model: "gpt-4o" - } - ) - expect(assistant).to include('id') - expect(assistant['name']).to eq("Math Tutor") - - assistants_list = client.assistants.list - expect(assistants_list['data']).to be_an(Array) - expect(assistants_list['data']).to include(include('id' => assistant['id'])) - - retrieved_assistant = client.assistants.retrieve(id: assistant['id']) - expect(retrieved_assistant).to eq(assistant) - - deleted_assistant = client.assistants.delete(id: assistant['id']) - expect(deleted_assistant['deleted']).to be true - expect(deleted_assistant['id']).to eq(assistant['id']) - end - - it 'performs streaming assistant operations' do - puts "\n=== Starting Streaming Assistant Test ===" - - assistant = client.assistants.create( - parameters: { - name: "Math Tutor", - instructions: "You are a personal math tutor. Write and run code to answer math questions.", - tools: [{ type: "code_interpreter" }], - model: "gpt-4o" - } - ) - puts "Created assistant: #{assistant['id']}" - expect(assistant).to include('id') - - thread = client.threads.create - puts "Created thread: #{thread['id']}" - expect(thread).to include('id') - - message = client.messages.create( - thread_id: thread['id'], - parameters: { - role: "user", - content: "I need to solve the equation `3x + 11 = 14`. Can you help me?" - } - ) - puts "Created message: #{message['id']}" - puts "User question: #{message['content']}" - expect(message).to include('id') - expect(message['role']).to eq('user') - - puts "\nStarting streaming response:" - puts "------------------------" - run = client.runs.create( - thread_id: thread['id'], - parameters: { - assistant_id: assistant['id'], - max_prompt_tokens: 256, - max_completion_tokens: 16, - stream: proc do |chunk, _bytesize| - puts "Received chunk: #{chunk.inspect}" # Debug: Print raw chunk - if chunk["object"] == "thread.message.delta" - content = chunk.dig("delta", "content") - puts "Content: #{content.inspect}" # Debug: Print content structure - if content && content[0] && content[0]["text"] - print content[0]["text"]["value"] - $stdout.flush # Ensure output is printed immediately - end - end - end - } - ) - puts "\n------------------------" - puts "Run completed: #{run['id']}" - expect(run).not_to be_nil - ensure - client.assistants.delete(id: assistant['id']) if assistant && assistant['id'] - client.threads.delete(id: thread['id']) if thread && thread['id'] - end -end \ No newline at end of file diff --git a/tests/pass_through_tests/test_openai_assistants_passthrough.py b/tests/pass_through_tests/test_openai_assistants_passthrough.py index 28568005fd6..9afd8b23b2f 100644 --- a/tests/pass_through_tests/test_openai_assistants_passthrough.py +++ b/tests/pass_through_tests/test_openai_assistants_passthrough.py @@ -1,141 +1,22 @@ -import pytest import openai -import aiohttp -import asyncio import tempfile -from typing_extensions import override -from openai import AssistantEventHandler client = openai.OpenAI(base_url="http://0.0.0.0:4000/openai", api_key="sk-1234") def test_pass_through_file_operations(): - # Create a temporary file with tempfile.NamedTemporaryFile( mode="w+", suffix=".txt", delete=False ) as temp_file: temp_file.write("This is a test file for the OpenAI Assistants API.") temp_file.flush() - # create a file file = client.files.create( file=open(temp_file.name, "rb"), purpose="assistants", ) print("file created", file) - # delete the file delete_file = client.files.delete(file.id) print("file deleted", delete_file) - - -def test_openai_assistants_e2e_operations(): - assistant = client.beta.assistants.create( - name="Math Tutor", - instructions="You are a personal math tutor. Write and run code to answer math questions.", - tools=[{"type": "code_interpreter"}], - model="gpt-4o", - ) - print("assistant created", assistant) - - get_assistant = client.beta.assistants.retrieve(assistant.id) - print(get_assistant) - - delete_assistant = client.beta.assistants.delete(assistant.id) - print(delete_assistant) - - -class EventHandler(AssistantEventHandler): - @override - def on_text_created(self, text) -> None: - print(f"\nassistant > ", end="", flush=True) - - @override - def on_text_delta(self, delta, snapshot): - print(delta.value, end="", flush=True) - - def on_tool_call_created(self, tool_call): - print(f"\nassistant > {tool_call.type}\n", flush=True) - - def on_tool_call_delta(self, delta, snapshot): - if delta.type == "code_interpreter": - if delta.code_interpreter.input: - print(delta.code_interpreter.input, end="", flush=True) - if delta.code_interpreter.outputs: - print(f"\n\noutput >", flush=True) - for output in delta.code_interpreter.outputs: - if output.type == "logs": - print(f"\n{output.logs}", flush=True) - - -def test_openai_assistants_e2e_operations_stream(): - - assistant = client.beta.assistants.create( - name="Math Tutor", - instructions="You are a personal math tutor. Write and run code to answer math questions.", - tools=[{"type": "code_interpreter"}], - model="gpt-4o", - ) - print("assistant created", assistant) - - thread = client.beta.threads.create() - print("thread created", thread) - - message = client.beta.threads.messages.create( - thread_id=thread.id, - role="user", - content="I need to solve the equation `3x + 11 = 14`. Can you help me?", - ) - print("message created", message) - - # Then, we use the `stream` SDK helper - # with the `EventHandler` class to create the Run - # and stream the response. - - with client.beta.threads.runs.stream( - thread_id=thread.id, - assistant_id=assistant.id, - instructions="Please address the user as Jane Doe. The user has a premium account.", - event_handler=EventHandler(), - ) as stream: - stream.until_done() - - -def test_azure_openai_assistants_e2e_operations_stream(): - from openai import AzureOpenAI - - client = AzureOpenAI( - base_url="http://0.0.0.0:4000/azure-config-passthrough/openai", - api_key="sk-1234", - api_version="2025-01-01-preview", - ) - assistant = client.beta.assistants.create( - name="Math Tutor", - instructions="You are a personal math tutor. Write and run code to answer math questions.", - tools=[{"type": "code_interpreter"}], - model="gpt-4o", - ) - print("assistant created", assistant) - - thread = client.beta.threads.create() - print("thread created", thread) - - message = client.beta.threads.messages.create( - thread_id=thread.id, - role="user", - content="I need to solve the equation `3x + 11 = 14`. Can you help me?", - ) - print("message created", message) - - # Then, we use the `stream` SDK helper - # with the `EventHandler` class to create the Run - # and stream the response. - - with client.beta.threads.runs.stream( - thread_id=thread.id, - assistant_id=assistant.id, - instructions="Please address the user as Jane Doe. The user has a premium account.", - event_handler=EventHandler(), - ) as stream: - stream.until_done() diff --git a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py index e70f2cf4430..70fc8f9ccf2 100644 --- a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py +++ b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py @@ -1,7 +1,6 @@ import json -import os -from datetime import datetime -from unittest.mock import AsyncMock, Mock, patch, MagicMock +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import Optional from fastapi import Request import pytest @@ -31,17 +30,62 @@ class TestCustomLogger(CustomLogger): self.logged_kwargs = kwargs +UPSTREAM_RESPONSE_BODY = { + "id": "modr-abc123", + "model": "omni-moderation-latest", + "results": [ + { + "flagged": False, + "categories": {"violence": False}, + "category_scores": {"violence": 1.2e-06}, + } + ], +} + + +@pytest.fixture +def upstream(): + received: dict = {} + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self): + body = self.rfile.read(int(self.headers.get("content-length", 0) or 0)) + received["path"] = self.path + received["body"] = json.loads(body or b"{}") + payload = json.dumps(UPSTREAM_RESPONSE_BODY).encode() + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(("127.0.0.1", 0), Handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + try: + yield f"http://127.0.0.1:{server.server_port}", received + finally: + server.shutdown() + server.server_close() + + @pytest.mark.asyncio -async def test_assistants_passthrough_logging(): +async def test_passthrough_logging_payload_for_a_route_no_provider_handler_claims( + upstream, +): + base_url, upstream_received = upstream + test_custom_logger = TestCustomLogger() litellm._async_success_callback = [test_custom_logger] - TARGET_URL = "https://api.openai.com/v1/assistants" + TARGET_URL = f"{base_url}/v1/moderations" REQUEST_BODY = { - "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.", - "name": "Math Tutor", - "tools": [{"type": "code_interpreter"}], - "model": "gpt-4.1-mini", + "model": "omni-moderation-latest", + "input": "I want to bake a cake for my friend's birthday.", } TARGET_METHOD = "POST" @@ -50,23 +94,18 @@ async def test_assistants_passthrough_logging(): scope={ "type": "http", "method": TARGET_METHOD, - "path": "/v1/assistants", + "path": "/v1/moderations", "query_string": b"", "headers": [ (b"content-type", b"application/json"), - ( - b"authorization", - f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(), - ), - (b"openai-beta", b"assistants=v2"), + (b"authorization", b"Bearer sk-test-passthrough"), ], }, ), target=TARGET_URL, custom_headers={ "Content-Type": "application/json", - "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}", - "OpenAI-Beta": "assistants=v2", + "Authorization": "Bearer sk-test-passthrough", }, user_api_key_dict=UserAPIKeyAuth( api_key="test", @@ -83,6 +122,10 @@ async def test_assistants_passthrough_logging(): print("result status code", result.status_code) print("result content", result.body) + assert upstream_received.get("path") == "/v1/moderations" + assert upstream_received.get("body") == REQUEST_BODY + assert result.status_code == 200 + await asyncio.sleep(1) assert test_custom_logger.logged_kwargs is not None @@ -92,79 +135,8 @@ async def test_assistants_passthrough_logging(): assert passthrough_logging_payload is not None assert passthrough_logging_payload["url"] == TARGET_URL assert passthrough_logging_payload["request_body"] == REQUEST_BODY - - # assert that the response body content matches the response body content - client_facing_response_body = json.loads(result.body) - assert passthrough_logging_payload["response_body"] == client_facing_response_body - - # assert that the request method is correct assert passthrough_logging_payload["request_method"] == TARGET_METHOD - -@pytest.mark.asyncio -async def test_threads_passthrough_logging(): - test_custom_logger = TestCustomLogger() - litellm._async_success_callback = [test_custom_logger] - - TARGET_URL = "https://api.openai.com/v1/threads" - REQUEST_BODY = {} - TARGET_METHOD = "POST" - - result = await pass_through_request( - request=Request( - scope={ - "type": "http", - "method": TARGET_METHOD, - "path": "/v1/threads", - "query_string": b"", - "headers": [ - (b"content-type", b"application/json"), - ( - b"authorization", - f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(), - ), - (b"openai-beta", b"assistants=v2"), - ], - }, - ), - target=TARGET_URL, - custom_headers={ - "Content-Type": "application/json", - "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}", - "OpenAI-Beta": "assistants=v2", - }, - user_api_key_dict=UserAPIKeyAuth( - api_key="test", - user_id="test", - team_id="test", - end_user_id="test", - ), - custom_body=REQUEST_BODY, - forward_headers=False, - merge_query_params=False, - ) - - print("got result", result) - print("result status code", result.status_code) - print("result content", result.body) - - await asyncio.sleep(1) - - assert test_custom_logger.logged_kwargs is not None - passthrough_logging_payload = test_custom_logger.logged_kwargs[ - "passthrough_logging_payload" - ] - assert passthrough_logging_payload is not None - - # Fix for TypedDict access errors - assert passthrough_logging_payload.get("url") == TARGET_URL - assert passthrough_logging_payload.get("request_body") == REQUEST_BODY - - # Fix for json.loads error with potential memoryview - response_body = result.body - client_facing_response_body = json.loads(response_body) - - assert ( - passthrough_logging_payload.get("response_body") == client_facing_response_body - ) - assert passthrough_logging_payload.get("request_method") == TARGET_METHOD + client_facing_response_body = json.loads(result.body) + assert client_facing_response_body == UPSTREAM_RESPONSE_BODY + assert passthrough_logging_payload["response_body"] == client_facing_response_body diff --git a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py index b72a1453576..ceb0dbf6749 100644 --- a/tests/proxy_admin_ui_tests/test_access_group_team_sync.py +++ b/tests/proxy_admin_ui_tests/test_access_group_team_sync.py @@ -23,9 +23,10 @@ from litellm.proxy.management_helpers.access_group_team_sync import ( sync_team_access_group_membership, ) -TEAM = "ags-team-a" -OTHER_TEAM = "ags-team-b" -GROUPS = ("ags-group-1", "ags-group-2", "ags-group-3") +_XDIST_WORKER = os.environ.get("PYTEST_XDIST_WORKER", "master") +TEAM = f"ags-team-a-{_XDIST_WORKER}" +OTHER_TEAM = f"ags-team-b-{_XDIST_WORKER}" +GROUPS = tuple(f"ags-group-{n}-{_XDIST_WORKER}" for n in (1, 2, 3)) _DELETE_SEEDED = 'DELETE FROM "LiteLLM_AccessGroupTable" WHERE access_group_id = ANY($1::TEXT[])' _DELETE_TEAMS = 'DELETE FROM "LiteLLM_TeamTable" WHERE team_id = ANY($1::TEXT[])'