diff --git a/.circleci/config.yml b/.circleci/config.yml index 4615a6a5a7e..55fa9410845 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2421,45 +2421,6 @@ jobs: - wait_for_service: url: http://localhost:4000 timeout: "300" - # Add Ruby installation and testing before the existing Node.js and Python tests - - run: - name: Install Ruby and Bundler - command: | - # Clone RVM at pinned tag and verify the commit SHA matches the - # published tag before running its install script. - RVM_VERSION="1.29.12" - RVM_EXPECTED_SHA="6bfc9213c9d6914fe756f524eb034a403d51db81" - git clone --depth 1 --branch "$RVM_VERSION" https://github.com/rvm/rvm.git /tmp/rvm - RVM_ACTUAL_SHA="$(git -C /tmp/rvm rev-parse HEAD)" - if [ "$RVM_ACTUAL_SHA" != "$RVM_EXPECTED_SHA" ]; then - echo "RVM tag $RVM_VERSION resolved to $RVM_ACTUAL_SHA; expected $RVM_EXPECTED_SHA" >&2 - exit 1 - fi - - # Import RVM signing keys (used by `rvm install` to verify Ruby tarballs) - gpg --keyserver hkp://keyserver.ubuntu.com --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3 7D2BAF1CF37B13E2069D6956105BD0E739499BDB - - # Install RVM from the verified checkout. The install script - # sources `scripts/functions/installer` using paths relative to - # its own working directory, so it must be run from /tmp/rvm. - (cd /tmp/rvm && ./install --path "$HOME/.rvm") - source "$HOME/.rvm/scripts/rvm" - - # Install Ruby 3.2.2 (RVM verifies the tarball PGP signature) - rvm install 3.2.2 - rvm use 3.2.2 --default - - # Install latest Bundler - gem install bundler - - - run: - name: Run Ruby tests - command: | - source $HOME/.rvm/scripts/rvm - cd tests/pass_through_tests/ruby_passthrough_tests - bundle install - bundle exec rspec - no_output_timeout: 30m # Install Node.js directly from nodejs.org with SHA256 verification, # instead of piping NodeSource's setup_24.x apt-repo installer into # sudo bash (which runs a mutable upstream script unattended). diff --git a/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py b/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py index 1b198623381..2346a5ee047 100644 --- a/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py +++ b/tests/logging_callback_tests/test_langfuse_dynamic_credentials.py @@ -94,11 +94,13 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch): langfuse_public_key=None, langfuse_secret=None, langfuse_host=None, + langfuse_environment=None, allow_env_credentials=True, ): captured["langfuse_public_key"] = langfuse_public_key captured["langfuse_secret"] = langfuse_secret captured["langfuse_host"] = langfuse_host + captured["langfuse_environment"] = langfuse_environment captured["allow_env_credentials"] = allow_env_credentials class FakeDynamicLoggingCache: @@ -117,6 +119,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch): "langfuse_public_key": "dynamic-public", "langfuse_secret_key": "dynamic-secret", "langfuse_host": "https://langfuse.example", + "langfuse_environment": "dynamic-environment", }, in_memory_dynamic_logger_cache=FakeDynamicLoggingCache(), ) @@ -124,6 +127,7 @@ def test_langfuse_handler_accepts_secret_key_alias(monkeypatch): assert captured["langfuse_public_key"] == "dynamic-public" assert captured["langfuse_secret"] == "dynamic-secret" assert captured["langfuse_host"] == "https://langfuse.example" + assert captured["langfuse_environment"] == "dynamic-environment" assert captured["allow_env_credentials"] is False assert captured["cached_service_name"] == "langfuse" assert captured["cached_logging_obj"] is logger diff --git a/tests/logging_callback_tests/test_langfuse_unit_tests.py b/tests/logging_callback_tests/test_langfuse_unit_tests.py index 1c25b169243..405b6e9e48e 100644 --- a/tests/logging_callback_tests/test_langfuse_unit_tests.py +++ b/tests/logging_callback_tests/test_langfuse_unit_tests.py @@ -123,16 +123,12 @@ def test_get_langfuse_logger_for_request_with_dynamic_params( assert result.secret_key == "test_secret" assert result.langfuse_host == "https://test.langfuse.com" - # Check if the logger is cached - cached_logger = dynamic_logging_cache.get_cache( - credentials={ - "langfuse_public_key": "test_public_key", - "langfuse_secret": "test_secret", - "langfuse_host": "https://test.langfuse.com", - }, - service_name="langfuse", + logger_for_identical_repeat_request = LangFuseHandler.get_langfuse_logger_for_request( + standard_callback_dynamic_params=standard_params, + in_memory_dynamic_logger_cache=dynamic_logging_cache, + globalLangfuseLogger=globalLangfuseLogger, ) - assert cached_logger is result + assert logger_for_identical_repeat_request is result @pytest.mark.parametrize("globalLangfuseLogger", [None, global_langfuse_logger]) diff --git a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile b/tests/pass_through_tests/ruby_passthrough_tests/Gemfile deleted file mode 100644 index 56860496b2b..00000000000 --- a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile +++ /dev/null @@ -1,4 +0,0 @@ -source 'https://rubygems.org' - -gem 'rspec' -gem 'ruby-openai' \ No newline at end of file diff --git a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock b/tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock deleted file mode 100644 index 2072798ccfc..00000000000 --- a/tests/pass_through_tests/ruby_passthrough_tests/Gemfile.lock +++ /dev/null @@ -1,42 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - base64 (0.2.0) - diff-lcs (1.6.0) - event_stream_parser (1.0.0) - faraday (2.8.1) - base64 - faraday-net_http (>= 2.0, < 3.1) - ruby2_keywords (>= 0.0.4) - faraday-multipart (1.1.0) - multipart-post (~> 2.0) - faraday-net_http (3.0.2) - multipart-post (2.4.1) - rspec (3.13.0) - rspec-core (~> 3.13.0) - rspec-expectations (~> 3.13.0) - rspec-mocks (~> 3.13.0) - rspec-core (3.13.3) - rspec-support (~> 3.13.0) - rspec-expectations (3.13.3) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-mocks (3.13.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.13.0) - rspec-support (3.13.2) - ruby-openai (7.4.0) - event_stream_parser (>= 0.3.0, < 2.0.0) - faraday (>= 1) - faraday-multipart (>= 1) - ruby2_keywords (0.0.5) - -PLATFORMS - ruby - -DEPENDENCIES - rspec - ruby-openai - -BUNDLED WITH - 2.6.5 diff --git a/tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb b/tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb deleted file mode 100644 index 5a4dc0395f8..00000000000 --- a/tests/pass_through_tests/ruby_passthrough_tests/spec/openai_assistants_passthrough_spec.rb +++ /dev/null @@ -1,96 +0,0 @@ -require 'openai' -require 'rspec' - -RSpec.describe 'OpenAI Assistants Passthrough' do - let(:client) do - OpenAI::Client.new( - access_token: "sk-1234", - uri_base: "http://0.0.0.0:4000/openai", - request_timeout: 600 - ) - end - - - it 'performs basic assistant operations' do - assistant = client.assistants.create( - parameters: { - name: "Math Tutor", - instructions: "You are a personal math tutor. Write and run code to answer math questions.", - tools: [{ type: "code_interpreter" }], - model: "gpt-4o" - } - ) - expect(assistant).to include('id') - expect(assistant['name']).to eq("Math Tutor") - - assistants_list = client.assistants.list - expect(assistants_list['data']).to be_an(Array) - expect(assistants_list['data']).to include(include('id' => assistant['id'])) - - retrieved_assistant = client.assistants.retrieve(id: assistant['id']) - expect(retrieved_assistant).to eq(assistant) - - deleted_assistant = client.assistants.delete(id: assistant['id']) - expect(deleted_assistant['deleted']).to be true - expect(deleted_assistant['id']).to eq(assistant['id']) - end - - it 'performs streaming assistant operations' do - puts "\n=== Starting Streaming Assistant Test ===" - - assistant = client.assistants.create( - parameters: { - name: "Math Tutor", - instructions: "You are a personal math tutor. Write and run code to answer math questions.", - tools: [{ type: "code_interpreter" }], - model: "gpt-4o" - } - ) - puts "Created assistant: #{assistant['id']}" - expect(assistant).to include('id') - - thread = client.threads.create - puts "Created thread: #{thread['id']}" - expect(thread).to include('id') - - message = client.messages.create( - thread_id: thread['id'], - parameters: { - role: "user", - content: "I need to solve the equation `3x + 11 = 14`. Can you help me?" - } - ) - puts "Created message: #{message['id']}" - puts "User question: #{message['content']}" - expect(message).to include('id') - expect(message['role']).to eq('user') - - puts "\nStarting streaming response:" - puts "------------------------" - run = client.runs.create( - thread_id: thread['id'], - parameters: { - assistant_id: assistant['id'], - max_prompt_tokens: 256, - max_completion_tokens: 16, - stream: proc do |chunk, _bytesize| - puts "Received chunk: #{chunk.inspect}" # Debug: Print raw chunk - if chunk["object"] == "thread.message.delta" - content = chunk.dig("delta", "content") - puts "Content: #{content.inspect}" # Debug: Print content structure - if content && content[0] && content[0]["text"] - print content[0]["text"]["value"] - $stdout.flush # Ensure output is printed immediately - end - end - end - } - ) - puts "\n------------------------" - puts "Run completed: #{run['id']}" - expect(run).not_to be_nil - ensure - client.assistants.delete(id: assistant['id']) if assistant && assistant['id'] - client.threads.delete(id: thread['id']) if thread && thread['id'] - end -end \ No newline at end of file diff --git a/tests/pass_through_tests/test_openai_assistants_passthrough.py b/tests/pass_through_tests/test_openai_assistants_passthrough.py index 28568005fd6..78833e85fa9 100644 --- a/tests/pass_through_tests/test_openai_assistants_passthrough.py +++ b/tests/pass_through_tests/test_openai_assistants_passthrough.py @@ -1,7 +1,4 @@ -import pytest import openai -import aiohttp -import asyncio import tempfile from typing_extensions import override from openai import AssistantEventHandler @@ -30,22 +27,6 @@ def test_pass_through_file_operations(): print("file deleted", delete_file) -def test_openai_assistants_e2e_operations(): - assistant = client.beta.assistants.create( - name="Math Tutor", - instructions="You are a personal math tutor. Write and run code to answer math questions.", - tools=[{"type": "code_interpreter"}], - model="gpt-4o", - ) - print("assistant created", assistant) - - get_assistant = client.beta.assistants.retrieve(assistant.id) - print(get_assistant) - - delete_assistant = client.beta.assistants.delete(assistant.id) - print(delete_assistant) - - class EventHandler(AssistantEventHandler): @override def on_text_created(self, text) -> None: @@ -69,39 +50,6 @@ class EventHandler(AssistantEventHandler): print(f"\n{output.logs}", flush=True) -def test_openai_assistants_e2e_operations_stream(): - - assistant = client.beta.assistants.create( - name="Math Tutor", - instructions="You are a personal math tutor. Write and run code to answer math questions.", - tools=[{"type": "code_interpreter"}], - model="gpt-4o", - ) - print("assistant created", assistant) - - thread = client.beta.threads.create() - print("thread created", thread) - - message = client.beta.threads.messages.create( - thread_id=thread.id, - role="user", - content="I need to solve the equation `3x + 11 = 14`. Can you help me?", - ) - print("message created", message) - - # Then, we use the `stream` SDK helper - # with the `EventHandler` class to create the Run - # and stream the response. - - with client.beta.threads.runs.stream( - thread_id=thread.id, - assistant_id=assistant.id, - instructions="Please address the user as Jane Doe. The user has a premium account.", - event_handler=EventHandler(), - ) as stream: - stream.until_done() - - def test_azure_openai_assistants_e2e_operations_stream(): from openai import AzureOpenAI diff --git a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py index e70f2cf4430..d7add68753b 100644 --- a/tests/pass_through_unit_tests/test_custom_logger_passthrough.py +++ b/tests/pass_through_unit_tests/test_custom_logger_passthrough.py @@ -1,7 +1,5 @@ import json import os -from datetime import datetime -from unittest.mock import AsyncMock, Mock, patch, MagicMock from typing import Optional from fastapi import Request import pytest @@ -32,16 +30,16 @@ class TestCustomLogger(CustomLogger): @pytest.mark.asyncio -async def test_assistants_passthrough_logging(): +async def test_untracked_openai_route_passthrough_logging(): + """Keep this on a route `_is_supported_openai_endpoint` does not claim, or the + OpenAI-specific handler takes over and the generic payload stops being exercised.""" test_custom_logger = TestCustomLogger() litellm._async_success_callback = [test_custom_logger] - TARGET_URL = "https://api.openai.com/v1/assistants" + TARGET_URL = "https://api.openai.com/v1/moderations" REQUEST_BODY = { - "instructions": "You are a personal math tutor. When asked a question, write and run Python code to answer the question.", - "name": "Math Tutor", - "tools": [{"type": "code_interpreter"}], - "model": "gpt-4.1-mini", + "model": "omni-moderation-latest", + "input": "I want to bake a cake for my friend's birthday.", } TARGET_METHOD = "POST" @@ -50,7 +48,7 @@ async def test_assistants_passthrough_logging(): scope={ "type": "http", "method": TARGET_METHOD, - "path": "/v1/assistants", + "path": "/v1/moderations", "query_string": b"", "headers": [ (b"content-type", b"application/json"), @@ -58,7 +56,6 @@ async def test_assistants_passthrough_logging(): b"authorization", f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(), ), - (b"openai-beta", b"assistants=v2"), ], }, ), @@ -66,7 +63,6 @@ async def test_assistants_passthrough_logging(): custom_headers={ "Content-Type": "application/json", "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}", - "OpenAI-Beta": "assistants=v2", }, user_api_key_dict=UserAPIKeyAuth( api_key="test", @@ -83,6 +79,8 @@ async def test_assistants_passthrough_logging(): print("result status code", result.status_code) print("result content", result.body) + assert result.status_code == 200 + await asyncio.sleep(1) assert test_custom_logger.logged_kwargs is not None @@ -92,79 +90,8 @@ async def test_assistants_passthrough_logging(): assert passthrough_logging_payload is not None assert passthrough_logging_payload["url"] == TARGET_URL assert passthrough_logging_payload["request_body"] == REQUEST_BODY - - # assert that the response body content matches the response body content - client_facing_response_body = json.loads(result.body) - assert passthrough_logging_payload["response_body"] == client_facing_response_body - - # assert that the request method is correct assert passthrough_logging_payload["request_method"] == TARGET_METHOD - -@pytest.mark.asyncio -async def test_threads_passthrough_logging(): - test_custom_logger = TestCustomLogger() - litellm._async_success_callback = [test_custom_logger] - - TARGET_URL = "https://api.openai.com/v1/threads" - REQUEST_BODY = {} - TARGET_METHOD = "POST" - - result = await pass_through_request( - request=Request( - scope={ - "type": "http", - "method": TARGET_METHOD, - "path": "/v1/threads", - "query_string": b"", - "headers": [ - (b"content-type", b"application/json"), - ( - b"authorization", - f"Bearer {os.getenv('OPENAI_API_KEY')}".encode(), - ), - (b"openai-beta", b"assistants=v2"), - ], - }, - ), - target=TARGET_URL, - custom_headers={ - "Content-Type": "application/json", - "Authorization": f"Bearer {os.getenv('OPENAI_API_KEY')}", - "OpenAI-Beta": "assistants=v2", - }, - user_api_key_dict=UserAPIKeyAuth( - api_key="test", - user_id="test", - team_id="test", - end_user_id="test", - ), - custom_body=REQUEST_BODY, - forward_headers=False, - merge_query_params=False, - ) - - print("got result", result) - print("result status code", result.status_code) - print("result content", result.body) - - await asyncio.sleep(1) - - assert test_custom_logger.logged_kwargs is not None - passthrough_logging_payload = test_custom_logger.logged_kwargs[ - "passthrough_logging_payload" - ] - assert passthrough_logging_payload is not None - - # Fix for TypedDict access errors - assert passthrough_logging_payload.get("url") == TARGET_URL - assert passthrough_logging_payload.get("request_body") == REQUEST_BODY - - # Fix for json.loads error with potential memoryview - response_body = result.body - client_facing_response_body = json.loads(response_body) - - assert ( - passthrough_logging_payload.get("response_body") == client_facing_response_body - ) - assert passthrough_logging_payload.get("request_method") == TARGET_METHOD + client_facing_response_body = json.loads(result.body) + assert client_facing_response_body["results"] + assert passthrough_logging_payload["response_body"] == client_facing_response_body