diff --git a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py index 54dd86eaba7..2fef5dee46f 100644 --- a/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py +++ b/litellm/llms/anthropic/experimental_pass_through/responses_adapters/transformation.py @@ -8,7 +8,6 @@ path used for OpenAI and Azure models. import json from typing import Any, Dict, List, Optional, Union, cast -import litellm from litellm.llms.anthropic.experimental_pass_through.utils import ( is_default_reasoning_summary_disabled, ) diff --git a/scripts/test_gpt54_reasoning_tools.py b/scripts/test_gpt54_reasoning_tools.py deleted file mode 100644 index df6dea9ce78..00000000000 --- a/scripts/test_gpt54_reasoning_tools.py +++ /dev/null @@ -1,77 +0,0 @@ -""" -Repro script: verify that gpt-5.4 drops reasoning_effort when tools are present. -Expected: the call succeeds (reasoning_effort is silently dropped). -If the bug were still present, OpenAI would return an error like: - "reasoning_effort is not supported with function calling" -""" - -import os -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -litellm.set_verbose = True - -tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather for a city", - "parameters": { - "type": "object", - "properties": { - "city": {"type": "string", "description": "City name"}, - }, - "required": ["city"], - }, - }, - } -] - -print("=== Test: gpt-5.4 + reasoning_effort='medium' + tools ===") -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "What's the weather in Buenos Aires?"}], - reasoning_effort="medium", - tools=tools, - drop_params=True, - ) - print(f"SUCCESS - model: {response.model}") - print(f"Choice: {response.choices[0].message}") - if response.choices[0].message.tool_calls: - print(f"Tool calls: {response.choices[0].message.tool_calls}") - print("\nreasoning_effort was correctly dropped (no error from OpenAI)") -except Exception as e: - print(f"FAILED: {e}") - -print("\n=== Test: gpt-5.4 + reasoning_effort='high' + tools ===") -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "What's 2+2?"}], - reasoning_effort="high", - tools=tools, - drop_params=True, - ) - print(f"SUCCESS - model: {response.model}") - print(f"reasoning_effort was correctly dropped (no error from OpenAI)") -except Exception as e: - print(f"FAILED: {e}") - -print("\n=== Test: gpt-5.4 + reasoning_effort='none' + tools (should KEEP reasoning_effort) ===") -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "Say hello"}], - reasoning_effort="none", - tools=tools, - drop_params=True, - ) - print(f"SUCCESS - model: {response.model}") - print(f"reasoning_effort='none' correctly kept (OpenAI allows this)") -except Exception as e: - print(f"FAILED: {e}") diff --git a/scripts/test_perplexity_regression.py b/scripts/test_perplexity_regression.py deleted file mode 100644 index 67a01e2dcd9..00000000000 --- a/scripts/test_perplexity_regression.py +++ /dev/null @@ -1,83 +0,0 @@ -""" -Simple regression test: call Perplexity through LiteLLM -to verify chat completions and responses API both work. -""" -import os -import sys -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -# Show which branch we're on -branch = os.popen("git rev-parse --abbrev-ref HEAD 2>/dev/null || echo unknown").read().strip() -print(f"=== Branch: {branch} ===\n") - -# 1. Chat completions -print("--- Test 1: Chat Completions ---") -try: - resp = litellm.completion( - model="perplexity/sonar", - messages=[{"role": "user", "content": "Say hello in 3 words"}], - max_tokens=20, - ) - print(f"OK: {resp.choices[0].message.content[:80]}") - print(f" model: {resp.model}") - print(f" usage: {resp.usage}") -except Exception as e: - print(f"FAIL: {e}") - -# 2. Responses API (string input) -print("\n--- Test 2: Responses API (string input) ---") -try: - resp = litellm.responses( - model="perplexity/sonar", - input="Say hello in 3 words", - max_output_tokens=20, - ) - print(f"OK: {resp.output[0].content[0].text[:80]}") - print(f" model: {resp.model}") -except Exception as e: - print(f"FAIL: {e}") - -# 3. Responses API (list input - the _format_input concern) -print("\n--- Test 3: Responses API (list input without type field) ---") -try: - resp = litellm.responses( - model="perplexity/sonar", - input=[{"role": "user", "content": "Say hello in 3 words"}], - max_output_tokens=20, - ) - print(f"OK: {resp.output[0].content[0].text[:80]}") -except Exception as e: - print(f"FAIL: {e}") - -# 4. Check which config class is resolved for chat -print("\n--- Test 4: Config class resolution ---") -from litellm.utils import ProviderConfigManager -from litellm.types.utils import LlmProviders - -chat_config = ProviderConfigManager.get_provider_chat_config( - model="perplexity/sonar", provider=LlmProviders.PERPLEXITY -) -print(f"Chat config class: {type(chat_config).__name__}") -print(f" module: {type(chat_config).__module__}") - -resp_config = ProviderConfigManager.get_provider_responses_api_config( - provider=LlmProviders.PERPLEXITY -) -print(f"Responses config class: {type(resp_config).__name__}") -print(f" module: {type(resp_config).__module__}") - -# 5. Check supported params include preset/models for responses -print("\n--- Test 5: Supported params ---") -if resp_config: - params = resp_config.get_supported_openai_params("sonar") - print(f"Responses supported params: {params}") - has_preset = "preset" in params - has_models = "models" in params - print(f" Has 'preset': {has_preset}") - print(f" Has 'models': {has_models}") - -print("\n=== Done ===") diff --git a/scripts/test_perplexity_responses.py b/scripts/test_perplexity_responses.py deleted file mode 100644 index 5616ada380f..00000000000 --- a/scripts/test_perplexity_responses.py +++ /dev/null @@ -1,179 +0,0 @@ -""" -Live test: Perplexity Responses API via LiteLLM. -Tests: non-streaming, streaming, preset models, models fallback param, -chat completions (regression check), and cost dict→float parsing. - -DO NOT COMMIT this file. -""" - -import os -import traceback - -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -# litellm.set_verbose = True - - -def test_non_streaming_preset(): - """Test non-streaming with preset model.""" - print("=" * 60) - print("TEST 1: Non-streaming preset/pro-search") - print("=" * 60) - - response = litellm.responses( - model="perplexity/preset/pro-search", - input="What is 2 + 2? Answer in one word.", - ) - - print(f" Response ID: {response.id}") - print(f" Model: {response.model}") - print(f" Status: {response.status}") - - assert response.status == "completed", f"FAIL: status={response.status}" - assert response.output, "FAIL: no output" - print(" PASS: non-streaming preset works") - - if response.usage and response.usage.cost is not None: - assert isinstance(response.usage.cost, (int, float)), ( - f"FAIL: cost is {type(response.usage.cost)}: {response.usage.cost}" - ) - print(f" PASS: cost={response.usage.cost} (float, not dict)") - print() - - -def test_streaming_preset(): - """Test streaming with preset model.""" - print("=" * 60) - print("TEST 2: Streaming preset/pro-search") - print("=" * 60) - - response = litellm.responses( - model="perplexity/preset/pro-search", - input="What is the capital of France? One word.", - stream=True, - ) - - chunks = 0 - completed = False - for chunk in response: - chunks += 1 - event_type = getattr(chunk, "type", "unknown") - if event_type == "response.output_text.delta": - print(f" delta: {chunk.delta}", end="", flush=True) - elif event_type == "response.completed": - completed = True - print(f"\n [completed] model={chunk.response.model}") - if chunk.response.usage and chunk.response.usage.cost is not None: - cost = chunk.response.usage.cost - assert isinstance(cost, (int, float)), ( - f"FAIL: streaming cost is {type(cost)}: {cost}" - ) - print(f" PASS: streaming cost={cost} (float)") - - assert chunks > 0, "FAIL: no chunks received" - assert completed, "FAIL: never got response.completed event" - print(f" Total chunks: {chunks}") - print(" PASS: streaming preset works") - print() - - -def test_models_fallback_param(): - """Test that 'models' param (Perplexity fallback chain) is forwarded.""" - print("=" * 60) - print("TEST 3: models param (fallback chain)") - print("=" * 60) - - response = litellm.responses( - model="perplexity/openai/gpt-5.1", - input="Say 'hello' and nothing else.", - models=["openai/gpt-5-mini", "openai/gpt-5.1"], - ) - - print(f" Response ID: {response.id}") - print(f" Model used: {response.model}") - print(f" Status: {response.status}") - - assert response.status == "completed", f"FAIL: status={response.status}" - print(" PASS: models fallback param works") - print() - - -def test_chat_completions_not_broken(): - """Regression: Perplexity chat completions must still use PerplexityChatConfig.""" - print("=" * 60) - print("TEST 4: Chat completions regression check") - print("=" * 60) - - response = litellm.completion( - model="perplexity/sonar", - messages=[{"role": "user", "content": "Say 'hi' and nothing else."}], - max_tokens=10, - ) - - print(f" Model: {response.model}") - print(f" Content: {response.choices[0].message.content[:50]}") - - assert response.choices, "FAIL: no choices" - assert response.choices[0].message.content, "FAIL: empty content" - print(" PASS: chat completions still work (no regression)") - print() - - -def test_with_instructions(): - """Test instructions param.""" - print("=" * 60) - print("TEST 5: instructions param") - print("=" * 60) - - response = litellm.responses( - model="perplexity/preset/pro-search", - input="What is Python?", - instructions="Answer in exactly 5 words.", - ) - - print(f" Status: {response.status}") - # Extract text from output - for item in response.output: - if hasattr(item, "content"): - for c in item.content: - if hasattr(c, "text"): - print(f" Answer: {c.text}") - break - - assert response.status == "completed", f"FAIL: status={response.status}" - print(" PASS: instructions param works") - print() - - -if __name__ == "__main__": - api_key = os.environ.get("PERPLEXITYAI_API_KEY", "NOT SET") - print(f"Using PERPLEXITYAI_API_KEY: {api_key[:10]}...") - print() - - tests = [ - test_non_streaming_preset, - test_streaming_preset, - test_models_fallback_param, - test_chat_completions_not_broken, - test_with_instructions, - ] - - passed = 0 - failed = 0 - for test in tests: - try: - test() - passed += 1 - except Exception as e: - failed += 1 - print(f" FAIL: {e}") - traceback.print_exc() - print() - - print("=" * 60) - print(f"Results: {passed} passed, {failed} failed out of {len(tests)}") - print("=" * 60) diff --git a/scripts/test_reasoning_none_tools.py b/scripts/test_reasoning_none_tools.py deleted file mode 100644 index 30d7bd1dabe..00000000000 --- a/scripts/test_reasoning_none_tools.py +++ /dev/null @@ -1,39 +0,0 @@ -""" -Repro: gpt-5.4 + reasoning_effort='none' + tools -Current behavior: reasoning_effort='none' is NOT dropped, but OpenAI rejects it. -""" - -import os -from dotenv import load_dotenv - -load_dotenv() - -import litellm - -tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get current weather", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, - }, - } -] - -print("=== gpt-5.4 + reasoning_effort='none' + tools ===") -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "What's the weather in Buenos Aires?"}], - reasoning_effort="none", - tools=tools, - ) - print(f"SUCCESS - model: {response.model}") - print(f"Choice: {response.choices[0].message}") -except Exception as e: - print(f"FAILED: {e}") diff --git a/scripts/test_reasoning_tools.py b/scripts/test_reasoning_tools.py deleted file mode 100644 index b925d0ade29..00000000000 --- a/scripts/test_reasoning_tools.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Test gpt-5.4 with reasoning_effort + tools to see OpenAI's response.""" -import os -from dotenv import load_dotenv -load_dotenv() - -import litellm - -try: - response = litellm.completion( - model="gpt-5.4", - messages=[{"role": "user", "content": "What's the weather in SF?"}], - tools=[{"type": "function", "function": {"name": "get_weather", "description": "Get weather", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}], - reasoning_effort="high", - ) - print("SUCCESS:") - print(response) -except Exception as e: - print(f"ERROR ({type(e).__name__}):") - print(e) diff --git a/scripts/test_tool_choice_responses.py b/scripts/test_tool_choice_responses.py deleted file mode 100644 index dab599933a6..00000000000 --- a/scripts/test_tool_choice_responses.py +++ /dev/null @@ -1,45 +0,0 @@ -"""Post-fix verification for #23423: tool_choice with responses/ prefix.""" -import os -from dotenv import load_dotenv -load_dotenv() - -import litellm - -# Verify supports_tool_choice resolves correctly -from litellm.utils import supports_tool_choice -print("supports_tool_choice('gpt-5.4'):", supports_tool_choice("gpt-5.4")) -print("supports_tool_choice('openai/responses/gpt-5.4'):", supports_tool_choice("openai/responses/gpt-5.4")) - -# Verify tool_choice is in supported params -params = litellm.get_supported_openai_params(model="openai/responses/gpt-5.4", custom_llm_provider="openai") -print("tool_choice in supported params:", "tool_choice" in params) - -# Real API call with tool_choice -tools = [ - { - "type": "function", - "function": { - "name": "get_weather", - "description": "Get the weather for a city", - "parameters": { - "type": "object", - "properties": {"city": {"type": "string"}}, - "required": ["city"], - }, - }, - } -] - -response = litellm.completion( - model="openai/responses/gpt-4.1-nano", # cheaper model - messages=[{"role": "user", "content": "What's the weather in Buenos Aires?"}], - tools=tools, - tool_choice="required", -) - -print("\nResponse:") -print(" tool_calls:", response.choices[0].message.tool_calls) -print(" finish_reason:", response.choices[0].finish_reason) - -has_tool_call = response.choices[0].message.tool_calls is not None -print("\nVERDICT:", "PASS - tool_choice works" if has_tool_call else "FAIL - tool_choice dropped") diff --git a/scripts/test_tool_search_chat.py b/scripts/test_tool_search_chat.py deleted file mode 100644 index 9cfecae9835..00000000000 --- a/scripts/test_tool_search_chat.py +++ /dev/null @@ -1,57 +0,0 @@ -"""Test the Chat Completions Bridge tool search example from docs (line 856-887)""" -import os -from dotenv import load_dotenv -load_dotenv() - -import litellm - -try: - response = litellm.completion( - model="openai/responses/gpt-5.4", - messages=[{"role": "user", "content": "Look up invoice INV-2024-001"}], - tools=[ - {"type": "tool_search"}, - { - "type": "namespace", - "name": "billing", - "description": "Billing and invoicing tools", - "tools": [ - { - "type": "function", - "name": "get_invoice", - "description": "Get an invoice by ID", - "parameters": { - "type": "object", - "properties": {"invoice_id": {"type": "string"}}, - "required": ["invoice_id"], - }, - "defer_loading": True, - }, - ], - }, - ], - ) - - print("=== Raw response ===") - print(f"tool_calls value: {response.choices[0].message.tool_calls}") - print(f"tool_calls is None? {response.choices[0].message.tool_calls is None}") - print() - - # Test the docs code exactly as written - print("=== Testing docs code (no None guard) ===") - try: - for tool_call in response.choices[0].message.tool_calls: - print(f"Called: {tool_call.function.name}({tool_call.function.arguments})") - except TypeError as e: - print(f" !!! TypeError: {e}") - print(f" Greptile was RIGHT - need 'or []' guard") - - # Test with the fix - print() - print("=== Testing with fix (or [] guard) ===") - for tool_call in (response.choices[0].message.tool_calls or []): - print(f"Called: {tool_call.function.name}({tool_call.function.arguments})") - print(" OK - no crash") - -except Exception as e: - print(f"API Error: {type(e).__name__}: {e}") diff --git a/scripts/test_tool_search_responses.py b/scripts/test_tool_search_responses.py deleted file mode 100644 index 58c7f1e2aca..00000000000 --- a/scripts/test_tool_search_responses.py +++ /dev/null @@ -1,101 +0,0 @@ -"""Test the Responses API tool search example from docs (line 705-783)""" -import os -from dotenv import load_dotenv -load_dotenv() - -import litellm -import json - -# Define namespaces with deferred tools -tools = [ - {"type": "tool_search"}, # Enable tool search - { - "type": "namespace", - "name": "crm", - "description": "CRM tools for customer management", - "tools": [ - { - "type": "function", - "name": "get_customer", - "description": "Get customer details by ID", - "parameters": { - "type": "object", - "properties": { - "customer_id": {"type": "string"} - }, - "required": ["customer_id"], - }, - "defer_loading": True, - }, - { - "type": "function", - "name": "list_customers", - "description": "List customers with optional filters", - "parameters": { - "type": "object", - "properties": { - "status": {"type": "string", "enum": ["active", "inactive"]}, - }, - }, - "defer_loading": True, - }, - ], - }, - { - "type": "namespace", - "name": "billing", - "description": "Billing and invoicing tools", - "tools": [ - { - "type": "function", - "name": "get_invoice", - "description": "Get an invoice by ID", - "parameters": { - "type": "object", - "properties": { - "invoice_id": {"type": "string"} - }, - "required": ["invoice_id"], - }, - "defer_loading": True, - }, - ], - }, -] - -try: - response = litellm.responses( - model="openai/gpt-5.4", - input="Look up invoice INV-2024-001 from the billing system", - tools=tools, - ) - - print("=== Raw response.output ===") - print(response.output) - print() - - # Test the parsing code from the docs - print("=== Parsing output items ===") - for item in response.output: - print(f" item type: {type(item)}") - if isinstance(item, dict): - print(f" dict keys: {item.keys()}") - if item["type"] == "tool_search_call": - print(f"Searched namespaces: {item['arguments']['paths']}") - elif item["type"] == "tool_search_output": - print(f"Loaded {len(item['tools'])} tool(s)") - elif item["type"] == "function_call": - print(f"Called: {item.get('namespace', '')}.{item['name']}({item['arguments']})") - else: - print(f" object attrs: {dir(item)}") - if item.type == "function_call": - # Greptile says this will fail if namespace is missing - print(f" Has 'namespace' attr? {hasattr(item, 'namespace')}") - try: - print(f"Called: {item.namespace}.{item.name}({item.arguments})") - except AttributeError as e: - print(f" !!! AttributeError: {e}") - print(f" Greptile was RIGHT - need getattr fallback") - -except Exception as e: - print(f"API Error: {type(e).__name__}: {e}")