mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
198906495f
* refactor(python-bridge): split non-streaming bridge modules * refactor(python-bridge): bring shared function tracing into route layer * feat(dev): list Python route functions and call sites * feat(dev): list Rust route functions and call sites * docs(dev): record OCR parity gaps across Python and Rust * feat(dev): list executed SDK calls with runtime tracing * feat(dev): report Python vs Rust SDK pipeline steps in one CLI * feat(dev): side-by-side pipeline step report in compare CLI * fix(dev): drop invalid Final annotations in compare cell loop * feat(dev): blue python-only and yellow rust-only steps in compare CLI * feat(dev): vertical layout with section spacing in compare CLI * fix(dev): validate SDK trace stages across sync and async routes * refactor(rust): align SDK route call structure with Python * refactor(python-bridge): share sync and async route call wrappers * refactor(dev): split compare CLI into fixtures, runtime, and report modules * fix(ci): run SDK trace tests and satisfy test lint
244 lines
10 KiB
Python
244 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
from typing import Final
|
|
|
|
import pytest
|
|
|
|
from tests.sdk_function_trace.profiler import FunctionTraceEvent
|
|
from tests.sdk_function_trace.runtime import trace_diff
|
|
from tests.sdk_function_trace.steps import pipeline_issues, pipeline_steps
|
|
|
|
|
|
def test_python_ocr_projection_keeps_pipeline_and_drops_noise() -> None:
|
|
events: Final = (
|
|
FunctionTraceEvent("utils.py:1747 client.<locals>.wrapper_async", 0),
|
|
FunctionTraceEvent("ocr/main.py:331 aocr", 1),
|
|
FunctionTraceEvent("ocr/main.py:70 _prepare_ocr_request", 2),
|
|
FunctionTraceEvent("litellm_core_utils/get_llm_provider_logic.py:142 get_llm_provider", 3),
|
|
FunctionTraceEvent("utils.py:9303 ProviderConfigManager.get_provider_ocr_config", 3),
|
|
FunctionTraceEvent("llms/mistral/ocr/transformation.py:34 MistralOCRConfig.get_supported_ocr_params", 4),
|
|
FunctionTraceEvent("llms/mistral/ocr/transformation.py:72 MistralOCRConfig.map_ocr_params", 4),
|
|
FunctionTraceEvent("llms/mistral/ocr/transformation.py:34 MistralOCRConfig.get_supported_ocr_params", 5),
|
|
FunctionTraceEvent("llms/custom_httpx/llm_http_handler.py:1705 BaseLLMHTTPHandler.async_ocr", 2),
|
|
FunctionTraceEvent("llms/mistral/ocr/transformation.py:94 MistralOCRConfig.validate_environment", 4),
|
|
FunctionTraceEvent("llms/mistral/ocr/transformation.py:124 MistralOCRConfig.get_complete_url", 4),
|
|
FunctionTraceEvent("llms/base_llm/ocr/transformation.py:209 BaseOCRConfig.async_transform_ocr_request", 5),
|
|
FunctionTraceEvent("llms/mistral/ocr/transformation.py:149 MistralOCRConfig.transform_ocr_request", 6),
|
|
FunctionTraceEvent("llms/custom_httpx/http_handler.py:654 AsyncHTTPHandler.post", 6),
|
|
FunctionTraceEvent("llms/base_llm/ocr/transformation.py:255 BaseOCRConfig.async_transform_ocr_response", 4),
|
|
FunctionTraceEvent("llms/mistral/ocr/transformation.py:200 MistralOCRConfig.transform_ocr_response", 5),
|
|
FunctionTraceEvent("cost_calculator.py:1874 ocr_cost", 6),
|
|
)
|
|
|
|
assert pipeline_steps("ocr", "python", events) == (
|
|
FunctionTraceEvent("ocr", 0),
|
|
FunctionTraceEvent("prepare_ocr_call", 1),
|
|
FunctionTraceEvent("get_provider_ocr_config", 2),
|
|
FunctionTraceEvent("supported_ocr_params", 3),
|
|
FunctionTraceEvent("map_ocr_params", 3),
|
|
FunctionTraceEvent("execute_ocr_provider_call", 1),
|
|
FunctionTraceEvent("validate_environment", 2),
|
|
FunctionTraceEvent("complete_url", 2),
|
|
FunctionTraceEvent("transform_ocr_request", 3),
|
|
FunctionTraceEvent("http_request", 3),
|
|
FunctionTraceEvent("transform_ocr_response", 2),
|
|
)
|
|
|
|
|
|
def test_rust_ocr_projection_reuses_step_names_and_keeps_unknown_spans() -> None:
|
|
events: Final = (
|
|
FunctionTraceEvent("ocr", 0),
|
|
FunctionTraceEvent("prepare_ocr_call", 1),
|
|
FunctionTraceEvent("map_ocr_params", 2),
|
|
FunctionTraceEvent("supported_ocr_params", 3),
|
|
FunctionTraceEvent("map_ocr_params", 2),
|
|
FunctionTraceEvent("transform_ocr_request", 2),
|
|
FunctionTraceEvent("execute_ocr_provider_call", 2),
|
|
FunctionTraceEvent("transform_ocr_response", 3),
|
|
FunctionTraceEvent("new_uninstrumented_span", 3),
|
|
)
|
|
|
|
assert pipeline_steps("ocr", "rust", events) == (
|
|
FunctionTraceEvent("ocr", 0),
|
|
FunctionTraceEvent("prepare_ocr_call", 1),
|
|
FunctionTraceEvent("map_ocr_params", 2),
|
|
FunctionTraceEvent("supported_ocr_params", 3),
|
|
FunctionTraceEvent("transform_ocr_request", 2),
|
|
FunctionTraceEvent("execute_ocr_provider_call", 2),
|
|
FunctionTraceEvent("transform_ocr_response", 3),
|
|
FunctionTraceEvent("new_uninstrumented_span", 3),
|
|
)
|
|
|
|
|
|
def test_projection_resets_depth_on_thread_root() -> None:
|
|
events: Final = (
|
|
FunctionTraceEvent("main.py:387 acompletion", 1),
|
|
FunctionTraceEvent("llms/anthropic/chat/handler.py:255 AnthropicChatCompletion.acompletion_function", 2),
|
|
FunctionTraceEvent(
|
|
"llms/anthropic/experimental_pass_through/messages/handler.py:416 anthropic_messages_handler", 0
|
|
),
|
|
FunctionTraceEvent(
|
|
"llms/anthropic/experimental_pass_through/messages/transformation.py:575"
|
|
" AnthropicMessagesConfig.transform_anthropic_messages_request",
|
|
4,
|
|
),
|
|
)
|
|
|
|
assert pipeline_steps("chat_completions", "python", events) == (
|
|
FunctionTraceEvent("chat_completions", 0),
|
|
FunctionTraceEvent("execute_chat_completions_provider_call", 1),
|
|
)
|
|
assert pipeline_steps("messages", "python", events) == (
|
|
FunctionTraceEvent("execute_messages_provider_call", 0),
|
|
FunctionTraceEvent("transform_request", 1),
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("function", ("completion", "completion_function", "acompletion_function"))
|
|
def test_chat_projection_includes_sync_and_async_handlers(function: str) -> None:
|
|
events: Final = (FunctionTraceEvent(f"llms/anthropic/chat/handler.py:100 AnthropicChatCompletion.{function}", 0),)
|
|
|
|
assert pipeline_steps("chat_completions", "python", events) == (
|
|
FunctionTraceEvent("execute_chat_completions_provider_call", 0),
|
|
)
|
|
|
|
|
|
def test_trace_diff_reports_no_difference_for_identical_steps() -> None:
|
|
steps: Final = (
|
|
FunctionTraceEvent("ocr", 0),
|
|
FunctionTraceEvent("transform_ocr_request", 1),
|
|
)
|
|
|
|
diff: Final = trace_diff(steps, steps)
|
|
|
|
assert diff.python_only == ()
|
|
assert diff.rust_only == ()
|
|
assert diff.shared_order_matches
|
|
assert diff.matches
|
|
|
|
|
|
def test_trace_diff_reports_exclusive_steps_and_reordered_shared_steps() -> None:
|
|
python: Final = (
|
|
FunctionTraceEvent("ocr", 0),
|
|
FunctionTraceEvent("supported_ocr_params", 1),
|
|
FunctionTraceEvent("map_ocr_params", 1),
|
|
FunctionTraceEvent("http_request", 2),
|
|
)
|
|
rust: Final = (
|
|
FunctionTraceEvent("ocr", 0),
|
|
FunctionTraceEvent("map_ocr_params", 1),
|
|
FunctionTraceEvent("supported_ocr_params", 2),
|
|
FunctionTraceEvent("transform_ocr_response", 2),
|
|
)
|
|
|
|
diff: Final = trace_diff(python, rust)
|
|
|
|
assert diff.python_only == ("http_request",)
|
|
assert diff.rust_only == ("transform_ocr_response",)
|
|
assert not diff.shared_order_matches
|
|
assert not diff.matches
|
|
|
|
|
|
def test_trace_diff_does_not_claim_empty_or_disjoint_traces_match() -> None:
|
|
assert not trace_diff((), ()).shared_order_matches
|
|
assert not trace_diff((FunctionTraceEvent("ocr", 0),), (FunctionTraceEvent("messages", 0),)).shared_order_matches
|
|
|
|
|
|
def test_projection_uses_actual_ancestors_after_coroutine_resumption() -> None:
|
|
entrypoint: Final = "main.py:387 acompletion"
|
|
handler: Final = "llms/anthropic/chat/handler.py:255 AnthropicChatCompletion.acompletion_function"
|
|
events: Final = (
|
|
FunctionTraceEvent(entrypoint, 0, ()),
|
|
FunctionTraceEvent(handler, 1, (entrypoint,)),
|
|
FunctionTraceEvent("utils.py:100 unrelated_worker", 0, ()),
|
|
FunctionTraceEvent("llms/anthropic/chat/transformation.py:100 transform_response", 1, (handler,)),
|
|
)
|
|
|
|
assert pipeline_steps("chat_completions", "python", events) == (
|
|
FunctionTraceEvent("chat_completions", 0),
|
|
FunctionTraceEvent("execute_chat_completions_provider_call", 1),
|
|
FunctionTraceEvent("transform_response", 2),
|
|
)
|
|
|
|
|
|
def test_projection_does_not_nest_siblings_under_a_returned_config_lookup() -> None:
|
|
events: Final = (
|
|
FunctionTraceEvent("main.py:387 completion", 0),
|
|
FunctionTraceEvent("utils.py:100 ProviderConfigManager.get_provider_chat_config", 1),
|
|
FunctionTraceEvent("utils.py:200 unrelated_helper", 1),
|
|
FunctionTraceEvent("llms/anthropic/chat/transformation.py:100 transform_request", 2),
|
|
)
|
|
|
|
assert pipeline_steps("chat_completions", "python", events) == (
|
|
FunctionTraceEvent("chat_completions", 0),
|
|
FunctionTraceEvent("get_provider_chat_config", 1),
|
|
FunctionTraceEvent("transform_request", 1),
|
|
)
|
|
|
|
|
|
CHAT_RUST_STEPS: Final = (
|
|
"chat_completions",
|
|
"get_provider_chat_config",
|
|
"supported_openai_params",
|
|
"execute_chat_completions_provider_call",
|
|
"validate_environment",
|
|
"transform_request",
|
|
"http_request",
|
|
"transform_response",
|
|
)
|
|
|
|
|
|
@pytest.mark.parametrize("missing", CHAT_RUST_STEPS)
|
|
def test_pipeline_check_rejects_missing_stages(missing: str) -> None:
|
|
steps: Final = tuple(FunctionTraceEvent(name, 0) for name in CHAT_RUST_STEPS if name != missing)
|
|
|
|
assert f"missing {missing}" in pipeline_issues("chat_completions", "rust", steps)
|
|
|
|
|
|
def test_pipeline_check_rejects_http_before_request_transformation() -> None:
|
|
steps: Final = tuple(
|
|
FunctionTraceEvent(name, 0)
|
|
for name in (
|
|
"chat_completions",
|
|
"get_provider_chat_config",
|
|
"supported_openai_params",
|
|
"execute_chat_completions_provider_call",
|
|
"validate_environment",
|
|
"http_request",
|
|
"transform_request",
|
|
"transform_response",
|
|
)
|
|
)
|
|
|
|
assert "transform_request must precede http_request" in pipeline_issues("chat_completions", "rust", steps)
|
|
|
|
|
|
def test_step_parity_rejects_different_handler_boundaries_even_with_valid_stages() -> None:
|
|
rust: Final = tuple(FunctionTraceEvent(name, 0) for name in CHAT_RUST_STEPS)
|
|
python: Final = tuple(
|
|
FunctionTraceEvent(name, 0)
|
|
for name in (
|
|
"chat_completions",
|
|
"get_provider_chat_config",
|
|
"supported_openai_params",
|
|
"validate_environment",
|
|
"transform_request",
|
|
"execute_chat_completions_provider_call",
|
|
"http_request",
|
|
"transform_response",
|
|
)
|
|
)
|
|
|
|
assert not trace_diff(python, rust).shared_order_matches
|
|
assert not trace_diff(python, rust).matches
|
|
assert pipeline_issues("chat_completions", "python", python) == ()
|
|
assert pipeline_issues("chat_completions", "rust", rust) == ()
|
|
|
|
|
|
def test_step_parity_rejects_an_exclusive_helper_with_matching_shared_order() -> None:
|
|
rust: Final = tuple(FunctionTraceEvent(name, 0) for name in CHAT_RUST_STEPS)
|
|
python: Final = (*rust, FunctionTraceEvent("unmatched_helper", 0))
|
|
diff: Final = trace_diff(python, rust)
|
|
|
|
assert diff.shared_order_matches
|
|
assert not diff.matches
|