mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-15 23:31:29 +00:00
ee08c36fc0
* wip * refactor(tests): move sdk function tracing into rust python harness * dead code * fix: handle harness keyboard interrupts * refactor(tests): deduplicate rust python harness helpers * fix(harness): expose validated strategy choices * wip * refactor(harness): let strategies own parity reports * docs(harness): update strategy structure * refactor(harness): localize strategy report views * wip * fix(harness): satisfy mapping runner type checks * fix(harness): clarify trace parity output * wip * fix(harness): clarify unit mapping report * fix(harness): finalize trace parity contracts * refactor(harness): structure parity contracts * feat: derive unit test mapping from traces * feat(harness): map rstest test families * feat(ocr): port Azure document intelligence tests * feat(harness): enforce complete unit mappings * feat(ocr): add reducto core transforms * feat(harness): classify host-only unit tests * fix(ocr): complete Rust provider plumbing * fix(harness): reuse OCR parity workers
251 lines
9.6 KiB
Python
251 lines
9.6 KiB
Python
from __future__ import annotations
|
|
|
|
from collections.abc import Sequence
|
|
from typing import Final, Literal
|
|
|
|
import pytest
|
|
|
|
from ...shared.reporting.models import CaseResult, Coverage, HarnessCase, ResultArtifact, RunStatus
|
|
from ...shared.reporting.strategy import ModuleCaseSpec, NotImplementedCaseSpec
|
|
from ...shared.tracing.steps import PipelineStep, TraceContract, TraceMapping, mapping
|
|
from . import reporting
|
|
from .reporting import TRACE_COMPARISON_ARTIFACT, TraceComparisonArtifact, render_trace_results
|
|
|
|
MAPPINGS: Final = (
|
|
mapping(rust_span="ocr", python_frame=r"ocr/main\.py:\d+ a?ocr$"),
|
|
mapping(rust_span="http_request", python_frame=r"AsyncHTTPHandler\.post$"),
|
|
)
|
|
|
|
|
|
def _result(comparison: TraceComparisonArtifact) -> CaseResult:
|
|
case: Final = HarnessCase(
|
|
strategy_id="trace_parity",
|
|
strategy_label="Trace parity",
|
|
sdk_function=comparison.sdk_function,
|
|
spec=ModuleCaseSpec(coverage=Coverage.PARTIAL, module="example"),
|
|
surface=comparison.surface,
|
|
)
|
|
result: Final = CaseResult(case=case)
|
|
nodeid: Final = f"trace:sdk:{comparison.sdk_function}:{comparison.scenario}:{comparison.mode}"
|
|
result.collected.add(nodeid)
|
|
result.record(
|
|
nodeid,
|
|
RunStatus.PASSED,
|
|
artifacts=(ResultArtifact(TRACE_COMPARISON_ARTIFACT, comparison.model_dump_json()),),
|
|
)
|
|
return result
|
|
|
|
|
|
def _comparison(
|
|
python: tuple[PipelineStep, ...],
|
|
rust: tuple[PipelineStep, ...],
|
|
*,
|
|
mappings: Sequence[TraceMapping] = MAPPINGS,
|
|
rust_error: str | None = None,
|
|
) -> TraceComparisonArtifact:
|
|
return TraceComparisonArtifact.from_traces(
|
|
surface="sdk",
|
|
sdk_function="ocr",
|
|
scenario="default",
|
|
mode="sync",
|
|
mappings=mappings,
|
|
contract=TraceContract(),
|
|
python=python,
|
|
rust=rust,
|
|
python_unmatched=796,
|
|
rust_error=rust_error,
|
|
)
|
|
|
|
|
|
def _events(*items: tuple[str, int, str | None]) -> tuple[PipelineStep, ...]:
|
|
parents: dict[int, int] = {}
|
|
steps: list[PipelineStep] = []
|
|
for event_id, (span, depth, raw) in enumerate(items):
|
|
parent_id = parents.get(depth - 1) if depth else None
|
|
steps.append(PipelineStep(event_id, parent_id, span, raw if raw is not None else span))
|
|
parents[depth] = event_id
|
|
return tuple(steps)
|
|
|
|
|
|
def test_renderer_shows_matching_python_and_rust_paths() -> None:
|
|
rust: Final = _events(("ocr", 0, None), ("http_request", 1, None))
|
|
python: Final = _events(
|
|
("ocr", 0, "ocr/main.py:88 aocr"),
|
|
("http_request", 1, "http_handler.py:673 AsyncHTTPHandler.post"),
|
|
)
|
|
|
|
section: Final = render_trace_results((_result(_comparison(python, rust)),))[0]
|
|
report: Final = "\n\n".join(section.blocks)
|
|
|
|
assert section.title == "SDK trace comparisons"
|
|
assert "Case: ocr" in report
|
|
assert "PYTHON (2 steps)\n1 aocr (ocr/main.py:88)\n2 AsyncHTTPHandler.post (http_handler.py:673)" in report
|
|
assert "RUST (2 steps)\nocr -> 1 aocr\n http_request -> 2 AsyncHTTPHandler.post" in report
|
|
assert "Mapping (identifier -> span)" not in report
|
|
assert "Trace: MATCH" in report
|
|
assert "Same steps, order, and nesting" in report
|
|
assert "Unseen mappings:" not in report
|
|
|
|
|
|
def test_renderer_reports_mappings_that_matched_nothing() -> None:
|
|
events: Final = _events(("ocr", 0, None))
|
|
|
|
section: Final = render_trace_results((_result(_comparison(events, events)),))[0]
|
|
report: Final = "\n\n".join(section.blocks)
|
|
|
|
assert "Unseen mappings: http_request" in report
|
|
assert "Contract: FAIL" in report
|
|
|
|
|
|
def test_renderer_numbers_repeated_span_occurrences() -> None:
|
|
mappings: Final = (MAPPINGS[0], MAPPINGS[1])
|
|
rust: Final = _events(("ocr", 0, None), ("http_request", 1, None), ("http_request", 1, None))
|
|
python: Final = _events(
|
|
("ocr", 0, "ocr/main.py:88 aocr"),
|
|
("http_request", 1, "http_handler.py:673 AsyncHTTPHandler.post"),
|
|
("http_request", 1, "http_handler.py:673 AsyncHTTPHandler.post"),
|
|
)
|
|
|
|
report: Final = "\n\n".join(render_trace_results((_result(_comparison(python, rust, mappings=mappings)),))[0].blocks)
|
|
|
|
assert "http_request#2" in report
|
|
|
|
|
|
def test_renderer_accepts_declared_engine_specific_steps() -> None:
|
|
mappings: Final = (
|
|
*MAPPINGS[:1],
|
|
mapping(span="python_prepare", python_frame=r"python_prepare$"),
|
|
mapping(rust_span="rust_prepare"),
|
|
)
|
|
python: Final = _events(("ocr", 0, None), ("python_prepare", 1, "prep.py:1 python_prepare"))
|
|
rust: Final = _events(("ocr", 0, None), ("rust_prepare", 1, None))
|
|
|
|
section: Final = render_trace_results((_result(_comparison(python, rust, mappings=mappings)),))[0]
|
|
report: Final = "\n\n".join(section.blocks)
|
|
|
|
assert "2 python_prepare (prep.py:1) [python only]" in report
|
|
assert "rust_prepare -> [rust only]" in report
|
|
assert "Trace: MATCH" in report
|
|
assert "Contract: PASS" in report
|
|
|
|
|
|
def test_unavailable_check_reports_mode_from_nodeid() -> None:
|
|
case: Final = HarnessCase(
|
|
strategy_id="trace_parity",
|
|
strategy_label="Trace parity",
|
|
sdk_function="ocr",
|
|
spec=ModuleCaseSpec(coverage=Coverage.PARTIAL, module="example"),
|
|
surface="sdk",
|
|
)
|
|
result: Final = CaseResult(case=case)
|
|
result.collected.add("trace:sdk:ocr:default:sync")
|
|
result.record("trace:sdk:ocr:default:sync", RunStatus.ERROR)
|
|
|
|
section: Final = render_trace_results((result,))[0]
|
|
report: Final = "\n\n".join(section.blocks)
|
|
|
|
assert "Case: ocr" in report
|
|
assert "Scenario: default / Mode: sync" in report
|
|
assert "Trace: NOT AVAILABLE\nTest outcome: error" in report
|
|
assert "unknown mode" not in report
|
|
|
|
|
|
def test_renderer_keeps_collected_trace_when_one_engine_errors() -> None:
|
|
python: Final = _events(
|
|
("ocr", 0, "ocr/main.py:88 aocr"),
|
|
("http_request", 1, "http_handler.py:673 AsyncHTTPHandler.post"),
|
|
)
|
|
|
|
section: Final = render_trace_results(
|
|
(_result(_comparison(python, (), rust_error="rust: native Rust bridge must include the trace-parity feature")),)
|
|
)[0]
|
|
report: Final = "\n\n".join(section.blocks)
|
|
|
|
assert "PYTHON (2 steps)\n1 aocr (ocr/main.py:88) [python only]" in report
|
|
assert "Rust error: rust: native Rust bridge must include the trace-parity feature" in report
|
|
assert "hint: rebuild the native bridge with the trace-parity feature" in report
|
|
assert "Contract: FAIL" in report
|
|
|
|
|
|
def test_renderer_groups_all_modes_under_one_case_header() -> None:
|
|
case: Final = HarnessCase(
|
|
strategy_id="trace_parity",
|
|
strategy_label="Trace parity",
|
|
sdk_function="ocr",
|
|
spec=ModuleCaseSpec(coverage=Coverage.PARTIAL, module="example"),
|
|
surface="sdk",
|
|
)
|
|
result: Final = CaseResult(case=case)
|
|
events: Final = _events(("ocr", 0, None))
|
|
modes: Final[tuple[Literal["sync", "async"], ...]] = ("sync", "async")
|
|
for mode in modes:
|
|
nodeid = f"trace:sdk:ocr:default:{mode}"
|
|
result.collected.add(nodeid)
|
|
comparison = TraceComparisonArtifact.from_traces(
|
|
surface="sdk",
|
|
sdk_function="ocr",
|
|
scenario="default",
|
|
mode=mode,
|
|
mappings=MAPPINGS,
|
|
contract=TraceContract(),
|
|
python=events,
|
|
rust=events,
|
|
python_unmatched=0,
|
|
)
|
|
result.record(
|
|
nodeid,
|
|
RunStatus.PASSED,
|
|
artifacts=(ResultArtifact(TRACE_COMPARISON_ARTIFACT, comparison.model_dump_json()),),
|
|
)
|
|
|
|
section: Final = render_trace_results((result,))[0]
|
|
|
|
assert len(section.blocks) == 1
|
|
report: Final = section.blocks[0]
|
|
assert report.count("Case: ocr") == 1
|
|
assert "Scenario: default / Mode: sync" in report
|
|
assert "Scenario: default / Mode: async" in report
|
|
|
|
|
|
def test_renderer_colors_every_trace_line_in_a_terminal(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
rust: Final = _events(("ocr", 0, None), ("http_request", 1, None))
|
|
python: Final = _events(
|
|
("ocr", 0, "ocr/main.py:88 aocr"),
|
|
("http_request", 1, "http_handler.py:673 AsyncHTTPHandler.post"),
|
|
)
|
|
monkeypatch.setattr(reporting.sys.stdout, "isatty", lambda: True)
|
|
monkeypatch.delenv("NO_COLOR", raising=False)
|
|
|
|
section: Final = render_trace_results((_result(_comparison(python, rust)),))[0]
|
|
report: Final = "\n\n".join(section.blocks)
|
|
|
|
assert "\033[36mPYTHON\033[0m (2 steps)" in report
|
|
assert "\033[36m1 aocr (ocr/main.py:88)\033[0m" in report
|
|
assert "\033[33mRUST\033[0m (2 steps)" in report
|
|
assert "\033[33mocr\033[0m -> \033[36m1 aocr\033[0m" in report
|
|
assert "\033[33mhttp_request\033[0m -> \033[36m2 AsyncHTTPHandler.post\033[0m" in report
|
|
|
|
|
|
def test_renderer_groups_cases_and_unavailable_entries_by_surface() -> None:
|
|
events: Final = _events(("ocr", 0, None))
|
|
gateway_results: Final = tuple(
|
|
CaseResult(
|
|
case=HarnessCase(
|
|
strategy_id="trace_parity",
|
|
strategy_label="Trace parity",
|
|
sdk_function=sdk_function,
|
|
spec=NotImplementedCaseSpec(reason=f"No {sdk_function} case is registered."),
|
|
surface="gateway",
|
|
),
|
|
status=RunStatus.NOT_IMPLEMENTED,
|
|
)
|
|
for sdk_function in ("ocr", "messages")
|
|
)
|
|
|
|
sections: Final = render_trace_results((_result(_comparison(events, events)), *gateway_results))
|
|
|
|
assert tuple(section.title for section in sections) == ("SDK trace comparisons", "GATEWAY trace comparisons")
|
|
gateway_report: Final = "\n\n".join(sections[1].blocks)
|
|
assert gateway_report.count("Not implemented") == 1
|
|
assert "- ocr: No ocr case is registered." in gateway_report
|
|
assert "- messages: No messages case is registered." in gateway_report
|