mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
ee08c36fc0
* wip * refactor(tests): move sdk function tracing into rust python harness * dead code * fix: handle harness keyboard interrupts * refactor(tests): deduplicate rust python harness helpers * fix(harness): expose validated strategy choices * wip * refactor(harness): let strategies own parity reports * docs(harness): update strategy structure * refactor(harness): localize strategy report views * wip * fix(harness): satisfy mapping runner type checks * fix(harness): clarify trace parity output * wip * fix(harness): clarify unit mapping report * fix(harness): finalize trace parity contracts * refactor(harness): structure parity contracts * feat: derive unit test mapping from traces * feat(harness): map rstest test families * feat(ocr): port Azure document intelligence tests * feat(harness): enforce complete unit mappings * feat(ocr): add reducto core transforms * feat(harness): classify host-only unit tests * fix(ocr): complete Rust provider plumbing * fix(harness): reuse OCR parity workers
103 lines
3.2 KiB
Python
103 lines
3.2 KiB
Python
from pathlib import Path
|
|
from typing import Final
|
|
|
|
from ...shared.reporting.models import SURFACES, Coverage
|
|
from ...shared.reporting.strategy import (
|
|
CaseDefinition,
|
|
ModuleCaseSpec,
|
|
NotImplementedCaseSpec,
|
|
StrategyDefinition,
|
|
)
|
|
from .reporting import render_e2e_results
|
|
from .runner import run_e2e_cases
|
|
|
|
CASES: Final[tuple[CaseDefinition, ...]] = (
|
|
CaseDefinition(
|
|
"ocr",
|
|
ModuleCaseSpec(
|
|
coverage=Coverage.PARTIAL,
|
|
module="tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.test_sdk_parity",
|
|
note=(
|
|
"Recorded sync/async SDK parity; invalid-model provider errors differ, "
|
|
"and Reducto lacks a Rust contract."
|
|
),
|
|
),
|
|
surface="sdk",
|
|
),
|
|
CaseDefinition(
|
|
"messages",
|
|
NotImplementedCaseSpec(
|
|
reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered."
|
|
),
|
|
surface="sdk",
|
|
),
|
|
CaseDefinition(
|
|
"responses",
|
|
NotImplementedCaseSpec(
|
|
reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered."
|
|
),
|
|
surface="sdk",
|
|
),
|
|
CaseDefinition(
|
|
"count_tokens",
|
|
NotImplementedCaseSpec(reason="No Rust count_tokens parity test is present yet."),
|
|
surface="sdk",
|
|
),
|
|
CaseDefinition(
|
|
"chat_completions",
|
|
NotImplementedCaseSpec(
|
|
reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered."
|
|
),
|
|
surface="sdk",
|
|
),
|
|
CaseDefinition(
|
|
"transcription",
|
|
NotImplementedCaseSpec(
|
|
reason="Bridge unit tests exist, but no standalone end-to-end parity case is registered."
|
|
),
|
|
surface="sdk",
|
|
),
|
|
CaseDefinition(
|
|
"ocr",
|
|
NotImplementedCaseSpec(reason="No gateway end-to-end OCR parity case is registered."),
|
|
surface="gateway",
|
|
),
|
|
CaseDefinition(
|
|
"messages",
|
|
NotImplementedCaseSpec(reason="No gateway end-to-end Messages parity case is registered."),
|
|
surface="gateway",
|
|
),
|
|
CaseDefinition(
|
|
"responses",
|
|
NotImplementedCaseSpec(reason="No gateway end-to-end Responses parity case is registered."),
|
|
surface="gateway",
|
|
),
|
|
CaseDefinition(
|
|
"count_tokens",
|
|
NotImplementedCaseSpec(reason="No gateway end-to-end token-count parity case is registered."),
|
|
surface="gateway",
|
|
),
|
|
CaseDefinition(
|
|
"chat_completions",
|
|
NotImplementedCaseSpec(reason="No gateway end-to-end chat parity case is registered."),
|
|
surface="gateway",
|
|
),
|
|
CaseDefinition(
|
|
"transcription",
|
|
NotImplementedCaseSpec(reason="No gateway end-to-end transcription parity case is registered."),
|
|
surface="gateway",
|
|
),
|
|
)
|
|
|
|
STRATEGY: Final = StrategyDefinition(
|
|
id="e2e_parity",
|
|
order=10,
|
|
label="End-to-end parity",
|
|
description="Compare observable Python and Rust behavior over generated and recorded inputs.",
|
|
directory=Path(__file__).parent,
|
|
runnable_spec=ModuleCaseSpec,
|
|
cases=CASES,
|
|
run=run_e2e_cases,
|
|
render=render_e2e_results,
|
|
surfaces=SURFACES,
|
|
)
|