From 120fa92e553900615ef778223de785f7217fb908 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 16 Oct 2026 14:42:24 -0700 Subject: [PATCH] docs(rust): design Python drift audits and review enforcement --- tests/rust-python-harness/AGENTS.md | 20 +- tests/rust-python-harness/cli/__init__.py | 34 +- tests/rust-python-harness/cli/catalog.py | 23 +- tests/rust-python-harness/cli/test_cli.py | 34 +- .../shared/reporting/strategy.py | 15 +- .../strategies/drift/AGENTS.md | 20 + .../strategies/drift/__init__.py | 14 + .../strategies/drift/command.py | 56 ++ .../strategies/drift/drift.json | 162 +++++ .../strategies/drift/runner.py | 492 ++++++++++++++ .../strategies/drift/test_command.py | 163 +++++ .../strategies/drift/test_runner.py | 640 ++++++++++++++++++ 12 files changed, 1637 insertions(+), 36 deletions(-) create mode 100644 tests/rust-python-harness/strategies/drift/AGENTS.md create mode 100644 tests/rust-python-harness/strategies/drift/__init__.py create mode 100644 tests/rust-python-harness/strategies/drift/command.py create mode 100644 tests/rust-python-harness/strategies/drift/drift.json create mode 100644 tests/rust-python-harness/strategies/drift/runner.py create mode 100644 tests/rust-python-harness/strategies/drift/test_command.py create mode 100644 tests/rust-python-harness/strategies/drift/test_runner.py diff --git a/tests/rust-python-harness/AGENTS.md b/tests/rust-python-harness/AGENTS.md index 3064960d66e..d63bfb73f11 100644 --- a/tests/rust-python-harness/AGENTS.md +++ b/tests/rust-python-harness/AGENTS.md @@ -9,6 +9,13 @@ tests/rust-python-harness/ │ └── commands.py │ ├── strategies/ +│ ├── drift/ +│ │ ├── __init__.py +│ │ ├── AGENTS.md +│ │ ├── command.py +│ │ ├── drift.json +│ │ └── runner.py +│ │ │ ├── e2e_parity/ │ │ ├── __init__.py │ │ ├── reporting.py @@ -45,11 +52,12 @@ tests/rust-python-harness/ └── suite_runner.py ``` -- A strategy is a folder under `strategies/` with a one-line `AGENTS.md` and an `__init__.py` exporting exactly one `STRATEGY: StrategyDefinition`. Its id must equal the folder name +- Every strategy is a folder under `strategies/` with an `AGENTS.md` and an `__init__.py` exporting exactly one `STRATEGY`. Its id must equal the folder name. SDK strategies export `StrategyDefinition`; the repository-source drift strategy exports `RepositoryStrategyDefinition` - `shared/reporting/strategy.py` is the contract: runnable module/suite specs, not-implemented/skipped specs, the runner protocol, and `StrategyDefinition` -- Every `STRATEGY` explicitly classifies every SDK function. Surface-aware strategies declare their surfaces and classify the complete surface-by-function matrix +- Every SDK `StrategyDefinition` explicitly classifies every SDK function. Surface-aware SDK strategies declare their surfaces and classify the complete surface-by-function matrix. Repository strategies use their own options rather than SDK `--function` or `--surface` selection - Run locally only. No CI integration -- `python -m tests.rust-python-harness run |all` runs the selected strategy. `--function` is common, while each strategy exposes only its supported options +- `uv run python -m tests.rust-python-harness run drift --target HEAD` inventories committed Python and Rust changes using `strategies/drift/drift.json`. See `strategies/drift/AGENTS.md` for usage, exploratory `--since` comparisons, and manual snapshot checkpoints with independent unknown, drifted, or aligned states +- `python -m tests.rust-python-harness run ` runs the selected strategy with its supported options. SDK strategies share `--function`. `run all` runs SDK execution tests; invoke `run drift` explicitly for the repository change inventory - Examples: `run e2e_parity --surface sdk --function ocr`, `run unit_tests_parity --function ocr --pytest-arg=-x`, or `run all --function ocr` - `cli/catalog.py` discovers strategies, validates their Python definitions, and orders them. `cli/__init__.py` builds the Click command tree. `cli/commands.py` runs selected cases - `e2e_parity/` compares SDK objects, exceptions, callbacks, and streams, or gateway HTTP responses @@ -59,8 +67,8 @@ tests/rust-python-harness/ - `unit_tests_parity/runner.py` runs each contract's `unit_parity_scope` with `LITELLM_RUST=0` and `LITELLM_RUST=1` in separate processes and requires matching outcomes, including failures. Exclusions require a reason in the contract - `unit_tests_rust/runner.py` runs each contract's focused Cargo test suite. Native Rust unit tests stay beside their implementation - `shared/unit_runners/suite_runner.py` runs typed suites registered in code with nodeids of the form `suite:::` -- Every strategy declares its report sections and presentation in its own `reporting.py`. Shared reporting code only provides reusable models and cell-formatting primitives +- Every SDK strategy declares its report sections and presentation in its own `reporting.py`. Shared reporting code only provides reusable models and cell-formatting primitives. Drift emits its repository change report directly - `shared/` contains reusable parity, tracing, reporting primitives, and unit-runner machinery - Keep fixtures with their owning API and existing Python tests in their current locations -- Each strategy folder carries an `AGENTS.md` one-liner stating what it should be doing -- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/trace_parity tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust -q` +- Each test strategy folder carries an `AGENTS.md` one-liner stating what it should be doing. The repository-source drift strategy keeps its usage guide in `strategies/drift/AGENTS.md` +- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/drift tests/rust-python-harness/strategies/trace_parity tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust -q` diff --git a/tests/rust-python-harness/cli/__init__.py b/tests/rust-python-harness/cli/__init__.py index 13b995825dd..944b9f2f0f5 100644 --- a/tests/rust-python-harness/cli/__init__.py +++ b/tests/rust-python-harness/cli/__init__.py @@ -2,15 +2,18 @@ from __future__ import annotations import sys from collections.abc import Sequence +from pathlib import Path from typing import Final, cast import click +from pydantic import TypeAdapter from ..shared.reporting.models import SDK_FUNCTIONS, SdkFunction, Strategy, Surface -from .catalog import load_catalog -from .commands import run_command, select_cases +from ..shared.reporting.strategy import RepositoryStrategyDefinition +from .catalog import CatalogStrategy, load_catalog +from .commands import REPO_ROOT, run_command, select_cases -__all__ = ["load_catalog", "main"] +__all__ = ["build_cli", "load_catalog", "main"] _INTERRUPTED_EXIT_CODE: Final = 130 @@ -34,7 +37,7 @@ def _run_all_command(strategies: Sequence[Strategy]) -> click.Command: "all", params=[_function_option()], callback=run_all, - help="run every strategy", + help="run every SDK test strategy", ) @@ -77,14 +80,18 @@ def _strategy_command(strategy: Strategy) -> click.Command: ) -def _build_cli(strategies: Sequence[Strategy]) -> click.Group: +def build_cli(strategies: Sequence[CatalogStrategy], repo_root: Path = REPO_ROOT) -> click.Group: root: Final = click.Group( "rust-python-harness", help="Run Rust/Python parity tests with raw progress and strategy reports.", ) - run: Final = click.Group("run", help="run one strategy or the complete harness") - run.add_command(_run_all_command(strategies)) + run: Final = click.Group("run", help="run one strategy or every SDK test strategy") + test_strategies: Final = tuple(strategy for strategy in strategies if isinstance(strategy, Strategy)) + run.add_command(_run_all_command(test_strategies)) for strategy in strategies: + if isinstance(strategy, RepositoryStrategyDefinition): + run.add_command(strategy.command(repo_root)) + continue run.add_command(_strategy_command(strategy)) root.add_command(run) return root @@ -93,12 +100,15 @@ def _build_cli(strategies: Sequence[Strategy]) -> click.Group: def main(argv: Sequence[str] | None = None) -> int: try: strategies: Final = load_catalog() - result: Final = _build_cli(strategies).main( - args=None if argv is None else list(argv), - prog_name="rust-python-harness", - standalone_mode=False, + result: Final = TypeAdapter[int | None](int | None).validate_python( + build_cli(strategies).main( + args=None if argv is None else list(argv), + prog_name="rust-python-harness", + standalone_mode=False, + ), + strict=True, ) - exit_code: Final = result if isinstance(result, int) else 0 + exit_code: Final = result if result is not None else 0 except click.ClickException as error: error.show() return error.exit_code diff --git a/tests/rust-python-harness/cli/catalog.py b/tests/rust-python-harness/cli/catalog.py index 073873d0121..c162e24eaa6 100644 --- a/tests/rust-python-harness/cli/catalog.py +++ b/tests/rust-python-harness/cli/catalog.py @@ -5,13 +5,16 @@ import importlib import importlib.util import pkgutil import sys +from itertools import product from pathlib import Path from types import ModuleType -from typing import Final +from typing import Final, TypeAlias from .. import strategies as _strategies_package from ..shared.reporting.models import SDK_FUNCTIONS, SURFACES, CaseDisposition, HarnessCase, Strategy -from ..shared.reporting.strategy import StrategyDefinition +from ..shared.reporting.strategy import RepositoryStrategyDefinition, StrategyDefinition + +CatalogStrategy: TypeAlias = Strategy | RepositoryStrategyDefinition _STRATEGIES_PACKAGE: Final = _strategies_package STRATEGIES_ROOT: Final = Path(_STRATEGIES_PACKAGE.__path__[0]) @@ -39,15 +42,19 @@ def _synthetic_module_name(folder: Path) -> str: return f"_harness_strategy_{folder.name}_{digest}" -def _load_strategy(name: str, folder: Path, prefix: str | None) -> Strategy: +def _load_strategy(name: str, folder: Path, prefix: str | None) -> CatalogStrategy: module: Final = _load_strategy_module(name, folder, prefix) definition: Final = getattr(module, "STRATEGY", None) - if not isinstance(definition, StrategyDefinition): - raise ValueError(f"{folder}: __init__.py must export STRATEGY: StrategyDefinition") + if not isinstance(definition, (StrategyDefinition, RepositoryStrategyDefinition)): + raise ValueError( + f"{folder}: __init__.py must export STRATEGY: StrategyDefinition or RepositoryStrategyDefinition" + ) if definition.id != name: raise ValueError(f"{folder}: strategy id {definition.id!r} must match folder name {name!r}") if definition.directory.resolve() != folder.resolve(): raise ValueError(f"{folder}: strategy directory must be {folder}") + if isinstance(definition, RepositoryStrategyDefinition): + return definition if len(set(definition.surfaces)) != len(definition.surfaces) or any( surface not in SURFACES for surface in definition.surfaces ): @@ -56,9 +63,7 @@ def _load_strategy(name: str, folder: Path, prefix: str | None) -> Strategy: duplicates: Final = tuple(sorted(key for key in set(keys) if keys.count(key) > 1)) if duplicates: raise ValueError(f"{folder}: duplicate strategy cases: {duplicates}") - expected: Final = frozenset( - (surface, function) for surface in (definition.surfaces or (None,)) for function in SDK_FUNCTIONS - ) + expected: Final = frozenset(product(definition.surfaces or (None,), SDK_FUNCTIONS)) actual: Final = frozenset(keys) if actual != expected: missing: Final = tuple(sorted(expected - actual)) @@ -94,7 +99,7 @@ def _load_strategy(name: str, folder: Path, prefix: str | None) -> Strategy: ) -def load_catalog(root: Path | None = None) -> tuple[Strategy, ...]: +def load_catalog(root: Path | None = None) -> tuple[CatalogStrategy, ...]: resolved: Final = STRATEGIES_ROOT if root is None else root prefix: Final = _STRATEGIES_PACKAGE.__name__ if resolved == STRATEGIES_ROOT else None folders: Final = tuple(info.name for info in pkgutil.iter_modules([str(resolved)]) if info.ispkg) diff --git a/tests/rust-python-harness/cli/test_cli.py b/tests/rust-python-harness/cli/test_cli.py index ebc4e2653d9..8e98ed50d44 100644 --- a/tests/rust-python-harness/cli/test_cli.py +++ b/tests/rust-python-harness/cli/test_cli.py @@ -22,10 +22,14 @@ from ..shared.reporting.ui import PlainDashboard, final_report, make_dashboard from ..strategies.unit_tests_parity import UNIT_PARITY_SUITES from ..strategies.unit_tests_rust import RUST_SUITES from . import main -from .catalog import STRATEGIES_ROOT, load_catalog +from .catalog import STRATEGIES_ROOT, CatalogStrategy, load_catalog from .commands import REPO_ROOT, select_cases +def _sdk_strategies(strategies: Sequence[CatalogStrategy]) -> tuple[Strategy, ...]: + return tuple(strategy for strategy in strategies if isinstance(strategy, Strategy)) + + def _strategy_source( *, strategy_id: str = "example", @@ -84,16 +88,17 @@ def _write_strategy_folder( return folder -def test_should_load_surface_aware_and_function_only_strategies() -> None: +def test_should_load_test_and_repository_strategies() -> None: strategies: Final = load_catalog() assert [strategy.id for strategy in strategies] == [ + "drift", "e2e_parity", "trace_parity", "unit_tests_parity", "unit_tests_rust", ] - for strategy in strategies: + for strategy in _sdk_strategies(strategies): expected: Final = tuple( (surface, function) for surface in (strategy.definition.surfaces or (None,)) for function in SDK_FUNCTIONS ) @@ -102,7 +107,9 @@ def test_should_load_surface_aware_and_function_only_strategies() -> None: def test_unit_strategies_use_function_only_cases() -> None: strategies: Final = { - strategy.id: strategy for strategy in load_catalog() if strategy.id in {"unit_tests_parity", "unit_tests_rust"} + strategy.id: strategy + for strategy in _sdk_strategies(load_catalog()) + if strategy.id in {"unit_tests_parity", "unit_tests_rust"} } for sdk_function in SDK_FUNCTIONS: @@ -122,7 +129,7 @@ def test_unit_strategies_use_function_only_cases() -> None: def test_raw_dashboard_is_always_the_default() -> None: - assert isinstance(make_dashboard(load_catalog()), PlainDashboard) + assert isinstance(make_dashboard(_sdk_strategies(load_catalog())), PlainDashboard) def test_every_strategy_folder_complies() -> None: @@ -132,7 +139,7 @@ def test_every_strategy_folder_complies() -> None: } assert folders == {strategy.id for strategy in strategies} - for strategy in strategies: + for strategy in _sdk_strategies(strategies): definition: Final = strategy.definition assert isinstance(definition, StrategyDefinition) assert definition.directory == strategy.directory @@ -201,7 +208,7 @@ def test_should_reject_an_unavailable_case_with_a_blank_reason( def test_should_select_functions_and_surfaces() -> None: - strategy: Final = next(strategy for strategy in load_catalog() if strategy.id == "e2e_parity") + strategy: Final = next(strategy for strategy in _sdk_strategies(load_catalog()) if strategy.id == "e2e_parity") assert tuple(case.key for case in select_cases((strategy,), {"messages"})) == ( "e2e_parity:messages", @@ -240,7 +247,7 @@ def test_every_unavailable_case_finishes_and_explains_itself() -> None: } unavailable: Final = tuple( (strategy, case) - for strategy in load_catalog() + for strategy in _sdk_strategies(load_catalog()) for case in strategy.cases if case.spec.disposition is not CaseDisposition.RUNNABLE ) @@ -283,6 +290,7 @@ def test_run_help_lists_all_and_every_strategy(capsys: pytest.CaptureFixture[str "trace_parity", "unit_tests_parity", "unit_tests_rust", + "drift", ): assert command in captured.out @@ -292,6 +300,8 @@ def test_run_help_lists_all_and_every_strategy(capsys: pytest.CaptureFixture[str ( ("list",), ("check",), + ("drift",), + ("run", "drift", "--function", "messages"), ("run", "--strategy", "unit_tests_parity"), ("run", "unit_tests_parity", "--surface", "sdk"), ("run", "unit_tests_parity", "--plain"), @@ -473,3 +483,11 @@ def test_strategy_subcommand_accepts_function_filter(capsys: pytest.CaptureFixtu assert exit_code == 0 assert "- messages: not_implemented" in captured.out assert "unit_tests_rust:messages: not_implemented" not in captured.out + + +def test_drift_help_exposes_inventory_options(capsys: pytest.CaptureFixture[str]) -> None: + exit_code: Final = main(("run", "drift", "--help")) + captured: Final = capsys.readouterr() + + assert exit_code == 0 + assert all(option in captured.out for option in ("--target", "--manifest", "--since", "--output")) diff --git a/tests/rust-python-harness/shared/reporting/strategy.py b/tests/rust-python-harness/shared/reporting/strategy.py index 7e76f035e20..e10735f2e54 100644 --- a/tests/rust-python-harness/shared/reporting/strategy.py +++ b/tests/rust-python-harness/shared/reporting/strategy.py @@ -3,13 +3,16 @@ from __future__ import annotations from collections.abc import Callable, Sequence from dataclasses import dataclass from pathlib import Path -from typing import Annotated, Literal, Protocol, TypeAlias +from typing import TYPE_CHECKING, Annotated, Literal, Protocol, TypeAlias from pydantic import BaseModel, ConfigDict, StringConstraints from .models import CaseDisposition, Coverage, HarnessCase, HarnessRun, SdkFunction, Surface from .rendering import StrategyRenderer +if TYPE_CHECKING: + from click import Command + UpdateCallback: TypeAlias = Callable[[HarnessRun], None] NonBlankString: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] @@ -90,3 +93,13 @@ class StrategyDefinition: render: StrategyRenderer surfaces: tuple[Surface, ...] = () runner_argument: RunnerArgumentDefinition | None = None + + +@dataclass(frozen=True, slots=True) +class RepositoryStrategyDefinition: + id: str + order: int + label: str + description: str + directory: Path + command: Callable[[Path], Command] diff --git a/tests/rust-python-harness/strategies/drift/AGENTS.md b/tests/rust-python-harness/strategies/drift/AGENTS.md new file mode 100644 index 00000000000..5398b337cfa --- /dev/null +++ b/tests/rust-python-harness/strategies/drift/AGENTS.md @@ -0,0 +1,20 @@ +# Drift + +Compare committed Python and Rust changes using this folder's `drift.json`. Run locally without provider calls or CI integration + +```sh +uv run python -m tests.rust-python-harness run drift --target HEAD --output /tmp/litellm-drift.json +``` + +Paths are repository-relative files or directory prefixes ending in `/`. Keep unmapped changes and uncommitted files visible in the report. Commit manifest renames separately from mapping edits so Git can follow the old watch paths + +`checkpoint_sha` is a comparison snapshot. `checkpoint_state` records `unknown`, `drifted`, or `aligned` at that snapshot. Never update either automatically. `no_changes` does not imply alignment + +A mapping may carry its own `checkpoint_sha` and `checkpoint_state`, recording the commit at which its Rust paths were reviewed against its Python paths. `mapping_checkpoints` in the report lists, for each such mapping, the commits and patches touching its paths since that checkpoint. Port those changes to the other side, then move the mapping's `checkpoint_sha` forward by hand in the same PR + +```sh +uv run python -m tests.rust-python-harness run drift --output /tmp/litellm-drift.json +jq '.mapping_checkpoints[] | select(.id == "")' /tmp/litellm-drift.json +``` + +Use `--since` for an exploratory comparison and `--manifest` for another mapping file. Omit `--output` to print JSON. Fetch remote refs before comparing them. This strategy has its own options and does not run through `run all` diff --git a/tests/rust-python-harness/strategies/drift/__init__.py b/tests/rust-python-harness/strategies/drift/__init__.py new file mode 100644 index 00000000000..dd743387f84 --- /dev/null +++ b/tests/rust-python-harness/strategies/drift/__init__.py @@ -0,0 +1,14 @@ +from pathlib import Path +from typing import Final + +from ...shared.reporting.strategy import RepositoryStrategyDefinition +from .command import drift_command + +STRATEGY: Final = RepositoryStrategyDefinition( + id="drift", + order=5, + label="Drift", + description="Inventory committed Python and Rust changes from an explicit checkpoint.", + directory=Path(__file__).parent, + command=drift_command, +) diff --git a/tests/rust-python-harness/strategies/drift/command.py b/tests/rust-python-harness/strategies/drift/command.py new file mode 100644 index 00000000000..23cb6506608 --- /dev/null +++ b/tests/rust-python-harness/strategies/drift/command.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Final + +import click + +from .runner import DriftError, inspect_drift + + +def drift_command(repo_root: Path) -> click.Command: + def inspect(target: str, manifest: Path, since: str | None, output: Path | None) -> int: + manifest_path: Final = repo_root / manifest + try: + overwrites_manifest: Final = output is not None and ( + output.resolve() == manifest_path.resolve() + or (output.exists() and manifest_path.exists() and output.samefile(manifest_path)) + ) + except (OSError, RuntimeError) as error: + raise click.ClickException(f"Cannot resolve drift output or manifest path, {error}") from error + if overwrites_manifest: + raise click.ClickException("Output path must differ from the drift manifest") + report: Final = inspect_drift(repo_root, manifest_path, target_ref=target, since_ref=since) + if isinstance(report, DriftError): + raise click.ClickException(report.message) + rendered: Final = report.model_dump_json(indent=2) + if output is None: + click.echo(rendered) + return 0 + try: + output.write_text(f"{rendered}\n", encoding="utf-8") + except OSError as error: + raise click.ClickException(f"Cannot write drift report to {output}, {error}") from error + return 0 + + return click.Command( + "drift", + params=[ + click.Option(("--target",), default="HEAD", show_default=True, help="Git revision to inspect"), + click.Option( + ("--manifest",), + default="tests/rust-python-harness/strategies/drift/drift.json", + type=click.Path(path_type=Path, dir_okay=False), + show_default=True, + help="Manifest path, relative to the repository root", + ), + click.Option(("--since",), help="Exploratory comparison revision, overriding the checkpoint"), + click.Option( + ("--output",), + type=click.Path(path_type=Path, dir_okay=False), + help="Write JSON to this path instead of stdout", + ), + ], + callback=inspect, + help="Inventory Python and Rust changes. Exit zero means a report was generated, without asserting parity.", + ) diff --git a/tests/rust-python-harness/strategies/drift/drift.json b/tests/rust-python-harness/strategies/drift/drift.json new file mode 100644 index 00000000000..58dca7c1363 --- /dev/null +++ b/tests/rust-python-harness/strategies/drift/drift.json @@ -0,0 +1,162 @@ +{ + "schema_version": 1, + "checkpoint_sha": "6530b329f43ef456c502ba2db7656f871d62474e", + "checkpoint_state": "unknown", + "candidate_paths": [ + "litellm/", + "litellm-rust/", + "tests/", + "model_prices_and_context_window.json", + "model_prices_and_context_window.schema.json", + "pyproject.toml", + "uv.lock", + "packaging/litellm-core/", + "Makefile", + ".github/workflows/", + ".github/CODEOWNERS", + "enterprise/litellm_enterprise/", + "enterprise/pyproject.toml" + ], + "mappings": [ + { + "id": "messages", + "python_paths": [ + "litellm/messages/", + "litellm/anthropic_interface/", + "litellm/llms/anthropic/pass_through/", + "litellm/llms/base_llm/anthropic_messages/", + "litellm/types/llms/anthropic_messages/", + "litellm/llms/azure_ai/anthropic/", + "litellm/llms/bedrock/claude_platform/", + "litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/", + "litellm/anthropic_beta_headers_manager.py", + "litellm/anthropic_beta_headers_config.json", + "litellm/proxy/anthropic_endpoints/", + "litellm/rust_bridge/messages/", + "litellm/rust_bridge/host/routes/messages.py", + "litellm/llms/bedrock/messages/", + "litellm/llms/bedrock_mantle/messages/", + "litellm/llms/deepseek/messages/", + "litellm/llms/edenai/messages/", + "litellm/llms/github_copilot/messages/", + "litellm/llms/minimax/messages/", + "litellm/llms/openai_like/messages/", + "litellm/llms/tencent/messages/", + "litellm/llms/anthropic/common_utils.py", + "litellm/types/llms/anthropic.py", + "litellm/types/llms/anthropic_tool_search.py" + ], + "rust_paths": [ + "litellm-rust/crates/inference-messages/", + "litellm-rust/crates/llms-types/src/formats/messages/", + "litellm-rust/crates/llms/src/base_llm/messages/", + "litellm-rust/crates/llms/src/bedrock/claude_platform/messages/", + "litellm-rust/crates/python-bridge/src/routes/messages/", + "litellm-rust/crates/gateway-inference/src/messages.rs", + "litellm-rust/crates/llms/src/anthropic/messages/", + "litellm-rust/crates/llms/src/azure_ai/messages/", + "litellm-rust/crates/llms/src/bedrock/messages/", + "litellm-rust/crates/llms/src/bedrock_mantle/messages/", + "litellm-rust/crates/llms/src/deepseek/messages/", + "litellm-rust/crates/llms/src/edenai/messages/", + "litellm-rust/crates/llms/src/minimax/messages/", + "litellm-rust/crates/llms/src/openai_like/messages/", + "litellm-rust/crates/llms/src/openrouter/messages/", + "litellm-rust/crates/llms/src/tencent/messages/", + "litellm-rust/crates/llms/src/vertex_ai/messages/", + "litellm-rust/crates/llms/src/anthropic/common_utils.rs", + "litellm-rust/crates/llms/src/anthropic/beta_headers.rs", + "litellm-rust/crates/llms-types/src/providers/anthropic/" + ] + }, + { + "id": "responses", + "python_paths": [ + "litellm/responses/", + "litellm/types/responses/", + "litellm/types/llms/openai.py", + "litellm/completion_extras/litellm_responses_transformation/", + "litellm/proxy/response_api_endpoints/", + "litellm/rust_bridge/responses/", + "litellm/rust_bridge/host/routes/responses.py", + "litellm/llms/azure/responses/", + "litellm/llms/azure_ai/responses/", + "litellm/llms/base_llm/responses/", + "litellm/llms/bedrock/responses/", + "litellm/llms/bedrock_mantle/responses/", + "litellm/llms/chatgpt/responses/", + "litellm/llms/databricks/responses/", + "litellm/llms/edenai/responses/", + "litellm/llms/fireworks_ai/responses/", + "litellm/llms/github_copilot/responses/", + "litellm/llms/hosted_vllm/responses/", + "litellm/llms/litellm_proxy/responses/", + "litellm/llms/manus/responses/", + "litellm/llms/openai/responses/", + "litellm/llms/openai_like/responses/", + "litellm/llms/openrouter/responses/", + "litellm/llms/perplexity/responses/", + "litellm/llms/sail/responses/", + "litellm/llms/volcengine/responses/", + "litellm/llms/xai/responses/", + "litellm/llms/anthropic/pass_through/responses_adapters/", + "litellm/llms/openai/common_utils.py", + "litellm/llms/openai/workload_identity.py", + "enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py", + "enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py" + ], + "rust_paths": [ + "litellm-rust/crates/inference-responses/", + "litellm-rust/crates/llms-types/src/formats/responses/", + "litellm-rust/crates/python-bridge/src/routes/responses/", + "litellm-rust/crates/gateway-inference/src/responses.rs", + "litellm-rust/crates/llms/src/base_llm/responses/", + "litellm-rust/crates/llms/src/openai/responses/", + "litellm-rust/crates/llms/src/openrouter/responses/" + ] + }, + { + "id": "shared-runtime", + "python_paths": [ + "litellm/__init__.py", + "litellm/main.py", + "litellm/utils.py", + "litellm/router.py", + "litellm/exceptions.py", + "litellm/types/utils.py", + "litellm/litellm_core_utils/", + "litellm/llms/custom_httpx/", + "litellm/llms/openai_like/", + "litellm/rust_bridge/", + "litellm/caching/", + "litellm/secret_managers/", + "litellm/model_prices_and_context_window_backup.json", + "model_prices_and_context_window.json", + "litellm/router_utils/common_utils.py", + "litellm/router_utils/cooldown_handlers.py", + "litellm/types/router.py", + "litellm/types/litellm_params.py", + "litellm/proxy/common_request_processing.py" + ], + "rust_paths": [ + "litellm-rust/crates/core-utils/", + "litellm-rust/crates/http/", + "litellm-rust/crates/framer/", + "litellm-rust/crates/inference/", + "litellm-rust/crates/host/", + "litellm-rust/crates/host-python/", + "litellm-rust/crates/host-http/", + "litellm-rust/crates/python-bridge/", + "litellm-rust/crates/callbacks-legacy-python/", + "litellm-rust/crates/cache-response/", + "litellm-rust/crates/router/", + "litellm-rust/crates/router-types/", + "litellm-rust/crates/gateway-inference/", + "litellm-rust/crates/model-catalog/", + "litellm-rust/crates/llms/src/base_llm/sse.rs", + "litellm-rust/crates/llms/src/base_llm/auth.rs", + "litellm-rust/crates/llms/src/error.rs" + ] + } + ] +} diff --git a/tests/rust-python-harness/strategies/drift/runner.py b/tests/rust-python-harness/strategies/drift/runner.py new file mode 100644 index 00000000000..bf5609a71ff --- /dev/null +++ b/tests/rust-python-harness/strategies/drift/runner.py @@ -0,0 +1,492 @@ +from __future__ import annotations + +import hashlib +import subprocess +from collections.abc import Iterator +from dataclasses import dataclass +from itertools import chain +from pathlib import Path, PurePosixPath +from typing import Annotated, Final, Literal, TypeAlias + +from pydantic import BaseModel, ConfigDict, Field, ValidationError + +_Paths: TypeAlias = Annotated[tuple[str, ...], Field(min_length=1)] +_Sha: TypeAlias = Annotated[str, Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")] +CheckpointState: TypeAlias = Literal["unknown", "drifted", "aligned"] + + +class _Mapping(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + id: Annotated[str, Field(min_length=1)] + python_paths: _Paths + rust_paths: _Paths + checkpoint_sha: _Sha | None = None + checkpoint_state: CheckpointState = "unknown" + + +class _Manifest(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: Literal[1] + checkpoint_sha: _Sha | None + checkpoint_state: CheckpointState = "unknown" + candidate_paths: _Paths + mappings: Annotated[tuple[_Mapping, ...], Field(min_length=1)] + + +@dataclass(frozen=True, slots=True) +class DriftError: + message: str + + +@dataclass(frozen=True, slots=True) +class _PathChange: + status: str + old_path: str | None + path: str + + @property + def paths(self) -> tuple[str, ...]: + return (self.path,) if self.old_path is None else (self.old_path, self.path) + + +class DriftChange(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: str + old_path: str | None + path: str + mapping_ids: tuple[str, ...] + sides: tuple[Literal["python", "rust"], ...] + patch: str + + +class DriftCommit(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + sha: str + subject: str + + +class MappingDrift(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + id: str + status: Literal["changes_found", "no_changes"] + checkpoint_sha: str + checkpoint_state: CheckpointState + commits: tuple[DriftCommit, ...] + changes: tuple[DriftChange, ...] + + +class WorkingTreeChange(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: str + old_path: str | None + path: str + + +class DriftReport(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + status: Literal["unreviewed", "changes_found", "no_changes"] + target_sha: str + checkpoint_sha: str | None + checkpoint_state: CheckpointState + comparison_sha: str | None + comparison_source: Literal["checkpoint", "override", "none"] + manifest_path: str + manifest_sha256: str + mappings: tuple[_Mapping, ...] + changes: tuple[DriftChange, ...] + unclassified: tuple[DriftChange, ...] + mapping_checkpoints: tuple[MappingDrift, ...] + working_tree_changes: tuple[WorkingTreeChange, ...] + + +def _git(repo_root: Path, *args: str) -> str | DriftError: + try: + result: Final = subprocess.run( + ("git", "--no-optional-locks", "--no-pager", "--literal-pathspecs", *args), + cwd=repo_root, + capture_output=True, + check=False, + timeout=60, + ) + stdout: Final = result.stdout.decode("utf-8") + stderr: Final = result.stderr.decode("utf-8") + except (OSError, subprocess.SubprocessError, UnicodeError) as error: + return DriftError(f"Git operation failed: {error}") + if result.returncode: + return DriftError(f"Git {' '.join(args)} failed: {stderr.strip()}") + return stdout + + +def _commit(repo_root: Path, ref: str) -> str | DriftError: + result: Final = _git(repo_root, "rev-parse", "--verify", "--end-of-options", f"{ref}^{{commit}}") + return result if isinstance(result, DriftError) else result.strip() + + +def _read_manifest(source: str, label: str) -> _Manifest | DriftError: + try: + manifest: Final = _Manifest.model_validate_json(source) + except ValidationError as error: + return DriftError(f"Invalid manifest {label}: {error}") + if manifest.checkpoint_sha is None and manifest.checkpoint_state != "unknown": + return DriftError(f"Invalid manifest {label}: checkpoint state requires a checkpoint SHA") + stateless: Final = next( + ( + mapping.id + for mapping in manifest.mappings + if mapping.checkpoint_sha is None and mapping.checkpoint_state != "unknown" + ), + None, + ) + if stateless is not None: + return DriftError(f"Invalid manifest {label}: mapping {stateless!r} checkpoint state requires a checkpoint SHA") + selectors: Final = tuple(chain(manifest.candidate_paths, _mapping_paths(manifest.mappings))) + invalid: Final = next((path for path in selectors if not _valid_selector(path)), None) + if invalid is not None: + return DriftError(f"Invalid manifest path {invalid!r}: use a relative file or directory prefix ending in /") + ids: Final = tuple(mapping.id for mapping in manifest.mappings) + if len(frozenset(ids)) != len(ids) or any(not name.strip() for name in ids): + return DriftError(f"Invalid manifest {label}: mapping IDs must be nonblank and unique") + return manifest + + +def _valid_selector(path: str) -> bool: + return bool( + path + and not path.startswith("/") + and not any(character in path for character in ("*", "?", "[", "]", "\\", "\0")) + and all(part not in {"", ".", ".."} for part in path.removesuffix("/").split("/")) + ) + + +def _mapping_paths(mappings: tuple[_Mapping, ...]) -> Iterator[str]: + return chain.from_iterable(chain(mapping.python_paths, mapping.rust_paths) for mapping in mappings) + + +def _manifest_at_path(repo_root: Path, sha: str, path: str) -> tuple[_Manifest, ...] | DriftError: + listing: Final = _git(repo_root, "ls-tree", "-z", sha, "--", path) + if isinstance(listing, DriftError): + return listing + if not listing: + return () + source: Final = _git(repo_root, "show", f"{sha}:{path}") + if isinstance(source, DriftError): + return source + manifest: Final = _read_manifest(source, f"{sha}:{path}") + return manifest if isinstance(manifest, DriftError) else (manifest,) + + +def _manifest_at(repo_root: Path, sha: str, path: str) -> tuple[_Manifest, ...] | DriftError: + existing: Final = _manifest_at_path(repo_root, sha, path) + if isinstance(existing, DriftError) or existing: + return existing + previous_path: Final = _historical_manifest_path(repo_root, sha, path) + if isinstance(previous_path, DriftError): + return previous_path + return () if previous_path == path else _manifest_at_path(repo_root, sha, previous_path) + + +def _matches(paths: tuple[str, ...], selectors: tuple[str, ...]) -> bool: + return any(_matches_path(path, selectors) for path in paths) + + +def _matches_path(path: str, selectors: tuple[str, ...]) -> bool: + return any(path == selector or (selector.endswith("/") and path.startswith(selector)) for selector in selectors) + + +def _parse_change(status: str, tokens: Iterator[str]) -> _PathChange: + first: Final = next(tokens) + if status.startswith(("R", "C")): + return _PathChange(status, first, next(tokens)) + return _PathChange(status, None, first) + + +def _parse_changes(source: str) -> Iterator[_PathChange]: + tokens: Final = iter(source.split("\0")[:-1]) + return (_parse_change(status, tokens) for status in tokens) + + +def _parse_working_change(record: str, tokens: Iterator[str]) -> _PathChange: + status: Final = record[:2] + path: Final = record[3:] + old_path: Final = next(tokens) if "R" in status or "C" in status else None + return _PathChange(status, old_path, path) + + +def _working_changes(repo_root: Path, selectors: tuple[str, ...]) -> tuple[WorkingTreeChange, ...] | DriftError: + source: Final = _git(repo_root, "status", "--porcelain=v1", "-z", "--untracked-files=all", "--renames") + if isinstance(source, DriftError): + return source + tokens: Final = iter(source.split("\0")[:-1]) + parsed: Final = (_parse_working_change(record, tokens) for record in tokens) + return tuple( + WorkingTreeChange(status=change.status, old_path=change.old_path, path=change.path) + for change in parsed + if _matches(change.paths, selectors) + ) + + +def _historical_manifest_path(repo_root: Path, sha: str, path: str) -> str | DriftError: + working: Final = _working_changes(repo_root, (path,)) + if isinstance(working, DriftError): + return working + staged_path: Final = next( + (change.old_path for change in working if change.path == path and change.old_path is not None), None + ) + head_path: Final = staged_path if staged_path is not None else path + head: Final = _commit(repo_root, "HEAD") + if isinstance(head, DriftError): + return head + endpoint: Final = _git( + repo_root, + "diff", + "--no-color", + "--no-ext-diff", + "--no-textconv", + "--find-renames=50%", + "--name-status", + "-z", + sha, + head, + ) + if isinstance(endpoint, DriftError): + return endpoint + endpoint_path: Final = next( + ( + change.old_path + for change in _parse_changes(endpoint) + if change.path == head_path and change.status.startswith("R") + ), + None, + ) + if endpoint_path is not None: + return endpoint_path + source: Final = _git( + repo_root, + "log", + "--follow", + "--format=", + "--no-color", + "--no-ext-diff", + "--no-textconv", + "--find-renames=50%", + "--diff-filter=R", + "--name-status", + "-z", + f"{sha}..{head}", + "--", + head_path, + ) + if isinstance(source, DriftError): + return source + previous: Final = tuple(change.old_path for change in _parse_changes(source) if change.old_path is not None) + return previous[-1] if previous else head_path + + +def _change_report( + repo_root: Path, baseline: str, target: str, change: _PathChange, mappings: tuple[_Mapping, ...] +) -> DriftChange | DriftError: + patch: Final = _git( + repo_root, + "diff", + "--no-color", + "--no-ext-diff", + "--no-textconv", + "--find-renames=50%", + baseline, + target, + "--", + *change.paths, + ) + if isinstance(patch, DriftError): + return patch + python: Final = any(_matches(change.paths, mapping.python_paths) for mapping in mappings) + rust: Final = any(_matches(change.paths, mapping.rust_paths) for mapping in mappings) + sides: Final[tuple[Literal["python", "rust"], ...]] = (("python",) if python else ()) + (("rust",) if rust else ()) + ids: Final = tuple( + sorted( + frozenset( + mapping.id + for mapping in mappings + if _matches(change.paths, (*mapping.python_paths, *mapping.rust_paths)) + ) + ) + ) + return DriftChange( + status=change.status, old_path=change.old_path, path=change.path, mapping_ids=ids, sides=sides, patch=patch + ) + + +def _ancestor(repo_root: Path, baseline: str, target: str) -> DriftError | None: + result: Final = _git(repo_root, "merge-base", "--is-ancestor", baseline, target) + if isinstance(result, DriftError): + return DriftError(f"Comparison commit must be an ancestor of the target: {result.message}") + return None + + +def _changed_files( + repo_root: Path, baseline: str | None, target: str, selectors: tuple[str, ...], mappings: tuple[_Mapping, ...] +) -> tuple[DriftChange, ...] | DriftError: + if baseline is None: + return () + ancestry: Final = _ancestor(repo_root, baseline, target) + if ancestry is not None: + return ancestry + source: Final = _git( + repo_root, + "diff", + "--no-color", + "--no-ext-diff", + "--no-textconv", + "--find-renames=50%", + "--name-status", + "-z", + baseline, + target, + ) + if isinstance(source, DriftError): + return source + reports: Final = tuple( + _change_report(repo_root, baseline, target, change, mappings) + for change in _parse_changes(source) + if _matches(change.paths, selectors) + ) + error: Final = next((report for report in reports if isinstance(report, DriftError)), None) + if error is not None: + return error + return tuple(report for report in reports if isinstance(report, DriftChange)) + + +def _commits( + repo_root: Path, baseline: str, target: str, paths: tuple[str, ...] +) -> tuple[DriftCommit, ...] | DriftError: + source: Final = _git( + repo_root, "log", "-z", "--no-color", "--format=%H%x00%s", f"{baseline}..{target}", "--", *paths + ) + if isinstance(source, DriftError): + return source + fields: Final = iter(source.split("\0")) + return tuple(DriftCommit(sha=sha, subject=subject) for sha, subject in zip(fields, fields) if sha) + + +def _mapping_drift(repo_root: Path, target: str, mapping: _Mapping) -> MappingDrift | DriftError | None: + if mapping.checkpoint_sha is None: + return None + checkpoint: Final = _commit(repo_root, mapping.checkpoint_sha) + if isinstance(checkpoint, DriftError): + return checkpoint + paths: Final = (*mapping.python_paths, *mapping.rust_paths) + changes: Final = _changed_files(repo_root, checkpoint, target, paths, (mapping,)) + if isinstance(changes, DriftError): + return changes + commits: Final = _commits(repo_root, checkpoint, target, paths) + if isinstance(commits, DriftError): + return commits + return MappingDrift( + id=mapping.id, + status="changes_found" if changes else "no_changes", + checkpoint_sha=checkpoint, + checkpoint_state=mapping.checkpoint_state, + commits=commits, + changes=changes, + ) + + +def _merged_mapping(identifier: str, mappings: tuple[_Mapping, ...]) -> _Mapping: + matching: Final = tuple(mapping for mapping in mappings if mapping.id == identifier) + python: Final = chain.from_iterable(mapping.python_paths for mapping in matching) + rust: Final = chain.from_iterable(mapping.rust_paths for mapping in matching) + return _Mapping( + id=identifier, + python_paths=tuple(sorted(frozenset(python))), + rust_paths=tuple(sorted(frozenset(rust))), + checkpoint_sha=matching[-1].checkpoint_sha, + checkpoint_state=matching[-1].checkpoint_state, + ) + + +def _report( + repo_root: Path, path: str, raw: str, manifest: _Manifest, target: str, baseline: str | None, override: bool +) -> DriftReport | DriftError: + previous: Final = () if baseline is None else _manifest_at(repo_root, baseline, path) + if isinstance(previous, DriftError): + return previous + current: Final = _manifest_at(repo_root, target, path) + if isinstance(current, DriftError): + return current + checkpoint_policy: Final = ( + () + if manifest.checkpoint_sha is None or manifest.checkpoint_sha in {baseline, target} + else _manifest_at(repo_root, manifest.checkpoint_sha, path) + ) + if isinstance(checkpoint_policy, DriftError): + return checkpoint_policy + policies: Final = (*previous, *checkpoint_policy, *current, manifest) + all_mappings: Final = tuple(chain.from_iterable(policy.mappings for policy in policies)) + ids: Final = tuple(sorted(frozenset(mapping.id for mapping in all_mappings))) + mappings: Final = tuple(_merged_mapping(identifier, all_mappings) for identifier in ids) + candidates: Final = chain.from_iterable(policy.candidate_paths for policy in policies) + selectors: Final = tuple(chain(candidates, _mapping_paths(mappings), (path,))) + changes: Final = _changed_files(repo_root, baseline, target, selectors, mappings) + if isinstance(changes, DriftError): + return changes + working: Final = _working_changes(repo_root, selectors) + if isinstance(working, DriftError): + return working + drifts: Final = tuple(_mapping_drift(repo_root, target, mapping) for mapping in manifest.mappings) + drift_error: Final = next((drift for drift in drifts if isinstance(drift, DriftError)), None) + if drift_error is not None: + return drift_error + status: Final = "unreviewed" if manifest.checkpoint_sha is None else "changes_found" if changes else "no_changes" + return DriftReport( + status=status, + target_sha=target, + checkpoint_sha=manifest.checkpoint_sha, + checkpoint_state=manifest.checkpoint_state, + comparison_sha=baseline, + comparison_source="override" if override else "checkpoint" if baseline else "none", + manifest_path=path, + manifest_sha256=hashlib.sha256(raw.encode("utf-8")).hexdigest(), + mappings=mappings, + changes=changes, + unclassified=tuple(change for change in changes if not change.mapping_ids), + mapping_checkpoints=tuple(drift for drift in drifts if isinstance(drift, MappingDrift)), + working_tree_changes=working, + ) + + +def inspect_drift( + repo_root: Path, manifest_path: Path, target_ref: str = "HEAD", since_ref: str | None = None +) -> DriftReport | DriftError: + root: Final = repo_root.resolve() + try: + resolved_path: Final = (root / manifest_path).resolve() + relative: Final = resolved_path.relative_to(root).as_posix() + raw: Final = resolved_path.read_bytes().decode("utf-8") + except (OSError, RuntimeError, ValueError) as error: + return DriftError(f"Cannot read manifest inside repository: {error}") + if not _valid_selector(relative) or PurePosixPath(relative).as_posix() != relative: + return DriftError("Manifest must be a repository-relative file") + manifest: Final = _read_manifest(raw, relative) + if isinstance(manifest, DriftError): + return manifest + target: Final = _commit(root, target_ref) + if isinstance(target, DriftError): + return target + checkpoint: Final = None if manifest.checkpoint_sha is None else _commit(root, manifest.checkpoint_sha) + if isinstance(checkpoint, DriftError): + return checkpoint + checkpoint_ancestry: Final = None if checkpoint is None else _ancestor(root, checkpoint, target) + if checkpoint_ancestry is not None: + return checkpoint_ancestry + reference: Final = since_ref if since_ref is not None else checkpoint + baseline: Final = None if reference is None else _commit(root, reference) + if isinstance(baseline, DriftError): + return baseline + return _report(root, relative, raw, manifest, target, baseline, since_ref is not None) diff --git a/tests/rust-python-harness/strategies/drift/test_command.py b/tests/rust-python-harness/strategies/drift/test_command.py new file mode 100644 index 00000000000..f30a961fd4a --- /dev/null +++ b/tests/rust-python-harness/strategies/drift/test_command.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import json +import os +import subprocess +from pathlib import Path +from typing import Final + +import pytest +from click.testing import CliRunner + +from ...cli import build_cli +from ...cli.catalog import load_catalog +from .command import drift_command +from .runner import DriftReport + + +def _drift_git(repo_root: Path, *args: str) -> str: + result: Final = subprocess.run(("git", "-C", str(repo_root), *args), check=True, capture_output=True, text=True) + return result.stdout.strip() + + +def _drift_manifest(checkpoint_sha: str | None = None) -> str: + return json.dumps( + { + "schema_version": 1, + "checkpoint_sha": checkpoint_sha, + "candidate_paths": ["litellm/", "litellm-rust/"], + "mappings": [ + { + "id": "messages", + "python_paths": ["litellm/messages/"], + "rust_paths": ["litellm-rust/crates/inference-messages/"], + } + ], + } + ) + + +def _drift_repo(tmp_path: Path) -> tuple[Path, str]: + repo_root: Final = tmp_path / "repo" + repo_root.mkdir() + _drift_git(repo_root, "init", "--quiet", "--initial-branch=main") + _drift_git(repo_root, "config", "user.name", "Harness Test") + _drift_git(repo_root, "config", "user.email", "harness@example.invalid") + source: Final = repo_root / "litellm/messages/dispatch.py" + source.parent.mkdir(parents=True) + source.write_text("VALUE = 'initial'\n", encoding="utf-8") + manifest: Final = repo_root / "tests/rust-python-harness/strategies/drift/drift.json" + manifest.parent.mkdir(parents=True) + manifest.write_text(_drift_manifest(), encoding="utf-8") + _drift_git(repo_root, "add", ".") + _drift_git(repo_root, "commit", "--quiet", "-m", "initial") + return repo_root, _drift_git(repo_root, "rev-parse", "HEAD") + + +def test_drift_command_reports_unreviewed_checkpoint_from_default_manifest(tmp_path: Path) -> None: + repo_root, initial_sha = _drift_repo(tmp_path) + result: Final = CliRunner().invoke(drift_command(repo_root)) + + assert result.exit_code == 0, result.output + report: Final = DriftReport.model_validate_json(result.output) + assert report.status == "unreviewed" + assert report.target_sha == initial_sha + assert report.checkpoint_sha is None + + +@pytest.mark.parametrize("absolute_path", (False, True)) +def test_drift_command_resolves_custom_manifest_paths(tmp_path: Path, absolute_path: bool) -> None: + repo_root, initial_sha = _drift_repo(tmp_path) + manifest: Final = repo_root / "custom.json" + manifest.write_text(_drift_manifest(initial_sha), encoding="utf-8") + manifest_arg: Final = str(manifest if absolute_path else manifest.relative_to(repo_root)) + result: Final = CliRunner().invoke(drift_command(repo_root), ("--manifest", manifest_arg)) + + assert result.exit_code == 0, result.output + report: Final = DriftReport.model_validate_json(result.output) + assert report.status == "no_changes" + assert report.checkpoint_sha == initial_sha + + +def test_drift_command_compares_requested_revisions_without_claiming_parity(tmp_path: Path) -> None: + repo_root, initial_sha = _drift_repo(tmp_path) + source: Final = repo_root / "litellm/messages/dispatch.py" + source.write_text("VALUE = 'second'\n", encoding="utf-8") + _drift_git(repo_root, "add", ".") + _drift_git(repo_root, "commit", "--quiet", "-m", "second") + second_sha: Final = _drift_git(repo_root, "rev-parse", "HEAD") + source.write_text("VALUE = 'third'\n", encoding="utf-8") + _drift_git(repo_root, "add", ".") + _drift_git(repo_root, "commit", "--quiet", "-m", "third") + result: Final = CliRunner().invoke(drift_command(repo_root), ("--since", initial_sha, "--target", second_sha)) + + assert result.exit_code == 0, result.output + report: Final = DriftReport.model_validate_json(result.output) + assert report.status == "unreviewed" + assert report.checkpoint_sha is None + assert report.comparison_sha == initial_sha + assert report.target_sha == second_sha + assert tuple(change.path for change in report.changes) == ("litellm/messages/dispatch.py",) + assert "+VALUE = 'second'" in report.changes[0].patch + assert "third" not in report.changes[0].patch + + +def test_drift_command_writes_report_to_file(tmp_path: Path) -> None: + repo_root, initial_sha = _drift_repo(tmp_path) + output: Final = tmp_path / "report.json" + result: Final = CliRunner().invoke(drift_command(repo_root), ("--output", str(output))) + + assert result.exit_code == 0, result.output + assert result.output == "" + report: Final = DriftReport.model_validate_json(output.read_text(encoding="utf-8")) + assert report.status == "unreviewed" + assert report.target_sha == initial_sha + + +def test_drift_command_surfaces_inventory_errors(tmp_path: Path) -> None: + repo_root, _ = _drift_repo(tmp_path) + result: Final = CliRunner().invoke(drift_command(repo_root), ("--manifest", "missing.json")) + + assert result.exit_code != 0 + assert "Error" in result.output + assert "missing.json" in result.output + + +def test_drift_command_reports_output_write_errors(tmp_path: Path) -> None: + repo_root, _ = _drift_repo(tmp_path) + output: Final = tmp_path / "missing/report.json" + result: Final = CliRunner().invoke(drift_command(repo_root), ("--output", str(output))) + + assert result.exit_code != 0 + assert "Cannot write drift report" in result.output + assert str(output) in result.output + + +@pytest.mark.parametrize("output_kind", ("absolute", "relative", "symlink", "hardlink")) +def test_drift_command_preserves_manifest_when_output_aliases_it(tmp_path: Path, output_kind: str) -> None: + repo_root, _ = _drift_repo(tmp_path) + manifest: Final = repo_root / "tests/rust-python-harness/strategies/drift/drift.json" + original: Final = manifest.read_bytes() + alias: Final = repo_root / "report.json" + if output_kind == "symlink": + alias.symlink_to(manifest) + if output_kind == "hardlink": + alias.hardlink_to(manifest) + destination: Final = alias if output_kind in {"symlink", "hardlink"} else manifest + output: Final = os.path.relpath(destination) if output_kind == "relative" else str(destination) + result: Final = CliRunner().invoke(drift_command(repo_root), ("--output", output)) + + assert result.exit_code != 0 + assert "Output path must differ from the drift manifest" in result.output + assert manifest.read_bytes() == original + + +def test_run_drift_uses_registered_command_with_repository_context(tmp_path: Path) -> None: + repo_root, initial_sha = _drift_repo(tmp_path) + result: Final = CliRunner().invoke(build_cli(load_catalog(), repo_root), ("run", "drift")) + + assert result.exit_code == 0, result.output + report: Final = DriftReport.model_validate_json(result.output) + assert report.target_sha == initial_sha + assert report.checkpoint_state == "unknown" + assert report.manifest_path == "tests/rust-python-harness/strategies/drift/drift.json" diff --git a/tests/rust-python-harness/strategies/drift/test_runner.py b/tests/rust-python-harness/strategies/drift/test_runner.py new file mode 100644 index 00000000000..d793fa3b29e --- /dev/null +++ b/tests/rust-python-harness/strategies/drift/test_runner.py @@ -0,0 +1,640 @@ +from __future__ import annotations + +import hashlib +import json +import subprocess +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest + +from .runner import DriftError, DriftReport, inspect_drift + + +@dataclass(frozen=True, slots=True) +class Repository: + root: Path + baseline: str + + +def _git(root: Path, *arguments: str) -> str: + result: Final = subprocess.run( + ( + "git", + "-c", + "user.name=Drift Test", + "-c", + "user.email=drift@example.invalid", + "-c", + "commit.gpgsign=false", + "-c", + "core.hooksPath=/dev/null", + *arguments, + ), + cwd=root, + check=True, + capture_output=True, + text=True, + ) + return result.stdout.strip() + + +def _write(root: Path, path: str, content: str) -> None: + destination: Final = root / path + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_text(content, encoding="utf-8") + + +def _commit(root: Path) -> str: + _git(root, "add", "--all") + _git(root, "commit", "--quiet", "-m", "test: update fixture behavior") + return _git(root, "rev-parse", "HEAD") + + +def _manifest( + root: Path, + checkpoint: str | None, + *, + checkpoint_state: str | None = None, + pretty: bool = False, + candidates: tuple[str, ...] = ("litellm/", "litellm-rust/"), + mappings: tuple[tuple[str, tuple[str, ...], tuple[str, ...]], ...] = ( + ("messages", ("litellm/messages/", "litellm/common.py"), ("litellm-rust/messages.rs",)), + ("responses", ("litellm/responses/", "litellm/common.py"), ("litellm-rust/responses.rs",)), + ), + mapping_checkpoints: MappingProxyType[str, dict[str, str]] = MappingProxyType({}), +) -> Path: + path: Final = root / "drift.json" + path.write_text( + json.dumps( + { + "schema_version": 1, + "checkpoint_sha": checkpoint, + **({"checkpoint_state": checkpoint_state} if checkpoint_state is not None else {}), + "candidate_paths": candidates, + "mappings": tuple( + { + "id": identifier, + "python_paths": python_paths, + "rust_paths": rust_paths, + **mapping_checkpoints.get(identifier, {}), + } + for identifier, python_paths, rust_paths in mappings + ), + }, + indent=2 if pretty else None, + ), + encoding="utf-8", + ) + return path + + +@pytest.fixture +def repository(tmp_path: Path) -> Repository: + root: Final = tmp_path / "repository" + root.mkdir() + _git(root, "init", "--quiet", "--initial-branch=main") + _write(root, "litellm/messages/handler.py", "def result():\n return 'before'\n") + _write(root, "litellm/common.py", "TIMEOUT = 10\n") + _write(root, "litellm-rust/messages.rs", 'pub const RESULT: &str = "before";\n') + return Repository(root=root, baseline=_commit(root)) + + +def test_reports_python_rust_shared_and_unmapped_changes_with_real_patches(repository: Repository) -> None: + _write(repository.root, "litellm/messages/handler.py", "def result():\n return 'after'\n") + _write(repository.root, "litellm/common.py", "TIMEOUT = 20\n") + _write(repository.root, "litellm-rust/messages.rs", 'pub const RESULT: &str = "after";\n') + _write(repository.root, "litellm/new_endpoint.py", "ENABLED = True\n") + target: Final = _commit(repository.root) + manifest: Final = _manifest(repository.root, repository.baseline) + original_manifest: Final = manifest.read_bytes() + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert result.status == "changes_found" + assert (result.target_sha, result.checkpoint_sha, result.comparison_sha) == ( + target, + repository.baseline, + repository.baseline, + ) + assert frozenset((change.path, change.status, change.mapping_ids, change.sides) for change in result.changes) == { + ("litellm/messages/handler.py", "M", ("messages",), ("python",)), + ("litellm/common.py", "M", ("messages", "responses"), ("python",)), + ("litellm-rust/messages.rs", "M", ("messages",), ("rust",)), + ("litellm/new_endpoint.py", "A", (), ()), + } + assert tuple(change.path for change in result.unclassified) == ("litellm/new_endpoint.py",) + python_change: Final = next(change for change in result.changes if change.path == "litellm/messages/handler.py") + assert "- return 'before'" in python_change.patch + assert "+ return 'after'" in python_change.patch + assert result.manifest_sha256 == hashlib.sha256(original_manifest).hexdigest() + assert manifest.read_bytes() == original_manifest + + +def test_mapping_checkpoint_reports_only_its_paths_and_commits_since_that_checkpoint(repository: Repository) -> None: + _write(repository.root, "litellm/messages/handler.py", "def result():\n return 'ported'\n") + checkpoint: Final = _commit(repository.root) + _write(repository.root, "litellm/responses/handler.py", "ENABLED = True\n") + _git(repository.root, "add", "--all") + _git(repository.root, "commit", "--quiet", "-m", "feat: unrelated responses work") + _write(repository.root, "litellm/messages/handler.py", "def result():\n return 'drifted'\n") + _git(repository.root, "add", "--all") + _git(repository.root, "commit", "--quiet", "-m", "fix: change messages result") + drifted: Final = _git(repository.root, "rev-parse", "HEAD") + manifest: Final = _manifest( + repository.root, + repository.baseline, + mapping_checkpoints=MappingProxyType( + {"messages": {"checkpoint_sha": checkpoint, "checkpoint_state": "aligned"}} + ), + ) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert len(result.mapping_checkpoints) == 1 + drift: Final = result.mapping_checkpoints[0] + assert (drift.id, drift.status, drift.checkpoint_sha, drift.checkpoint_state) == ( + "messages", + "changes_found", + checkpoint, + "aligned", + ) + assert tuple((commit.sha, commit.subject) for commit in drift.commits) == ( + (drifted, "fix: change messages result"), + ) + assert tuple((change.path, change.sides) for change in drift.changes) == ( + ("litellm/messages/handler.py", ("python",)), + ) + assert "- return 'ported'" in drift.changes[0].patch + assert "+ return 'drifted'" in drift.changes[0].patch + assert "before" not in drift.changes[0].patch + + +def test_mapping_checkpoint_at_target_has_no_changes(repository: Repository) -> None: + manifest: Final = _manifest( + repository.root, + repository.baseline, + mapping_checkpoints=MappingProxyType({"responses": {"checkpoint_sha": repository.baseline}}), + ) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert tuple((drift.id, drift.status, drift.commits, drift.changes) for drift in result.mapping_checkpoints) == ( + ("responses", "no_changes", (), ()), + ) + + +def test_mapping_checkpoint_state_requires_a_checkpoint_sha(repository: Repository) -> None: + manifest: Final = _manifest( + repository.root, + repository.baseline, + mapping_checkpoints=MappingProxyType({"messages": {"checkpoint_state": "aligned"}}), + ) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftError), result + assert "'messages'" in result.message + + +def test_patch_is_usable_when_git_diff_colors_are_forced(repository: Repository) -> None: + _write(repository.root, "litellm/common.py", "TIMEOUT = 20\n") + _commit(repository.root) + _git(repository.root, "config", "color.ui", "always") + _git(repository.root, "config", "color.diff", "always") + manifest: Final = _manifest(repository.root, repository.baseline) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert len(result.changes) == 1 + patch: Final = result.changes[0].patch + assert "-TIMEOUT = 10\n+TIMEOUT = 20\n" in patch + patch_path: Final = repository.root.parent / "change.patch" + patch_path.write_text(patch, encoding="utf-8") + _git(repository.root, "apply", "--check", "--reverse", str(patch_path)) + + +def test_detects_deletions_and_renames_outside_candidate_directories(repository: Repository) -> None: + _git(repository.root, "mv", "litellm/messages/handler.py", "renamed_handler.py") + (repository.root / "litellm-rust/messages.rs").unlink() + _commit(repository.root) + manifest: Final = _manifest(repository.root, repository.baseline) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert frozenset((change.status, change.old_path, change.path) for change in result.changes) == { + ("R100", "litellm/messages/handler.py", "renamed_handler.py"), + ("D", None, "litellm-rust/messages.rs"), + } + rename: Final = next(change for change in result.changes if change.old_path is not None) + assert rename.mapping_ids == ("messages",) + assert rename.sides == ("python",) + assert "rename from litellm/messages/handler.py" in rename.patch + assert "rename to renamed_handler.py" in rename.patch + + +def test_rust_only_changes_still_require_review(repository: Repository) -> None: + _write(repository.root, "litellm-rust/messages.rs", 'pub const RESULT: &str = "changed independently";\n') + _commit(repository.root) + manifest: Final = _manifest(repository.root, repository.baseline) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert result.status == "changes_found" + assert tuple((change.path, change.sides) for change in result.changes) == (("litellm-rust/messages.rs", ("rust",)),) + + +def test_preserves_spaces_and_newlines_in_changed_filenames(repository: Repository) -> None: + filename: Final = "litellm/messages/handler with space\nand newline.py" + _write(repository.root, filename, "ENABLED = True\n") + _commit(repository.root) + manifest: Final = _manifest(repository.root, repository.baseline) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert tuple((change.path, change.mapping_ids) for change in result.changes) == ((filename, ("messages",)),) + assert "+ENABLED = True" in result.changes[0].patch + + +def test_exact_file_selectors_do_not_capture_files_with_the_same_prefix(repository: Repository) -> None: + _write(repository.root, "litellm/common.py.backup", "unrelated copy\n") + _commit(repository.root) + manifest: Final = _manifest(repository.root, repository.baseline, candidates=("litellm/common.py",)) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert result.status == "no_changes" + assert result.changes == () + + +def test_null_checkpoint_cannot_claim_no_changes(repository: Repository) -> None: + manifest: Final = _manifest(repository.root, None) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert result.status == "unreviewed" + assert result.checkpoint_sha is None + assert result.checkpoint_state == "unknown" + assert result.comparison_sha is None + assert result.changes == () + + +@pytest.mark.parametrize("state", ("unknown", "drifted", "aligned")) +def test_no_changes_preserves_recorded_checkpoint_state(repository: Repository, state: str) -> None: + manifest: Final = _manifest(repository.root, repository.baseline, checkpoint_state=state) + original_manifest: Final = manifest.read_bytes() + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert result.status == "no_changes" + assert result.checkpoint_state == state + assert manifest.read_bytes() == original_manifest + + +@pytest.mark.parametrize(("state", "with_checkpoint"), (("invalid", True), ("aligned", False), ("drifted", False))) +def test_rejects_invalid_checkpoint_state_without_rewriting_manifest( + repository: Repository, state: str, with_checkpoint: bool +) -> None: + manifest: Final = _manifest( + repository.root, repository.baseline if with_checkpoint else None, checkpoint_state=state + ) + original_manifest: Final = manifest.read_bytes() + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftError), result + assert result.message + assert manifest.read_bytes() == original_manifest + + +def test_manifest_digest_preserves_crlf_bytes(repository: Repository) -> None: + manifest: Final = _manifest(repository.root, repository.baseline) + raw: Final = manifest.read_bytes().replace(b", ", b",\r\n") + manifest.write_bytes(raw) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert result.manifest_sha256 == hashlib.sha256(raw).hexdigest() + assert manifest.read_bytes() == raw + + +def test_explicit_target_ignores_later_committed_work(repository: Repository) -> None: + _write(repository.root, "litellm/common.py", "TIMEOUT = 30\n") + _commit(repository.root) + manifest: Final = _manifest(repository.root, repository.baseline) + + result: Final = inspect_drift(repository.root, manifest, target_ref=repository.baseline) + + assert isinstance(result, DriftReport), result + assert result.status == "no_changes" + assert result.target_sha == repository.baseline + assert result.changes == () + + +def test_since_override_does_not_create_or_advance_checkpoint(repository: Repository) -> None: + _write(repository.root, "litellm/common.py", "TIMEOUT = 30\n") + _commit(repository.root) + manifest: Final = _manifest(repository.root, None) + original_manifest: Final = manifest.read_bytes() + + result: Final = inspect_drift(repository.root, manifest, since_ref=repository.baseline[:12]) + + assert isinstance(result, DriftReport), result + assert result.status == "unreviewed" + assert result.checkpoint_sha is None + assert result.comparison_sha == repository.baseline + assert tuple(change.path for change in result.changes) == ("litellm/common.py",) + assert manifest.read_bytes() == original_manifest + + +@pytest.mark.parametrize("checkpoint", ("f" * 40, "HEAD", "1234567")) +@pytest.mark.parametrize("use_override", (False, True)) +def test_missing_or_nonfull_checkpoint_is_an_error(repository: Repository, checkpoint: str, use_override: bool) -> None: + manifest: Final = _manifest(repository.root, checkpoint) + + result: Final = inspect_drift(repository.root, manifest, since_ref=repository.baseline if use_override else None) + + assert isinstance(result, DriftError), result + assert result.message + + +@pytest.mark.parametrize("use_override", (False, True)) +def test_unrelated_checkpoint_is_an_error(repository: Repository, use_override: bool) -> None: + _git(repository.root, "checkout", "--quiet", "--orphan", "unrelated") + _git(repository.root, "rm", "--quiet", "-rf", ".") + _write(repository.root, "other.py", "unrelated = True\n") + unrelated: Final = _commit(repository.root) + _git(repository.root, "checkout", "--quiet", "main") + manifest: Final = _manifest(repository.root, unrelated) + + result: Final = inspect_drift(repository.root, manifest, since_ref=repository.baseline if use_override else None) + + assert isinstance(result, DriftError), result + assert result.message + + +def test_removed_policy_paths_stay_visible_from_checkpoint_and_target(repository: Repository) -> None: + _manifest( + repository.root, + None, + candidates=("litellm/",), + mappings=(("previous", ("litellm/messages/",), ("legacy-native/",)),), + ) + checkpoint: Final = _commit(repository.root) + _write(repository.root, "litellm/messages/handler.py", "def result():\n return 'after'\n") + _write(repository.root, "litellm-rust/messages.rs", 'pub const RESULT: &str = "after";\n') + _manifest( + repository.root, + checkpoint, + candidates=("litellm-rust/",), + mappings=(("target", ("litellm/responses/",), ("litellm-rust/messages.rs",)),), + ) + _commit(repository.root) + manifest: Final = _manifest( + repository.root, + checkpoint, + candidates=("unrelated/",), + mappings=(("local", ("unrelated/",), ("unrelated-native/",)),), + ) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert result.status == "changes_found" + assert ("litellm/messages/handler.py", ("previous",)) in tuple( + (change.path, change.mapping_ids) for change in result.changes + ) + assert ("litellm-rust/messages.rs", ("target",)) in tuple( + (change.path, change.mapping_ids) for change in result.changes + ) + assert "drift.json" in tuple(change.path for change in result.changes) + + +def test_report_exposes_merged_selectors_for_the_same_mapping_id(repository: Repository) -> None: + _manifest( + repository.root, + None, + mappings=(("messages", ("litellm/messages/",), ("old-native/",)),), + ) + checkpoint: Final = _commit(repository.root) + _write(repository.root, "litellm/messages/handler.py", "def result():\n return 'after'\n") + _manifest( + repository.root, + checkpoint, + mappings=(("messages", ("litellm/common.py",), ("litellm-rust/messages.rs",)),), + ) + _commit(repository.root) + manifest: Final = _manifest( + repository.root, + checkpoint, + mappings=(("messages", ("litellm/new-messages/",), ("new-native/",)),), + ) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert tuple((mapping.id, mapping.python_paths, mapping.rust_paths) for mapping in result.mappings) == ( + ( + "messages", + ("litellm/common.py", "litellm/messages/", "litellm/new-messages/"), + ("litellm-rust/messages.rs", "new-native/", "old-native/"), + ), + ) + assert ("litellm/messages/handler.py", ("messages",)) in tuple( + (change.path, change.mapping_ids) for change in result.changes + ) + + +def test_since_override_keeps_policy_from_the_stored_checkpoint(repository: Repository) -> None: + _manifest( + repository.root, + None, + candidates=("litellm/",), + mappings=(("previous", ("litellm/messages/",), ("old-native/",)),), + ) + checkpoint: Final = _commit(repository.root) + manifest: Final = _manifest( + repository.root, + checkpoint, + candidates=("unrelated/",), + mappings=(("current", ("unrelated/",), ("new-native/",)),), + ) + comparison: Final = _commit(repository.root) + _write(repository.root, "litellm/messages/handler.py", "def result():\n return 'after'\n") + _commit(repository.root) + + result: Final = inspect_drift(repository.root, manifest, since_ref=comparison) + + assert isinstance(result, DriftReport), result + assert result.checkpoint_sha == checkpoint + assert result.comparison_sha == comparison + assert tuple((change.path, change.mapping_ids) for change in result.changes) == ( + ("litellm/messages/handler.py", ("previous",)), + ) + + +@pytest.mark.parametrize("rename_state", ("none", "committed", "staged")) +@pytest.mark.parametrize("use_override", (False, True)) +def test_renamed_manifest_keeps_historical_watch_paths( + repository: Repository, rename_state: str, use_override: bool +) -> None: + _write(repository.root, "legacy-owned/handler.py", "ENABLED = False\n") + _manifest( + repository.root, + None, + candidates=("legacy-owned/",), + mappings=(("messages", ("legacy-owned/",), ("native/",)),), + pretty=True, + ) + checkpoint: Final = _commit(repository.root) + _manifest( + repository.root, + checkpoint, + candidates=("unrelated/",), + mappings=(("messages", ("unrelated/",), ("native/",)),), + pretty=True, + ) + (repository.root / "policy").mkdir() + _git(repository.root, "mv", "drift.json", "policy/drift.json") + comparison: Final = _commit(repository.root) + _write(repository.root, "legacy-owned/handler.py", "ENABLED = True\n") + _commit(repository.root) + if rename_state != "none": + _git(repository.root, "mv", "policy/drift.json", "policy/current-drift.json") + if rename_state == "committed": + _commit(repository.root) + if rename_state == "staged": + _git(repository.root, "config", "status.renames", "false") + manifest: Final = repository.root / ("policy/drift.json" if rename_state == "none" else "policy/current-drift.json") + + result: Final = inspect_drift(repository.root, manifest, since_ref=comparison if use_override else None) + + assert isinstance(result, DriftReport), result + assert result.checkpoint_sha == checkpoint + assert result.comparison_sha == (comparison if use_override else checkpoint) + assert ("legacy-owned/handler.py", ("messages",)) in tuple( + (change.path, change.mapping_ids) for change in result.changes + ) + mapping: Final = next(mapping for mapping in result.mappings if mapping.id == "messages") + assert mapping.python_paths == ("legacy-owned/", "unrelated/") + if rename_state == "staged": + assert ("policy/drift.json", "policy/current-drift.json") in tuple( + (change.old_path, change.path) for change in result.working_tree_changes + ) + + +def test_manifest_rename_history_survives_later_large_policy_rewrites(repository: Repository) -> None: + _write(repository.root, "legacy-owned/topic0/handler.py", "ENABLED = False\n") + _manifest( + repository.root, + None, + candidates=("legacy-owned/",), + mappings=tuple( + (f"topic{index}", (f"legacy-owned/topic{index}/",), (f"native/topic{index}/",)) for index in range(10) + ), + pretty=True, + ) + checkpoint: Final = _commit(repository.root) + _git(repository.root, "mv", "drift.json", "intermediate-drift.json") + comparison: Final = _commit(repository.root) + replacement: Final = _manifest( + repository.root, + checkpoint, + candidates=("unrelated/",), + mappings=(("current", ("unrelated/",), ("native/",)),), + pretty=True, + ) + (repository.root / "intermediate-drift.json").write_bytes(replacement.read_bytes()) + replacement.unlink() + _write(repository.root, "legacy-owned/topic0/handler.py", "ENABLED = True\n") + _commit(repository.root) + _git(repository.root, "mv", "intermediate-drift.json", "current-drift.json") + _commit(repository.root) + + result: Final = inspect_drift(repository.root, repository.root / "current-drift.json", since_ref=comparison) + + assert isinstance(result, DriftReport), result + assert ("legacy-owned/topic0/handler.py", ("topic0",)) in tuple( + (change.path, change.mapping_ids) for change in result.changes + ) + assert tuple(mapping.id for mapping in result.mappings) == ("current", *(f"topic{index}" for index in range(10))) + + +def test_future_target_recovers_renamed_policy_from_an_older_checkout(repository: Repository) -> None: + _manifest( + repository.root, + None, + candidates=("legacy-owned/",), + mappings=(("legacy", ("legacy-owned/",), ("native/",)),), + pretty=True, + ) + checkpoint: Final = _commit(repository.root) + _manifest( + repository.root, + checkpoint, + candidates=("future-owned/",), + mappings=(("future", ("future-owned/",), ("native/",)),), + pretty=True, + ) + _git(repository.root, "mv", "drift.json", "future-drift.json") + _write(repository.root, "future-owned/handler.py", "ENABLED = True\n") + target: Final = _commit(repository.root) + _git(repository.root, "checkout", "--quiet", "--detach", checkpoint) + manifest: Final = _manifest( + repository.root, + checkpoint, + candidates=("legacy-owned/",), + mappings=(("legacy", ("legacy-owned/",), ("native/",)),), + pretty=True, + ) + + result: Final = inspect_drift(repository.root, manifest, target_ref=target) + + assert isinstance(result, DriftReport), result + assert result.target_sha == target + assert ("future-owned/handler.py", ("future",)) in tuple( + (change.path, change.mapping_ids) for change in result.changes + ) + + +def test_reports_relevant_staged_unstaged_and_untracked_changes_without_diffing_them(repository: Repository) -> None: + _git(repository.root, "mv", "litellm/messages/handler.py", "renamed_handler.py") + _write(repository.root, "litellm/common.py", "TIMEOUT = 99\n") + _write(repository.root, "litellm/new_endpoint.py", "ENABLED = True\n") + _write(repository.root, "unrelated.txt", "outside the candidate paths\n") + manifest: Final = _manifest(repository.root, repository.baseline) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftReport), result + assert result.target_sha == repository.baseline + assert result.changes == () + assert frozenset((change.status, change.old_path, change.path) for change in result.working_tree_changes) == { + ("R ", "litellm/messages/handler.py", "renamed_handler.py"), + (" M", None, "litellm/common.py"), + ("??", None, "litellm/new_endpoint.py"), + ("??", None, "drift.json"), + } + + +@pytest.mark.parametrize("selector", ("../litellm/", "/litellm/", "litellm/**", "litellm/../outside/")) +def test_rejects_ambiguous_or_outside_repository_selectors(repository: Repository, selector: str) -> None: + manifest: Final = _manifest(repository.root, repository.baseline, candidates=(selector,)) + + result: Final = inspect_drift(repository.root, manifest) + + assert isinstance(result, DriftError), result + assert result.message