mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-24 05:45:11 +00:00
docs(rust): design Python drift audits and review enforcement
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
Some checks failed
LiteLLM Rust / rust-lint (push) Has been cancelled
LiteLLM Rust / rust-test (push) Has been cancelled
LiteLLM Rust / rust-wheel (push) Has been cancelled
Terraform Modules / fmt, validate, test (aws) (push) Has been cancelled
Terraform Modules / fmt, validate, test (gcp) (push) Has been cancelled
This commit is contained in:
parent
e3163aa98b
commit
120fa92e55
12 changed files with 1637 additions and 36 deletions
|
|
@ -9,6 +9,13 @@ tests/rust-python-harness/
|
|||
│ └── commands.py
|
||||
│
|
||||
├── strategies/
|
||||
│ ├── drift/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── AGENTS.md
|
||||
│ │ ├── command.py
|
||||
│ │ ├── drift.json
|
||||
│ │ └── runner.py
|
||||
│ │
|
||||
│ ├── e2e_parity/
|
||||
│ │ ├── __init__.py
|
||||
│ │ ├── reporting.py
|
||||
|
|
@ -45,11 +52,12 @@ tests/rust-python-harness/
|
|||
└── suite_runner.py
|
||||
```
|
||||
|
||||
- A strategy is a folder under `strategies/` with a one-line `AGENTS.md` and an `__init__.py` exporting exactly one `STRATEGY: StrategyDefinition`. Its id must equal the folder name
|
||||
- Every strategy is a folder under `strategies/` with an `AGENTS.md` and an `__init__.py` exporting exactly one `STRATEGY`. Its id must equal the folder name. SDK strategies export `StrategyDefinition`; the repository-source drift strategy exports `RepositoryStrategyDefinition`
|
||||
- `shared/reporting/strategy.py` is the contract: runnable module/suite specs, not-implemented/skipped specs, the runner protocol, and `StrategyDefinition`
|
||||
- Every `STRATEGY` explicitly classifies every SDK function. Surface-aware strategies declare their surfaces and classify the complete surface-by-function matrix
|
||||
- Every SDK `StrategyDefinition` explicitly classifies every SDK function. Surface-aware SDK strategies declare their surfaces and classify the complete surface-by-function matrix. Repository strategies use their own options rather than SDK `--function` or `--surface` selection
|
||||
- Run locally only. No CI integration
|
||||
- `python -m tests.rust-python-harness run <strategy>|all` runs the selected strategy. `--function` is common, while each strategy exposes only its supported options
|
||||
- `uv run python -m tests.rust-python-harness run drift --target HEAD` inventories committed Python and Rust changes using `strategies/drift/drift.json`. See `strategies/drift/AGENTS.md` for usage, exploratory `--since` comparisons, and manual snapshot checkpoints with independent unknown, drifted, or aligned states
|
||||
- `python -m tests.rust-python-harness run <strategy>` runs the selected strategy with its supported options. SDK strategies share `--function`. `run all` runs SDK execution tests; invoke `run drift` explicitly for the repository change inventory
|
||||
- Examples: `run e2e_parity --surface sdk --function ocr`, `run unit_tests_parity --function ocr --pytest-arg=-x`, or `run all --function ocr`
|
||||
- `cli/catalog.py` discovers strategies, validates their Python definitions, and orders them. `cli/__init__.py` builds the Click command tree. `cli/commands.py` runs selected cases
|
||||
- `e2e_parity/` compares SDK objects, exceptions, callbacks, and streams, or gateway HTTP responses
|
||||
|
|
@ -59,8 +67,8 @@ tests/rust-python-harness/
|
|||
- `unit_tests_parity/runner.py` runs each contract's `unit_parity_scope` with `LITELLM_RUST=0` and `LITELLM_RUST=1` in separate processes and requires matching outcomes, including failures. Exclusions require a reason in the contract
|
||||
- `unit_tests_rust/runner.py` runs each contract's focused Cargo test suite. Native Rust unit tests stay beside their implementation
|
||||
- `shared/unit_runners/suite_runner.py` runs typed suites registered in code with nodeids of the form `suite:<strategy_id>:<function>:<suite>`
|
||||
- Every strategy declares its report sections and presentation in its own `reporting.py`. Shared reporting code only provides reusable models and cell-formatting primitives
|
||||
- Every SDK strategy declares its report sections and presentation in its own `reporting.py`. Shared reporting code only provides reusable models and cell-formatting primitives. Drift emits its repository change report directly
|
||||
- `shared/` contains reusable parity, tracing, reporting primitives, and unit-runner machinery
|
||||
- Keep fixtures with their owning API and existing Python tests in their current locations
|
||||
- Each strategy folder carries an `AGENTS.md` one-liner stating what it should be doing
|
||||
- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/trace_parity tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust -q`
|
||||
- Each test strategy folder carries an `AGENTS.md` one-liner stating what it should be doing. The repository-source drift strategy keeps its usage guide in `strategies/drift/AGENTS.md`
|
||||
- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/drift tests/rust-python-harness/strategies/trace_parity tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust -q`
|
||||
|
|
|
|||
|
|
@ -2,15 +2,18 @@ from __future__ import annotations
|
|||
|
||||
import sys
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Final, cast
|
||||
|
||||
import click
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
from ..shared.reporting.models import SDK_FUNCTIONS, SdkFunction, Strategy, Surface
|
||||
from .catalog import load_catalog
|
||||
from .commands import run_command, select_cases
|
||||
from ..shared.reporting.strategy import RepositoryStrategyDefinition
|
||||
from .catalog import CatalogStrategy, load_catalog
|
||||
from .commands import REPO_ROOT, run_command, select_cases
|
||||
|
||||
__all__ = ["load_catalog", "main"]
|
||||
__all__ = ["build_cli", "load_catalog", "main"]
|
||||
|
||||
_INTERRUPTED_EXIT_CODE: Final = 130
|
||||
|
||||
|
|
@ -34,7 +37,7 @@ def _run_all_command(strategies: Sequence[Strategy]) -> click.Command:
|
|||
"all",
|
||||
params=[_function_option()],
|
||||
callback=run_all,
|
||||
help="run every strategy",
|
||||
help="run every SDK test strategy",
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -77,14 +80,18 @@ def _strategy_command(strategy: Strategy) -> click.Command:
|
|||
)
|
||||
|
||||
|
||||
def _build_cli(strategies: Sequence[Strategy]) -> click.Group:
|
||||
def build_cli(strategies: Sequence[CatalogStrategy], repo_root: Path = REPO_ROOT) -> click.Group:
|
||||
root: Final = click.Group(
|
||||
"rust-python-harness",
|
||||
help="Run Rust/Python parity tests with raw progress and strategy reports.",
|
||||
)
|
||||
run: Final = click.Group("run", help="run one strategy or the complete harness")
|
||||
run.add_command(_run_all_command(strategies))
|
||||
run: Final = click.Group("run", help="run one strategy or every SDK test strategy")
|
||||
test_strategies: Final = tuple(strategy for strategy in strategies if isinstance(strategy, Strategy))
|
||||
run.add_command(_run_all_command(test_strategies))
|
||||
for strategy in strategies:
|
||||
if isinstance(strategy, RepositoryStrategyDefinition):
|
||||
run.add_command(strategy.command(repo_root))
|
||||
continue
|
||||
run.add_command(_strategy_command(strategy))
|
||||
root.add_command(run)
|
||||
return root
|
||||
|
|
@ -93,12 +100,15 @@ def _build_cli(strategies: Sequence[Strategy]) -> click.Group:
|
|||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
try:
|
||||
strategies: Final = load_catalog()
|
||||
result: Final = _build_cli(strategies).main(
|
||||
args=None if argv is None else list(argv),
|
||||
prog_name="rust-python-harness",
|
||||
standalone_mode=False,
|
||||
result: Final = TypeAdapter[int | None](int | None).validate_python(
|
||||
build_cli(strategies).main(
|
||||
args=None if argv is None else list(argv),
|
||||
prog_name="rust-python-harness",
|
||||
standalone_mode=False,
|
||||
),
|
||||
strict=True,
|
||||
)
|
||||
exit_code: Final = result if isinstance(result, int) else 0
|
||||
exit_code: Final = result if result is not None else 0
|
||||
except click.ClickException as error:
|
||||
error.show()
|
||||
return error.exit_code
|
||||
|
|
|
|||
|
|
@ -5,13 +5,16 @@ import importlib
|
|||
import importlib.util
|
||||
import pkgutil
|
||||
import sys
|
||||
from itertools import product
|
||||
from pathlib import Path
|
||||
from types import ModuleType
|
||||
from typing import Final
|
||||
from typing import Final, TypeAlias
|
||||
|
||||
from .. import strategies as _strategies_package
|
||||
from ..shared.reporting.models import SDK_FUNCTIONS, SURFACES, CaseDisposition, HarnessCase, Strategy
|
||||
from ..shared.reporting.strategy import StrategyDefinition
|
||||
from ..shared.reporting.strategy import RepositoryStrategyDefinition, StrategyDefinition
|
||||
|
||||
CatalogStrategy: TypeAlias = Strategy | RepositoryStrategyDefinition
|
||||
|
||||
_STRATEGIES_PACKAGE: Final = _strategies_package
|
||||
STRATEGIES_ROOT: Final = Path(_STRATEGIES_PACKAGE.__path__[0])
|
||||
|
|
@ -39,15 +42,19 @@ def _synthetic_module_name(folder: Path) -> str:
|
|||
return f"_harness_strategy_{folder.name}_{digest}"
|
||||
|
||||
|
||||
def _load_strategy(name: str, folder: Path, prefix: str | None) -> Strategy:
|
||||
def _load_strategy(name: str, folder: Path, prefix: str | None) -> CatalogStrategy:
|
||||
module: Final = _load_strategy_module(name, folder, prefix)
|
||||
definition: Final = getattr(module, "STRATEGY", None)
|
||||
if not isinstance(definition, StrategyDefinition):
|
||||
raise ValueError(f"{folder}: __init__.py must export STRATEGY: StrategyDefinition")
|
||||
if not isinstance(definition, (StrategyDefinition, RepositoryStrategyDefinition)):
|
||||
raise ValueError(
|
||||
f"{folder}: __init__.py must export STRATEGY: StrategyDefinition or RepositoryStrategyDefinition"
|
||||
)
|
||||
if definition.id != name:
|
||||
raise ValueError(f"{folder}: strategy id {definition.id!r} must match folder name {name!r}")
|
||||
if definition.directory.resolve() != folder.resolve():
|
||||
raise ValueError(f"{folder}: strategy directory must be {folder}")
|
||||
if isinstance(definition, RepositoryStrategyDefinition):
|
||||
return definition
|
||||
if len(set(definition.surfaces)) != len(definition.surfaces) or any(
|
||||
surface not in SURFACES for surface in definition.surfaces
|
||||
):
|
||||
|
|
@ -56,9 +63,7 @@ def _load_strategy(name: str, folder: Path, prefix: str | None) -> Strategy:
|
|||
duplicates: Final = tuple(sorted(key for key in set(keys) if keys.count(key) > 1))
|
||||
if duplicates:
|
||||
raise ValueError(f"{folder}: duplicate strategy cases: {duplicates}")
|
||||
expected: Final = frozenset(
|
||||
(surface, function) for surface in (definition.surfaces or (None,)) for function in SDK_FUNCTIONS
|
||||
)
|
||||
expected: Final = frozenset(product(definition.surfaces or (None,), SDK_FUNCTIONS))
|
||||
actual: Final = frozenset(keys)
|
||||
if actual != expected:
|
||||
missing: Final = tuple(sorted(expected - actual))
|
||||
|
|
@ -94,7 +99,7 @@ def _load_strategy(name: str, folder: Path, prefix: str | None) -> Strategy:
|
|||
)
|
||||
|
||||
|
||||
def load_catalog(root: Path | None = None) -> tuple[Strategy, ...]:
|
||||
def load_catalog(root: Path | None = None) -> tuple[CatalogStrategy, ...]:
|
||||
resolved: Final = STRATEGIES_ROOT if root is None else root
|
||||
prefix: Final = _STRATEGIES_PACKAGE.__name__ if resolved == STRATEGIES_ROOT else None
|
||||
folders: Final = tuple(info.name for info in pkgutil.iter_modules([str(resolved)]) if info.ispkg)
|
||||
|
|
|
|||
|
|
@ -22,10 +22,14 @@ from ..shared.reporting.ui import PlainDashboard, final_report, make_dashboard
|
|||
from ..strategies.unit_tests_parity import UNIT_PARITY_SUITES
|
||||
from ..strategies.unit_tests_rust import RUST_SUITES
|
||||
from . import main
|
||||
from .catalog import STRATEGIES_ROOT, load_catalog
|
||||
from .catalog import STRATEGIES_ROOT, CatalogStrategy, load_catalog
|
||||
from .commands import REPO_ROOT, select_cases
|
||||
|
||||
|
||||
def _sdk_strategies(strategies: Sequence[CatalogStrategy]) -> tuple[Strategy, ...]:
|
||||
return tuple(strategy for strategy in strategies if isinstance(strategy, Strategy))
|
||||
|
||||
|
||||
def _strategy_source(
|
||||
*,
|
||||
strategy_id: str = "example",
|
||||
|
|
@ -84,16 +88,17 @@ def _write_strategy_folder(
|
|||
return folder
|
||||
|
||||
|
||||
def test_should_load_surface_aware_and_function_only_strategies() -> None:
|
||||
def test_should_load_test_and_repository_strategies() -> None:
|
||||
strategies: Final = load_catalog()
|
||||
|
||||
assert [strategy.id for strategy in strategies] == [
|
||||
"drift",
|
||||
"e2e_parity",
|
||||
"trace_parity",
|
||||
"unit_tests_parity",
|
||||
"unit_tests_rust",
|
||||
]
|
||||
for strategy in strategies:
|
||||
for strategy in _sdk_strategies(strategies):
|
||||
expected: Final = tuple(
|
||||
(surface, function) for surface in (strategy.definition.surfaces or (None,)) for function in SDK_FUNCTIONS
|
||||
)
|
||||
|
|
@ -102,7 +107,9 @@ def test_should_load_surface_aware_and_function_only_strategies() -> None:
|
|||
|
||||
def test_unit_strategies_use_function_only_cases() -> None:
|
||||
strategies: Final = {
|
||||
strategy.id: strategy for strategy in load_catalog() if strategy.id in {"unit_tests_parity", "unit_tests_rust"}
|
||||
strategy.id: strategy
|
||||
for strategy in _sdk_strategies(load_catalog())
|
||||
if strategy.id in {"unit_tests_parity", "unit_tests_rust"}
|
||||
}
|
||||
|
||||
for sdk_function in SDK_FUNCTIONS:
|
||||
|
|
@ -122,7 +129,7 @@ def test_unit_strategies_use_function_only_cases() -> None:
|
|||
|
||||
|
||||
def test_raw_dashboard_is_always_the_default() -> None:
|
||||
assert isinstance(make_dashboard(load_catalog()), PlainDashboard)
|
||||
assert isinstance(make_dashboard(_sdk_strategies(load_catalog())), PlainDashboard)
|
||||
|
||||
|
||||
def test_every_strategy_folder_complies() -> None:
|
||||
|
|
@ -132,7 +139,7 @@ def test_every_strategy_folder_complies() -> None:
|
|||
}
|
||||
|
||||
assert folders == {strategy.id for strategy in strategies}
|
||||
for strategy in strategies:
|
||||
for strategy in _sdk_strategies(strategies):
|
||||
definition: Final = strategy.definition
|
||||
assert isinstance(definition, StrategyDefinition)
|
||||
assert definition.directory == strategy.directory
|
||||
|
|
@ -201,7 +208,7 @@ def test_should_reject_an_unavailable_case_with_a_blank_reason(
|
|||
|
||||
|
||||
def test_should_select_functions_and_surfaces() -> None:
|
||||
strategy: Final = next(strategy for strategy in load_catalog() if strategy.id == "e2e_parity")
|
||||
strategy: Final = next(strategy for strategy in _sdk_strategies(load_catalog()) if strategy.id == "e2e_parity")
|
||||
|
||||
assert tuple(case.key for case in select_cases((strategy,), {"messages"})) == (
|
||||
"e2e_parity:messages",
|
||||
|
|
@ -240,7 +247,7 @@ def test_every_unavailable_case_finishes_and_explains_itself() -> None:
|
|||
}
|
||||
unavailable: Final = tuple(
|
||||
(strategy, case)
|
||||
for strategy in load_catalog()
|
||||
for strategy in _sdk_strategies(load_catalog())
|
||||
for case in strategy.cases
|
||||
if case.spec.disposition is not CaseDisposition.RUNNABLE
|
||||
)
|
||||
|
|
@ -283,6 +290,7 @@ def test_run_help_lists_all_and_every_strategy(capsys: pytest.CaptureFixture[str
|
|||
"trace_parity",
|
||||
"unit_tests_parity",
|
||||
"unit_tests_rust",
|
||||
"drift",
|
||||
):
|
||||
assert command in captured.out
|
||||
|
||||
|
|
@ -292,6 +300,8 @@ def test_run_help_lists_all_and_every_strategy(capsys: pytest.CaptureFixture[str
|
|||
(
|
||||
("list",),
|
||||
("check",),
|
||||
("drift",),
|
||||
("run", "drift", "--function", "messages"),
|
||||
("run", "--strategy", "unit_tests_parity"),
|
||||
("run", "unit_tests_parity", "--surface", "sdk"),
|
||||
("run", "unit_tests_parity", "--plain"),
|
||||
|
|
@ -473,3 +483,11 @@ def test_strategy_subcommand_accepts_function_filter(capsys: pytest.CaptureFixtu
|
|||
assert exit_code == 0
|
||||
assert "- messages: not_implemented" in captured.out
|
||||
assert "unit_tests_rust:messages: not_implemented" not in captured.out
|
||||
|
||||
|
||||
def test_drift_help_exposes_inventory_options(capsys: pytest.CaptureFixture[str]) -> None:
|
||||
exit_code: Final = main(("run", "drift", "--help"))
|
||||
captured: Final = capsys.readouterr()
|
||||
|
||||
assert exit_code == 0
|
||||
assert all(option in captured.out for option in ("--target", "--manifest", "--since", "--output"))
|
||||
|
|
|
|||
|
|
@ -3,13 +3,16 @@ from __future__ import annotations
|
|||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Literal, Protocol, TypeAlias
|
||||
from typing import TYPE_CHECKING, Annotated, Literal, Protocol, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, StringConstraints
|
||||
|
||||
from .models import CaseDisposition, Coverage, HarnessCase, HarnessRun, SdkFunction, Surface
|
||||
from .rendering import StrategyRenderer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from click import Command
|
||||
|
||||
UpdateCallback: TypeAlias = Callable[[HarnessRun], None]
|
||||
NonBlankString: TypeAlias = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)]
|
||||
|
||||
|
|
@ -90,3 +93,13 @@ class StrategyDefinition:
|
|||
render: StrategyRenderer
|
||||
surfaces: tuple[Surface, ...] = ()
|
||||
runner_argument: RunnerArgumentDefinition | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RepositoryStrategyDefinition:
|
||||
id: str
|
||||
order: int
|
||||
label: str
|
||||
description: str
|
||||
directory: Path
|
||||
command: Callable[[Path], Command]
|
||||
|
|
|
|||
20
tests/rust-python-harness/strategies/drift/AGENTS.md
Normal file
20
tests/rust-python-harness/strategies/drift/AGENTS.md
Normal file
|
|
@ -0,0 +1,20 @@
|
|||
# Drift
|
||||
|
||||
Compare committed Python and Rust changes using this folder's `drift.json`. Run locally without provider calls or CI integration
|
||||
|
||||
```sh
|
||||
uv run python -m tests.rust-python-harness run drift --target HEAD --output /tmp/litellm-drift.json
|
||||
```
|
||||
|
||||
Paths are repository-relative files or directory prefixes ending in `/`. Keep unmapped changes and uncommitted files visible in the report. Commit manifest renames separately from mapping edits so Git can follow the old watch paths
|
||||
|
||||
`checkpoint_sha` is a comparison snapshot. `checkpoint_state` records `unknown`, `drifted`, or `aligned` at that snapshot. Never update either automatically. `no_changes` does not imply alignment
|
||||
|
||||
A mapping may carry its own `checkpoint_sha` and `checkpoint_state`, recording the commit at which its Rust paths were reviewed against its Python paths. `mapping_checkpoints` in the report lists, for each such mapping, the commits and patches touching its paths since that checkpoint. Port those changes to the other side, then move the mapping's `checkpoint_sha` forward by hand in the same PR
|
||||
|
||||
```sh
|
||||
uv run python -m tests.rust-python-harness run drift --output /tmp/litellm-drift.json
|
||||
jq '.mapping_checkpoints[] | select(.id == "<mapping id>")' /tmp/litellm-drift.json
|
||||
```
|
||||
|
||||
Use `--since` for an exploratory comparison and `--manifest` for another mapping file. Omit `--output` to print JSON. Fetch remote refs before comparing them. This strategy has its own options and does not run through `run all`
|
||||
14
tests/rust-python-harness/strategies/drift/__init__.py
Normal file
14
tests/rust-python-harness/strategies/drift/__init__.py
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
from ...shared.reporting.strategy import RepositoryStrategyDefinition
|
||||
from .command import drift_command
|
||||
|
||||
STRATEGY: Final = RepositoryStrategyDefinition(
|
||||
id="drift",
|
||||
order=5,
|
||||
label="Drift",
|
||||
description="Inventory committed Python and Rust changes from an explicit checkpoint.",
|
||||
directory=Path(__file__).parent,
|
||||
command=drift_command,
|
||||
)
|
||||
56
tests/rust-python-harness/strategies/drift/command.py
Normal file
56
tests/rust-python-harness/strategies/drift/command.py
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import click
|
||||
|
||||
from .runner import DriftError, inspect_drift
|
||||
|
||||
|
||||
def drift_command(repo_root: Path) -> click.Command:
|
||||
def inspect(target: str, manifest: Path, since: str | None, output: Path | None) -> int:
|
||||
manifest_path: Final = repo_root / manifest
|
||||
try:
|
||||
overwrites_manifest: Final = output is not None and (
|
||||
output.resolve() == manifest_path.resolve()
|
||||
or (output.exists() and manifest_path.exists() and output.samefile(manifest_path))
|
||||
)
|
||||
except (OSError, RuntimeError) as error:
|
||||
raise click.ClickException(f"Cannot resolve drift output or manifest path, {error}") from error
|
||||
if overwrites_manifest:
|
||||
raise click.ClickException("Output path must differ from the drift manifest")
|
||||
report: Final = inspect_drift(repo_root, manifest_path, target_ref=target, since_ref=since)
|
||||
if isinstance(report, DriftError):
|
||||
raise click.ClickException(report.message)
|
||||
rendered: Final = report.model_dump_json(indent=2)
|
||||
if output is None:
|
||||
click.echo(rendered)
|
||||
return 0
|
||||
try:
|
||||
output.write_text(f"{rendered}\n", encoding="utf-8")
|
||||
except OSError as error:
|
||||
raise click.ClickException(f"Cannot write drift report to {output}, {error}") from error
|
||||
return 0
|
||||
|
||||
return click.Command(
|
||||
"drift",
|
||||
params=[
|
||||
click.Option(("--target",), default="HEAD", show_default=True, help="Git revision to inspect"),
|
||||
click.Option(
|
||||
("--manifest",),
|
||||
default="tests/rust-python-harness/strategies/drift/drift.json",
|
||||
type=click.Path(path_type=Path, dir_okay=False),
|
||||
show_default=True,
|
||||
help="Manifest path, relative to the repository root",
|
||||
),
|
||||
click.Option(("--since",), help="Exploratory comparison revision, overriding the checkpoint"),
|
||||
click.Option(
|
||||
("--output",),
|
||||
type=click.Path(path_type=Path, dir_okay=False),
|
||||
help="Write JSON to this path instead of stdout",
|
||||
),
|
||||
],
|
||||
callback=inspect,
|
||||
help="Inventory Python and Rust changes. Exit zero means a report was generated, without asserting parity.",
|
||||
)
|
||||
162
tests/rust-python-harness/strategies/drift/drift.json
Normal file
162
tests/rust-python-harness/strategies/drift/drift.json
Normal file
|
|
@ -0,0 +1,162 @@
|
|||
{
|
||||
"schema_version": 1,
|
||||
"checkpoint_sha": "6530b329f43ef456c502ba2db7656f871d62474e",
|
||||
"checkpoint_state": "unknown",
|
||||
"candidate_paths": [
|
||||
"litellm/",
|
||||
"litellm-rust/",
|
||||
"tests/",
|
||||
"model_prices_and_context_window.json",
|
||||
"model_prices_and_context_window.schema.json",
|
||||
"pyproject.toml",
|
||||
"uv.lock",
|
||||
"packaging/litellm-core/",
|
||||
"Makefile",
|
||||
".github/workflows/",
|
||||
".github/CODEOWNERS",
|
||||
"enterprise/litellm_enterprise/",
|
||||
"enterprise/pyproject.toml"
|
||||
],
|
||||
"mappings": [
|
||||
{
|
||||
"id": "messages",
|
||||
"python_paths": [
|
||||
"litellm/messages/",
|
||||
"litellm/anthropic_interface/",
|
||||
"litellm/llms/anthropic/pass_through/",
|
||||
"litellm/llms/base_llm/anthropic_messages/",
|
||||
"litellm/types/llms/anthropic_messages/",
|
||||
"litellm/llms/azure_ai/anthropic/",
|
||||
"litellm/llms/bedrock/claude_platform/",
|
||||
"litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/",
|
||||
"litellm/anthropic_beta_headers_manager.py",
|
||||
"litellm/anthropic_beta_headers_config.json",
|
||||
"litellm/proxy/anthropic_endpoints/",
|
||||
"litellm/rust_bridge/messages/",
|
||||
"litellm/rust_bridge/host/routes/messages.py",
|
||||
"litellm/llms/bedrock/messages/",
|
||||
"litellm/llms/bedrock_mantle/messages/",
|
||||
"litellm/llms/deepseek/messages/",
|
||||
"litellm/llms/edenai/messages/",
|
||||
"litellm/llms/github_copilot/messages/",
|
||||
"litellm/llms/minimax/messages/",
|
||||
"litellm/llms/openai_like/messages/",
|
||||
"litellm/llms/tencent/messages/",
|
||||
"litellm/llms/anthropic/common_utils.py",
|
||||
"litellm/types/llms/anthropic.py",
|
||||
"litellm/types/llms/anthropic_tool_search.py"
|
||||
],
|
||||
"rust_paths": [
|
||||
"litellm-rust/crates/inference-messages/",
|
||||
"litellm-rust/crates/llms-types/src/formats/messages/",
|
||||
"litellm-rust/crates/llms/src/base_llm/messages/",
|
||||
"litellm-rust/crates/llms/src/bedrock/claude_platform/messages/",
|
||||
"litellm-rust/crates/python-bridge/src/routes/messages/",
|
||||
"litellm-rust/crates/gateway-inference/src/messages.rs",
|
||||
"litellm-rust/crates/llms/src/anthropic/messages/",
|
||||
"litellm-rust/crates/llms/src/azure_ai/messages/",
|
||||
"litellm-rust/crates/llms/src/bedrock/messages/",
|
||||
"litellm-rust/crates/llms/src/bedrock_mantle/messages/",
|
||||
"litellm-rust/crates/llms/src/deepseek/messages/",
|
||||
"litellm-rust/crates/llms/src/edenai/messages/",
|
||||
"litellm-rust/crates/llms/src/minimax/messages/",
|
||||
"litellm-rust/crates/llms/src/openai_like/messages/",
|
||||
"litellm-rust/crates/llms/src/openrouter/messages/",
|
||||
"litellm-rust/crates/llms/src/tencent/messages/",
|
||||
"litellm-rust/crates/llms/src/vertex_ai/messages/",
|
||||
"litellm-rust/crates/llms/src/anthropic/common_utils.rs",
|
||||
"litellm-rust/crates/llms/src/anthropic/beta_headers.rs",
|
||||
"litellm-rust/crates/llms-types/src/providers/anthropic/"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "responses",
|
||||
"python_paths": [
|
||||
"litellm/responses/",
|
||||
"litellm/types/responses/",
|
||||
"litellm/types/llms/openai.py",
|
||||
"litellm/completion_extras/litellm_responses_transformation/",
|
||||
"litellm/proxy/response_api_endpoints/",
|
||||
"litellm/rust_bridge/responses/",
|
||||
"litellm/rust_bridge/host/routes/responses.py",
|
||||
"litellm/llms/azure/responses/",
|
||||
"litellm/llms/azure_ai/responses/",
|
||||
"litellm/llms/base_llm/responses/",
|
||||
"litellm/llms/bedrock/responses/",
|
||||
"litellm/llms/bedrock_mantle/responses/",
|
||||
"litellm/llms/chatgpt/responses/",
|
||||
"litellm/llms/databricks/responses/",
|
||||
"litellm/llms/edenai/responses/",
|
||||
"litellm/llms/fireworks_ai/responses/",
|
||||
"litellm/llms/github_copilot/responses/",
|
||||
"litellm/llms/hosted_vllm/responses/",
|
||||
"litellm/llms/litellm_proxy/responses/",
|
||||
"litellm/llms/manus/responses/",
|
||||
"litellm/llms/openai/responses/",
|
||||
"litellm/llms/openai_like/responses/",
|
||||
"litellm/llms/openrouter/responses/",
|
||||
"litellm/llms/perplexity/responses/",
|
||||
"litellm/llms/sail/responses/",
|
||||
"litellm/llms/volcengine/responses/",
|
||||
"litellm/llms/xai/responses/",
|
||||
"litellm/llms/anthropic/pass_through/responses_adapters/",
|
||||
"litellm/llms/openai/common_utils.py",
|
||||
"litellm/llms/openai/workload_identity.py",
|
||||
"enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py",
|
||||
"enterprise/litellm_enterprise/proxy/common_utils/managed_object_attribution.py"
|
||||
],
|
||||
"rust_paths": [
|
||||
"litellm-rust/crates/inference-responses/",
|
||||
"litellm-rust/crates/llms-types/src/formats/responses/",
|
||||
"litellm-rust/crates/python-bridge/src/routes/responses/",
|
||||
"litellm-rust/crates/gateway-inference/src/responses.rs",
|
||||
"litellm-rust/crates/llms/src/base_llm/responses/",
|
||||
"litellm-rust/crates/llms/src/openai/responses/",
|
||||
"litellm-rust/crates/llms/src/openrouter/responses/"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "shared-runtime",
|
||||
"python_paths": [
|
||||
"litellm/__init__.py",
|
||||
"litellm/main.py",
|
||||
"litellm/utils.py",
|
||||
"litellm/router.py",
|
||||
"litellm/exceptions.py",
|
||||
"litellm/types/utils.py",
|
||||
"litellm/litellm_core_utils/",
|
||||
"litellm/llms/custom_httpx/",
|
||||
"litellm/llms/openai_like/",
|
||||
"litellm/rust_bridge/",
|
||||
"litellm/caching/",
|
||||
"litellm/secret_managers/",
|
||||
"litellm/model_prices_and_context_window_backup.json",
|
||||
"model_prices_and_context_window.json",
|
||||
"litellm/router_utils/common_utils.py",
|
||||
"litellm/router_utils/cooldown_handlers.py",
|
||||
"litellm/types/router.py",
|
||||
"litellm/types/litellm_params.py",
|
||||
"litellm/proxy/common_request_processing.py"
|
||||
],
|
||||
"rust_paths": [
|
||||
"litellm-rust/crates/core-utils/",
|
||||
"litellm-rust/crates/http/",
|
||||
"litellm-rust/crates/framer/",
|
||||
"litellm-rust/crates/inference/",
|
||||
"litellm-rust/crates/host/",
|
||||
"litellm-rust/crates/host-python/",
|
||||
"litellm-rust/crates/host-http/",
|
||||
"litellm-rust/crates/python-bridge/",
|
||||
"litellm-rust/crates/callbacks-legacy-python/",
|
||||
"litellm-rust/crates/cache-response/",
|
||||
"litellm-rust/crates/router/",
|
||||
"litellm-rust/crates/router-types/",
|
||||
"litellm-rust/crates/gateway-inference/",
|
||||
"litellm-rust/crates/model-catalog/",
|
||||
"litellm-rust/crates/llms/src/base_llm/sse.rs",
|
||||
"litellm-rust/crates/llms/src/base_llm/auth.rs",
|
||||
"litellm-rust/crates/llms/src/error.rs"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
492
tests/rust-python-harness/strategies/drift/runner.py
Normal file
492
tests/rust-python-harness/strategies/drift/runner.py
Normal file
|
|
@ -0,0 +1,492 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import subprocess
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from itertools import chain
|
||||
from pathlib import Path, PurePosixPath
|
||||
from typing import Annotated, Final, Literal, TypeAlias
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, ValidationError
|
||||
|
||||
_Paths: TypeAlias = Annotated[tuple[str, ...], Field(min_length=1)]
|
||||
_Sha: TypeAlias = Annotated[str, Field(pattern=r"^(?:[0-9a-f]{40}|[0-9a-f]{64})$")]
|
||||
CheckpointState: TypeAlias = Literal["unknown", "drifted", "aligned"]
|
||||
|
||||
|
||||
class _Mapping(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
id: Annotated[str, Field(min_length=1)]
|
||||
python_paths: _Paths
|
||||
rust_paths: _Paths
|
||||
checkpoint_sha: _Sha | None = None
|
||||
checkpoint_state: CheckpointState = "unknown"
|
||||
|
||||
|
||||
class _Manifest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
schema_version: Literal[1]
|
||||
checkpoint_sha: _Sha | None
|
||||
checkpoint_state: CheckpointState = "unknown"
|
||||
candidate_paths: _Paths
|
||||
mappings: Annotated[tuple[_Mapping, ...], Field(min_length=1)]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DriftError:
|
||||
message: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _PathChange:
|
||||
status: str
|
||||
old_path: str | None
|
||||
path: str
|
||||
|
||||
@property
|
||||
def paths(self) -> tuple[str, ...]:
|
||||
return (self.path,) if self.old_path is None else (self.old_path, self.path)
|
||||
|
||||
|
||||
class DriftChange(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
status: str
|
||||
old_path: str | None
|
||||
path: str
|
||||
mapping_ids: tuple[str, ...]
|
||||
sides: tuple[Literal["python", "rust"], ...]
|
||||
patch: str
|
||||
|
||||
|
||||
class DriftCommit(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
sha: str
|
||||
subject: str
|
||||
|
||||
|
||||
class MappingDrift(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
id: str
|
||||
status: Literal["changes_found", "no_changes"]
|
||||
checkpoint_sha: str
|
||||
checkpoint_state: CheckpointState
|
||||
commits: tuple[DriftCommit, ...]
|
||||
changes: tuple[DriftChange, ...]
|
||||
|
||||
|
||||
class WorkingTreeChange(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
status: str
|
||||
old_path: str | None
|
||||
path: str
|
||||
|
||||
|
||||
class DriftReport(BaseModel):
|
||||
model_config = ConfigDict(frozen=True, extra="forbid")
|
||||
|
||||
status: Literal["unreviewed", "changes_found", "no_changes"]
|
||||
target_sha: str
|
||||
checkpoint_sha: str | None
|
||||
checkpoint_state: CheckpointState
|
||||
comparison_sha: str | None
|
||||
comparison_source: Literal["checkpoint", "override", "none"]
|
||||
manifest_path: str
|
||||
manifest_sha256: str
|
||||
mappings: tuple[_Mapping, ...]
|
||||
changes: tuple[DriftChange, ...]
|
||||
unclassified: tuple[DriftChange, ...]
|
||||
mapping_checkpoints: tuple[MappingDrift, ...]
|
||||
working_tree_changes: tuple[WorkingTreeChange, ...]
|
||||
|
||||
|
||||
def _git(repo_root: Path, *args: str) -> str | DriftError:
|
||||
try:
|
||||
result: Final = subprocess.run(
|
||||
("git", "--no-optional-locks", "--no-pager", "--literal-pathspecs", *args),
|
||||
cwd=repo_root,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=60,
|
||||
)
|
||||
stdout: Final = result.stdout.decode("utf-8")
|
||||
stderr: Final = result.stderr.decode("utf-8")
|
||||
except (OSError, subprocess.SubprocessError, UnicodeError) as error:
|
||||
return DriftError(f"Git operation failed: {error}")
|
||||
if result.returncode:
|
||||
return DriftError(f"Git {' '.join(args)} failed: {stderr.strip()}")
|
||||
return stdout
|
||||
|
||||
|
||||
def _commit(repo_root: Path, ref: str) -> str | DriftError:
|
||||
result: Final = _git(repo_root, "rev-parse", "--verify", "--end-of-options", f"{ref}^{{commit}}")
|
||||
return result if isinstance(result, DriftError) else result.strip()
|
||||
|
||||
|
||||
def _read_manifest(source: str, label: str) -> _Manifest | DriftError:
|
||||
try:
|
||||
manifest: Final = _Manifest.model_validate_json(source)
|
||||
except ValidationError as error:
|
||||
return DriftError(f"Invalid manifest {label}: {error}")
|
||||
if manifest.checkpoint_sha is None and manifest.checkpoint_state != "unknown":
|
||||
return DriftError(f"Invalid manifest {label}: checkpoint state requires a checkpoint SHA")
|
||||
stateless: Final = next(
|
||||
(
|
||||
mapping.id
|
||||
for mapping in manifest.mappings
|
||||
if mapping.checkpoint_sha is None and mapping.checkpoint_state != "unknown"
|
||||
),
|
||||
None,
|
||||
)
|
||||
if stateless is not None:
|
||||
return DriftError(f"Invalid manifest {label}: mapping {stateless!r} checkpoint state requires a checkpoint SHA")
|
||||
selectors: Final = tuple(chain(manifest.candidate_paths, _mapping_paths(manifest.mappings)))
|
||||
invalid: Final = next((path for path in selectors if not _valid_selector(path)), None)
|
||||
if invalid is not None:
|
||||
return DriftError(f"Invalid manifest path {invalid!r}: use a relative file or directory prefix ending in /")
|
||||
ids: Final = tuple(mapping.id for mapping in manifest.mappings)
|
||||
if len(frozenset(ids)) != len(ids) or any(not name.strip() for name in ids):
|
||||
return DriftError(f"Invalid manifest {label}: mapping IDs must be nonblank and unique")
|
||||
return manifest
|
||||
|
||||
|
||||
def _valid_selector(path: str) -> bool:
|
||||
return bool(
|
||||
path
|
||||
and not path.startswith("/")
|
||||
and not any(character in path for character in ("*", "?", "[", "]", "\\", "\0"))
|
||||
and all(part not in {"", ".", ".."} for part in path.removesuffix("/").split("/"))
|
||||
)
|
||||
|
||||
|
||||
def _mapping_paths(mappings: tuple[_Mapping, ...]) -> Iterator[str]:
|
||||
return chain.from_iterable(chain(mapping.python_paths, mapping.rust_paths) for mapping in mappings)
|
||||
|
||||
|
||||
def _manifest_at_path(repo_root: Path, sha: str, path: str) -> tuple[_Manifest, ...] | DriftError:
|
||||
listing: Final = _git(repo_root, "ls-tree", "-z", sha, "--", path)
|
||||
if isinstance(listing, DriftError):
|
||||
return listing
|
||||
if not listing:
|
||||
return ()
|
||||
source: Final = _git(repo_root, "show", f"{sha}:{path}")
|
||||
if isinstance(source, DriftError):
|
||||
return source
|
||||
manifest: Final = _read_manifest(source, f"{sha}:{path}")
|
||||
return manifest if isinstance(manifest, DriftError) else (manifest,)
|
||||
|
||||
|
||||
def _manifest_at(repo_root: Path, sha: str, path: str) -> tuple[_Manifest, ...] | DriftError:
|
||||
existing: Final = _manifest_at_path(repo_root, sha, path)
|
||||
if isinstance(existing, DriftError) or existing:
|
||||
return existing
|
||||
previous_path: Final = _historical_manifest_path(repo_root, sha, path)
|
||||
if isinstance(previous_path, DriftError):
|
||||
return previous_path
|
||||
return () if previous_path == path else _manifest_at_path(repo_root, sha, previous_path)
|
||||
|
||||
|
||||
def _matches(paths: tuple[str, ...], selectors: tuple[str, ...]) -> bool:
|
||||
return any(_matches_path(path, selectors) for path in paths)
|
||||
|
||||
|
||||
def _matches_path(path: str, selectors: tuple[str, ...]) -> bool:
|
||||
return any(path == selector or (selector.endswith("/") and path.startswith(selector)) for selector in selectors)
|
||||
|
||||
|
||||
def _parse_change(status: str, tokens: Iterator[str]) -> _PathChange:
|
||||
first: Final = next(tokens)
|
||||
if status.startswith(("R", "C")):
|
||||
return _PathChange(status, first, next(tokens))
|
||||
return _PathChange(status, None, first)
|
||||
|
||||
|
||||
def _parse_changes(source: str) -> Iterator[_PathChange]:
|
||||
tokens: Final = iter(source.split("\0")[:-1])
|
||||
return (_parse_change(status, tokens) for status in tokens)
|
||||
|
||||
|
||||
def _parse_working_change(record: str, tokens: Iterator[str]) -> _PathChange:
|
||||
status: Final = record[:2]
|
||||
path: Final = record[3:]
|
||||
old_path: Final = next(tokens) if "R" in status or "C" in status else None
|
||||
return _PathChange(status, old_path, path)
|
||||
|
||||
|
||||
def _working_changes(repo_root: Path, selectors: tuple[str, ...]) -> tuple[WorkingTreeChange, ...] | DriftError:
|
||||
source: Final = _git(repo_root, "status", "--porcelain=v1", "-z", "--untracked-files=all", "--renames")
|
||||
if isinstance(source, DriftError):
|
||||
return source
|
||||
tokens: Final = iter(source.split("\0")[:-1])
|
||||
parsed: Final = (_parse_working_change(record, tokens) for record in tokens)
|
||||
return tuple(
|
||||
WorkingTreeChange(status=change.status, old_path=change.old_path, path=change.path)
|
||||
for change in parsed
|
||||
if _matches(change.paths, selectors)
|
||||
)
|
||||
|
||||
|
||||
def _historical_manifest_path(repo_root: Path, sha: str, path: str) -> str | DriftError:
|
||||
working: Final = _working_changes(repo_root, (path,))
|
||||
if isinstance(working, DriftError):
|
||||
return working
|
||||
staged_path: Final = next(
|
||||
(change.old_path for change in working if change.path == path and change.old_path is not None), None
|
||||
)
|
||||
head_path: Final = staged_path if staged_path is not None else path
|
||||
head: Final = _commit(repo_root, "HEAD")
|
||||
if isinstance(head, DriftError):
|
||||
return head
|
||||
endpoint: Final = _git(
|
||||
repo_root,
|
||||
"diff",
|
||||
"--no-color",
|
||||
"--no-ext-diff",
|
||||
"--no-textconv",
|
||||
"--find-renames=50%",
|
||||
"--name-status",
|
||||
"-z",
|
||||
sha,
|
||||
head,
|
||||
)
|
||||
if isinstance(endpoint, DriftError):
|
||||
return endpoint
|
||||
endpoint_path: Final = next(
|
||||
(
|
||||
change.old_path
|
||||
for change in _parse_changes(endpoint)
|
||||
if change.path == head_path and change.status.startswith("R")
|
||||
),
|
||||
None,
|
||||
)
|
||||
if endpoint_path is not None:
|
||||
return endpoint_path
|
||||
source: Final = _git(
|
||||
repo_root,
|
||||
"log",
|
||||
"--follow",
|
||||
"--format=",
|
||||
"--no-color",
|
||||
"--no-ext-diff",
|
||||
"--no-textconv",
|
||||
"--find-renames=50%",
|
||||
"--diff-filter=R",
|
||||
"--name-status",
|
||||
"-z",
|
||||
f"{sha}..{head}",
|
||||
"--",
|
||||
head_path,
|
||||
)
|
||||
if isinstance(source, DriftError):
|
||||
return source
|
||||
previous: Final = tuple(change.old_path for change in _parse_changes(source) if change.old_path is not None)
|
||||
return previous[-1] if previous else head_path
|
||||
|
||||
|
||||
def _change_report(
|
||||
repo_root: Path, baseline: str, target: str, change: _PathChange, mappings: tuple[_Mapping, ...]
|
||||
) -> DriftChange | DriftError:
|
||||
patch: Final = _git(
|
||||
repo_root,
|
||||
"diff",
|
||||
"--no-color",
|
||||
"--no-ext-diff",
|
||||
"--no-textconv",
|
||||
"--find-renames=50%",
|
||||
baseline,
|
||||
target,
|
||||
"--",
|
||||
*change.paths,
|
||||
)
|
||||
if isinstance(patch, DriftError):
|
||||
return patch
|
||||
python: Final = any(_matches(change.paths, mapping.python_paths) for mapping in mappings)
|
||||
rust: Final = any(_matches(change.paths, mapping.rust_paths) for mapping in mappings)
|
||||
sides: Final[tuple[Literal["python", "rust"], ...]] = (("python",) if python else ()) + (("rust",) if rust else ())
|
||||
ids: Final = tuple(
|
||||
sorted(
|
||||
frozenset(
|
||||
mapping.id
|
||||
for mapping in mappings
|
||||
if _matches(change.paths, (*mapping.python_paths, *mapping.rust_paths))
|
||||
)
|
||||
)
|
||||
)
|
||||
return DriftChange(
|
||||
status=change.status, old_path=change.old_path, path=change.path, mapping_ids=ids, sides=sides, patch=patch
|
||||
)
|
||||
|
||||
|
||||
def _ancestor(repo_root: Path, baseline: str, target: str) -> DriftError | None:
|
||||
result: Final = _git(repo_root, "merge-base", "--is-ancestor", baseline, target)
|
||||
if isinstance(result, DriftError):
|
||||
return DriftError(f"Comparison commit must be an ancestor of the target: {result.message}")
|
||||
return None
|
||||
|
||||
|
||||
def _changed_files(
|
||||
repo_root: Path, baseline: str | None, target: str, selectors: tuple[str, ...], mappings: tuple[_Mapping, ...]
|
||||
) -> tuple[DriftChange, ...] | DriftError:
|
||||
if baseline is None:
|
||||
return ()
|
||||
ancestry: Final = _ancestor(repo_root, baseline, target)
|
||||
if ancestry is not None:
|
||||
return ancestry
|
||||
source: Final = _git(
|
||||
repo_root,
|
||||
"diff",
|
||||
"--no-color",
|
||||
"--no-ext-diff",
|
||||
"--no-textconv",
|
||||
"--find-renames=50%",
|
||||
"--name-status",
|
||||
"-z",
|
||||
baseline,
|
||||
target,
|
||||
)
|
||||
if isinstance(source, DriftError):
|
||||
return source
|
||||
reports: Final = tuple(
|
||||
_change_report(repo_root, baseline, target, change, mappings)
|
||||
for change in _parse_changes(source)
|
||||
if _matches(change.paths, selectors)
|
||||
)
|
||||
error: Final = next((report for report in reports if isinstance(report, DriftError)), None)
|
||||
if error is not None:
|
||||
return error
|
||||
return tuple(report for report in reports if isinstance(report, DriftChange))
|
||||
|
||||
|
||||
def _commits(
|
||||
repo_root: Path, baseline: str, target: str, paths: tuple[str, ...]
|
||||
) -> tuple[DriftCommit, ...] | DriftError:
|
||||
source: Final = _git(
|
||||
repo_root, "log", "-z", "--no-color", "--format=%H%x00%s", f"{baseline}..{target}", "--", *paths
|
||||
)
|
||||
if isinstance(source, DriftError):
|
||||
return source
|
||||
fields: Final = iter(source.split("\0"))
|
||||
return tuple(DriftCommit(sha=sha, subject=subject) for sha, subject in zip(fields, fields) if sha)
|
||||
|
||||
|
||||
def _mapping_drift(repo_root: Path, target: str, mapping: _Mapping) -> MappingDrift | DriftError | None:
|
||||
if mapping.checkpoint_sha is None:
|
||||
return None
|
||||
checkpoint: Final = _commit(repo_root, mapping.checkpoint_sha)
|
||||
if isinstance(checkpoint, DriftError):
|
||||
return checkpoint
|
||||
paths: Final = (*mapping.python_paths, *mapping.rust_paths)
|
||||
changes: Final = _changed_files(repo_root, checkpoint, target, paths, (mapping,))
|
||||
if isinstance(changes, DriftError):
|
||||
return changes
|
||||
commits: Final = _commits(repo_root, checkpoint, target, paths)
|
||||
if isinstance(commits, DriftError):
|
||||
return commits
|
||||
return MappingDrift(
|
||||
id=mapping.id,
|
||||
status="changes_found" if changes else "no_changes",
|
||||
checkpoint_sha=checkpoint,
|
||||
checkpoint_state=mapping.checkpoint_state,
|
||||
commits=commits,
|
||||
changes=changes,
|
||||
)
|
||||
|
||||
|
||||
def _merged_mapping(identifier: str, mappings: tuple[_Mapping, ...]) -> _Mapping:
|
||||
matching: Final = tuple(mapping for mapping in mappings if mapping.id == identifier)
|
||||
python: Final = chain.from_iterable(mapping.python_paths for mapping in matching)
|
||||
rust: Final = chain.from_iterable(mapping.rust_paths for mapping in matching)
|
||||
return _Mapping(
|
||||
id=identifier,
|
||||
python_paths=tuple(sorted(frozenset(python))),
|
||||
rust_paths=tuple(sorted(frozenset(rust))),
|
||||
checkpoint_sha=matching[-1].checkpoint_sha,
|
||||
checkpoint_state=matching[-1].checkpoint_state,
|
||||
)
|
||||
|
||||
|
||||
def _report(
|
||||
repo_root: Path, path: str, raw: str, manifest: _Manifest, target: str, baseline: str | None, override: bool
|
||||
) -> DriftReport | DriftError:
|
||||
previous: Final = () if baseline is None else _manifest_at(repo_root, baseline, path)
|
||||
if isinstance(previous, DriftError):
|
||||
return previous
|
||||
current: Final = _manifest_at(repo_root, target, path)
|
||||
if isinstance(current, DriftError):
|
||||
return current
|
||||
checkpoint_policy: Final = (
|
||||
()
|
||||
if manifest.checkpoint_sha is None or manifest.checkpoint_sha in {baseline, target}
|
||||
else _manifest_at(repo_root, manifest.checkpoint_sha, path)
|
||||
)
|
||||
if isinstance(checkpoint_policy, DriftError):
|
||||
return checkpoint_policy
|
||||
policies: Final = (*previous, *checkpoint_policy, *current, manifest)
|
||||
all_mappings: Final = tuple(chain.from_iterable(policy.mappings for policy in policies))
|
||||
ids: Final = tuple(sorted(frozenset(mapping.id for mapping in all_mappings)))
|
||||
mappings: Final = tuple(_merged_mapping(identifier, all_mappings) for identifier in ids)
|
||||
candidates: Final = chain.from_iterable(policy.candidate_paths for policy in policies)
|
||||
selectors: Final = tuple(chain(candidates, _mapping_paths(mappings), (path,)))
|
||||
changes: Final = _changed_files(repo_root, baseline, target, selectors, mappings)
|
||||
if isinstance(changes, DriftError):
|
||||
return changes
|
||||
working: Final = _working_changes(repo_root, selectors)
|
||||
if isinstance(working, DriftError):
|
||||
return working
|
||||
drifts: Final = tuple(_mapping_drift(repo_root, target, mapping) for mapping in manifest.mappings)
|
||||
drift_error: Final = next((drift for drift in drifts if isinstance(drift, DriftError)), None)
|
||||
if drift_error is not None:
|
||||
return drift_error
|
||||
status: Final = "unreviewed" if manifest.checkpoint_sha is None else "changes_found" if changes else "no_changes"
|
||||
return DriftReport(
|
||||
status=status,
|
||||
target_sha=target,
|
||||
checkpoint_sha=manifest.checkpoint_sha,
|
||||
checkpoint_state=manifest.checkpoint_state,
|
||||
comparison_sha=baseline,
|
||||
comparison_source="override" if override else "checkpoint" if baseline else "none",
|
||||
manifest_path=path,
|
||||
manifest_sha256=hashlib.sha256(raw.encode("utf-8")).hexdigest(),
|
||||
mappings=mappings,
|
||||
changes=changes,
|
||||
unclassified=tuple(change for change in changes if not change.mapping_ids),
|
||||
mapping_checkpoints=tuple(drift for drift in drifts if isinstance(drift, MappingDrift)),
|
||||
working_tree_changes=working,
|
||||
)
|
||||
|
||||
|
||||
def inspect_drift(
|
||||
repo_root: Path, manifest_path: Path, target_ref: str = "HEAD", since_ref: str | None = None
|
||||
) -> DriftReport | DriftError:
|
||||
root: Final = repo_root.resolve()
|
||||
try:
|
||||
resolved_path: Final = (root / manifest_path).resolve()
|
||||
relative: Final = resolved_path.relative_to(root).as_posix()
|
||||
raw: Final = resolved_path.read_bytes().decode("utf-8")
|
||||
except (OSError, RuntimeError, ValueError) as error:
|
||||
return DriftError(f"Cannot read manifest inside repository: {error}")
|
||||
if not _valid_selector(relative) or PurePosixPath(relative).as_posix() != relative:
|
||||
return DriftError("Manifest must be a repository-relative file")
|
||||
manifest: Final = _read_manifest(raw, relative)
|
||||
if isinstance(manifest, DriftError):
|
||||
return manifest
|
||||
target: Final = _commit(root, target_ref)
|
||||
if isinstance(target, DriftError):
|
||||
return target
|
||||
checkpoint: Final = None if manifest.checkpoint_sha is None else _commit(root, manifest.checkpoint_sha)
|
||||
if isinstance(checkpoint, DriftError):
|
||||
return checkpoint
|
||||
checkpoint_ancestry: Final = None if checkpoint is None else _ancestor(root, checkpoint, target)
|
||||
if checkpoint_ancestry is not None:
|
||||
return checkpoint_ancestry
|
||||
reference: Final = since_ref if since_ref is not None else checkpoint
|
||||
baseline: Final = None if reference is None else _commit(root, reference)
|
||||
if isinstance(baseline, DriftError):
|
||||
return baseline
|
||||
return _report(root, relative, raw, manifest, target, baseline, since_ref is not None)
|
||||
163
tests/rust-python-harness/strategies/drift/test_command.py
Normal file
163
tests/rust-python-harness/strategies/drift/test_command.py
Normal file
|
|
@ -0,0 +1,163 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from click.testing import CliRunner
|
||||
|
||||
from ...cli import build_cli
|
||||
from ...cli.catalog import load_catalog
|
||||
from .command import drift_command
|
||||
from .runner import DriftReport
|
||||
|
||||
|
||||
def _drift_git(repo_root: Path, *args: str) -> str:
|
||||
result: Final = subprocess.run(("git", "-C", str(repo_root), *args), check=True, capture_output=True, text=True)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _drift_manifest(checkpoint_sha: str | None = None) -> str:
|
||||
return json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"checkpoint_sha": checkpoint_sha,
|
||||
"candidate_paths": ["litellm/", "litellm-rust/"],
|
||||
"mappings": [
|
||||
{
|
||||
"id": "messages",
|
||||
"python_paths": ["litellm/messages/"],
|
||||
"rust_paths": ["litellm-rust/crates/inference-messages/"],
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _drift_repo(tmp_path: Path) -> tuple[Path, str]:
|
||||
repo_root: Final = tmp_path / "repo"
|
||||
repo_root.mkdir()
|
||||
_drift_git(repo_root, "init", "--quiet", "--initial-branch=main")
|
||||
_drift_git(repo_root, "config", "user.name", "Harness Test")
|
||||
_drift_git(repo_root, "config", "user.email", "harness@example.invalid")
|
||||
source: Final = repo_root / "litellm/messages/dispatch.py"
|
||||
source.parent.mkdir(parents=True)
|
||||
source.write_text("VALUE = 'initial'\n", encoding="utf-8")
|
||||
manifest: Final = repo_root / "tests/rust-python-harness/strategies/drift/drift.json"
|
||||
manifest.parent.mkdir(parents=True)
|
||||
manifest.write_text(_drift_manifest(), encoding="utf-8")
|
||||
_drift_git(repo_root, "add", ".")
|
||||
_drift_git(repo_root, "commit", "--quiet", "-m", "initial")
|
||||
return repo_root, _drift_git(repo_root, "rev-parse", "HEAD")
|
||||
|
||||
|
||||
def test_drift_command_reports_unreviewed_checkpoint_from_default_manifest(tmp_path: Path) -> None:
|
||||
repo_root, initial_sha = _drift_repo(tmp_path)
|
||||
result: Final = CliRunner().invoke(drift_command(repo_root))
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
report: Final = DriftReport.model_validate_json(result.output)
|
||||
assert report.status == "unreviewed"
|
||||
assert report.target_sha == initial_sha
|
||||
assert report.checkpoint_sha is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("absolute_path", (False, True))
|
||||
def test_drift_command_resolves_custom_manifest_paths(tmp_path: Path, absolute_path: bool) -> None:
|
||||
repo_root, initial_sha = _drift_repo(tmp_path)
|
||||
manifest: Final = repo_root / "custom.json"
|
||||
manifest.write_text(_drift_manifest(initial_sha), encoding="utf-8")
|
||||
manifest_arg: Final = str(manifest if absolute_path else manifest.relative_to(repo_root))
|
||||
result: Final = CliRunner().invoke(drift_command(repo_root), ("--manifest", manifest_arg))
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
report: Final = DriftReport.model_validate_json(result.output)
|
||||
assert report.status == "no_changes"
|
||||
assert report.checkpoint_sha == initial_sha
|
||||
|
||||
|
||||
def test_drift_command_compares_requested_revisions_without_claiming_parity(tmp_path: Path) -> None:
|
||||
repo_root, initial_sha = _drift_repo(tmp_path)
|
||||
source: Final = repo_root / "litellm/messages/dispatch.py"
|
||||
source.write_text("VALUE = 'second'\n", encoding="utf-8")
|
||||
_drift_git(repo_root, "add", ".")
|
||||
_drift_git(repo_root, "commit", "--quiet", "-m", "second")
|
||||
second_sha: Final = _drift_git(repo_root, "rev-parse", "HEAD")
|
||||
source.write_text("VALUE = 'third'\n", encoding="utf-8")
|
||||
_drift_git(repo_root, "add", ".")
|
||||
_drift_git(repo_root, "commit", "--quiet", "-m", "third")
|
||||
result: Final = CliRunner().invoke(drift_command(repo_root), ("--since", initial_sha, "--target", second_sha))
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
report: Final = DriftReport.model_validate_json(result.output)
|
||||
assert report.status == "unreviewed"
|
||||
assert report.checkpoint_sha is None
|
||||
assert report.comparison_sha == initial_sha
|
||||
assert report.target_sha == second_sha
|
||||
assert tuple(change.path for change in report.changes) == ("litellm/messages/dispatch.py",)
|
||||
assert "+VALUE = 'second'" in report.changes[0].patch
|
||||
assert "third" not in report.changes[0].patch
|
||||
|
||||
|
||||
def test_drift_command_writes_report_to_file(tmp_path: Path) -> None:
|
||||
repo_root, initial_sha = _drift_repo(tmp_path)
|
||||
output: Final = tmp_path / "report.json"
|
||||
result: Final = CliRunner().invoke(drift_command(repo_root), ("--output", str(output)))
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
assert result.output == ""
|
||||
report: Final = DriftReport.model_validate_json(output.read_text(encoding="utf-8"))
|
||||
assert report.status == "unreviewed"
|
||||
assert report.target_sha == initial_sha
|
||||
|
||||
|
||||
def test_drift_command_surfaces_inventory_errors(tmp_path: Path) -> None:
|
||||
repo_root, _ = _drift_repo(tmp_path)
|
||||
result: Final = CliRunner().invoke(drift_command(repo_root), ("--manifest", "missing.json"))
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert "Error" in result.output
|
||||
assert "missing.json" in result.output
|
||||
|
||||
|
||||
def test_drift_command_reports_output_write_errors(tmp_path: Path) -> None:
|
||||
repo_root, _ = _drift_repo(tmp_path)
|
||||
output: Final = tmp_path / "missing/report.json"
|
||||
result: Final = CliRunner().invoke(drift_command(repo_root), ("--output", str(output)))
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert "Cannot write drift report" in result.output
|
||||
assert str(output) in result.output
|
||||
|
||||
|
||||
@pytest.mark.parametrize("output_kind", ("absolute", "relative", "symlink", "hardlink"))
|
||||
def test_drift_command_preserves_manifest_when_output_aliases_it(tmp_path: Path, output_kind: str) -> None:
|
||||
repo_root, _ = _drift_repo(tmp_path)
|
||||
manifest: Final = repo_root / "tests/rust-python-harness/strategies/drift/drift.json"
|
||||
original: Final = manifest.read_bytes()
|
||||
alias: Final = repo_root / "report.json"
|
||||
if output_kind == "symlink":
|
||||
alias.symlink_to(manifest)
|
||||
if output_kind == "hardlink":
|
||||
alias.hardlink_to(manifest)
|
||||
destination: Final = alias if output_kind in {"symlink", "hardlink"} else manifest
|
||||
output: Final = os.path.relpath(destination) if output_kind == "relative" else str(destination)
|
||||
result: Final = CliRunner().invoke(drift_command(repo_root), ("--output", output))
|
||||
|
||||
assert result.exit_code != 0
|
||||
assert "Output path must differ from the drift manifest" in result.output
|
||||
assert manifest.read_bytes() == original
|
||||
|
||||
|
||||
def test_run_drift_uses_registered_command_with_repository_context(tmp_path: Path) -> None:
|
||||
repo_root, initial_sha = _drift_repo(tmp_path)
|
||||
result: Final = CliRunner().invoke(build_cli(load_catalog(), repo_root), ("run", "drift"))
|
||||
|
||||
assert result.exit_code == 0, result.output
|
||||
report: Final = DriftReport.model_validate_json(result.output)
|
||||
assert report.target_sha == initial_sha
|
||||
assert report.checkpoint_state == "unknown"
|
||||
assert report.manifest_path == "tests/rust-python-harness/strategies/drift/drift.json"
|
||||
640
tests/rust-python-harness/strategies/drift/test_runner.py
Normal file
640
tests/rust-python-harness/strategies/drift/test_runner.py
Normal file
|
|
@ -0,0 +1,640 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import subprocess
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from .runner import DriftError, DriftReport, inspect_drift
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Repository:
|
||||
root: Path
|
||||
baseline: str
|
||||
|
||||
|
||||
def _git(root: Path, *arguments: str) -> str:
|
||||
result: Final = subprocess.run(
|
||||
(
|
||||
"git",
|
||||
"-c",
|
||||
"user.name=Drift Test",
|
||||
"-c",
|
||||
"user.email=drift@example.invalid",
|
||||
"-c",
|
||||
"commit.gpgsign=false",
|
||||
"-c",
|
||||
"core.hooksPath=/dev/null",
|
||||
*arguments,
|
||||
),
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
return result.stdout.strip()
|
||||
|
||||
|
||||
def _write(root: Path, path: str, content: str) -> None:
|
||||
destination: Final = root / path
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
destination.write_text(content, encoding="utf-8")
|
||||
|
||||
|
||||
def _commit(root: Path) -> str:
|
||||
_git(root, "add", "--all")
|
||||
_git(root, "commit", "--quiet", "-m", "test: update fixture behavior")
|
||||
return _git(root, "rev-parse", "HEAD")
|
||||
|
||||
|
||||
def _manifest(
|
||||
root: Path,
|
||||
checkpoint: str | None,
|
||||
*,
|
||||
checkpoint_state: str | None = None,
|
||||
pretty: bool = False,
|
||||
candidates: tuple[str, ...] = ("litellm/", "litellm-rust/"),
|
||||
mappings: tuple[tuple[str, tuple[str, ...], tuple[str, ...]], ...] = (
|
||||
("messages", ("litellm/messages/", "litellm/common.py"), ("litellm-rust/messages.rs",)),
|
||||
("responses", ("litellm/responses/", "litellm/common.py"), ("litellm-rust/responses.rs",)),
|
||||
),
|
||||
mapping_checkpoints: MappingProxyType[str, dict[str, str]] = MappingProxyType({}),
|
||||
) -> Path:
|
||||
path: Final = root / "drift.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": 1,
|
||||
"checkpoint_sha": checkpoint,
|
||||
**({"checkpoint_state": checkpoint_state} if checkpoint_state is not None else {}),
|
||||
"candidate_paths": candidates,
|
||||
"mappings": tuple(
|
||||
{
|
||||
"id": identifier,
|
||||
"python_paths": python_paths,
|
||||
"rust_paths": rust_paths,
|
||||
**mapping_checkpoints.get(identifier, {}),
|
||||
}
|
||||
for identifier, python_paths, rust_paths in mappings
|
||||
),
|
||||
},
|
||||
indent=2 if pretty else None,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def repository(tmp_path: Path) -> Repository:
|
||||
root: Final = tmp_path / "repository"
|
||||
root.mkdir()
|
||||
_git(root, "init", "--quiet", "--initial-branch=main")
|
||||
_write(root, "litellm/messages/handler.py", "def result():\n return 'before'\n")
|
||||
_write(root, "litellm/common.py", "TIMEOUT = 10\n")
|
||||
_write(root, "litellm-rust/messages.rs", 'pub const RESULT: &str = "before";\n')
|
||||
return Repository(root=root, baseline=_commit(root))
|
||||
|
||||
|
||||
def test_reports_python_rust_shared_and_unmapped_changes_with_real_patches(repository: Repository) -> None:
|
||||
_write(repository.root, "litellm/messages/handler.py", "def result():\n return 'after'\n")
|
||||
_write(repository.root, "litellm/common.py", "TIMEOUT = 20\n")
|
||||
_write(repository.root, "litellm-rust/messages.rs", 'pub const RESULT: &str = "after";\n')
|
||||
_write(repository.root, "litellm/new_endpoint.py", "ENABLED = True\n")
|
||||
target: Final = _commit(repository.root)
|
||||
manifest: Final = _manifest(repository.root, repository.baseline)
|
||||
original_manifest: Final = manifest.read_bytes()
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.status == "changes_found"
|
||||
assert (result.target_sha, result.checkpoint_sha, result.comparison_sha) == (
|
||||
target,
|
||||
repository.baseline,
|
||||
repository.baseline,
|
||||
)
|
||||
assert frozenset((change.path, change.status, change.mapping_ids, change.sides) for change in result.changes) == {
|
||||
("litellm/messages/handler.py", "M", ("messages",), ("python",)),
|
||||
("litellm/common.py", "M", ("messages", "responses"), ("python",)),
|
||||
("litellm-rust/messages.rs", "M", ("messages",), ("rust",)),
|
||||
("litellm/new_endpoint.py", "A", (), ()),
|
||||
}
|
||||
assert tuple(change.path for change in result.unclassified) == ("litellm/new_endpoint.py",)
|
||||
python_change: Final = next(change for change in result.changes if change.path == "litellm/messages/handler.py")
|
||||
assert "- return 'before'" in python_change.patch
|
||||
assert "+ return 'after'" in python_change.patch
|
||||
assert result.manifest_sha256 == hashlib.sha256(original_manifest).hexdigest()
|
||||
assert manifest.read_bytes() == original_manifest
|
||||
|
||||
|
||||
def test_mapping_checkpoint_reports_only_its_paths_and_commits_since_that_checkpoint(repository: Repository) -> None:
|
||||
_write(repository.root, "litellm/messages/handler.py", "def result():\n return 'ported'\n")
|
||||
checkpoint: Final = _commit(repository.root)
|
||||
_write(repository.root, "litellm/responses/handler.py", "ENABLED = True\n")
|
||||
_git(repository.root, "add", "--all")
|
||||
_git(repository.root, "commit", "--quiet", "-m", "feat: unrelated responses work")
|
||||
_write(repository.root, "litellm/messages/handler.py", "def result():\n return 'drifted'\n")
|
||||
_git(repository.root, "add", "--all")
|
||||
_git(repository.root, "commit", "--quiet", "-m", "fix: change messages result")
|
||||
drifted: Final = _git(repository.root, "rev-parse", "HEAD")
|
||||
manifest: Final = _manifest(
|
||||
repository.root,
|
||||
repository.baseline,
|
||||
mapping_checkpoints=MappingProxyType(
|
||||
{"messages": {"checkpoint_sha": checkpoint, "checkpoint_state": "aligned"}}
|
||||
),
|
||||
)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert len(result.mapping_checkpoints) == 1
|
||||
drift: Final = result.mapping_checkpoints[0]
|
||||
assert (drift.id, drift.status, drift.checkpoint_sha, drift.checkpoint_state) == (
|
||||
"messages",
|
||||
"changes_found",
|
||||
checkpoint,
|
||||
"aligned",
|
||||
)
|
||||
assert tuple((commit.sha, commit.subject) for commit in drift.commits) == (
|
||||
(drifted, "fix: change messages result"),
|
||||
)
|
||||
assert tuple((change.path, change.sides) for change in drift.changes) == (
|
||||
("litellm/messages/handler.py", ("python",)),
|
||||
)
|
||||
assert "- return 'ported'" in drift.changes[0].patch
|
||||
assert "+ return 'drifted'" in drift.changes[0].patch
|
||||
assert "before" not in drift.changes[0].patch
|
||||
|
||||
|
||||
def test_mapping_checkpoint_at_target_has_no_changes(repository: Repository) -> None:
|
||||
manifest: Final = _manifest(
|
||||
repository.root,
|
||||
repository.baseline,
|
||||
mapping_checkpoints=MappingProxyType({"responses": {"checkpoint_sha": repository.baseline}}),
|
||||
)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert tuple((drift.id, drift.status, drift.commits, drift.changes) for drift in result.mapping_checkpoints) == (
|
||||
("responses", "no_changes", (), ()),
|
||||
)
|
||||
|
||||
|
||||
def test_mapping_checkpoint_state_requires_a_checkpoint_sha(repository: Repository) -> None:
|
||||
manifest: Final = _manifest(
|
||||
repository.root,
|
||||
repository.baseline,
|
||||
mapping_checkpoints=MappingProxyType({"messages": {"checkpoint_state": "aligned"}}),
|
||||
)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftError), result
|
||||
assert "'messages'" in result.message
|
||||
|
||||
|
||||
def test_patch_is_usable_when_git_diff_colors_are_forced(repository: Repository) -> None:
|
||||
_write(repository.root, "litellm/common.py", "TIMEOUT = 20\n")
|
||||
_commit(repository.root)
|
||||
_git(repository.root, "config", "color.ui", "always")
|
||||
_git(repository.root, "config", "color.diff", "always")
|
||||
manifest: Final = _manifest(repository.root, repository.baseline)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert len(result.changes) == 1
|
||||
patch: Final = result.changes[0].patch
|
||||
assert "-TIMEOUT = 10\n+TIMEOUT = 20\n" in patch
|
||||
patch_path: Final = repository.root.parent / "change.patch"
|
||||
patch_path.write_text(patch, encoding="utf-8")
|
||||
_git(repository.root, "apply", "--check", "--reverse", str(patch_path))
|
||||
|
||||
|
||||
def test_detects_deletions_and_renames_outside_candidate_directories(repository: Repository) -> None:
|
||||
_git(repository.root, "mv", "litellm/messages/handler.py", "renamed_handler.py")
|
||||
(repository.root / "litellm-rust/messages.rs").unlink()
|
||||
_commit(repository.root)
|
||||
manifest: Final = _manifest(repository.root, repository.baseline)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert frozenset((change.status, change.old_path, change.path) for change in result.changes) == {
|
||||
("R100", "litellm/messages/handler.py", "renamed_handler.py"),
|
||||
("D", None, "litellm-rust/messages.rs"),
|
||||
}
|
||||
rename: Final = next(change for change in result.changes if change.old_path is not None)
|
||||
assert rename.mapping_ids == ("messages",)
|
||||
assert rename.sides == ("python",)
|
||||
assert "rename from litellm/messages/handler.py" in rename.patch
|
||||
assert "rename to renamed_handler.py" in rename.patch
|
||||
|
||||
|
||||
def test_rust_only_changes_still_require_review(repository: Repository) -> None:
|
||||
_write(repository.root, "litellm-rust/messages.rs", 'pub const RESULT: &str = "changed independently";\n')
|
||||
_commit(repository.root)
|
||||
manifest: Final = _manifest(repository.root, repository.baseline)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.status == "changes_found"
|
||||
assert tuple((change.path, change.sides) for change in result.changes) == (("litellm-rust/messages.rs", ("rust",)),)
|
||||
|
||||
|
||||
def test_preserves_spaces_and_newlines_in_changed_filenames(repository: Repository) -> None:
|
||||
filename: Final = "litellm/messages/handler with space\nand newline.py"
|
||||
_write(repository.root, filename, "ENABLED = True\n")
|
||||
_commit(repository.root)
|
||||
manifest: Final = _manifest(repository.root, repository.baseline)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert tuple((change.path, change.mapping_ids) for change in result.changes) == ((filename, ("messages",)),)
|
||||
assert "+ENABLED = True" in result.changes[0].patch
|
||||
|
||||
|
||||
def test_exact_file_selectors_do_not_capture_files_with_the_same_prefix(repository: Repository) -> None:
|
||||
_write(repository.root, "litellm/common.py.backup", "unrelated copy\n")
|
||||
_commit(repository.root)
|
||||
manifest: Final = _manifest(repository.root, repository.baseline, candidates=("litellm/common.py",))
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.status == "no_changes"
|
||||
assert result.changes == ()
|
||||
|
||||
|
||||
def test_null_checkpoint_cannot_claim_no_changes(repository: Repository) -> None:
|
||||
manifest: Final = _manifest(repository.root, None)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.status == "unreviewed"
|
||||
assert result.checkpoint_sha is None
|
||||
assert result.checkpoint_state == "unknown"
|
||||
assert result.comparison_sha is None
|
||||
assert result.changes == ()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("state", ("unknown", "drifted", "aligned"))
|
||||
def test_no_changes_preserves_recorded_checkpoint_state(repository: Repository, state: str) -> None:
|
||||
manifest: Final = _manifest(repository.root, repository.baseline, checkpoint_state=state)
|
||||
original_manifest: Final = manifest.read_bytes()
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.status == "no_changes"
|
||||
assert result.checkpoint_state == state
|
||||
assert manifest.read_bytes() == original_manifest
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("state", "with_checkpoint"), (("invalid", True), ("aligned", False), ("drifted", False)))
|
||||
def test_rejects_invalid_checkpoint_state_without_rewriting_manifest(
|
||||
repository: Repository, state: str, with_checkpoint: bool
|
||||
) -> None:
|
||||
manifest: Final = _manifest(
|
||||
repository.root, repository.baseline if with_checkpoint else None, checkpoint_state=state
|
||||
)
|
||||
original_manifest: Final = manifest.read_bytes()
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftError), result
|
||||
assert result.message
|
||||
assert manifest.read_bytes() == original_manifest
|
||||
|
||||
|
||||
def test_manifest_digest_preserves_crlf_bytes(repository: Repository) -> None:
|
||||
manifest: Final = _manifest(repository.root, repository.baseline)
|
||||
raw: Final = manifest.read_bytes().replace(b", ", b",\r\n")
|
||||
manifest.write_bytes(raw)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.manifest_sha256 == hashlib.sha256(raw).hexdigest()
|
||||
assert manifest.read_bytes() == raw
|
||||
|
||||
|
||||
def test_explicit_target_ignores_later_committed_work(repository: Repository) -> None:
|
||||
_write(repository.root, "litellm/common.py", "TIMEOUT = 30\n")
|
||||
_commit(repository.root)
|
||||
manifest: Final = _manifest(repository.root, repository.baseline)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest, target_ref=repository.baseline)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.status == "no_changes"
|
||||
assert result.target_sha == repository.baseline
|
||||
assert result.changes == ()
|
||||
|
||||
|
||||
def test_since_override_does_not_create_or_advance_checkpoint(repository: Repository) -> None:
|
||||
_write(repository.root, "litellm/common.py", "TIMEOUT = 30\n")
|
||||
_commit(repository.root)
|
||||
manifest: Final = _manifest(repository.root, None)
|
||||
original_manifest: Final = manifest.read_bytes()
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest, since_ref=repository.baseline[:12])
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.status == "unreviewed"
|
||||
assert result.checkpoint_sha is None
|
||||
assert result.comparison_sha == repository.baseline
|
||||
assert tuple(change.path for change in result.changes) == ("litellm/common.py",)
|
||||
assert manifest.read_bytes() == original_manifest
|
||||
|
||||
|
||||
@pytest.mark.parametrize("checkpoint", ("f" * 40, "HEAD", "1234567"))
|
||||
@pytest.mark.parametrize("use_override", (False, True))
|
||||
def test_missing_or_nonfull_checkpoint_is_an_error(repository: Repository, checkpoint: str, use_override: bool) -> None:
|
||||
manifest: Final = _manifest(repository.root, checkpoint)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest, since_ref=repository.baseline if use_override else None)
|
||||
|
||||
assert isinstance(result, DriftError), result
|
||||
assert result.message
|
||||
|
||||
|
||||
@pytest.mark.parametrize("use_override", (False, True))
|
||||
def test_unrelated_checkpoint_is_an_error(repository: Repository, use_override: bool) -> None:
|
||||
_git(repository.root, "checkout", "--quiet", "--orphan", "unrelated")
|
||||
_git(repository.root, "rm", "--quiet", "-rf", ".")
|
||||
_write(repository.root, "other.py", "unrelated = True\n")
|
||||
unrelated: Final = _commit(repository.root)
|
||||
_git(repository.root, "checkout", "--quiet", "main")
|
||||
manifest: Final = _manifest(repository.root, unrelated)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest, since_ref=repository.baseline if use_override else None)
|
||||
|
||||
assert isinstance(result, DriftError), result
|
||||
assert result.message
|
||||
|
||||
|
||||
def test_removed_policy_paths_stay_visible_from_checkpoint_and_target(repository: Repository) -> None:
|
||||
_manifest(
|
||||
repository.root,
|
||||
None,
|
||||
candidates=("litellm/",),
|
||||
mappings=(("previous", ("litellm/messages/",), ("legacy-native/",)),),
|
||||
)
|
||||
checkpoint: Final = _commit(repository.root)
|
||||
_write(repository.root, "litellm/messages/handler.py", "def result():\n return 'after'\n")
|
||||
_write(repository.root, "litellm-rust/messages.rs", 'pub const RESULT: &str = "after";\n')
|
||||
_manifest(
|
||||
repository.root,
|
||||
checkpoint,
|
||||
candidates=("litellm-rust/",),
|
||||
mappings=(("target", ("litellm/responses/",), ("litellm-rust/messages.rs",)),),
|
||||
)
|
||||
_commit(repository.root)
|
||||
manifest: Final = _manifest(
|
||||
repository.root,
|
||||
checkpoint,
|
||||
candidates=("unrelated/",),
|
||||
mappings=(("local", ("unrelated/",), ("unrelated-native/",)),),
|
||||
)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.status == "changes_found"
|
||||
assert ("litellm/messages/handler.py", ("previous",)) in tuple(
|
||||
(change.path, change.mapping_ids) for change in result.changes
|
||||
)
|
||||
assert ("litellm-rust/messages.rs", ("target",)) in tuple(
|
||||
(change.path, change.mapping_ids) for change in result.changes
|
||||
)
|
||||
assert "drift.json" in tuple(change.path for change in result.changes)
|
||||
|
||||
|
||||
def test_report_exposes_merged_selectors_for_the_same_mapping_id(repository: Repository) -> None:
|
||||
_manifest(
|
||||
repository.root,
|
||||
None,
|
||||
mappings=(("messages", ("litellm/messages/",), ("old-native/",)),),
|
||||
)
|
||||
checkpoint: Final = _commit(repository.root)
|
||||
_write(repository.root, "litellm/messages/handler.py", "def result():\n return 'after'\n")
|
||||
_manifest(
|
||||
repository.root,
|
||||
checkpoint,
|
||||
mappings=(("messages", ("litellm/common.py",), ("litellm-rust/messages.rs",)),),
|
||||
)
|
||||
_commit(repository.root)
|
||||
manifest: Final = _manifest(
|
||||
repository.root,
|
||||
checkpoint,
|
||||
mappings=(("messages", ("litellm/new-messages/",), ("new-native/",)),),
|
||||
)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert tuple((mapping.id, mapping.python_paths, mapping.rust_paths) for mapping in result.mappings) == (
|
||||
(
|
||||
"messages",
|
||||
("litellm/common.py", "litellm/messages/", "litellm/new-messages/"),
|
||||
("litellm-rust/messages.rs", "new-native/", "old-native/"),
|
||||
),
|
||||
)
|
||||
assert ("litellm/messages/handler.py", ("messages",)) in tuple(
|
||||
(change.path, change.mapping_ids) for change in result.changes
|
||||
)
|
||||
|
||||
|
||||
def test_since_override_keeps_policy_from_the_stored_checkpoint(repository: Repository) -> None:
|
||||
_manifest(
|
||||
repository.root,
|
||||
None,
|
||||
candidates=("litellm/",),
|
||||
mappings=(("previous", ("litellm/messages/",), ("old-native/",)),),
|
||||
)
|
||||
checkpoint: Final = _commit(repository.root)
|
||||
manifest: Final = _manifest(
|
||||
repository.root,
|
||||
checkpoint,
|
||||
candidates=("unrelated/",),
|
||||
mappings=(("current", ("unrelated/",), ("new-native/",)),),
|
||||
)
|
||||
comparison: Final = _commit(repository.root)
|
||||
_write(repository.root, "litellm/messages/handler.py", "def result():\n return 'after'\n")
|
||||
_commit(repository.root)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest, since_ref=comparison)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.checkpoint_sha == checkpoint
|
||||
assert result.comparison_sha == comparison
|
||||
assert tuple((change.path, change.mapping_ids) for change in result.changes) == (
|
||||
("litellm/messages/handler.py", ("previous",)),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rename_state", ("none", "committed", "staged"))
|
||||
@pytest.mark.parametrize("use_override", (False, True))
|
||||
def test_renamed_manifest_keeps_historical_watch_paths(
|
||||
repository: Repository, rename_state: str, use_override: bool
|
||||
) -> None:
|
||||
_write(repository.root, "legacy-owned/handler.py", "ENABLED = False\n")
|
||||
_manifest(
|
||||
repository.root,
|
||||
None,
|
||||
candidates=("legacy-owned/",),
|
||||
mappings=(("messages", ("legacy-owned/",), ("native/",)),),
|
||||
pretty=True,
|
||||
)
|
||||
checkpoint: Final = _commit(repository.root)
|
||||
_manifest(
|
||||
repository.root,
|
||||
checkpoint,
|
||||
candidates=("unrelated/",),
|
||||
mappings=(("messages", ("unrelated/",), ("native/",)),),
|
||||
pretty=True,
|
||||
)
|
||||
(repository.root / "policy").mkdir()
|
||||
_git(repository.root, "mv", "drift.json", "policy/drift.json")
|
||||
comparison: Final = _commit(repository.root)
|
||||
_write(repository.root, "legacy-owned/handler.py", "ENABLED = True\n")
|
||||
_commit(repository.root)
|
||||
if rename_state != "none":
|
||||
_git(repository.root, "mv", "policy/drift.json", "policy/current-drift.json")
|
||||
if rename_state == "committed":
|
||||
_commit(repository.root)
|
||||
if rename_state == "staged":
|
||||
_git(repository.root, "config", "status.renames", "false")
|
||||
manifest: Final = repository.root / ("policy/drift.json" if rename_state == "none" else "policy/current-drift.json")
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest, since_ref=comparison if use_override else None)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.checkpoint_sha == checkpoint
|
||||
assert result.comparison_sha == (comparison if use_override else checkpoint)
|
||||
assert ("legacy-owned/handler.py", ("messages",)) in tuple(
|
||||
(change.path, change.mapping_ids) for change in result.changes
|
||||
)
|
||||
mapping: Final = next(mapping for mapping in result.mappings if mapping.id == "messages")
|
||||
assert mapping.python_paths == ("legacy-owned/", "unrelated/")
|
||||
if rename_state == "staged":
|
||||
assert ("policy/drift.json", "policy/current-drift.json") in tuple(
|
||||
(change.old_path, change.path) for change in result.working_tree_changes
|
||||
)
|
||||
|
||||
|
||||
def test_manifest_rename_history_survives_later_large_policy_rewrites(repository: Repository) -> None:
|
||||
_write(repository.root, "legacy-owned/topic0/handler.py", "ENABLED = False\n")
|
||||
_manifest(
|
||||
repository.root,
|
||||
None,
|
||||
candidates=("legacy-owned/",),
|
||||
mappings=tuple(
|
||||
(f"topic{index}", (f"legacy-owned/topic{index}/",), (f"native/topic{index}/",)) for index in range(10)
|
||||
),
|
||||
pretty=True,
|
||||
)
|
||||
checkpoint: Final = _commit(repository.root)
|
||||
_git(repository.root, "mv", "drift.json", "intermediate-drift.json")
|
||||
comparison: Final = _commit(repository.root)
|
||||
replacement: Final = _manifest(
|
||||
repository.root,
|
||||
checkpoint,
|
||||
candidates=("unrelated/",),
|
||||
mappings=(("current", ("unrelated/",), ("native/",)),),
|
||||
pretty=True,
|
||||
)
|
||||
(repository.root / "intermediate-drift.json").write_bytes(replacement.read_bytes())
|
||||
replacement.unlink()
|
||||
_write(repository.root, "legacy-owned/topic0/handler.py", "ENABLED = True\n")
|
||||
_commit(repository.root)
|
||||
_git(repository.root, "mv", "intermediate-drift.json", "current-drift.json")
|
||||
_commit(repository.root)
|
||||
|
||||
result: Final = inspect_drift(repository.root, repository.root / "current-drift.json", since_ref=comparison)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert ("legacy-owned/topic0/handler.py", ("topic0",)) in tuple(
|
||||
(change.path, change.mapping_ids) for change in result.changes
|
||||
)
|
||||
assert tuple(mapping.id for mapping in result.mappings) == ("current", *(f"topic{index}" for index in range(10)))
|
||||
|
||||
|
||||
def test_future_target_recovers_renamed_policy_from_an_older_checkout(repository: Repository) -> None:
|
||||
_manifest(
|
||||
repository.root,
|
||||
None,
|
||||
candidates=("legacy-owned/",),
|
||||
mappings=(("legacy", ("legacy-owned/",), ("native/",)),),
|
||||
pretty=True,
|
||||
)
|
||||
checkpoint: Final = _commit(repository.root)
|
||||
_manifest(
|
||||
repository.root,
|
||||
checkpoint,
|
||||
candidates=("future-owned/",),
|
||||
mappings=(("future", ("future-owned/",), ("native/",)),),
|
||||
pretty=True,
|
||||
)
|
||||
_git(repository.root, "mv", "drift.json", "future-drift.json")
|
||||
_write(repository.root, "future-owned/handler.py", "ENABLED = True\n")
|
||||
target: Final = _commit(repository.root)
|
||||
_git(repository.root, "checkout", "--quiet", "--detach", checkpoint)
|
||||
manifest: Final = _manifest(
|
||||
repository.root,
|
||||
checkpoint,
|
||||
candidates=("legacy-owned/",),
|
||||
mappings=(("legacy", ("legacy-owned/",), ("native/",)),),
|
||||
pretty=True,
|
||||
)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest, target_ref=target)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.target_sha == target
|
||||
assert ("future-owned/handler.py", ("future",)) in tuple(
|
||||
(change.path, change.mapping_ids) for change in result.changes
|
||||
)
|
||||
|
||||
|
||||
def test_reports_relevant_staged_unstaged_and_untracked_changes_without_diffing_them(repository: Repository) -> None:
|
||||
_git(repository.root, "mv", "litellm/messages/handler.py", "renamed_handler.py")
|
||||
_write(repository.root, "litellm/common.py", "TIMEOUT = 99\n")
|
||||
_write(repository.root, "litellm/new_endpoint.py", "ENABLED = True\n")
|
||||
_write(repository.root, "unrelated.txt", "outside the candidate paths\n")
|
||||
manifest: Final = _manifest(repository.root, repository.baseline)
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftReport), result
|
||||
assert result.target_sha == repository.baseline
|
||||
assert result.changes == ()
|
||||
assert frozenset((change.status, change.old_path, change.path) for change in result.working_tree_changes) == {
|
||||
("R ", "litellm/messages/handler.py", "renamed_handler.py"),
|
||||
(" M", None, "litellm/common.py"),
|
||||
("??", None, "litellm/new_endpoint.py"),
|
||||
("??", None, "drift.json"),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("selector", ("../litellm/", "/litellm/", "litellm/**", "litellm/../outside/"))
|
||||
def test_rejects_ambiguous_or_outside_repository_selectors(repository: Repository, selector: str) -> None:
|
||||
manifest: Final = _manifest(repository.root, repository.baseline, candidates=(selector,))
|
||||
|
||||
result: Final = inspect_drift(repository.root, manifest)
|
||||
|
||||
assert isinstance(result, DriftError), result
|
||||
assert result.message
|
||||
Loading…
Add table
Reference in a new issue