ci: report a shadow-mode PR risk tier from changed paths, diff size, and test delta

.github/risk-tiers.yml

@@ -0,0 +1,62 @@
+paths:
+  high:
+    - "litellm/proxy/auth/**"
+    - "litellm/proxy/management_endpoints/**"
+    - "litellm/proxy/spend_tracking/**"
+    - "litellm/proxy/db/**"
+    - "**/migrations/**"
+    - "**/schema.prisma"
+    - "litellm/caching/**"
+    - "litellm/router.py"
+    - "litellm/router_utils/**"
+    - "litellm/proxy/proxy_server.py"
+    - "litellm/__init__.py"
+    - "litellm/utils.py"
+    - "litellm/proxy/guardrails/**"
+    - "litellm/proxy/hooks/**"
+    - "litellm/proxy/custom_hooks/**"
+    - "enterprise/enterprise_hooks/**"
+    - "enterprise/litellm_enterprise/proxy/hooks/**"
+    - "litellm/secret_managers/**"
+    - "litellm/proxy/pass_through_endpoints/**"
+    - "litellm/passthrough/**"
+    - ".github/**"
+    - ".circleci/**"
+    - "pyproject.toml"
+    - "uv.lock"
+    - "**/package*.json"
+    - "**/Dockerfile*"
+    - "scripts/*gate*"
+  low:
+    - "cookbook/**"
+    - "model_prices_and_context_window.json"
+    - "litellm/model_prices_and_context_window_backup.json"
+    - "**/*.md"
+    - "**/*.mdx"
+
+modules:
+  medium_from: 2
+  high_from: 4
+
+size:
+  low:
+    lines_under: 100
+    files_up_to: 3
+  medium:
+    lines_under: 400
+    files_up_to: 10
+  ignore:
+    - "uv.lock"
+    - "**/package-lock.json"
+    - "ui/litellm-dashboard/src/lib/http/schema.d.ts"
+
+tests:
+  files:
+    - "tests/**"
+    - "**/*.test.*"
+    - "**/*.spec.*"
+    - "**/__tests__/**"
+
+authors:
+  low:
+    - "devin-ai-integration[bot]"

.github/scripts/risk_tier.py

@@ -0,0 +1,342 @@
+from __future__ import annotations
+
+import argparse
+import json
+import re
+import subprocess
+import sys
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Final, Literal
+
+import yaml
+from pydantic import BaseModel, ConfigDict, Field
+
+Tier = Literal["low", "medium", "high"]
+
+TEST_DEF_RE: Final = re.compile(r"^\s*(?:async\s+)?def\s+test_|^\s*(?:it|test)\(")
+SKIP_RE: Final = re.compile(r"pytest\.(?:mark\.)?(?:skip|xfail)|unittest\.skip|\b(?:it|test|describe)\.skip\(|\bxit\(")
+ASSERT_RE: Final = re.compile(r"^\s*assert\b|\bexpect\(")
+GLOB_TOKEN_RE: Final = re.compile(r"(\*\*/|\*\*|\*|\?)")
+DIFF_BLOCK_SEPARATOR: Final = "\ndiff --git a/"
+MAX_LISTED_PATHS: Final = 5
+
+
+class SizeLimit(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    lines_under: int = Field(gt=0)
+    files_up_to: int = Field(gt=0)
+
+
+class SizeRules(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    low: SizeLimit
+    medium: SizeLimit
+    ignore: tuple[str, ...] = ()
+
+
+class ModuleRules(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    medium_from: int = Field(gt=1)
+    high_from: int = Field(gt=1)
+
+
+class PathRules(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    high: tuple[str, ...]
+    low: tuple[str, ...]
+
+
+class TestRules(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    files: tuple[str, ...]
+
+
+class AuthorRules(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    low: tuple[str, ...]
+
+
+class RiskConfig(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    paths: PathRules
+    modules: ModuleRules
+    size: SizeRules
+    tests: TestRules
+    authors: AuthorRules
+
+
+@dataclass(frozen=True, slots=True)
+class PathMatcher:
+    patterns: tuple[re.Pattern[str], ...]
+
+    @staticmethod
+    def from_globs(globs: Sequence[str]) -> PathMatcher:
+        return PathMatcher(tuple(glob_to_regex(glob) for glob in globs))
+
+    def matches(self, path: str) -> bool:
+        return any(pattern.fullmatch(path) for pattern in self.patterns)
+
+
+@dataclass(frozen=True, slots=True)
+class Rules:
+    config: RiskConfig
+    high_paths: PathMatcher
+    low_paths: PathMatcher
+    test_files: PathMatcher
+    size_ignored: PathMatcher
+
+    @staticmethod
+    def from_config(config: RiskConfig) -> Rules:
+        return Rules(
+            config=config,
+            high_paths=PathMatcher.from_globs(config.paths.high),
+            low_paths=PathMatcher.from_globs(config.paths.low),
+            test_files=PathMatcher.from_globs(config.tests.files),
+            size_ignored=PathMatcher.from_globs(config.size.ignore),
+        )
+
+    def path_tier(self, path: str) -> Tier:
+        if self.high_paths.matches(path):
+            return "high"
+        if self.test_files.matches(path) or self.low_paths.matches(path):
+            return "low"
+        return "medium"
+
+    def is_production(self, path: str) -> bool:
+        return self.path_tier(path) != "low"
+
+
+@dataclass(frozen=True, slots=True)
+class FileChange:
+    path: str
+    added_lines: tuple[str, ...]
+    deleted_lines: tuple[str, ...]
+
+    @property
+    def line_count(self) -> int:
+        return len(self.added_lines) + len(self.deleted_lines)
+
+
+@dataclass(frozen=True, slots=True)
+class Factor:
+    name: str
+    tier: Tier
+    reason: str
+
+
+@dataclass(frozen=True, slots=True)
+class Verdict:
+    tier: Tier
+    factors: tuple[Factor, ...]
+
+    def summary_markdown(self) -> str:
+        rows = "\n".join(f"| {factor.name} | {factor.tier} | {factor.reason} |" for factor in self.factors)
+        return f"risk: {self.tier} (shadow mode, nothing is blocked)\n\n| factor | tier | why |\n| --- | --- | --- |\n{rows}\n"
+
+    def to_json(self) -> str:
+        payload = {
+            "tier": self.tier,
+            "factors": [{"name": f.name, "tier": f.tier, "reason": f.reason} for f in self.factors],
+            "summary": self.summary_markdown(),
+        }
+        return json.dumps(payload, indent=2)
+
+
+def glob_to_regex(pattern: str) -> re.Pattern[str]:
+    return re.compile("".join(_translate_glob_token(token) for token in GLOB_TOKEN_RE.split(pattern)))
+
+
+def _translate_glob_token(token: str) -> str:
+    match token:
+        case "**/":
+            return "(?:.*/)?"
+        case "**":
+            return ".*"
+        case "*":
+            return "[^/]*"
+        case "?":
+            return "[^/]"
+        case _:
+            return re.escape(token)
+
+
+def load_config(path: Path) -> RiskConfig:
+    return RiskConfig.model_validate(yaml.safe_load(path.read_text(encoding="utf-8")))
+
+
+def parse_diff(diff_text: str) -> tuple[FileChange, ...]:
+    blocks = ("\n" + diff_text).split(DIFF_BLOCK_SEPARATOR)[1:]
+    return tuple(_parse_block(block) for block in blocks)
+
+
+def _parse_block(block: str) -> FileChange:
+    header, _, body = block.partition("\n")
+    path = header[: (len(header) - 3) // 2]
+    lines = body.split("\n")
+    first_hunk = next((index for index, line in enumerate(lines) if line.startswith("@@")), len(lines))
+    hunk_lines = lines[first_hunk:]
+    return FileChange(
+        path=path,
+        added_lines=tuple(line[1:] for line in hunk_lines if line.startswith("+")),
+        deleted_lines=tuple(line[1:] for line in hunk_lines if line.startswith("-")),
+    )
+
+
+def module_key(path: str) -> str:
+    parts = path.split("/")
+    if parts[0] != "litellm":
+        return parts[0]
+    depth = 3 if len(parts) > 3 else 2
+    return "/".join(parts[:depth])
+
+
+def highest(tiers: Sequence[Tier]) -> Tier:
+    if "high" in tiers:
+        return "high"
+    if "medium" in tiers:
+        return "medium"
+    return "low"
+
+
+def _listed(paths: Sequence[str]) -> str:
+    shown = ", ".join(f"`{path}`" for path in paths[:MAX_LISTED_PATHS])
+    rest = len(paths) - MAX_LISTED_PATHS
+    return f"{shown} and {rest} more" if rest > 0 else shown
+
+
+def paths_factor(changes: Sequence[FileChange], rules: Rules) -> Factor:
+    tier = highest([rules.path_tier(change.path) for change in changes])
+    matching = [change.path for change in changes if rules.path_tier(change.path) == tier]
+    match tier:
+        case "high":
+            return Factor("paths", "high", f"always-human: {_listed(matching)}")
+        case "medium":
+            return Factor("paths", "medium", f"{len(matching)} file(s) outside the docs, tests, and model map tiers")
+        case "low":
+            return Factor("paths", "low", "docs, tests, cookbook, or model map only")
+
+
+def modules_factor(changes: Sequence[FileChange], rules: Rules) -> Factor:
+    modules = sorted({module_key(change.path) for change in changes if rules.is_production(change.path)})
+    count = len(modules)
+    tier: Tier = (
+        "high"
+        if count >= rules.config.modules.high_from
+        else "medium"
+        if count >= rules.config.modules.medium_from
+        else "low"
+    )
+    return Factor("modules", tier, f"{count} module(s): {_listed(modules)}" if modules else "no production module")
+
+
+def size_factor(changes: Sequence[FileChange], rules: Rules) -> Factor:
+    counted = [change for change in changes if not rules.size_ignored.matches(change.path)]
+    lines = sum(change.line_count for change in counted)
+    files = len(counted)
+    limits = rules.config.size
+    tier: Tier = (
+        "low"
+        if lines < limits.low.lines_under and files <= limits.low.files_up_to
+        else "medium"
+        if lines < limits.medium.lines_under and files <= limits.medium.files_up_to
+        else "high"
+    )
+    return Factor("size", tier, f"{lines} line(s) across {files} file(s)")
+
+
+def _net(changes: Sequence[FileChange], pattern: re.Pattern[str]) -> int:
+    added = sum(1 for change in changes for line in change.added_lines if pattern.search(line))
+    deleted = sum(1 for change in changes for line in change.deleted_lines if pattern.search(line))
+    return added - deleted
+
+
+def tests_factor(changes: Sequence[FileChange], rules: Rules) -> Factor:
+    test_changes = [change for change in changes if rules.test_files.matches(change.path)]
+    net_tests = _net(test_changes, TEST_DEF_RE)
+    net_skips = _net(test_changes, SKIP_RE)
+    net_asserts = _net(test_changes, ASSERT_RE)
+    production_changed = any(rules.is_production(change.path) for change in changes)
+    if net_tests < 0:
+        return Factor("tests", "high", f"{-net_tests} test(s) removed")
+    if net_skips > 0:
+        return Factor("tests", "high", f"{net_skips} skip marker(s) added")
+    if net_asserts < 0:
+        return Factor("tests", "high", f"{-net_asserts} assertion(s) removed")
+    if not production_changed:
+        return Factor("tests", "low", "no production code changed")
+    if net_tests > 0:
+        return Factor("tests", "low", f"{net_tests} test(s) added")
+    if test_changes:
+        return Factor("tests", "medium", "tests edited, none added")
+    return Factor("tests", "medium", "production code changed with no test touched")
+
+
+def author_factor(author: str, from_fork: bool, rules: Rules) -> Factor:
+    if from_fork:
+        return Factor("author", "high", f"`{author}` from a fork")
+    if author in rules.config.authors.low:
+        return Factor("author", "low", f"`{author}` on an internal branch")
+    return Factor("author", "medium", f"`{author}` on an internal branch, human-opened")
+
+
+def classify(changes: Sequence[FileChange], author: str, from_fork: bool, rules: Rules) -> Verdict:
+    factors = (
+        paths_factor(changes, rules),
+        modules_factor(changes, rules),
+        size_factor(changes, rules),
+        tests_factor(changes, rules),
+        author_factor(author, from_fork, rules),
+    )
+    return Verdict(highest([factor.tier for factor in factors]), factors)
+
+
+def git_diff(repo: Path, base: str, head: str) -> str:
+    completed = subprocess.run(
+        ["git", "-c", "core.quotePath=false", "diff", "--no-renames", "--no-ext-diff", "-U0", base, head],
+        cwd=repo,
+        capture_output=True,
+        encoding="utf-8",
+        errors="replace",
+        check=True,
+    )
+    return completed.stdout
+
+
+def build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description="Compute a pull request's floor risk tier from its diff")
+    parser.add_argument("--config", type=Path, required=True)
+    parser.add_argument("--base", required=True)
+    parser.add_argument("--head", required=True)
+    parser.add_argument("--author", required=True)
+    parser.add_argument("--from-fork", action="store_true")
+    parser.add_argument("--repo", type=Path, default=Path.cwd())
+    parser.add_argument("--json-out", type=Path)
+    return parser
+
+
+class CliArgs(BaseModel):
+    model_config = ConfigDict(extra="forbid", frozen=True)
+    config: Path
+    base: str
+    head: str
+    author: str
+    from_fork: bool
+    repo: Path
+    json_out: Path | None
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    args = CliArgs.model_validate(vars(build_parser().parse_args(argv)))
+    rules = Rules.from_config(load_config(args.config))
+    changes = parse_diff(git_diff(args.repo, args.base, args.head))
+    verdict = classify(changes, args.author, args.from_fork, rules)
+    if args.json_out is not None:
+        args.json_out.write_text(verdict.to_json(), encoding="utf-8")
+    sys.stdout.write(verdict.summary_markdown())
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/risk-gate.yml

@@ -0,0 +1,83 @@
+name: risk-gate
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+    branches:
+      - litellm_internal_staging
+
+permissions: {}
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  floor:
+    name: risk-gate floor (shadow)
+    if: github.event.pull_request.head.repo.full_name == github.repository
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      contents: read
+      pull-requests: write
+      checks: write
+    steps:
+      - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+        with:
+          python-version: "3.12"
+
+      - name: Compute the floor tier with the base branch's copy of the gate
+        shell: bash
+        env:
+          PR_AUTHOR: ${{ github.event.pull_request.user.login }}
+        run: |
+          python -m pip install "pyyaml==6.0.3" "pydantic==2.13.4"
+          for gate_file in scripts/risk_tier.py risk-tiers.yml; do
+            target="${RUNNER_TEMP}/$(basename "${gate_file}")"
+            git show "HEAD^1:.github/${gate_file}" > "${target}" 2>/dev/null || cp ".github/${gate_file}" "${target}"
+          done
+          python "${RUNNER_TEMP}/risk_tier.py" \
+            --config "${RUNNER_TEMP}/risk-tiers.yml" \
+            --base HEAD^1 \
+            --head HEAD \
+            --author "${PR_AUTHOR}" \
+            --json-out "${RUNNER_TEMP}/risk.json" \
+            | tee -a "${GITHUB_STEP_SUMMARY}"
+
+      - name: Publish the risk-gate check run and the risk label
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        env:
+          RISK_JSON: ${{ runner.temp }}/risk.json
+        with:
+          script: |
+            const fs = require('fs');
+            const risk = JSON.parse(fs.readFileSync(process.env.RISK_JSON, 'utf8'));
+            const pr = context.payload.pull_request;
+            const repo = { owner: context.repo.owner, repo: context.repo.repo };
+            await github.rest.checks.create({
+              ...repo,
+              name: 'risk-gate',
+              head_sha: pr.head.sha,
+              status: 'completed',
+              conclusion: 'neutral',
+              output: { title: `risk: ${risk.tier} (shadow)`, summary: risk.summary },
+            });
+            const wanted = `risk:${risk.tier}`;
+            const { data: labels } = await github.rest.issues.listLabelsOnIssue({
+              ...repo,
+              issue_number: pr.number,
+              per_page: 100,
+            });
+            const current = labels.map((label) => label.name).filter((name) => name.startsWith('risk:'));
+            for (const name of current.filter((name) => name !== wanted)) {
+              await github.rest.issues.removeLabel({ ...repo, issue_number: pr.number, name });
+            }
+            if (!current.includes(wanted)) {
+              await github.rest.issues.addLabels({ ...repo, issue_number: pr.number, labels: [wanted] });
+            }

tests/test_litellm/test_github_risk_tier.py

@@ -0,0 +1,351 @@
+from __future__ import annotations
+
+import importlib.util
+import json
+import subprocess
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+import pytest
+from pydantic import ValidationError
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+SCRIPT_PATH = REPO_ROOT / ".github" / "scripts" / "risk_tier.py"
+CONFIG_PATH = REPO_ROOT / ".github" / "risk-tiers.yml"
+DEVIN = "devin-ai-integration[bot]"
+
+
+@pytest.fixture(scope="module")
+def risk_tier():
+    spec = importlib.util.spec_from_file_location("risk_tier", SCRIPT_PATH)
+    assert spec and spec.loader, f"Could not load spec for {SCRIPT_PATH}"
+    module = importlib.util.module_from_spec(spec)
+    sys.modules["risk_tier"] = module
+    spec.loader.exec_module(module)
+    return module
+
+
+@pytest.fixture(scope="module")
+def rules(risk_tier):
+    return risk_tier.Rules.from_config(risk_tier.load_config(CONFIG_PATH))
+
+
+def _file_diff(path: str, added: Sequence[str] = (), deleted: Sequence[str] = ()) -> str:
+    header = (
+        f"diff --git a/{path} b/{path}\n"
+        f"index 0000000..1111111 100644\n"
+        f"--- a/{path}\n"
+        f"+++ b/{path}\n"
+        f"@@ -1,{len(deleted)} +1,{len(added)} @@\n"
+    )
+    body = "".join(f"-{line}\n" for line in deleted) + "".join(f"+{line}\n" for line in added)
+    return header + body
+
+
+def _lines(count: int, prefix: str = "x = ") -> tuple[str, ...]:
+    return tuple(f"{prefix}{index}" for index in range(count))
+
+
+NEW_TEST = ("def test_regression():", "    assert True")
+
+
+def _factor(verdict, name: str):
+    return next(factor for factor in verdict.factors if factor.name == name)
+
+
+def _verdict(risk_tier, rules, diff: str, author: str = DEVIN, from_fork: bool = False):
+    return risk_tier.classify(risk_tier.parse_diff(diff), author, from_fork, rules)
+
+
+def test_always_human_path_is_high_even_when_tiny_and_tested(risk_tier, rules):
+    diff = _file_diff("litellm/proxy/auth/user_api_key_auth.py", added=("x = 1",)) + _file_diff(
+        "tests/test_litellm/proxy/auth/test_user_api_key_auth.py", added=NEW_TEST
+    )
+    verdict = _verdict(risk_tier, rules, diff)
+    assert verdict.tier == "high"
+    assert _factor(verdict, "paths").tier == "high"
+    assert "litellm/proxy/auth/user_api_key_auth.py" in _factor(verdict, "paths").reason
+    assert _factor(verdict, "size").tier == "low"
+    assert _factor(verdict, "tests").tier == "low"
+
+
+def test_docs_and_tests_only_change_is_low_on_every_factor(risk_tier, rules):
+    diff = _file_diff("README.md", added=("hello",)) + _file_diff("tests/test_litellm/test_docs.py", added=NEW_TEST)
+    verdict = _verdict(risk_tier, rules, diff)
+    assert verdict.tier == "low"
+    assert {factor.tier for factor in verdict.factors} == {"low"}
+
+
+def test_provider_change_with_regression_test_is_medium_by_path_only(risk_tier, rules):
+    diff = _file_diff("litellm/llms/anthropic/chat/transformation.py", added=_lines(20)) + _file_diff(
+        "tests/test_litellm/llms/anthropic/test_chat_transformation.py", added=NEW_TEST
+    )
+    verdict = _verdict(risk_tier, rules, diff)
+    assert verdict.tier == "medium"
+    assert {factor.name: factor.tier for factor in verdict.factors} == {
+        "paths": "medium",
+        "modules": "low",
+        "size": "low",
+        "tests": "low",
+        "author": "low",
+    }
+
+
+@pytest.mark.parametrize(
+    ("paths", "expected"),
+    [
+        (("litellm/llms/anthropic/chat/x.py",), "low"),
+        (("litellm/llms/anthropic/chat/x.py", "litellm/types/llms/anthropic.py"), "medium"),
+        (("litellm/llms/anthropic/chat/x.py", "litellm/llms/openai/chat/x.py", "litellm/types/llms/a.py"), "medium"),
+        (
+            (
+                "litellm/llms/anthropic/chat/x.py",
+                "litellm/llms/openai/chat/x.py",
+                "litellm/types/llms/a.py",
+                "ui/litellm-dashboard/src/a.tsx",
+            ),
+            "high",
+        ),
+    ],
+)
+def test_modules_factor_counts_distinct_production_modules(risk_tier, rules, paths, expected):
+    diff = "".join(_file_diff(path, added=("x = 1",)) for path in paths)
+    diff_with_tests = diff + _file_diff("tests/test_litellm/test_a.py", added=NEW_TEST)
+    assert _factor(_verdict(risk_tier, rules, diff_with_tests), "modules").tier == expected
+
+
+def test_same_module_touched_twice_counts_once(risk_tier, rules):
+    diff = _file_diff("litellm/llms/anthropic/chat/a.py", added=("x = 1",)) + _file_diff(
+        "litellm/llms/anthropic/common_utils.py", added=("x = 1",)
+    )
+    factor = _factor(_verdict(risk_tier, rules, diff), "modules")
+    assert factor.tier == "low"
+    assert factor.reason == "1 module(s): `litellm/llms/anthropic`"
+
+
+@pytest.mark.parametrize(
+    ("line_counts", "expected"),
+    [
+        ((33, 33, 33), "low"),
+        ((34, 33, 33), "medium"),
+        ((1, 1, 1, 1), "medium"),
+        ((100, 100, 100, 99), "medium"),
+        ((100, 100, 100, 100), "high"),
+        ((1,) * 11, "high"),
+    ],
+)
+def test_size_factor_thresholds(risk_tier, rules, line_counts, expected):
+    diff = "".join(
+        _file_diff(f"litellm/llms/provider{index}/chat/x.py", added=_lines(count))
+        for index, count in enumerate(line_counts)
+    )
+    assert _factor(_verdict(risk_tier, rules, diff), "size").tier == expected
+
+
+def test_generated_files_do_not_count_toward_size(risk_tier, rules):
+    diff = _file_diff("ui/litellm-dashboard/src/lib/http/schema.d.ts", added=_lines(5000)) + _file_diff(
+        "litellm/llms/anthropic/chat/x.py", added=("x = 1",)
+    )
+    factor = _factor(_verdict(risk_tier, rules, diff), "size")
+    assert factor.tier == "low"
+    assert factor.reason == "1 line(s) across 1 file(s)"
+
+
+def test_removed_test_function_is_high(risk_tier, rules):
+    diff = _file_diff("tests/test_litellm/test_a.py", deleted=("def test_gone():", "    assert 1 == 1"))
+    verdict = _verdict(risk_tier, rules, diff)
+    assert verdict.tier == "high"
+    assert _factor(verdict, "tests").reason == "1 test(s) removed"
+
+
+def test_added_skip_marker_is_high(risk_tier, rules):
+    diff = _file_diff("tests/test_litellm/test_a.py", added=('@pytest.mark.skip(reason="flaky")',))
+    assert _factor(_verdict(risk_tier, rules, diff), "tests").tier == "high"
+
+
+def test_weakened_assertions_are_high(risk_tier, rules):
+    diff = _file_diff(
+        "tests/test_litellm/test_a.py",
+        added=("    assert result",),
+        deleted=("    assert result.status == 200", "    assert result.body == expected"),
+    )
+    factor = _factor(_verdict(risk_tier, rules, diff), "tests")
+    assert factor.tier == "high"
+    assert factor.reason == "1 assertion(s) removed"
+
+
+def test_renamed_test_file_is_not_a_deletion(risk_tier, rules):
+    body = ("def test_kept():", "    assert kept()")
+    diff = (
+        _file_diff("tests/test_litellm/test_old.py", deleted=body)
+        + _file_diff("tests/test_litellm/test_new.py", added=body)
+        + _file_diff("litellm/llms/anthropic/chat/x.py", added=("x = 1",))
+    )
+    factor = _factor(_verdict(risk_tier, rules, diff), "tests")
+    assert factor.tier == "medium"
+    assert factor.reason == "tests edited, none added"
+
+
+def test_production_change_without_any_test_is_medium(risk_tier, rules):
+    diff = _file_diff("litellm/llms/anthropic/chat/x.py", added=("x = 1",))
+    factor = _factor(_verdict(risk_tier, rules, diff), "tests")
+    assert factor.tier == "medium"
+    assert factor.reason == "production code changed with no test touched"
+
+
+@pytest.mark.parametrize(
+    ("added", "expected"),
+    [
+        (('it("hides the notice", () => {', "  expect(screen.queryByText(notice)).toBeNull();", "});"), "low"),
+        (('it.skip("hides the notice", () => {', "});"), "high"),
+    ],
+)
+def test_typescript_tests_count_like_python_ones(risk_tier, rules, added, expected):
+    diff = _file_diff("ui/litellm-dashboard/src/app/login/LoginPage.test.tsx", added=added) + _file_diff(
+        "ui/litellm-dashboard/src/app/login/LoginPage.tsx", added=("const x = 1;",)
+    )
+    assert _factor(_verdict(risk_tier, rules, diff), "tests").tier == expected
+
+
+@pytest.mark.parametrize(
+    ("author", "from_fork", "expected"),
+    [
+        (DEVIN, False, "low"),
+        ("mateo-berri", False, "medium"),
+        (DEVIN, True, "high"),
+        ("jairandresdiazp", True, "high"),
+    ],
+)
+def test_author_factor(risk_tier, rules, author, from_fork, expected):
+    diff = _file_diff("README.md", added=("hello",))
+    verdict = _verdict(risk_tier, rules, diff, author=author, from_fork=from_fork)
+    assert _factor(verdict, "author").tier == expected
+    assert verdict.tier == expected
+
+
+@pytest.mark.parametrize(
+    ("path", "expected"),
+    [
+        ("litellm/proxy/auth/user_api_key_auth.py", "high"),
+        ("litellm/proxy/schema.prisma", "high"),
+        ("litellm-proxy-extras/litellm_proxy_extras/migrations/20260901_x/migration.sql", "high"),
+        ("docker/Dockerfile.database", "high"),
+        ("ui/litellm-dashboard/package.json", "high"),
+        ("scripts/type_check_gate.py", "high"),
+        (".github/risk-tiers.yml", "high"),
+        ("enterprise/litellm_enterprise/proxy/hooks/x.py", "high"),
+        ("litellm/proxy/pass_through_endpoints/x.py", "high"),
+        ("litellm/llms/bedrock/passthrough/x.py", "medium"),
+        ("ui/litellm-dashboard/src/components/networking.tsx", "medium"),
+        ("litellm/types/proxy/x.py", "medium"),
+        ("ui/litellm-dashboard/src/app/login/LoginPage.test.tsx", "low"),
+        ("tests/e2e/x.py", "low"),
+        ("cookbook/x.ipynb", "low"),
+        ("model_prices_and_context_window.json", "low"),
+        ("litellm/model_prices_and_context_window_backup.json", "low"),
+        ("litellm/proxy/README.md", "low"),
+    ],
+)
+def test_path_tier_from_the_checked_in_config(rules, path, expected):
+    assert rules.path_tier(path) == expected
+
+
+def test_single_star_does_not_cross_directories(risk_tier):
+    assert risk_tier.glob_to_regex("Dockerfile*").fullmatch("Dockerfile.database")
+    assert risk_tier.glob_to_regex("Dockerfile*").fullmatch("docker/Dockerfile.database") is None
+    assert risk_tier.glob_to_regex("scripts/*gate*").fullmatch("scripts/nested/type_check_gate.py") is None
+    assert risk_tier.glob_to_regex("**/migrations/**").fullmatch("migrations/x.sql")
+
+
+def test_parse_diff_handles_binary_deleted_and_dash_prefixed_lines(risk_tier):
+    diff = (
+        "diff --git a/img.png b/img.png\n"
+        "index 0000000..1111111 100644\n"
+        "Binary files a/img.png and b/img.png differ\n"
+        "diff --git a/gone.sql b/gone.sql\n"
+        "deleted file mode 100644\n"
+        "index 1111111..0000000\n"
+        "--- a/gone.sql\n"
+        "+++ /dev/null\n"
+        "@@ -1,2 +0,0 @@\n"
+        "--- a sql comment\n"
+        "-SELECT 1;\n"
+    )
+    changes = risk_tier.parse_diff(diff)
+    assert [change.path for change in changes] == ["img.png", "gone.sql"]
+    assert changes[0].line_count == 0
+    assert changes[1].deleted_lines == ("-- a sql comment", "SELECT 1;")
+    assert changes[1].added_lines == ()
+
+
+def test_parse_diff_of_an_empty_diff_is_empty(risk_tier):
+    assert risk_tier.parse_diff("") == ()
+
+
+def test_config_rejects_unknown_keys(risk_tier, tmp_path):
+    bad = tmp_path / "risk-tiers.yml"
+    bad.write_text(CONFIG_PATH.read_text() + "\nprompt: judge.md\n")
+    with pytest.raises(ValidationError, match="Extra inputs"):
+        risk_tier.load_config(bad)
+
+
+def _git(repo: Path, *args: str) -> str:
+    return subprocess.run(
+        ["git", *args],
+        cwd=repo,
+        check=True,
+        capture_output=True,
+        text=True,
+        env={
+            "GIT_AUTHOR_NAME": "t",
+            "GIT_AUTHOR_EMAIL": "t@x",
+            "GIT_COMMITTER_NAME": "t",
+            "GIT_COMMITTER_EMAIL": "t@x",
+        },
+    ).stdout.strip()
+
+
+def test_main_end_to_end_against_a_git_repo(risk_tier, tmp_path, capsys):
+    repo = tmp_path / "repo"
+    (repo / "litellm" / "caching").mkdir(parents=True)
+    _git(tmp_path, "init", "-q", "-b", "main", "repo")
+    (repo / "litellm" / "caching" / "caching.py").write_text("x = 1\n")
+    _git(repo, "add", ".")
+    _git(repo, "commit", "-q", "-m", "base")
+    base = _git(repo, "rev-parse", "HEAD")
+    (repo / "litellm" / "caching" / "caching.py").write_text("x = 2\n")
+    _git(repo, "commit", "-q", "-am", "head")
+    head = _git(repo, "rev-parse", "HEAD")
+    json_out = tmp_path / "risk.json"
+
+    exit_code = risk_tier.main(
+        [
+            "--config",
+            str(CONFIG_PATH),
+            "--base",
+            base,
+            "--head",
+            head,
+            "--author",
+            DEVIN,
+            "--repo",
+            str(repo),
+            "--json-out",
+            str(json_out),
+        ]
+    )
+
+    assert exit_code == 0
+    payload = json.loads(json_out.read_text())
+    assert payload["tier"] == "high"
+    assert {factor["name"]: factor["tier"] for factor in payload["factors"]} == {
+        "paths": "high",
+        "modules": "low",
+        "size": "low",
+        "tests": "medium",
+        "author": "low",
+    }
+    printed = capsys.readouterr().out
+    assert printed.startswith("risk: high (shadow mode, nothing is blocked)")
+    assert payload["summary"] == printed