fix(ci): pass the live base tip to the cost map guard through env and re-run every guard run on a pin change

main now runs the guard on pull_request with the base branch's copy of the script, so the workflow passes the
live tip in COST_MAP_GUARD_TARGET, which the older script ignores. The recheck reads past GraphQL's 100-file
cap, re-runs every guard run for a shared head, and fails when a re-run cannot start. The guard tells a merge
conflict apart from a failed merge-tree call.
This commit is contained in:
mateo-berri 2026-10-28 18:57:56 -07:00
parent 28c3a27265
commit e4e03547e0
4 changed files with 112 additions and 18 deletions

View file

@ -40,8 +40,8 @@ jobs:
- name: Run the guard
env:
MERGE_BASE: ${{ steps.revisions.outputs.merge_base }}
TARGET: ${{ steps.revisions.outputs.target }}
COST_MAP_GUARD_TARGET: ${{ steps.revisions.outputs.target }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
HEAD_REF: ${{ github.event.pull_request.head.ref }}
run: |
uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --target "$TARGET" --head-ref "$HEAD_REF"
uv run --no-project python ci_cd/cost_map_guard.py --base "$MERGE_BASE" --head "$HEAD_SHA" --head-ref "$HEAD_REF"

View file

@ -22,22 +22,50 @@ jobs:
env:
GH_TOKEN: ${{ github.token }}
BASE_REF: ${{ github.ref_name }}
COST_MAP_PATHS: '["model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"]'
run: |
gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
candidates="$(gh api graphql --paginate -F owner="${GITHUB_REPOSITORY_OWNER}" -F name="${GITHUB_REPOSITORY#*/}" -F base="$BASE_REF" \
-f query='query($owner: String!, $name: String!, $base: String!, $endCursor: String) {
repository(owner: $owner, name: $name) {
pullRequests(states: OPEN, baseRefName: $base, first: 25, after: $endCursor) {
nodes { headRefOid files(first: 100) { nodes { path } } }
nodes { number headRefOid changedFiles files(first: 100) { nodes { path } } }
pageInfo { hasNextPage endCursor }
}
}
}' \
--jq '.data.repository.pullRequests.nodes[] | select(any(.files.nodes[].path; IN("model_prices_and_context_window.json", "litellm/model_prices_and_context_window_backup.json", "ci_cd/cost_map_pins.json"))) | .headRefOid' |
while read -r head_sha; do
run_id="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 1 --json databaseId --jq '.[0].databaseId // empty')"
if [ -z "$run_id" ]; then
echo "::warning::no cost map guard run found for $head_sha"
--jq '(env.COST_MAP_PATHS | fromjson) as $paths | .data.repository.pullRequests.nodes[]
| {number, headRefOid, listed: any(.files.nodes[].path; IN($paths[])), unlisted: (.changedFiles > 100)}
| select(.listed or .unlisted) | "\(.number) \(.headRefOid) \(.listed)"')"
failed=0
declare -A rerun_heads
while read -r number head_sha listed; do
[ -n "$number" ] || continue
if [ "$listed" = false ]; then
if ! changed="$(gh api --paginate "repos/${GITHUB_REPOSITORY}/pulls/${number}/files?per_page=100" \
--jq '(env.COST_MAP_PATHS | fromjson) as $paths | .[].filename | select(IN($paths[]))')"; then
echo "::error::could not list the files of #$number"
failed=1
continue
fi
[ -n "$changed" ] || continue
fi
[ -z "${rerun_heads[$head_sha]:-}" ] || continue
rerun_heads[$head_sha]=1
if ! run_ids="$(gh run list --repo "$GITHUB_REPOSITORY" --workflow cost-map-guard.yml --commit "$head_sha" --limit 100 --json databaseId --jq '.[].databaseId')"; then
echo "::error::could not list the cost map guard runs of #$number"
failed=1
continue
fi
gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY" || echo "::warning::could not re-run cost map guard run $run_id for $head_sha"
done
if [ -z "$run_ids" ]; then
echo "::warning::no cost map guard run found for #$number"
continue
fi
for run_id in $run_ids; do
gh run watch "$run_id" --repo "$GITHUB_REPOSITORY" > /dev/null || true
if ! gh run rerun "$run_id" --repo "$GITHUB_REPOSITORY"; then
echo "::error::could not re-run cost map guard run $run_id for #$number"
failed=1
fi
done
done <<< "$candidates"
exit "$failed"

View file

@ -16,6 +16,7 @@ from __future__ import annotations
import argparse
import json
import os
import subprocess
import sys
from collections.abc import Iterator, Mapping, Sequence
@ -30,6 +31,8 @@ GUARDED_PATHS: Final = (COST_MAP_PATH, BACKUP_PATH)
CHECKED_PATHS: Final = (*GUARDED_PATHS, PINS_PATH)
BOT_BRANCH_PREFIX: Final = "litellm_cost_map_sync_"
PRICE_SYNC_BRANCH_PREFIX: Final = "litellm-providers/price-sync"
TARGET_ENV: Final = "COST_MAP_GUARD_TARGET"
MERGE_CONFLICT_EXIT_CODE: Final = 1
SPECIAL_ROOT_KEYS: Final = frozenset({"sample_spec", "fallback_generalizations"})
EVALUATION_OUTPUT_PRICE_SOURCES: Final[Mapping[str, str]] = MappingProxyType({})
@ -44,6 +47,11 @@ class Snapshot:
pins: str
@dataclass(frozen=True, slots=True)
class MergeFailed:
stderr: str
@dataclass(frozen=True, slots=True)
class Pin:
key: str
@ -240,9 +248,19 @@ def snapshot(revision: str) -> Snapshot:
return Snapshot(*(_git("show", f"{revision}:{path}") or "" for path in CHECKED_PATHS))
def merged_tree(merge_base: str, target: str, head: str) -> str | None:
output: Final = _git("merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head)
return None if output is None else output.split("\n", 1)[0]
def merged_tree(merge_base: str, target: str, head: str) -> str | MergeFailed | None:
result: Final = subprocess.run(
("git", "merge-tree", "--write-tree", f"--merge-base={merge_base}", target, head),
check=False,
capture_output=True,
text=True,
)
tree: Final = result.stdout.split("\n", 1)[0]
if result.returncode == 0:
return tree
if result.returncode == MERGE_CONFLICT_EXIT_CODE and tree:
return None
return MergeFailed(result.stderr.strip())
def changed_files(base: str, head: str) -> tuple[str, ...] | None:
@ -254,7 +272,13 @@ def main(argv: Sequence[str]) -> int:
parser: Final = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--base", required=True, help="merge base of the pull request")
parser.add_argument("--head", required=True, help="head commit of the pull request")
parser.add_argument("--target", required=True, help="tip of the base branch the pull request merges into")
env_target: Final = os.environ.get(TARGET_ENV)
parser.add_argument(
"--target",
default=env_target,
required=env_target is None,
help=f"tip of the base branch the pull request merges into, defaulting to ${TARGET_ENV}",
)
parser.add_argument("--head-ref", required=True, help="head branch name of the pull request")
args: Final = parser.parse_args(argv)
bot: Final = args.head_ref.startswith(BOT_BRANCH_PREFIX)
@ -263,6 +287,9 @@ def main(argv: Sequence[str]) -> int:
print(f"cost map guard failed: git diff {args.base} {args.head} failed, so the changed files are unknown")
return 1
merged: Final = merged_tree(args.base, args.target, args.head)
if isinstance(merged, MergeFailed):
print(f"cost map guard failed: git merge-tree into {args.target} failed: {merged.stderr}")
return 1
failures: Final = (
*guard_failures(
snapshot(args.base), snapshot(args.head), None if merged is None else snapshot(merged), changed, bot

View file

@ -1,5 +1,6 @@
import importlib.util
import json
import os
import subprocess
import sys
from collections.abc import Mapping
@ -379,7 +380,10 @@ def _git_commit(repo: Path, message: str) -> str:
).stdout.strip()
def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.CompletedProcess[str]:
def _run_guard(
repo: Path, base: str, head: str, head_ref: str, *, target: str | None = None, target_env: str | None = None
) -> subprocess.CompletedProcess[str]:
target_args: Final = () if target_env is not None else ("--target", target or base)
return subprocess.run(
(
sys.executable,
@ -388,8 +392,7 @@ def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.Co
base,
"--head",
head,
"--target",
base,
*target_args,
"--head-ref",
head_ref,
),
@ -397,6 +400,7 @@ def _run_guard(repo: Path, base: str, head: str, head_ref: str) -> subprocess.Co
capture_output=True,
text=True,
check=False,
env=None if target_env is None else {**os.environ, guard.TARGET_ENV: target_env},
)
@ -469,3 +473,38 @@ def test_main_reads_the_pins_from_the_head_revision(tmp_path: Path, head_ref: st
assert result.returncode == 1, result.stdout + result.stderr
pin_line: Final = f"- {guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: "
assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)]
@pytest.mark.parametrize("head_ref", [BOT_REF, "litellm_fix_pricing"])
def test_main_holds_a_pin_the_base_branch_added_after_the_pr_branched(tmp_path: Path, head_ref: str) -> None:
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
merge_base: Final = _commit(tmp_path, BASE_MAP, "merge base")
target: Final = _commit(tmp_path, BASE_MAP, "pin on the base branch", pins=PINS)
subprocess.run(("git", "checkout", "-q", merge_base), cwd=tmp_path, check=True)
head: Final = _commit(tmp_path, NO_VISION_MAP, "branch cut before the pin lowers it", pins=NO_PINS)
result: Final = _run_guard(tmp_path, merge_base, head, head_ref, target_env=target)
assert result.returncode == 1, result.stdout + result.stderr
pin_line: Final = f"- {guard.COST_MAP_PATH}: openrouter/a.supports_vision must stay true (verified 2026-09-26: "
assert [line for line in result.stdout.splitlines() if line.startswith(pin_line)]
def test_main_reports_a_failed_merge_apart_from_a_conflict(tmp_path: Path) -> None:
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
base: Final = _commit(tmp_path, BASE_MAP, "base")
head: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/c": _entry()}, "head")
result: Final = _run_guard(tmp_path, base, head, "litellm_fix_pricing", target="0" * 40)
assert result.returncode == 1, result.stdout + result.stderr
assert result.stdout.startswith(f"cost map guard failed: git merge-tree into {'0' * 40} failed: ")
def test_main_tells_a_conflicting_pr_to_merge_the_base_branch_in(tmp_path: Path) -> None:
subprocess.run(("git", "init", "-q", str(tmp_path)), check=True)
merge_base: Final = _commit(tmp_path, BASE_MAP, "merge base")
target: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/b": _entry(3e-06)}, "base branch reprices b")
subprocess.run(("git", "checkout", "-q", merge_base), cwd=tmp_path, check=True)
head: Final = _commit(tmp_path, {**BASE_MAP, "openrouter/b": _entry(4e-06)}, "pr reprices b differently")
result: Final = _run_guard(tmp_path, merge_base, head, "litellm_fix_pricing", target=target)
assert result.returncode == 1, result.stdout + result.stderr
conflict_line: Final = "- the pull request does not merge cleanly into its base branch; merge the base branch in first"
assert conflict_line in result.stdout.splitlines()