diff --git a/.github/workflows/test-cost-map-independence.yml b/.github/workflows/test-cost-map-independence.yml deleted file mode 100644 index b7b5b941cc3..00000000000 --- a/.github/workflows/test-cost-map-independence.yml +++ /dev/null @@ -1,85 +0,0 @@ -name: "Cost map independence" - -on: - pull_request: - branches: - - main - - litellm_internal_staging - - litellm_oss_staging - - "litellm_**" - -permissions: - contents: read - -concurrency: - group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }} - cancel-in-progress: ${{ github.event_name == 'pull_request' }} - -env: - UV_PYTHON: "3.12" - UV_CACHE_DIR: "${{ github.workspace }}/.uv-cache" - LITELLM_LOCAL_MODEL_COST_MAP: "True" - -jobs: - run: - name: Run cost map mutation gate - runs-on: ubuntu-latest - timeout-minutes: 30 - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - ref: ${{ github.event.pull_request.head.sha }} - fetch-depth: 1 - clean: true - persist-credentials: false - - - name: Fetch gate base (merge-base with target branch) - env: - GH_TOKEN: ${{ github.token }} - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - run: | - retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; } - MERGE_BASE=$(retry gh api "repos/${{ github.repository }}/compare/${BASE_SHA}...${HEAD_SHA}?per_page=1" --jq '.merge_base_commit.sha') - test -n "$MERGE_BASE" - retry git fetch --no-tags --depth=1 origin "$MERGE_BASE" - echo "GATE_BASE_SHA=$MERGE_BASE" >> "$GITHUB_ENV" - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: ${{ env.UV_PYTHON }} - - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - - name: Cache uv dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 - with: - path: ${{ env.UV_CACHE_DIR }} - key: ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}-${{ hashFiles('uv.lock') }} - restore-keys: | - ${{ runner.os }}-uv-downloads-py${{ env.UV_PYTHON }}- - - - name: Cache the Rust build - uses: ./.github/actions/cache-cargo-build - - - name: Install dependencies - timeout-minutes: 8 - run: | - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml - uv run --no-sync python -c 'import os, sys; print(sys.version); assert f"{sys.version_info.major}.{sys.version_info.minor}" == os.environ["UV_PYTHON"]' - - - name: Cache Prisma binaries - uses: ./.github/actions/cache-prisma-binaries - - - name: Generate Prisma client - run: | - uv run --no-sync prisma generate --schema litellm/proxy/schema.prisma - - - name: Run cost map mutation gate - timeout-minutes: 20 - run: | - uv run --no-sync python scripts/cost_map_mutation_gate.py --base "$GATE_BASE_SHA" diff --git a/CLAUDE.md b/CLAUDE.md index 26504f953d0..5b0c8e33d67 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -25,7 +25,7 @@ Same thing for bug fixes. The tests should make it so that this specific bug can Never test structure of code only function of it -A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken. `uv run python scripts/cost_map_mutation_gate.py --base origin/main` runs your changed test files against a cost map with every price, limit and deprecation date rewritten, which is what the provider sync does; CI runs the same gate +A test must only fail when litellm code changes. Never pin facts we don't own (a vendor's price, a third party's field, an upstream default, today's date) as literals or as "X must be absent"; assert the invariant our code guarantees instead, e.g. two rows agree, a value is within range, a field is derived from another. If an outside fact is truly load-bearing, cite its source and date next to the assertion so a reader can tell stale from broken. `uv run python scripts/cost_map_mutation_gate.py --base origin/main` runs your changed test files against a cost map with every price, limit and deprecation date rewritten, which is what the provider sync does `tests/test_litellm/` mirrors `litellm/` in a parallel path (see `tests/test_litellm/readme.md`). Name tests `test_.py`, but always match the existing test file in the directory you touch — many provider dirs use longer descriptive names (e.g. `test_anthropic_chat_transformation.py`) to avoid ambiguity across sibling folders. For bug fixes, extend the existing mapped test file rather than creating a new one. Only create a new test file for a new feature (provider, endpoint, or transformation module) that has no mapped test yet, following that directory's naming convention (or `test_.py` if you're the first test there). One focused regression test beats many shallow ones