mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-03 07:23:33 +00:00
08f04fbc97
* test(interactions): assert the live status enum is parsable instead of pinning it * test(interactions): read the spec from a vendored snapshot and canary the live one daily * test(interactions): run the canary on PRs touching its inputs, file an issue on a red run, pin the regenerate command * test(interactions): take the live spec URL as a pytest option, assert its source, split the canary's tracking-issue job * test(interactions): file the canary issue whenever its compliance step did not pass, match the issue title exactly, cover UrlContextResult
91 lines
3.8 KiB
YAML
91 lines
3.8 KiB
YAML
name: Interactions spec canary
|
|
|
|
on:
|
|
schedule:
|
|
- cron: "17 7 * * *"
|
|
pull_request:
|
|
paths:
|
|
- .github/workflows/interactions-spec-canary.yml
|
|
- tests/unit/interactions/**
|
|
- litellm/types/interactions/**
|
|
workflow_dispatch:
|
|
|
|
permissions: {}
|
|
|
|
concurrency: interactions-spec-canary
|
|
|
|
jobs:
|
|
interactions-spec-canary:
|
|
name: interactions-spec-canary
|
|
if: github.repository == 'BerriAI/litellm'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 15
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
compliance: ${{ steps.compliance.outcome }}
|
|
steps:
|
|
- uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0
|
|
with:
|
|
persist-credentials: false
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
|
|
with:
|
|
python-version: "3.12"
|
|
|
|
- name: Set up uv
|
|
uses: ./.github/actions/setup-uv-with-retries
|
|
with:
|
|
version: "0.10.9"
|
|
|
|
- name: Install dependencies
|
|
run: .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra proxy --no-install-project
|
|
|
|
- name: Run the compliance tests against Google's live spec
|
|
id: compliance
|
|
run: |
|
|
uv run --no-sync pytest tests/unit/interactions/test_openapi_compliance.py \
|
|
--interactions-spec-url=https://ai.google.dev/static/api/interactions.openapi.json \
|
|
--tb=short -vv -p no:rerunfailures -p no:pytest-retry \
|
|
--allow-hosts=127.0.0.1,::1,localhost,ai.google.dev
|
|
|
|
tracking-issue:
|
|
name: tracking-issue
|
|
needs: interactions-spec-canary
|
|
if: always() && github.event_name != 'pull_request' && github.ref_name == github.event.repository.default_branch && github.repository == 'BerriAI/litellm'
|
|
runs-on: ubuntu-latest
|
|
timeout-minutes: 5
|
|
permissions:
|
|
issues: write
|
|
steps:
|
|
- name: File the tracking issue unless the compliance step passed, close it when it did
|
|
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1
|
|
env:
|
|
COMPLIANCE_OUTCOME: ${{ needs.interactions-spec-canary.outputs.compliance }}
|
|
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
|
with:
|
|
script: |
|
|
const title = "Interactions spec canary is red";
|
|
const outcome = process.env.COMPLIANCE_OUTCOME;
|
|
const runUrl = process.env.RUN_URL;
|
|
const query = `repo:${context.repo.owner}/${context.repo.repo} is:issue is:open in:title "${title}"`;
|
|
const { data: found } = await github.rest.search.issuesAndPullRequests({ q: query, per_page: 20 });
|
|
const open = found.items.find((issue) => issue.title === title);
|
|
if (outcome === "success") {
|
|
if (open === undefined) return;
|
|
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Green again: ${runUrl}` });
|
|
await github.rest.issues.update({ ...context.repo, issue_number: open.number, state: "closed" });
|
|
return;
|
|
}
|
|
const state = outcome === "failure" ? "red" : `not run (compliance step outcome: ${outcome || "none"})`;
|
|
if (open !== undefined) {
|
|
await github.rest.issues.createComment({ ...context.repo, issue_number: open.number, body: `Still ${state}: ${runUrl}` });
|
|
return;
|
|
}
|
|
await github.rest.issues.create({
|
|
...context.repo,
|
|
title,
|
|
assignees: ["mateo-berri"],
|
|
body: `The scheduled run of tests/unit/interactions/test_openapi_compliance.py against Google's live Interactions spec is ${state}: ${runUrl}\n\nlitellm/types/interactions/README.md says how to refresh the snapshot and regenerate the types.`,
|
|
});
|