diff --git a/.github/workflows/test-litellm-ui-build.yml b/.github/workflows/test-litellm-ui-build.yml index b83119712a7..0fd8093949d 100644 --- a/.github/workflows/test-litellm-ui-build.yml +++ b/.github/workflows/test-litellm-ui-build.yml @@ -111,4 +111,4 @@ jobs: if: ${{ !cancelled() && steps.changed.outputs.has_files == 'true' }} run: | npx eslint . -f json -o "$RUNNER_TEMP/lint-report.json" || true - node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json + node scripts/check-lint-budgets.mjs "$RUNNER_TEMP/lint-report.json" eslint-budgets.json --check eslint-metrics.json diff --git a/ui/litellm-dashboard/eslint-metrics.json b/ui/litellm-dashboard/eslint-metrics.json new file mode 100644 index 00000000000..28f9fc3a6af --- /dev/null +++ b/ui/litellm-dashboard/eslint-metrics.json @@ -0,0 +1,5 @@ +{ + "@typescript-eslint/no-explicit-any": 2027, + "complexity": 128, + "max-depth": 61 +} diff --git a/ui/litellm-dashboard/package.json b/ui/litellm-dashboard/package.json index 23da0bcd636..a8948f4be34 100644 --- a/ui/litellm-dashboard/package.json +++ b/ui/litellm-dashboard/package.json @@ -8,6 +8,7 @@ "build": "next build", "start": "next start", "lint": "eslint .", + "lint:metrics": "node scripts/update-lint-metrics.mjs", "test": "vitest", "test:dot": "vitest --reporter=dot", "test:watch": "vitest -w", diff --git a/ui/litellm-dashboard/scripts/check-lint-budgets.mjs b/ui/litellm-dashboard/scripts/check-lint-budgets.mjs index f6208f012bb..a7aba18ae76 100644 --- a/ui/litellm-dashboard/scripts/check-lint-budgets.mjs +++ b/ui/litellm-dashboard/scripts/check-lint-budgets.mjs @@ -1,22 +1,25 @@ import { readFileSync } from "fs"; +import { countBudgetViolations, findDrift } from "./lint-budget-lib.mjs"; -const [, , reportPath, budgetsPath] = process.argv; - -const report = JSON.parse(readFileSync(reportPath, "utf8")); -const budgets = JSON.parse(readFileSync(budgetsPath, "utf8")); - -const counts = {}; -for (const file of report) { - for (const message of file.messages) { - if (message.ruleId in budgets) { - counts[message.ruleId] = (counts[message.ruleId] || 0) + 1; - } +const argv = process.argv.slice(2); +const positional = []; +const flags = {}; +for (let i = 0; i < argv.length; i += 1) { + if (argv[i] === "--check") { + flags.check = argv[(i += 1)]; + } else { + positional.push(argv[i]); } } +const [reportPath, budgetsPath] = positional; +const report = JSON.parse(readFileSync(reportPath, "utf8")); +const budgets = JSON.parse(readFileSync(budgetsPath, "utf8")); +const counts = countBudgetViolations(report, budgets); + let failed = false; for (const [rule, { max, target }] of Object.entries(budgets)) { - const count = counts[rule] || 0; + const count = counts[rule]; const note = count > max ? "OVER BUDGET" : count <= target ? "at target" : `${max - count} of headroom`; console.log(`${rule}: ${count} | max: ${max} | target: ${target} | ${note}`); if (count > max) { @@ -27,4 +30,20 @@ for (const [rule, { max, target }] of Object.entries(budgets)) { } } +if (flags.check) { + const committed = JSON.parse(readFileSync(flags.check, "utf8")); + const drift = findDrift(committed, counts); + for (const { rule, committed: was, actual } of drift) { + console.error( + `::error::${flags.check} is stale for ${rule}: committed ${was ?? "missing"}, actual ${actual ?? "not a tracked rule"}.`, + ); + } + if (drift.length > 0) { + console.error(`::error::Run \`npm run lint:metrics\` and commit ${flags.check}.`); + failed = true; + } else { + console.log(`${flags.check} is up to date.`); + } +} + process.exit(failed ? 1 : 0); diff --git a/ui/litellm-dashboard/scripts/lint-budget-lib.mjs b/ui/litellm-dashboard/scripts/lint-budget-lib.mjs new file mode 100644 index 00000000000..a43305fc4ed --- /dev/null +++ b/ui/litellm-dashboard/scripts/lint-budget-lib.mjs @@ -0,0 +1,22 @@ +export function countBudgetViolations(report, budgets) { + const counts = {}; + for (const file of report) { + for (const message of file.messages) { + if (message.ruleId in budgets) { + counts[message.ruleId] = (counts[message.ruleId] || 0) + 1; + } + } + } + return Object.fromEntries( + Object.keys(budgets) + .sort() + .map((rule) => [rule, counts[rule] || 0]), + ); +} + +export function findDrift(committed, actual) { + const rules = [...new Set([...Object.keys(actual), ...Object.keys(committed)])].sort(); + return rules + .filter((rule) => committed[rule] !== actual[rule]) + .map((rule) => ({ rule, committed: committed[rule] ?? null, actual: actual[rule] ?? null })); +} diff --git a/ui/litellm-dashboard/scripts/update-lint-metrics.mjs b/ui/litellm-dashboard/scripts/update-lint-metrics.mjs new file mode 100644 index 00000000000..16704d1f7a2 --- /dev/null +++ b/ui/litellm-dashboard/scripts/update-lint-metrics.mjs @@ -0,0 +1,25 @@ +import { execSync } from "child_process"; +import { mkdtempSync, readFileSync, writeFileSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { countBudgetViolations } from "./lint-budget-lib.mjs"; + +const ESLINT_EXIT_LINT_ERRORS = 1; + +const budgets = JSON.parse(readFileSync("eslint-budgets.json", "utf8")); +const dir = mkdtempSync(join(tmpdir(), "litellm-lint-")); +const reportPath = join(dir, "report.json"); + +try { + execSync(`npx eslint . -f json -o "${reportPath}"`, { stdio: "inherit" }); +} catch (err) { + if (err.status !== ESLINT_EXIT_LINT_ERRORS) throw err; +} + +const report = JSON.parse(readFileSync(reportPath, "utf8")); +rmSync(dir, { recursive: true, force: true }); + +const metrics = countBudgetViolations(report, budgets); +writeFileSync("eslint-metrics.json", JSON.stringify(metrics, null, 2) + "\n"); +console.log("Updated eslint-metrics.json"); +console.table(metrics); diff --git a/ui/litellm-dashboard/tests/lint-budget-lib.test.ts b/ui/litellm-dashboard/tests/lint-budget-lib.test.ts new file mode 100644 index 00000000000..75b5efe6d1d --- /dev/null +++ b/ui/litellm-dashboard/tests/lint-budget-lib.test.ts @@ -0,0 +1,55 @@ +import { describe, it, expect } from "vitest"; +import { countBudgetViolations, findDrift } from "../scripts/lint-budget-lib.mjs"; + +const budgets = { + "@typescript-eslint/no-explicit-any": { max: 10, target: 5 }, + complexity: { max: 10, target: 5 }, +}; + +const file = (...ruleIds: (string | null)[]) => ({ messages: ruleIds.map((ruleId) => ({ ruleId })) }); + +describe("countBudgetViolations", () => { + it("counts only budgeted rules and sums across files", () => { + const report = [ + file("@typescript-eslint/no-explicit-any", "complexity", "max-depth"), + file("@typescript-eslint/no-explicit-any", "no-var", null), + ]; + expect(countBudgetViolations(report, budgets)).toEqual({ + "@typescript-eslint/no-explicit-any": 2, + complexity: 1, + }); + }); + + it("reports 0 for a budgeted rule with no violations", () => { + expect(countBudgetViolations([file("complexity")], budgets)).toEqual({ + "@typescript-eslint/no-explicit-any": 0, + complexity: 1, + }); + }); + + it("emits keys in sorted order so the committed snapshot diffs stably", () => { + const unsorted = { complexity: { max: 1, target: 1 }, "@typescript-eslint/no-explicit-any": { max: 1, target: 1 } }; + expect(Object.keys(countBudgetViolations([], unsorted))).toEqual([ + "@typescript-eslint/no-explicit-any", + "complexity", + ]); + }); +}); + +describe("findDrift", () => { + it("reports no drift when the snapshot matches the actual counts", () => { + expect(findDrift({ complexity: 5 }, { complexity: 5 })).toEqual([]); + }); + + it("detects a changed count", () => { + expect(findDrift({ complexity: 5 }, { complexity: 7 })).toEqual([{ rule: "complexity", committed: 5, actual: 7 }]); + }); + + it("detects a rule missing from the committed snapshot", () => { + expect(findDrift({}, { complexity: 7 })).toEqual([{ rule: "complexity", committed: null, actual: 7 }]); + }); + + it("detects a phantom rule the committed snapshot still carries", () => { + expect(findDrift({ "removed-rule": 3 }, {})).toEqual([{ rule: "removed-rule", committed: 3, actual: null }]); + }); +});