scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-09 22:31:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 87

ci: measure enterprise/ coverage (#37788)

Browse source 
codecov.yaml has carried an `Enterprise` component scoped to `enterprise/**`
since it was written, and it has never received a line of data. Every one of
the 19 coverage invocations across the unit base, the MCP workflow and the
CircleCI config passes `--cov=./litellm` and nothing else, so 11,203 lines of
paid-customer code sat outside the measured universe while the reported number
described only the rest.

litellm-enterprise is a uv workspace member and a direct dependency, so every
job that syncs already has it installed and importable; only the measurement
was missing.

Measured on tests/test_litellm/enterprise, the shard that exercises this code:
0 enterprise files in the report before, 142 after, at `enterprise/...` paths
that match the component's existing glob. That shard alone puts enterprise at
30.8%, which nudged its own total from 24.09% to 24.20% rather than down. The
aggregate direction across every shard is not knowable until they all report,
and a drop there is the instrument working, not a regression.
This commit is contained in:
yuneng-jiang 2026-08-22 22:55:01 -07:00 • committed by GitHub
parent a734afca32
commit 6c30b4331d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 19 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
.circleci/config.yml
View file

@ -430,7 +430,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \ -vv \
--cov=./litellm \ --cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \ --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=20 \ --durations=20 \
@ -504,7 +504,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \ -vv \
--cov=./litellm \ --cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml \ --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=20 \ --durations=20 \
@ -631,7 +631,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \ -v -x \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
-n 2" -n 2"
@ -738,7 +738,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-v -x \ -v -x \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
-n 4" -n 4"
@ -865,7 +865,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \ -vv \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=20 \ --durations=20 \
-n 4 \ -n 4 \
@ -910,7 +910,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \ -vv -x -s \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5" --durations=5"
no_output_timeout: 15m no_output_timeout: 15m
@ -954,7 +954,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \ -vv \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
-n 2 \ -n 2 \
@ -1000,7 +1000,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \ -vv -x -s \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
--retries 3 --retry-delay 5" --retries 3 --retry-delay 5"
@ -1091,7 +1091,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \ -vv -x \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
-n 4" -n 4"
@ -1135,7 +1135,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \ -vv -x \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
-n 4" -n 4"
@ -1213,7 +1213,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \ -vv -x -s \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
-n 2" -n 2"
@ -1257,7 +1257,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \ -vv -x -s \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
-n 2" -n 2"
@ -1302,7 +1302,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x \ -vv -x \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
-n 4" -n 4"
@ -1381,7 +1381,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv \ -vv \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
-n 4 \ -n 4 \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 \ --durations=5 \
@ -1426,7 +1426,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \ -vv -x -s \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5" --durations=5"
no_output_timeout: 15m no_output_timeout: 15m
@ -1479,7 +1479,7 @@ jobs:
--verbose \ --verbose \
--command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \ --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest \
-vv -x -s \ -vv -x -s \
--cov=./litellm --cov-report=xml \ --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml \
--junitxml=test-results/junit.xml \ --junitxml=test-results/junit.xml \
--durations=5 -n 2 \ --durations=5 -n 2 \
--reruns 2 --reruns-delay 1" --reruns 2 --reruns-delay 1"

4
.github/workflows/_test-unit-base.yml vendored
View file

@ -149,7 +149,7 @@ jobs:
--reruns "${RERUNS}" \ --reruns "${RERUNS}" \
--reruns-delay 1 \ --reruns-delay 1 \
--durations=20 \ --durations=20 \
--cov=./litellm \ --cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \ --cov-report=xml:coverage.xml \
--cov-config=pyproject.toml --cov-config=pyproject.toml
else else
@ -161,7 +161,7 @@ jobs:
--reruns-delay 1 \ --reruns-delay 1 \
--dist="${DIST}" \ --dist="${DIST}" \
--durations=20 \ --durations=20 \
--cov=./litellm \ --cov=./litellm --cov=./enterprise/litellm_enterprise \
--cov-report=xml:coverage.xml \ --cov-report=xml:coverage.xml \
--cov-config=pyproject.toml --cov-config=pyproject.toml
fi fi

2
.github/workflows/test-mcp.yml vendored
View file

@ -60,4 +60,4 @@ jobs:
- name: Run MCP tests - name: Run MCP tests
if: steps.changes.outputs.decision != 'skip' if: steps.changes.outputs.decision != 'skip'
run: | run: |
uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov-report=xml --durations=5 uv run --no-sync pytest tests/mcp_tests -x -vv -n 4 --cov=./litellm --cov=./enterprise/litellm_enterprise --cov-report=xml --durations=5