scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

feat(terraform): coverage-enforcing CI gate against the latest OpenAPI spec (#38710)

Browse source 
Co-authored-by: yassin <yassin@berri.ai>
Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
devin-ai-integration[bot] 2026-08-28 17:11:43 -07:00 • committed by GitHub
parent 592518202c
commit f7fb3694f8
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 389 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.github/workflows/test-terraform-provider.yml vendored
View file

@ -114,4 +114,4 @@ jobs:
- name: Audit provider endpoints against the schema
working-directory: terraform/provider
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json"
run: go run ./tools/endpointaudit -provider-dir ./litellm -spec "${RUNNER_TEMP}/openapi.json" -coverage-allowlist ./tools/endpointaudit/coverage_allowlist.txt

2
terraform/provider/README.md
View file

@ -4,7 +4,7 @@ This Terraform provider allows you to manage LiteLLM resources through Infrastru
## Source of truth
This directory (`terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm)) is the source of truth for the provider. [BerriAI/terraform-provider-litellm](https://github.com/BerriAI/terraform-provider-litellm) is a thin release mirror that the public Terraform Registry ingests from; do not open PRs there. Changes land here, where CI builds the provider, runs its tests, and statically audits every endpoint the provider calls against the proxy's generated OpenAPI schema (`tools/endpointaudit/`), so the provider cannot drift from the LiteLLM API silently. Releases are published by mirroring this directory into the split repo and tagging it, which triggers the goreleaser workflow there (see `RELEASING.md`)
This directory (`terraform/provider/` in [BerriAI/litellm](https://github.com/BerriAI/litellm)) is the source of truth for the provider. [BerriAI/terraform-provider-litellm](https://github.com/BerriAI/terraform-provider-litellm) is a thin release mirror that the public Terraform Registry ingests from; do not open PRs there. Changes land here, where CI builds the provider, runs its tests, and statically audits every endpoint the provider calls against the proxy's generated OpenAPI schema (`tools/endpointaudit/`), so the provider cannot drift from the LiteLLM API silently. The same audit runs in reverse as a coverage gate: every management endpoint in the schema must be covered by a resource or data source, or carry a documented entry in `tools/endpointaudit/coverage_allowlist.txt`, and stale allowlist entries fail CI. Releases are published by mirroring this directory into the split repo and tagging it, which triggers the goreleaser workflow there (see `RELEASING.md`)
## Versioning

106
terraform/provider/tools/endpointaudit/coverage.go Normal file
View file

@ -0,0 +1,106 @@
package main
import (
"bufio"
"encoding/json"
"fmt"
"os"
"sort"
"strings"
)
var managementPrefixes = map[string]bool{
"access_group": true,
"agent": true,
"budget": true,
"cache": true,
"config": true,
"coordination_redis": true,
"credentials": true,
"customer": true,
"fallback": true,
"guardrails": true,
"jwt": true,
"key": true,
"model": true,
"organization": true,
"project": true,
"prompts": true,
"router": true,
"search_tools": true,
"tag": true,
"team": true,
"user": true,
"vector_store": true,
}
func isManagementPath(path string) bool {
segments := strings.SplitN(strings.TrimPrefix(path, "/"), "/", 2)
return len(segments) > 0 && managementPrefixes[segments[0]]
}
func parseAllowlist(path string) (map[string]bool, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
entries := make(map[string]bool)
scanner := bufio.NewScanner(file)
line := 0
for scanner.Scan() {
line++
text := strings.TrimSpace(scanner.Text())
if text == "" || strings.HasPrefix(text, "#") {
continue
}
if idx := strings.Index(text, "#"); idx >= 0 {
text = strings.TrimSpace(text[:idx])
}
fields := strings.Fields(text)
if len(fields) != 2 || !strings.HasPrefix(fields[1], "/") {
return nil, fmt.Errorf("%s:%d: allowlist entries must be \"METHOD /path\", got %q", path, line, text)
}
entries[strings.ToUpper(fields[0])+" "+fields[1]] = true
}
return entries, scanner.Err()
}
func specCallCovered(calls []endpointCall, specMethod, specPath string) bool {
for _, call := range calls {
if strings.EqualFold(call.Method, specMethod) && pathMatches(call.Path, specPath) {
return true
}
}
return false
}
func auditCoverage(calls []endpointCall, specPaths map[string]map[string]json.RawMessage, allowlist map[string]bool) []string {
var violations []string
seen := make(map[string]bool)
for specPath, operations := range specPaths {
if !isManagementPath(specPath) {
continue
}
for method := range operations {
entry := strings.ToUpper(method) + " " + specPath
covered := specCallCovered(calls, method, specPath)
switch {
case allowlist[entry]:
seen[entry] = true
if covered {
violations = append(violations, fmt.Sprintf("stale allowlist entry: %s is covered by the provider; remove it from the allowlist", entry))
}
case !covered:
violations = append(violations, fmt.Sprintf("uncovered management endpoint: %s has no provider resource or data source; add coverage or allowlist it with a reason", entry))
}
}
}
for entry := range allowlist {
if !seen[entry] {
violations = append(violations, fmt.Sprintf("stale allowlist entry: %s is not a management endpoint in the proxy schema; remove it from the allowlist", entry))
}
}
sort.Strings(violations)
return violations
}

132
terraform/provider/tools/endpointaudit/coverage_allowlist.txt Normal file
View file

@ -0,0 +1,132 @@
# Management endpoints deliberately not covered by a Terraform resource or data source.
#
# Format: one "METHOD /path" per line, matching the proxy OpenAPI schema exactly;
# "#" starts a comment. The coverage gate (endpointaudit -coverage-allowlist) fails
# when a management endpoint is neither covered nor listed here, and also when an
# entry goes stale (the provider now covers it, or the endpoint left the schema),
# so this file can only shrink relative to the schema over time.
#
# Every entry needs a reason. Endpoints that are analytics, UI helpers, or
# imperative one-shot operations never get a resource. Entries marked "known gap"
# are real coverage gaps awaiting a resource; remove them when the resource lands.
# Read-only analytics and spend reporting; observability, not Terraform-managed state
GET /agent/daily/activity
GET /customer/daily/activity
GET /guardrails/usage/detail/{guardrail_id}
GET /guardrails/usage/logs
GET /guardrails/usage/overview
GET /key/spend/report
GET /organization/daily/activity
GET /organization/spend/report
GET /tag/daily/activity
GET /tag/dau
GET /tag/distinct
GET /tag/mau
GET /tag/summary
GET /tag/user-agent/per-user-analytics
GET /tag/wau
GET /team/daily/activity
GET /team/daily/activity/aggregated
GET /team/spend/report
GET /user/daily/activity
GET /user/daily/activity/aggregated
GET /user/spend/report
# Admin UI helper endpoints; serve UI forms and caller-scoped views, not desired state
GET /budget/settings
GET /router/fields
GET /guardrails/ui/add_guardrail_settings
GET /guardrails/ui/category_yaml/{category_name}
GET /guardrails/ui/major_airlines
GET /guardrails/ui/provider_specific_params
GET /key/aliases
GET /model/deprecations
GET /search_tools/ui/available_providers
GET /team/available
GET /team/metadata_schema
GET /team/{team_id}/members/me
GET /user/available_users
# Imperative one-shot operations: bulk edits, rotation, health probes, test hooks,
# migrations, and approval workflows; procedural, not declarative state
GET /cache/ping
GET /cache/redis/info
GET /credentials/migrate-encryption/check
POST /cache/delete
POST /cache/flushall
POST /cache/settings/test
POST /coordination_redis/settings/test
GET /guardrails/submissions
GET /guardrails/submissions/{guardrail_id}
POST /credentials/migrate-encryption
POST /customer/block
POST /customer/unblock
POST /guardrails/apply_guardrail
POST /guardrails/register
POST /guardrails/submissions/{guardrail_id}/approve
POST /guardrails/submissions/{guardrail_id}/reject
POST /guardrails/test_custom_code
POST /guardrails/validate_blocked_words_file
POST /key/bulk_update
POST /key/health
POST /key/regenerate
POST /key/service-account/generate
POST /key/{key}/regenerate
POST /key/{key}/reset_spend
POST /model/block
POST /model/unblock
POST /prompts/test
POST /search_tools/test_connection
POST /team/bulk_member_add
POST /team/{team_id}/member/{user_id}/reset_spend
POST /team/key/bulk_update
POST /team/permissions_bulk_update
POST /team/{team_id}/disable_logging
POST /user/bulk_update
# Alternate method or path for functionality the provider already manages elsewhere
GET /credentials/by_model/{model_id}
GET /guardrails/{guardrail_id}
GET /prompts/{prompt_id}
GET /prompts/{prompt_id}/versions
PATCH /guardrails/{guardrail_id}
PATCH /model/{model_id}/update
PATCH /prompts/{prompt_id}
PATCH /team/{team_id}
POST /team/model/add
POST /team/model/delete
# Known gaps awaiting a resource or data source; remove the entry when it lands
GET /credentials # known gap: plural credentials data source
GET /cache/settings # known gap: cache settings resource
POST /cache/settings # known gap: cache settings resource
GET /coordination_redis/settings # known gap: coordination redis settings resource
POST /coordination_redis/settings # known gap: coordination redis settings resource
GET /router/settings # known gap: router settings data source
GET /router/fields # known gap: router settings data source
GET /config/block_requests_for_models_without_pricing # known gap: proxy config resource
PATCH /config/block_requests_for_models_without_pricing # known gap: proxy config resource
GET /config/cost_discount_config # known gap: proxy config resource
PATCH /config/cost_discount_config # known gap: proxy config resource
GET /config/cost_margin_config # known gap: proxy config resource
PATCH /config/cost_margin_config # known gap: proxy config resource
GET /config/pass_through_endpoint # known gap: pass-through endpoint resource
POST /config/pass_through_endpoint # known gap: pass-through endpoint resource
DELETE /config/pass_through_endpoint # known gap: pass-through endpoint resource
POST /config/pass_through_endpoint/{endpoint_id} # known gap: pass-through endpoint resource
GET /config/pass_through_endpoint/team/{team_id} # known gap: pass-through endpoint resource
GET /vector_store/list # known gap: plural vector stores data source
GET /customer/info # known gap: litellm_customer resource
GET /customer/list # known gap: litellm_customer resource
POST /customer/new # known gap: litellm_customer resource
POST /customer/update # known gap: litellm_customer resource
POST /customer/delete # known gap: litellm_customer resource
GET /team/{team_id}/callback # known gap: team callback resource
POST /team/{team_id}/callback # known gap: team callback resource
DELETE /team/{team_id}/callback/{callback_name} # known gap: team callback resource
GET /jwt/key/mapping/info # known gap: litellm_jwt_key_mapping, in review (PR #36096)
GET /jwt/key/mapping/list # known gap: litellm_jwt_key_mapping, in review (PR #36096)
POST /jwt/key/mapping/new # known gap: litellm_jwt_key_mapping, in review (PR #36096)
POST /jwt/key/mapping/update # known gap: litellm_jwt_key_mapping, in review (PR #36096)
POST /jwt/key/mapping/delete # known gap: litellm_jwt_key_mapping, in review (PR #36096)

136
terraform/provider/tools/endpointaudit/coverage_test.go Normal file
View file

@ -0,0 +1,136 @@
package main
import (
"encoding/json"
"os"
"path/filepath"
"strings"
"testing"
)
func coverageSpecFixture(paths map[string][]string) map[string]map[string]json.RawMessage {
spec := make(map[string]map[string]json.RawMessage)
for path, methods := range paths {
operations := make(map[string]json.RawMessage)
for _, method := range methods {
operations[method] = json.RawMessage(`{}`)
}
spec[path] = operations
}
return spec
}
func writeAllowlist(t *testing.T, body string) string {
t.Helper()
path := filepath.Join(t.TempDir(), "allowlist.txt")
if err := os.WriteFile(path, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return path
}
func TestParseAllowlist(t *testing.T) {
path := writeAllowlist(t, `# comment
GET /team/spend/report
post /key/regenerate # inline reason
`)
entries, err := parseAllowlist(path)
if err != nil {
t.Fatal(err)
}
if len(entries) != 2 || !entries["GET /team/spend/report"] || !entries["POST /key/regenerate"] {
t.Fatalf("unexpected entries: %v", entries)
}
}
func TestParseAllowlistRejectsMalformedLines(t *testing.T) {
path := writeAllowlist(t, "GET\n")
if _, err := parseAllowlist(path); err == nil {
t.Fatal("expected error for malformed line")
}
}
func TestAuditCoverageFailsOnUncoveredManagementEndpoint(t *testing.T) {
spec := coverageSpecFixture(map[string][]string{
"/team/new": {"post"},
"/team/spend/report": {"get"},
"/chat/completions": {"post"},
"/health/liveliness": {"get"},
"/v1/chat/completions": {"post"},
})
calls := []endpointCall{{Method: "POST", Path: "/team/new"}}
violations := auditCoverage(calls, spec, nil)
if len(violations) != 1 || !strings.Contains(violations[0], "GET /team/spend/report") {
t.Fatalf("unexpected violations: %v", violations)
}
}
func TestAuditCoverageAllowlistSuppressesUncovered(t *testing.T) {
spec := coverageSpecFixture(map[string][]string{"/team/spend/report": {"get"}})
violations := auditCoverage(nil, spec, map[string]bool{"GET /team/spend/report": true})
if len(violations) != 0 {
t.Fatalf("unexpected violations: %v", violations)
}
}
func TestAuditCoverageFailsOnStaleCoveredEntry(t *testing.T) {
spec := coverageSpecFixture(map[string][]string{"/team/new": {"post"}})
calls := []endpointCall{{Method: "POST", Path: "/team/new"}}
violations := auditCoverage(calls, spec, map[string]bool{"POST /team/new": true})
if len(violations) != 1 || !strings.Contains(violations[0], "stale allowlist entry: POST /team/new is covered") {
t.Fatalf("unexpected violations: %v", violations)
}
}
func TestAuditCoverageFailsOnEntryMissingFromSchema(t *testing.T) {
spec := coverageSpecFixture(map[string][]string{"/team/new": {"post"}})
calls := []endpointCall{{Method: "POST", Path: "/team/new"}}
violations := auditCoverage(calls, spec, map[string]bool{"POST /team/removed": true})
if len(violations) != 1 || !strings.Contains(violations[0], "POST /team/removed is not a management endpoint") {
t.Fatalf("unexpected violations: %v", violations)
}
}
func TestAuditCoverageMatchesPathParams(t *testing.T) {
spec := coverageSpecFixture(map[string][]string{"/team/{team_id}/callback": {"get"}})
calls := []endpointCall{{Method: "GET", Path: "/team/{param}/callback"}}
violations := auditCoverage(calls, spec, nil)
if len(violations) != 0 {
t.Fatalf("unexpected violations: %v", violations)
}
}
func TestMountedDeclarativeAPIsAreManagementPaths(t *testing.T) {
for _, path := range []string{
"/cache/settings",
"/config/cost_discount_config",
"/coordination_redis/settings",
"/router/settings",
} {
if !isManagementPath(path) {
t.Fatalf("%s should be classified as a management path", path)
}
}
for _, path := range []string{"/chat/completions", "/health/liveliness"} {
if isManagementPath(path) {
t.Fatalf("%s should not be classified as a management path", path)
}
}
}
func TestBundledAllowlistEntriesAreManagementPaths(t *testing.T) {
entries, err := parseAllowlist("coverage_allowlist.txt")
if err != nil {
t.Fatal(err)
}
if len(entries) == 0 {
t.Fatal("bundled allowlist parsed to zero entries")
}
for entry := range entries {
fields := strings.Fields(entry)
if !isManagementPath(fields[1]) {
t.Fatalf("allowlist entry %q is not under a management prefix", entry)
}
}
}

15
terraform/provider/tools/endpointaudit/main.go
View file

@ -306,7 +306,7 @@ func auditCalls(calls []endpointCall, specPaths map[string]map[string]json.RawMe
return violations
}
func run(providerDir, specPath string) error {
func run(providerDir, specPath, coverageAllowlistPath string) error {
extracted, err := extractProviderCalls(providerDir)
if err != nil {
return err
@ -326,6 +326,16 @@ func run(providerDir, specPath string) error {
sort.Strings(violations)
return fmt.Errorf("provider/proxy endpoint drift:\n %s", strings.Join(violations, "\n "))
}
if coverageAllowlistPath != "" {
allowlist, err := parseAllowlist(coverageAllowlistPath)
if err != nil {
return err
}
coverageViolations := auditCoverage(extracted.Calls, specPaths, allowlist)
if len(coverageViolations) > 0 {
return fmt.Errorf("provider coverage gaps:\n %s", strings.Join(coverageViolations, "\n "))
}
}
fmt.Printf("OK: %d request call sites verified against %d proxy OpenAPI paths\n", len(extracted.Calls), len(specPaths))
return nil
}
@ -333,12 +343,13 @@ func run(providerDir, specPath string) error {
func main() {
providerDir := flag.String("provider-dir", "./litellm", "directory containing the provider Go source")
specPath := flag.String("spec", "", "path to the proxy OpenAPI schema JSON")
coverageAllowlist := flag.String("coverage-allowlist", "", "path to the coverage allowlist; when set, also fail on management endpoints with no provider coverage")
flag.Parse()
if *specPath == "" {
fmt.Fprintln(os.Stderr, "error: -spec is required")
os.Exit(2)
}
if err := run(*providerDir, *specPath); err != nil {
if err := run(*providerDir, *specPath, *coverageAllowlist); err != nil {
fmt.Fprintf(os.Stderr, "error: %v\n", err)
os.Exit(1)
}