perf(proxy): cache the public provider and agent field endpoints (#46617)

/public/providers/fields re-read and re-parsed the bundled 85K
provider_create_fields.json on every request, and /public/agents/fields did the
same for that file plus agent_create_fields.json. Both are unauthenticated
public endpoints, and both did the work inside an `async def`, so the read and
the parse ran on the event loop and serialized against every other in-flight
request. Measured against the bundled files, that is roughly 0.40 ms of blocking
work per request for the provider endpoint and 0.45 ms for the agent endpoint

Both now memoize with `lru_cache`, which is what `/public/endpoints` in the same
module already does through its `_cached_endpoints` global, so each file is read
once per process rather than once per request

The agent endpoint mutated the dicts it had just loaded, appending inherited
provider credential fields and popping `inherit_credentials_from_provider`. That
is only safe while the data is re-read every time, so caching it meant making the
merge pure: `_agent_with_inherited_credentials` builds a new mapping per agent,
and the cached value is the merged result rather than the raw file contents.
Responses are byte identical before and after, checked on both endpoints against
a live proxy

The bundled records are typed as Mapping[str, object] and narrowed with
isinstance at the point of use, rather than typing.Any, which ruff-strict bans

Side effects: 3 of the repo's ASYNC230 violations go away, and basedpyright
errors in the file drop from 63 to 52

Rebased onto main after /public/providers/fields started validating the bundled
JSON through a pydantic adapter on every request. That validation now runs once
and the validated records are cached; the agent endpoint keeps merging the raw
provider records, so both responses stay byte identical to main.

Co-authored-by: Alok Sharma <alok44170@gmail.com>

litellm/proxy/public_endpoints/public_endpoints.py

@@ -3,11 +3,12 @@ import json
 import os
 import re
 from collections.abc import Awaitable, Callable, Mapping, Sequence
+from functools import cache
 from importlib.resources import files
 from typing import TYPE_CHECKING, Final, Protocol
 
 from fastapi import APIRouter, HTTPException, Request
-from pydantic import TypeAdapter
+from pydantic import ConfigDict, TypeAdapter
 from typing_extensions import ReadOnly, TypedDict
 
 import litellm
@@ -31,6 +32,7 @@ from litellm.types.proxy.management_endpoints.model_management_endpoints import
 )
 from litellm.types.proxy.public_endpoints.public_endpoints import (
     AgentCreateInfo,
+    AgentCredentialField,
     AutoRouterPresetRecord,
     ComplexityScorerDefaults,
     ProviderCreateInfo,
@@ -210,6 +212,54 @@ def _load_endpoints() -> list[_EndpointEntry]:
     return _build_endpoints(raw)
 
 
+class _BundledAgentCreateInfo(AgentCreateInfo):
+    model_config = ConfigDict(frozen=True)
+
+    inherit_credentials_from_provider: str | None = None
+
+
+_BUNDLED_AGENTS_ADAPTER: Final = TypeAdapter(list[_BundledAgentCreateInfo])
+
+
+def _read_bundled_file(filename: str) -> bytes:
+    return files("litellm.proxy.public_endpoints").joinpath(filename).read_bytes()
+
+
+@cache
+def _provider_create_fields() -> tuple[ProviderCreateInfo, ...]:
+    return tuple(_PROVIDER_CREATE_FIELDS_ADAPTER.validate_json(_read_bundled_file("provider_create_fields.json")))
+
+
+def _inherited_credential_fields(provider: ProviderCreateInfo) -> tuple[AgentCredentialField, ...]:
+    return tuple(
+        AgentCredentialField.model_validate({**field.model_dump(), "include_in_litellm_params": True})
+        for field in provider.credential_fields
+    )
+
+
+def _with_inherited_credentials(
+    agent: _BundledAgentCreateInfo, providers: Mapping[str, ProviderCreateInfo]
+) -> AgentCreateInfo:
+    provider_name: Final = agent.inherit_credentials_from_provider
+    provider: Final = providers.get(provider_name) if provider_name else None
+    inherited: Final = _inherited_credential_fields(provider) if provider else ()
+    return AgentCreateInfo.model_validate(
+        {
+            **agent.model_dump(exclude={"inherit_credentials_from_provider"}),
+            "credential_fields": [*agent.credential_fields, *inherited],
+        }
+    )
+
+
+@cache
+def _agent_create_fields() -> tuple[AgentCreateInfo, ...]:
+    providers: Final = {provider.provider: provider for provider in _provider_create_fields()}
+    return tuple(
+        _with_inherited_credentials(agent, providers)
+        for agent in _BUNDLED_AGENTS_ADAPTER.validate_json(_read_bundled_file("agent_create_fields.json"))
+    )
+
+
 # ---------------------------------------------------------------------------
 
 
@@ -404,22 +454,15 @@ async def get_supported_providers() -> list[str]:
     tags=["public", "providers"],
     response_model=list[ProviderCreateInfo],
 )
-async def get_provider_fields() -> list[ProviderCreateInfo]:
+async def get_provider_fields() -> tuple[ProviderCreateInfo, ...]:
     """
     Return provider metadata required by the dashboard create-model flow.
+
+    Reads from the bundled local file. Result is cached in-process for the
+    lifetime of the server process.
     """
 
-    provider_create_fields_path: Final = os.path.join(
-        os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
-        "proxy",
-        "public_endpoints",
-        "provider_create_fields.json",
-    )
-
-    with open(provider_create_fields_path, "r") as f:
-        provider_create_fields: Final = _PROVIDER_CREATE_FIELDS_ADAPTER.validate_python(json.load(f))
-
-    return provider_create_fields
+    return _provider_create_fields()
 
 
 @router.get(
@@ -601,45 +644,14 @@ async def get_supported_endpoints() -> SupportedEndpointsResponse:
     tags=["public", "[beta] Agents"],
     response_model=list[AgentCreateInfo],
 )
-async def get_agent_fields() -> list[AgentCreateInfo]:
+async def get_agent_fields() -> tuple[AgentCreateInfo, ...]:
     """
     Return agent type metadata required by the dashboard create-agent flow.
 
     If an agent has `inherit_credentials_from_provider`, the provider's credential
     fields are automatically appended to the agent's credential_fields.
+
+    Reads from the bundled local files. Result is cached in-process for the
+    lifetime of the server process.
     """
-    base_path: Final = os.path.join(
-        os.path.dirname(os.path.dirname(os.path.dirname(__file__))),
-        "proxy",
-        "public_endpoints",
-    )
-
-    agent_create_fields_path: Final = os.path.join(base_path, "agent_create_fields.json")
-    provider_create_fields_path: Final = os.path.join(base_path, "provider_create_fields.json")
-
-    with open(agent_create_fields_path, "r") as f:
-        agent_create_fields: Final = json.load(f)
-
-    with open(provider_create_fields_path, "r") as f:
-        provider_create_fields: Final = json.load(f)
-
-    # Build a lookup map for providers by name
-    provider_map: Final = {p["provider"]: p for p in provider_create_fields}
-
-    # Merge inherited credential fields
-    for agent in agent_create_fields:
-        inherit_from = agent.get("inherit_credentials_from_provider")
-        if inherit_from and inherit_from in provider_map:
-            provider = provider_map[inherit_from]
-            # Copy provider fields and mark them for inclusion in litellm_params
-            inherited_fields = []
-            for field in provider.get("credential_fields", []):
-                field_copy = field.copy()
-                field_copy["include_in_litellm_params"] = True
-                inherited_fields.append(field_copy)
-            # Append provider credential fields after agent's own fields
-            agent["credential_fields"] = agent.get("credential_fields", []) + inherited_fields
-        # Remove the inherit field from response (not needed by frontend)
-        agent.pop("inherit_credentials_from_provider", None)
-
-    return agent_create_fields
+    return _agent_create_fields()

tests/unit/proxy/public_endpoints/test_public_endpoints.py

@@ -2,6 +2,8 @@ import asyncio
 import json
 import re
 from datetime import datetime, timezone
+from importlib.resources import files
+from itertools import chain
 from types import SimpleNamespace
 from typing import Final
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -18,7 +20,7 @@ from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_pres
 from litellm.types.proxy.management_endpoints.model_management_endpoints import (
     ModelGroupInfoProxy,
 )
-from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreateInfo
+from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreateInfo, ProviderCredentialField
 from litellm.types.utils import LlmProviders
 
 
@@ -1563,6 +1565,59 @@ async def test_fetch_remote_autorouter_presets_parses_and_rejects_empty(monkeypa
         await _fetch_remote_autorouter_presets("https://example.test/presets.json")
 
 
+# ---------------------------------------------------------------------------
+# /public/agents/fields inherited provider credentials
+# ---------------------------------------------------------------------------
+
+
+def test_get_agent_fields_merge_does_not_compound_across_requests():
+    """The inherited-credentials merge runs once, so credential_fields cannot grow per request.
+
+    Caching the raw file contents instead of the merged result would let the merge
+    append inherited fields again on every request.
+    """
+    client = _make_client()
+
+    def counts():
+        return {
+            agent["agent_type"]: len(agent.get("credential_fields") or [])
+            for agent in client.get("/public/agents/fields").json()
+        }
+
+    first = counts()
+    assert first, "expected at least one agent type"
+    assert first == counts() == counts()
+
+
+def test_get_agent_fields_appends_inherited_provider_credential_fields():
+    client: Final = _make_client()
+    providers: Final = {p["provider"]: p for p in client.get("/public/providers/fields").json()}
+    agents: Final = client.get("/public/agents/fields").json()
+    bundled: Final = json.loads(
+        files("litellm.proxy.public_endpoints").joinpath("agent_create_fields.json").read_text(encoding="utf-8")
+    )
+    own_counts: Final = {agent["agent_type"]: len(agent["credential_fields"]) for agent in bundled}
+    expected_inherited: Final = {
+        agent["agent_type"]: providers.get(agent.get("inherit_credentials_from_provider"), {}).get(
+            "credential_fields", []
+        )
+        for agent in bundled
+    }
+    inherited: Final = {
+        agent["agent_type"]: agent["credential_fields"][own_counts[agent["agent_type"]] :] for agent in agents
+    }
+
+    assert any(expected_inherited.values()), "expected at least one agent to inherit provider credential fields"
+    assert all("inherit_credentials_from_provider" not in agent for agent in agents)
+    assert {
+        agent_type: [
+            {key: field[key] for key in field if key in ProviderCredentialField.model_fields} for field in fields
+        ]
+        for agent_type, fields in inherited.items()
+    } == expected_inherited
+    assert all(field["include_in_litellm_params"] is True for field in chain.from_iterable(inherited.values()))
+
+
 @pytest.mark.parametrize("preference", ["auto_install", None])
 def test_public_skill_hub_echoes_registered_installation_preference(preference):
     from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -13438,6 +13438,9 @@ export interface paths {
          *
          *     If an agent has `inherit_credentials_from_provider`, the provider's credential
          *     fields are automatically appended to the agent's credential_fields.
+         *
+         *     Reads from the bundled local files. Result is cached in-process for the
+         *     lifetime of the server process.
          */
         get: operations["get_agent_fields_public_agents_fields_get"];
         put?: never;
@@ -13659,6 +13662,9 @@ export interface paths {
         /**
          * Get Provider Fields
          * @description Return provider metadata required by the dashboard create-model flow.
+         *
+         *     Reads from the bundled local file. Result is cached in-process for the
+         *     lifetime of the server process.
          */
         get: operations["get_provider_fields_public_providers_fields_get"];
         put?: never;