From 9440dbac74a72865525dbfdc6ebf4a74f53e146e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 6 Sep 2026 03:20:37 -0700 Subject: [PATCH] perf: cache repacked Agent Skills archives per skill version The well-known index needs every stored upload repacked to publish its digest, and both routes are unauthenticated, so each request was rebuilding every archive on the event loop. With 21 stored skills the index took ~2s and /health/liveliness on the same worker went from 1ms to 1.7s under two concurrent index requests. Repacking now runs off the event loop and each result is cached per skill version, so a worker builds an archive once until the skill changes. The archive route also declares application/zip in OpenAPI rather than JSON. --- litellm/proxy/_lazy_openapi_snapshot.json | 6 ++- .../agent_skills_endpoints.py | 51 +++++++++++++++---- .../test_agent_skills_endpoints.py | 47 +++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 4 files changed, 92 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 1e53049a887..94fa0b67caf 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -5084,8 +5084,10 @@ "responses": { "200": { "content": { - "application/json": { - "schema": {} + "application/zip": { + "schema": { + "type": "string" + } } }, "description": "Successful Response" diff --git a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py index d3c3b62fbfa..3084cbfd84f 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py @@ -6,6 +6,7 @@ routes are unauthenticated and stay off until ``litellm_settings.public_skills_i is enabled, which publishes every stored skill to anyone who can reach the proxy. """ +import asyncio import re from collections.abc import Sequence from itertools import groupby @@ -17,6 +18,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response import litellm from litellm._logging import verbose_proxy_logger +from litellm.caching.in_memory_cache import InMemoryCache from litellm.models.skills import LiteLLM_SkillsTable from litellm.proxy.discovery_endpoints.agent_skills_archive import SkillArchive, build_skill_archive from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import ( @@ -27,6 +29,15 @@ from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import ( ) MAX_INDEXED_SKILLS: Final = 1000 +MAX_CACHED_ARCHIVES: Final = 128 +MAX_CACHED_ARCHIVE_BYTES: Final = 512 * 1024 +ARCHIVE_CACHE_TTL_SECONDS: Final = 3600 + +_ARCHIVE_CACHE: Final = InMemoryCache( + max_size_in_memory=MAX_CACHED_ARCHIVES, + default_ttl=ARCHIVE_CACHE_TTL_SECONDS, + max_size_per_item=MAX_CACHED_ARCHIVE_BYTES // 1024, +) _NON_SLUG_PATTERN: Final = re.compile(r"[^a-z0-9]+") _FALLBACK_SKILL_NAME: Final = "skill" @@ -34,6 +45,12 @@ _FALLBACK_SKILL_NAME: Final = "skill" router: Final = APIRouter(tags=["public", "skills"]) # mutable-ok: fastapi types tags as list[str | Enum] +class ZipArchiveResponse(Response): + """Response whose OpenAPI entry declares an application/zip download rather than JSON.""" + + media_type = "application/zip" + + def ensure_index_enabled() -> None: if litellm.public_skills_index is not True: raise HTTPException(status_code=404, detail="Not Found") @@ -72,11 +89,7 @@ async def agent_skills_index( """Agent Skills v0.2.0 discovery index over every skill stored on this proxy.""" from litellm.proxy.utils import get_custom_url - installable: Final = tuple( - (skill, archive) - for skill, archive in ((skill, _archive_for(skill)) for skill in reversed(skills)) - if archive is not None - ) + installable: Final = await _installable(skills) names: Final = _deduplicated(tuple(_base_name(skill, archive) for skill, archive in installable)) return AgentSkillsIndex( @@ -99,34 +112,50 @@ async def agent_skills_index( @router.get( "/v1/skills/{skill_id}/archive", dependencies=(Depends(ensure_index_enabled),), + response_class=ZipArchiveResponse, ) async def agent_skills_archive( skill_id: str, skill: LiteLLM_SkillsTable | None = Depends(stored_skill), -) -> Response: +) -> ZipArchiveResponse: """Stored skill upload, repacked so SKILL.md sits at the archive root.""" - archive: Final = _archive_for(skill) if skill is not None else None + archive: Final = await _archive_for(skill) if skill is not None else None if archive is None: raise HTTPException(status_code=404, detail=f"No installable skill archive for: {skill_id}") - return Response( + return ZipArchiveResponse( content=archive.content, - media_type="application/zip", headers=MappingProxyType({"Content-Disposition": f'attachment; filename="{skill_id}.zip"'}), ) -def _archive_for(skill: LiteLLM_SkillsTable) -> SkillArchive | None: +async def _installable( + skills: Sequence[LiteLLM_SkillsTable], +) -> tuple[tuple[LiteLLM_SkillsTable, SkillArchive], ...]: + built: Final = tuple([(skill, await _archive_for(skill)) for skill in reversed(skills)]) + return tuple((skill, archive) for skill, archive in built if archive is not None) + + +async def _archive_for(skill: LiteLLM_SkillsTable) -> SkillArchive | None: if skill.file_content is None: return None - archive: Final = build_skill_archive(skill.file_content) + cache_key: Final = None if skill.updated_at is None else f"{skill.skill_id}:{skill.updated_at.isoformat()}" + cached: Final = None if cache_key is None else _ARCHIVE_CACHE.get_cache(cache_key) + if isinstance(cached, SkillArchive): + return cached + + archive: Final = await asyncio.to_thread(build_skill_archive, skill.file_content) if archive is None: verbose_proxy_logger.warning( "Agent Skills index: skipping skill %s, its upload is not a zip holding SKILL.md at the root of a " "single top-level folder", skill.skill_id, ) + return None + + if cache_key is not None and len(archive.content) <= MAX_CACHED_ARCHIVE_BYTES: + _ARCHIVE_CACHE.set_cache(cache_key, archive) return archive diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py b/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py index 4fbb243a2b3..ea889e75ae8 100644 --- a/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py +++ b/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py @@ -1,6 +1,7 @@ import hashlib import io import zipfile +from datetime import datetime, timezone import pytest from fastapi import FastAPI @@ -42,12 +43,14 @@ def skill( display_title: str | None = "PDF Summarizer", description: str | None = None, files: dict[str, bytes] | None = None, + updated_at: datetime | None = None, ) -> LiteLLM_SkillsTable: return LiteLLM_SkillsTable( skill_id=skill_id, display_title=display_title, description=description, file_content=zip_bytes(files if files is not None else {"pdf-summarizer/SKILL.md": MANIFEST}), + updated_at=updated_at, ) @@ -162,3 +165,47 @@ def test_archive_route_404s_for_a_skill_that_does_not_exist(index_enabled): client = client_for(skill("litellm_skill_1")) assert client.get("/v1/skills/litellm_skill_missing/archive").status_code == 404 + + +def test_a_stored_skill_is_repacked_once_per_version(index_enabled): + stamp = datetime(2026, 9, 6, 9, 0, tzinfo=timezone.utc) + first = client_for(skill("litellm_skill_cached", files={"s/SKILL.md": MANIFEST}, updated_at=stamp)) + published = first.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["digest"] + + unchanged_row = client_for( + skill("litellm_skill_cached", files={"s/SKILL.md": MANIFEST, "s/extra.md": b"rewritten"}, updated_at=stamp) + ) + + assert unchanged_row.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["digest"] == published + assert hashlib.sha256(unchanged_row.get("/v1/skills/litellm_skill_cached/archive").content).hexdigest() == ( + published.removeprefix("sha256:") + ) + + +def test_a_skill_edited_since_the_last_read_is_republished(index_enabled): + stamp = datetime(2026, 9, 6, 9, 0, tzinfo=timezone.utc) + before = client_for(skill("litellm_skill_edited", files={"s/SKILL.md": MANIFEST}, updated_at=stamp)) + published = before.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["digest"] + + after = client_for( + skill( + "litellm_skill_edited", + files={"s/SKILL.md": MANIFEST, "s/extra.md": b"rewritten"}, + updated_at=datetime(2026, 9, 6, 10, 0, tzinfo=timezone.utc), + ) + ) + republished = after.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["digest"] + + assert republished != published + assert hashlib.sha256(after.get("/v1/skills/litellm_skill_edited/archive").content).hexdigest() == ( + republished.removeprefix("sha256:") + ) + + +def test_openapi_declares_the_archive_route_as_a_zip_download(index_enabled): + schema = client_for(skill("litellm_skill_1")).get("/openapi.json").json() + + content = schema["paths"]["/v1/skills/{skill_id}/archive"]["get"]["responses"]["200"]["content"] + + assert "application/zip" in content + assert "application/json" not in content diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 03496f9a507..137e3e31c08 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -64820,7 +64820,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/zip": string; }; }; /** @description Validation Error */