From 6d993b50932941525f3b20fecd72fd81219541b5 Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 6 Sep 2026 02:45:23 -0700 Subject: [PATCH 1/2] feat(proxy): serve registered skills as an Agent Skills well-known index Adds an opt-in Agent Skills discovery index at /.well-known/agent-skills/index.json (and the /.well-known/skills/index.json alias) plus GET /v1/skills/{skill_id}/archive, so `npx skills add ` installs skills uploaded through the Skills Gateway into any agent the CLI supports. The archive route repacks the stored upload so SKILL.md sits at the archive root, with fixed entry timestamps so the SHA-256 digest published in the index reproduces. Both routes are unauthenticated, since discovery clients send no credentials, and stay 404 until an admin sets `litellm_settings.public_skills_index: true`. --- litellm/__init__.py | 1 + litellm/proxy/_lazy_openapi_snapshot.json | 41 +++++ litellm/proxy/discovery_endpoints/__init__.py | 3 +- .../agent_skills_archive.py | 130 +++++++++++++ .../agent_skills_endpoints.py | 171 ++++++++++++++++++ litellm/proxy/proxy_server.py | 6 +- .../agent_skills_endpoints.py | 25 +++ .../test_agent_skills_archive.py | 106 +++++++++++ .../test_agent_skills_endpoints.py | 164 +++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 157 ++++++++++++++++ 10 files changed, 802 insertions(+), 2 deletions(-) create mode 100644 litellm/proxy/discovery_endpoints/agent_skills_archive.py create mode 100644 litellm/proxy/discovery_endpoints/agent_skills_endpoints.py create mode 100644 litellm/types/proxy/discovery_endpoints/agent_skills_endpoints.py create mode 100644 tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_archive.py create mode 100644 tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 42c0ea881fd..33840c66b7c 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -492,6 +492,7 @@ disable_copilot_system_to_assistant: bool = False # If false (default), convert public_mcp_servers: Optional[List[str]] = None public_mcp_hub_strict_whitelist: bool = True public_model_groups: Optional[List[str]] = None +public_skills_index: bool = False public_agent_groups: Optional[List[str]] = None agent_search_embedding_model: Optional[str] = None mcp_tool_search: Optional[Mapping[str, object]] = None diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 91a97ad6544..1e53049a887 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -5065,6 +5065,47 @@ "anthropic_skills" ] } + }, + "/v1/skills/{skill_id}/archive": { + "get": { + "description": "Stored skill upload, repacked so SKILL.md sits at the archive root.", + "operationId": "agent_skills_archive_v1_skills__skill_id__archive_get", + "parameters": [ + { + "in": "path", + "name": "skill_id", + "required": true, + "schema": { + "title": "Skill Id", + "type": "string" + } + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/HTTPValidationError" + } + } + }, + "description": "Validation Error" + } + }, + "summary": "Agent Skills Archive", + "tags": [ + "anthropic_skills" + ] + } } } }, diff --git a/litellm/proxy/discovery_endpoints/__init__.py b/litellm/proxy/discovery_endpoints/__init__.py index a6401c2f1b4..52602f30b77 100644 --- a/litellm/proxy/discovery_endpoints/__init__.py +++ b/litellm/proxy/discovery_endpoints/__init__.py @@ -1,3 +1,4 @@ +from .agent_skills_endpoints import router as agent_skills_discovery_router from .ui_discovery_endpoints import router as ui_discovery_endpoints_router -__all__ = ["ui_discovery_endpoints_router"] +__all__ = ["agent_skills_discovery_router", "ui_discovery_endpoints_router"] diff --git a/litellm/proxy/discovery_endpoints/agent_skills_archive.py b/litellm/proxy/discovery_endpoints/agent_skills_archive.py new file mode 100644 index 00000000000..1f2fca3992e --- /dev/null +++ b/litellm/proxy/discovery_endpoints/agent_skills_archive.py @@ -0,0 +1,130 @@ +"""Repack a stored skill upload into the archive shape Agent Skills clients install from. + +Uploads follow the Anthropic Skills API layout, where every file sits under a single +top-level folder. Discovery clients read ``SKILL.md`` from the archive root, so that +folder is stripped and the zip is rebuilt with fixed entry timestamps, which keeps the +SHA-256 digest published in the index reproducible for identical uploads. +""" + +import hashlib +import io +import re +import zipfile +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +import yaml + +MAX_ARCHIVE_UNPACKED_BYTES: Final = 50 * 1024 * 1024 +MAX_ARCHIVE_ENTRIES: Final = 1000 +SKILL_MANIFEST_FILENAME: Final = "SKILL.md" + +_ZIP_ENTRY_TIMESTAMP: Final = (1980, 1, 1, 0, 0, 0) +_ZIP_ENTRY_PERMISSIONS: Final = 0o644 << 16 +_FRONTMATTER_PATTERN: Final = re.compile(r"^---\s*\n(.*?)\n---\s*(?:\n|$)", re.DOTALL) +_WINDOWS_DRIVE_PATTERN: Final = re.compile(r"^[A-Za-z]:") +_EMPTY_FRONTMATTER: Final[Mapping[str, object]] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class SkillArchive: + content: bytes + digest: str + declared_name: str | None + declared_description: str | None + + +def build_skill_archive(stored_content: bytes) -> SkillArchive | None: + """Return the installable archive for an upload, or None when it holds no root SKILL.md.""" + try: + with zipfile.ZipFile(io.BytesIO(stored_content)) as uploaded: + members: Final = _flattened_members(uploaded) + except (zipfile.BadZipFile, OSError, RuntimeError): + return None + + if members is None: + return None + + frontmatter: Final = _manifest_frontmatter(next(data for name, data in members if name == SKILL_MANIFEST_FILENAME)) + content: Final = _repack(members) + return SkillArchive( + content=content, + digest=f"sha256:{hashlib.sha256(content).hexdigest()}", + declared_name=_frontmatter_text(frontmatter, "name"), + declared_description=_frontmatter_text(frontmatter, "description"), + ) + + +def _flattened_members(uploaded: zipfile.ZipFile) -> tuple[tuple[str, bytes], ...] | None: + infos: Final = tuple(info for info in uploaded.infolist() if not info.is_dir()) + if not infos or len(infos) > MAX_ARCHIVE_ENTRIES: + return None + if sum(info.file_size for info in infos) > MAX_ARCHIVE_UNPACKED_BYTES: + return None + + normalized: Final = tuple((info, _normalized_path(info.filename)) for info in infos) + if any(path is None for _, path in normalized): + return None + + prefix: Final = _common_root_prefix(tuple(path for _, path in normalized if path is not None)) + flattened: Final = tuple((info, path[len(prefix) :]) for info, path in normalized if path is not None) + names: Final = frozenset(name for _, name in flattened) + if SKILL_MANIFEST_FILENAME not in names or len(names) != len(flattened): + return None + + return tuple((name, uploaded.read(info)) for info, name in sorted(flattened, key=lambda member: member[1])) + + +def _common_root_prefix(paths: tuple[str, ...]) -> str: + roots: Final = frozenset(path.split("/", 1)[0] for path in paths) + if len(roots) != 1 or not all("/" in path for path in paths): + return "" + return f"{next(iter(roots))}/" + + +def _normalized_path(raw_path: str) -> str | None: + if not raw_path or "\0" in raw_path or "\\" in raw_path: + return None + if raw_path.startswith("/") or _WINDOWS_DRIVE_PATTERN.match(raw_path): + return None + parts: Final = tuple(part for part in raw_path.split("/") if part) + if not parts or any(part in (".", "..") for part in parts): + return None + return "/".join(parts) + + +def _manifest_frontmatter(manifest: bytes) -> Mapping[str, object]: + match: Final = _FRONTMATTER_PATTERN.match(manifest.decode("utf-8", errors="replace")) + if match is None: + return _EMPTY_FRONTMATTER + try: + parsed: Final = yaml.safe_load(match.group(1)) + except yaml.YAMLError: + return _EMPTY_FRONTMATTER + if not isinstance(parsed, dict): + return _EMPTY_FRONTMATTER + return parsed + + +def _frontmatter_text(frontmatter: Mapping[str, object], key: str) -> str | None: + value: Final = frontmatter.get(key) + if not isinstance(value, str): + return None + return value.strip() or None + + +def _zip_entry(name: str) -> zipfile.ZipInfo: + entry: Final = zipfile.ZipInfo(filename=name, date_time=_ZIP_ENTRY_TIMESTAMP) + entry.compress_type = zipfile.ZIP_DEFLATED + entry.external_attr = _ZIP_ENTRY_PERMISSIONS + return entry + + +def _repack(members: tuple[tuple[str, bytes], ...]) -> bytes: + buffer: Final = io.BytesIO() + with zipfile.ZipFile(buffer, mode="w", compression=zipfile.ZIP_DEFLATED) as repacked: + for name, data in members: + repacked.writestr(_zip_entry(name), data) + return buffer.getvalue() diff --git a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py new file mode 100644 index 00000000000..d3c3b62fbfa --- /dev/null +++ b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py @@ -0,0 +1,171 @@ +"""Serve skills stored on the proxy as an Agent Skills well-known discovery index. + +``npx skills add -a `` reads ``/.well-known/agent-skills/index.json`` +and downloads each entry's archive. Discovery clients send no credentials, so both +routes are unauthenticated and stay off until ``litellm_settings.public_skills_index`` +is enabled, which publishes every stored skill to anyone who can reach the proxy. +""" + +import re +from collections.abc import Sequence +from itertools import groupby +from operator import itemgetter +from types import MappingProxyType +from typing import Final + +from fastapi import APIRouter, Depends, HTTPException, Request, Response + +import litellm +from litellm._logging import verbose_proxy_logger +from litellm.models.skills import LiteLLM_SkillsTable +from litellm.proxy.discovery_endpoints.agent_skills_archive import SkillArchive, build_skill_archive +from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import ( + MAX_SKILL_DESCRIPTION_LENGTH, + MAX_SKILL_NAME_LENGTH, + AgentSkillsIndex, + AgentSkillsIndexEntry, +) + +MAX_INDEXED_SKILLS: Final = 1000 + +_NON_SLUG_PATTERN: Final = re.compile(r"[^a-z0-9]+") +_FALLBACK_SKILL_NAME: Final = "skill" + +router: Final = APIRouter(tags=["public", "skills"]) # mutable-ok: fastapi types tags as list[str | Enum] + + +def ensure_index_enabled() -> None: + if litellm.public_skills_index is not True: + raise HTTPException(status_code=404, detail="Not Found") + + +async def stored_skills() -> Sequence[LiteLLM_SkillsTable]: + from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler + + return await LiteLLMSkillsHandler.list_skills(limit=MAX_INDEXED_SKILLS) + + +async def stored_skill(skill_id: str) -> LiteLLM_SkillsTable | None: + from litellm.llms.litellm_proxy.skills.handler import LiteLLMSkillsHandler + + try: + return await LiteLLMSkillsHandler.get_skill(skill_id) + except ValueError: + return None + + +@router.get( + "/.well-known/agent-skills/index.json", + response_model=AgentSkillsIndex, + dependencies=(Depends(ensure_index_enabled),), +) +@router.get( + "/.well-known/skills/index.json", + response_model=AgentSkillsIndex, + dependencies=(Depends(ensure_index_enabled),), + include_in_schema=False, +) +async def agent_skills_index( + request: Request, + skills: Sequence[LiteLLM_SkillsTable] = Depends(stored_skills), +) -> AgentSkillsIndex: + """Agent Skills v0.2.0 discovery index over every skill stored on this proxy.""" + from litellm.proxy.utils import get_custom_url + + installable: Final = tuple( + (skill, archive) + for skill, archive in ((skill, _archive_for(skill)) for skill in reversed(skills)) + if archive is not None + ) + names: Final = _deduplicated(tuple(_base_name(skill, archive) for skill, archive in installable)) + + return AgentSkillsIndex( + skills=tuple( + AgentSkillsIndexEntry( + name=name, + type="archive", + description=_description(skill, archive, name), + url=get_custom_url( + request_base_url=str(request.base_url), + route=f"v1/skills/{skill.skill_id}/archive", + ), + digest=archive.digest, + ) + for (skill, archive), name in zip(installable, names, strict=True) + ) + ) + + +@router.get( + "/v1/skills/{skill_id}/archive", + dependencies=(Depends(ensure_index_enabled),), +) +async def agent_skills_archive( + skill_id: str, + skill: LiteLLM_SkillsTable | None = Depends(stored_skill), +) -> Response: + """Stored skill upload, repacked so SKILL.md sits at the archive root.""" + archive: Final = _archive_for(skill) if skill is not None else None + if archive is None: + raise HTTPException(status_code=404, detail=f"No installable skill archive for: {skill_id}") + + return Response( + content=archive.content, + media_type="application/zip", + headers=MappingProxyType({"Content-Disposition": f'attachment; filename="{skill_id}.zip"'}), + ) + + +def _archive_for(skill: LiteLLM_SkillsTable) -> SkillArchive | None: + if skill.file_content is None: + return None + + archive: Final = build_skill_archive(skill.file_content) + if archive is None: + verbose_proxy_logger.warning( + "Agent Skills index: skipping skill %s, its upload is not a zip holding SKILL.md at the root of a " + "single top-level folder", + skill.skill_id, + ) + return archive + + +def _base_name(skill: LiteLLM_SkillsTable, archive: SkillArchive) -> str: + candidates: Final = (archive.declared_name, skill.display_title, skill.skill_id) + return next( + (slug for slug in (_slugify(candidate) for candidate in candidates) if slug is not None), + _FALLBACK_SKILL_NAME, + ) + + +def _slugify(raw: str | None) -> str | None: + if raw is None: + return None + return _NON_SLUG_PATTERN.sub("-", raw.lower()).strip("-")[:MAX_SKILL_NAME_LENGTH].rstrip("-") or None + + +def _deduplicated(names: Sequence[str]) -> tuple[str, ...]: + ordinals: Final = MappingProxyType( + { + position: ordinal + for _, duplicates in groupby(sorted(enumerate(names), key=itemgetter(1)), key=itemgetter(1)) + for ordinal, (position, _) in enumerate(duplicates) + } + ) + return tuple(_with_ordinal(name, ordinals[position]) for position, name in enumerate(names)) + + +def _with_ordinal(name: str, ordinal: int) -> str: + if ordinal == 0: + return name + suffix: Final = f"-{ordinal + 1}" + return f"{name[: MAX_SKILL_NAME_LENGTH - len(suffix)].rstrip('-')}{suffix}" + + +def _description(skill: LiteLLM_SkillsTable, archive: SkillArchive, name: str) -> str: + candidates: Final = (archive.declared_description, skill.description, skill.display_title) + chosen: Final = next( + (candidate.strip() for candidate in candidates if candidate is not None and candidate.strip()), + name, + ) + return chosen[:MAX_SKILL_DESCRIPTION_LENGTH] diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index ba5714fe950..4f0e5860b84 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -431,7 +431,10 @@ from litellm.proxy.db.proxy_worker_heartbeat import ( ProxyWorkerHeartbeat, ) from litellm.proxy.db.spend_counter_reseed import END_USER_COUNTER_PREFIX, SpendCounterReseed -from litellm.proxy.discovery_endpoints import ui_discovery_endpoints_router +from litellm.proxy.discovery_endpoints import ( + agent_skills_discovery_router, + ui_discovery_endpoints_router, +) from litellm.proxy.fine_tuning_endpoints.endpoints import router as fine_tuning_router from litellm.proxy.fine_tuning_endpoints.endpoints import set_fine_tuning_config from litellm.proxy.google_endpoints.endpoints import router as google_router @@ -18370,6 +18373,7 @@ app.include_router(user_agent_analytics_router) app.include_router(gateway_request_router) app.include_router(enterprise_router) app.include_router(ui_discovery_endpoints_router) +app.include_router(agent_skills_discovery_router) # Eager: /models/{name}:method overlaps with the OpenAI /models endpoint. app.include_router(google_router) diff --git a/litellm/types/proxy/discovery_endpoints/agent_skills_endpoints.py b/litellm/types/proxy/discovery_endpoints/agent_skills_endpoints.py new file mode 100644 index 00000000000..0d8bb29e172 --- /dev/null +++ b/litellm/types/proxy/discovery_endpoints/agent_skills_endpoints.py @@ -0,0 +1,25 @@ +"""Agent Skills discovery index, version 0.2.0. + +Schema: https://schemas.agentskills.io/discovery/0.2.0/schema.json +""" + +from typing import Final, Literal + +from pydantic import BaseModel, Field + +AGENT_SKILLS_DISCOVERY_SCHEMA_URL: Final = "https://schemas.agentskills.io/discovery/0.2.0/schema.json" +MAX_SKILL_NAME_LENGTH: Final = 64 +MAX_SKILL_DESCRIPTION_LENGTH: Final = 1024 + + +class AgentSkillsIndexEntry(BaseModel): + name: str + type: Literal["archive"] + description: str + url: str + digest: str + + +class AgentSkillsIndex(BaseModel): + discovery_schema: str = Field(default=AGENT_SKILLS_DISCOVERY_SCHEMA_URL, alias="$schema") + skills: tuple[AgentSkillsIndexEntry, ...] diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_archive.py b/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_archive.py new file mode 100644 index 00000000000..dd5ac230aac --- /dev/null +++ b/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_archive.py @@ -0,0 +1,106 @@ +import hashlib +import io +import zipfile + +from litellm.proxy.discovery_endpoints.agent_skills_archive import ( + MAX_ARCHIVE_ENTRIES, + build_skill_archive, +) + +MANIFEST = b"""--- +name: pdf-summarizer +description: Summarize a PDF into an executive brief. +--- + +Read the PDF, then write the brief. +""" + + +def zip_bytes(files: dict[str, bytes]) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in files.items(): + archive.writestr(name, content) + return buffer.getvalue() + + +def entries_of(content: bytes) -> dict[str, bytes]: + with zipfile.ZipFile(io.BytesIO(content)) as archive: + return {name: archive.read(name) for name in archive.namelist()} + + +def test_single_top_level_folder_is_stripped_so_skill_md_sits_at_the_root(): + archive = build_skill_archive( + zip_bytes( + { + "pdf-summarizer/SKILL.md": MANIFEST, + "pdf-summarizer/reference.md": b"page citations", + "pdf-summarizer/scripts/extract.py": b"print('hi')", + } + ) + ) + + assert archive is not None + assert entries_of(archive.content) == { + "SKILL.md": MANIFEST, + "reference.md": b"page citations", + "scripts/extract.py": b"print('hi')", + } + + +def test_digest_covers_the_repacked_bytes_and_is_stable_across_builds(): + upload = zip_bytes({"pdf-summarizer/SKILL.md": MANIFEST, "pdf-summarizer/reference.md": b"page citations"}) + + first = build_skill_archive(upload) + second = build_skill_archive(upload) + + assert first is not None and second is not None + assert first.digest == f"sha256:{hashlib.sha256(first.content).hexdigest()}" + assert first.content == second.content + + +def test_an_upload_that_is_already_flat_keeps_every_file_where_it_is(): + archive = build_skill_archive(zip_bytes({"SKILL.md": MANIFEST, "reference.md": b"page citations"})) + + assert archive is not None + assert sorted(entries_of(archive.content)) == ["SKILL.md", "reference.md"] + + +def test_manifest_frontmatter_supplies_the_declared_name_and_description(): + archive = build_skill_archive(zip_bytes({"pdf-summarizer/SKILL.md": MANIFEST})) + + assert archive is not None + assert archive.declared_name == "pdf-summarizer" + assert archive.declared_description == "Summarize a PDF into an executive brief." + + +def test_a_manifest_without_frontmatter_declares_nothing(): + archive = build_skill_archive(zip_bytes({"pdf-summarizer/SKILL.md": b"just prose, no frontmatter"})) + + assert archive is not None + assert archive.declared_name is None + assert archive.declared_description is None + + +def test_a_manifest_buried_below_the_stripped_folder_is_not_installable(): + assert build_skill_archive(zip_bytes({"pdf-summarizer/nested/SKILL.md": MANIFEST})) is None + + +def test_an_upload_with_no_manifest_is_not_installable(): + assert build_skill_archive(zip_bytes({"pdf-summarizer/reference.md": b"page citations"})) is None + + +def test_a_non_zip_upload_is_not_installable(): + assert build_skill_archive(MANIFEST) is None + + +def test_a_path_traversal_entry_is_not_installable(): + assert build_skill_archive(zip_bytes({"SKILL.md": MANIFEST, "../escape.md": b"nope"})) is None + + +def test_an_upload_over_the_entry_cap_is_not_installable(): + files = {"pdf-summarizer/SKILL.md": MANIFEST} | { + f"pdf-summarizer/file-{index}.md": b"x" for index in range(MAX_ARCHIVE_ENTRIES) + } + + assert build_skill_archive(zip_bytes(files)) is None diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py b/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py new file mode 100644 index 00000000000..4fbb243a2b3 --- /dev/null +++ b/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py @@ -0,0 +1,164 @@ +import hashlib +import io +import zipfile + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +import litellm +from litellm.models.skills import LiteLLM_SkillsTable +from litellm.proxy.discovery_endpoints.agent_skills_endpoints import ( + router, + stored_skill, + stored_skills, +) +from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import ( + AGENT_SKILLS_DISCOVERY_SCHEMA_URL, +) + +WELL_KNOWN_PATHS = ("/.well-known/agent-skills/index.json", "/.well-known/skills/index.json") + +MANIFEST = b"""--- +name: pdf-summarizer +description: Summarize a PDF into an executive brief. +--- + +Read the PDF, then write the brief. +""" + + +def zip_bytes(files: dict[str, bytes]) -> bytes: + buffer = io.BytesIO() + with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: + for name, content in files.items(): + archive.writestr(name, content) + return buffer.getvalue() + + +def skill( + skill_id: str, + *, + display_title: str | None = "PDF Summarizer", + description: str | None = None, + files: dict[str, bytes] | None = None, +) -> LiteLLM_SkillsTable: + return LiteLLM_SkillsTable( + skill_id=skill_id, + display_title=display_title, + description=description, + file_content=zip_bytes(files if files is not None else {"pdf-summarizer/SKILL.md": MANIFEST}), + ) + + +def client_for(*skills: LiteLLM_SkillsTable) -> TestClient: + app = FastAPI() + app.include_router(router) + + def _skills() -> tuple[LiteLLM_SkillsTable, ...]: + return skills + + def _skill(skill_id: str) -> LiteLLM_SkillsTable | None: + return next((candidate for candidate in skills if candidate.skill_id == skill_id), None) + + app.dependency_overrides[stored_skills] = _skills + app.dependency_overrides[stored_skill] = _skill + return TestClient(app) + + +@pytest.fixture +def index_enabled(monkeypatch): + monkeypatch.setattr(litellm, "public_skills_index", True) + + +def test_discovery_is_absent_until_public_skills_index_is_enabled(monkeypatch): + monkeypatch.setattr(litellm, "public_skills_index", False) + client = client_for(skill("litellm_skill_1")) + + for path in WELL_KNOWN_PATHS: + assert client.get(path).status_code == 404 + assert client.get("/v1/skills/litellm_skill_1/archive").status_code == 404 + + +@pytest.mark.parametrize("path", WELL_KNOWN_PATHS) +def test_index_publishes_each_stored_skill_in_the_v0_2_0_shape(index_enabled, path): + client = client_for(skill("litellm_skill_1")) + + body = client.get(path).json() + + assert body["$schema"] == AGENT_SKILLS_DISCOVERY_SCHEMA_URL + assert len(body["skills"]) == 1 + entry = body["skills"][0] + assert entry["name"] == "pdf-summarizer" + assert entry["type"] == "archive" + assert entry["description"] == "Summarize a PDF into an executive brief." + assert entry["url"].endswith("/v1/skills/litellm_skill_1/archive") + assert entry["digest"].startswith("sha256:") + + +def test_index_digest_matches_the_bytes_the_archive_route_serves(index_enabled): + client = client_for(skill("litellm_skill_1")) + + entry = client.get(WELL_KNOWN_PATHS[0]).json()["skills"][0] + downloaded = client.get(entry["url"]) + + assert downloaded.status_code == 200 + assert downloaded.headers["content-type"] == "application/zip" + assert entry["digest"] == f"sha256:{hashlib.sha256(downloaded.content).hexdigest()}" + + +def test_install_name_falls_back_to_the_manifest_name_without_a_display_title(index_enabled): + client = client_for(skill("litellm_skill_1", display_title=None)) + + assert client.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["name"] == "pdf-summarizer" + + +@pytest.mark.parametrize( + "manifest, stored_description, expected", + [ + (MANIFEST, "registry copy", "Summarize a PDF into an executive brief."), + (b"no frontmatter here", "registry copy", "registry copy"), + (b"no frontmatter here", None, "PDF Summarizer"), + ], +) +def test_description_prefers_the_manifest_then_the_registry_then_the_title( + index_enabled, manifest, stored_description, expected +): + client = client_for( + skill( + "litellm_skill_1", + description=stored_description, + files={"pdf-summarizer/SKILL.md": manifest}, + ) + ) + + assert client.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["description"] == expected + + +def test_skills_sharing_a_title_get_distinct_install_names(index_enabled): + client = client_for( + skill("litellm_skill_2", files={"pdf-summarizer/SKILL.md": b"second"}), + skill("litellm_skill_1", files={"pdf-summarizer/SKILL.md": b"first"}), + ) + + names = [entry["name"] for entry in client.get(WELL_KNOWN_PATHS[0]).json()["skills"]] + + assert names == ["pdf-summarizer", "pdf-summarizer-2"] + + +def test_uploads_without_a_root_manifest_are_left_out_of_the_index(index_enabled): + client = client_for( + skill("litellm_skill_1"), + skill("litellm_skill_2", files={"pdf-summarizer/reference.md": b"no manifest"}), + ) + + body = client.get(WELL_KNOWN_PATHS[0]).json() + + assert [entry["url"].split("/")[-2] for entry in body["skills"]] == ["litellm_skill_1"] + assert client.get("/v1/skills/litellm_skill_2/archive").status_code == 404 + + +def test_archive_route_404s_for_a_skill_that_does_not_exist(index_enabled): + client = client_for(skill("litellm_skill_1")) + + assert client.get("/v1/skills/litellm_skill_missing/archive").status_code == 404 diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index b1534c19670..03496f9a507 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -21,6 +21,26 @@ export interface paths { patch?: never; trace?: never; }; + "/.well-known/agent-skills/index.json": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Agent Skills Index + * @description Agent Skills v0.2.0 discovery index over every skill stored on this proxy. + */ + get: operations["agent_skills_index__well_known_agent_skills_index_json_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/.well-known/jwks.json": { parameters: { query?: never; @@ -319,6 +339,26 @@ export interface paths { patch?: never; trace?: never; }; + "/.well-known/skills/index.json": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Agent Skills Index + * @description Agent Skills v0.2.0 discovery index over every skill stored on this proxy. + */ + get: operations["agent_skills_index__well_known_skills_index_json_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/a2a/{agent_id}": { parameters: { query?: never; @@ -19962,6 +20002,26 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/skills/{skill_id}/archive": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** + * Agent Skills Archive + * @description Stored skill upload, repacked so SKILL.md sits at the archive root. + */ + get: operations["agent_skills_archive_v1_skills__skill_id__archive_get"]; + put?: never; + post?: never; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/threads": { parameters: { query?: never; @@ -23162,6 +23222,32 @@ export interface components { /** Tags */ tags?: string[]; }; + /** AgentSkillsIndex */ + AgentSkillsIndex: { + /** + * $Schema + * @default https://schemas.agentskills.io/discovery/0.2.0/schema.json + */ + $schema: string; + /** Skills */ + skills: components["schemas"]["AgentSkillsIndexEntry"][]; + }; + /** AgentSkillsIndexEntry */ + AgentSkillsIndexEntry: { + /** Description */ + description: string; + /** Digest */ + digest: string; + /** Name */ + name: string; + /** + * Type + * @constant + */ + type: "archive"; + /** Url */ + url: string; + }; /** * AlertType * @description Enum for alert types and management event types @@ -39831,6 +39917,26 @@ export interface operations { }; }; }; + agent_skills_index__well_known_agent_skills_index_json_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AgentSkillsIndex"]; + }; + }; + }; + }; jwks_json__well_known_jwks_json_get: { parameters: { query?: never; @@ -40168,6 +40274,26 @@ export interface operations { }; }; }; + agent_skills_index__well_known_skills_index_json_get: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["AgentSkillsIndex"]; + }; + }; + }; + }; invoke_agent_a2a_a2a__agent_id__post: { parameters: { query?: never; @@ -64677,6 +64803,37 @@ export interface operations { }; }; }; + agent_skills_archive_v1_skills__skill_id__archive_get: { + parameters: { + query?: never; + header?: never; + path: { + skill_id: string; + }; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; create_threads_v1_threads_post: { parameters: { query?: never; From 9440dbac74a72865525dbfdc6ebf4a74f53e146e Mon Sep 17 00:00:00 2001 From: mateo-berri <277851410+mateo-berri@users.noreply.github.com> Date: Sun, 6 Sep 2026 03:20:37 -0700 Subject: [PATCH 2/2] perf: cache repacked Agent Skills archives per skill version The well-known index needs every stored upload repacked to publish its digest, and both routes are unauthenticated, so each request was rebuilding every archive on the event loop. With 21 stored skills the index took ~2s and /health/liveliness on the same worker went from 1ms to 1.7s under two concurrent index requests. Repacking now runs off the event loop and each result is cached per skill version, so a worker builds an archive once until the skill changes. The archive route also declares application/zip in OpenAPI rather than JSON. --- litellm/proxy/_lazy_openapi_snapshot.json | 6 ++- .../agent_skills_endpoints.py | 51 +++++++++++++++---- .../test_agent_skills_endpoints.py | 47 +++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 +- 4 files changed, 92 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 1e53049a887..94fa0b67caf 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -5084,8 +5084,10 @@ "responses": { "200": { "content": { - "application/json": { - "schema": {} + "application/zip": { + "schema": { + "type": "string" + } } }, "description": "Successful Response" diff --git a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py index d3c3b62fbfa..3084cbfd84f 100644 --- a/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py +++ b/litellm/proxy/discovery_endpoints/agent_skills_endpoints.py @@ -6,6 +6,7 @@ routes are unauthenticated and stay off until ``litellm_settings.public_skills_i is enabled, which publishes every stored skill to anyone who can reach the proxy. """ +import asyncio import re from collections.abc import Sequence from itertools import groupby @@ -17,6 +18,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response import litellm from litellm._logging import verbose_proxy_logger +from litellm.caching.in_memory_cache import InMemoryCache from litellm.models.skills import LiteLLM_SkillsTable from litellm.proxy.discovery_endpoints.agent_skills_archive import SkillArchive, build_skill_archive from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import ( @@ -27,6 +29,15 @@ from litellm.types.proxy.discovery_endpoints.agent_skills_endpoints import ( ) MAX_INDEXED_SKILLS: Final = 1000 +MAX_CACHED_ARCHIVES: Final = 128 +MAX_CACHED_ARCHIVE_BYTES: Final = 512 * 1024 +ARCHIVE_CACHE_TTL_SECONDS: Final = 3600 + +_ARCHIVE_CACHE: Final = InMemoryCache( + max_size_in_memory=MAX_CACHED_ARCHIVES, + default_ttl=ARCHIVE_CACHE_TTL_SECONDS, + max_size_per_item=MAX_CACHED_ARCHIVE_BYTES // 1024, +) _NON_SLUG_PATTERN: Final = re.compile(r"[^a-z0-9]+") _FALLBACK_SKILL_NAME: Final = "skill" @@ -34,6 +45,12 @@ _FALLBACK_SKILL_NAME: Final = "skill" router: Final = APIRouter(tags=["public", "skills"]) # mutable-ok: fastapi types tags as list[str | Enum] +class ZipArchiveResponse(Response): + """Response whose OpenAPI entry declares an application/zip download rather than JSON.""" + + media_type = "application/zip" + + def ensure_index_enabled() -> None: if litellm.public_skills_index is not True: raise HTTPException(status_code=404, detail="Not Found") @@ -72,11 +89,7 @@ async def agent_skills_index( """Agent Skills v0.2.0 discovery index over every skill stored on this proxy.""" from litellm.proxy.utils import get_custom_url - installable: Final = tuple( - (skill, archive) - for skill, archive in ((skill, _archive_for(skill)) for skill in reversed(skills)) - if archive is not None - ) + installable: Final = await _installable(skills) names: Final = _deduplicated(tuple(_base_name(skill, archive) for skill, archive in installable)) return AgentSkillsIndex( @@ -99,34 +112,50 @@ async def agent_skills_index( @router.get( "/v1/skills/{skill_id}/archive", dependencies=(Depends(ensure_index_enabled),), + response_class=ZipArchiveResponse, ) async def agent_skills_archive( skill_id: str, skill: LiteLLM_SkillsTable | None = Depends(stored_skill), -) -> Response: +) -> ZipArchiveResponse: """Stored skill upload, repacked so SKILL.md sits at the archive root.""" - archive: Final = _archive_for(skill) if skill is not None else None + archive: Final = await _archive_for(skill) if skill is not None else None if archive is None: raise HTTPException(status_code=404, detail=f"No installable skill archive for: {skill_id}") - return Response( + return ZipArchiveResponse( content=archive.content, - media_type="application/zip", headers=MappingProxyType({"Content-Disposition": f'attachment; filename="{skill_id}.zip"'}), ) -def _archive_for(skill: LiteLLM_SkillsTable) -> SkillArchive | None: +async def _installable( + skills: Sequence[LiteLLM_SkillsTable], +) -> tuple[tuple[LiteLLM_SkillsTable, SkillArchive], ...]: + built: Final = tuple([(skill, await _archive_for(skill)) for skill in reversed(skills)]) + return tuple((skill, archive) for skill, archive in built if archive is not None) + + +async def _archive_for(skill: LiteLLM_SkillsTable) -> SkillArchive | None: if skill.file_content is None: return None - archive: Final = build_skill_archive(skill.file_content) + cache_key: Final = None if skill.updated_at is None else f"{skill.skill_id}:{skill.updated_at.isoformat()}" + cached: Final = None if cache_key is None else _ARCHIVE_CACHE.get_cache(cache_key) + if isinstance(cached, SkillArchive): + return cached + + archive: Final = await asyncio.to_thread(build_skill_archive, skill.file_content) if archive is None: verbose_proxy_logger.warning( "Agent Skills index: skipping skill %s, its upload is not a zip holding SKILL.md at the root of a " "single top-level folder", skill.skill_id, ) + return None + + if cache_key is not None and len(archive.content) <= MAX_CACHED_ARCHIVE_BYTES: + _ARCHIVE_CACHE.set_cache(cache_key, archive) return archive diff --git a/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py b/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py index 4fbb243a2b3..ea889e75ae8 100644 --- a/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py +++ b/tests/test_litellm/proxy/discovery_endpoints/test_agent_skills_endpoints.py @@ -1,6 +1,7 @@ import hashlib import io import zipfile +from datetime import datetime, timezone import pytest from fastapi import FastAPI @@ -42,12 +43,14 @@ def skill( display_title: str | None = "PDF Summarizer", description: str | None = None, files: dict[str, bytes] | None = None, + updated_at: datetime | None = None, ) -> LiteLLM_SkillsTable: return LiteLLM_SkillsTable( skill_id=skill_id, display_title=display_title, description=description, file_content=zip_bytes(files if files is not None else {"pdf-summarizer/SKILL.md": MANIFEST}), + updated_at=updated_at, ) @@ -162,3 +165,47 @@ def test_archive_route_404s_for_a_skill_that_does_not_exist(index_enabled): client = client_for(skill("litellm_skill_1")) assert client.get("/v1/skills/litellm_skill_missing/archive").status_code == 404 + + +def test_a_stored_skill_is_repacked_once_per_version(index_enabled): + stamp = datetime(2026, 9, 6, 9, 0, tzinfo=timezone.utc) + first = client_for(skill("litellm_skill_cached", files={"s/SKILL.md": MANIFEST}, updated_at=stamp)) + published = first.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["digest"] + + unchanged_row = client_for( + skill("litellm_skill_cached", files={"s/SKILL.md": MANIFEST, "s/extra.md": b"rewritten"}, updated_at=stamp) + ) + + assert unchanged_row.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["digest"] == published + assert hashlib.sha256(unchanged_row.get("/v1/skills/litellm_skill_cached/archive").content).hexdigest() == ( + published.removeprefix("sha256:") + ) + + +def test_a_skill_edited_since_the_last_read_is_republished(index_enabled): + stamp = datetime(2026, 9, 6, 9, 0, tzinfo=timezone.utc) + before = client_for(skill("litellm_skill_edited", files={"s/SKILL.md": MANIFEST}, updated_at=stamp)) + published = before.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["digest"] + + after = client_for( + skill( + "litellm_skill_edited", + files={"s/SKILL.md": MANIFEST, "s/extra.md": b"rewritten"}, + updated_at=datetime(2026, 9, 6, 10, 0, tzinfo=timezone.utc), + ) + ) + republished = after.get(WELL_KNOWN_PATHS[0]).json()["skills"][0]["digest"] + + assert republished != published + assert hashlib.sha256(after.get("/v1/skills/litellm_skill_edited/archive").content).hexdigest() == ( + republished.removeprefix("sha256:") + ) + + +def test_openapi_declares_the_archive_route_as_a_zip_download(index_enabled): + schema = client_for(skill("litellm_skill_1")).get("/openapi.json").json() + + content = schema["paths"]["/v1/skills/{skill_id}/archive"]["get"]["responses"]["200"]["content"] + + assert "application/zip" in content + assert "application/json" not in content diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 03496f9a507..137e3e31c08 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -64820,7 +64820,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/zip": string; }; }; /** @description Validation Error */