diff --git a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace_sync.py b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace_sync.py index 00bfe29dd96..b788738626b 100644 --- a/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace_sync.py +++ b/litellm/proxy/anthropic_endpoints/claude_code_endpoints/claude_code_marketplace_sync.py @@ -141,18 +141,26 @@ def _parse_source_ref(raw: str) -> ResolvedSource: return ResolvedSource(host="url", repo_or_url=stripped) -def _build_manifest_url(resolved: ResolvedSource, branch: str) -> str: +def _build_raw_file_url(resolved: ResolvedSource, branch: str, path: str) -> str: match resolved.host: case "github": - return f"https://raw.githubusercontent.com/{resolved.repo_or_url}/{branch}/.claude-plugin/marketplace.json" + return f"https://raw.githubusercontent.com/{resolved.repo_or_url}/{branch}/{path}" case "gitlab": - return f"https://gitlab.com/{resolved.repo_or_url}/-/raw/{branch}/.claude-plugin/marketplace.json" + return f"https://gitlab.com/{resolved.repo_or_url}/-/raw/{branch}/{path}" case "bitbucket": - return f"https://bitbucket.org/{resolved.repo_or_url}/raw/{branch}/.claude-plugin/marketplace.json" + return f"https://bitbucket.org/{resolved.repo_or_url}/raw/{branch}/{path}" case "url": return resolved.repo_or_url +def _build_manifest_url(resolved: ResolvedSource, branch: str) -> str: + return _build_raw_file_url(resolved, branch, ".claude-plugin/marketplace.json") + + +def _build_plugin_json_url(resolved: ResolvedSource, branch: str) -> str: + return _build_raw_file_url(resolved, branch, ".claude-plugin/plugin.json") + + def _git_clone_url(resolved: ResolvedSource) -> str: match resolved.host: case "github": @@ -217,6 +225,24 @@ def _parse_marketplace_manifest(response: httpx.Response) -> _ExternalMarketplac raise MarketplaceSyncError(reason="invalid_schema", detail=str(exc)) from exc +class _ExternalPluginManifest(BaseModel): + """Shape of a single-plugin ``.claude-plugin/plugin.json`` (as opposed to + a multi-plugin ``.claude-plugin/marketplace.json``): one plugin's own + metadata, with an explicit list of its skills' SKILL.md paths.""" + + name: str + description: str | None = None + skills: list[str] = Field(default_factory=list) + + +def _parse_plugin_manifest(response: httpx.Response) -> _ExternalPluginManifest: + body = _parse_json_body(response) + try: + return _ExternalPluginManifest.model_validate(body) + except ValidationError as exc: + raise MarketplaceSyncError(reason="invalid_schema", detail=str(exc)) from exc + + def _normalize_relative_path(raw: str) -> str | None: """Normalize a marketplace.json ``source`` relative path, rejecting traversal. @@ -416,6 +442,52 @@ async def _discover_github_skill_docs( return tuple(doc for docs in grouped for doc in docs) +async def _check_root_skill_doc( + client: AsyncHTTPHandler, repo: str, branch: str, entry: _GithubContentsEntry, *, timeout: float +) -> _DiscoveredSkillDoc | None: + if await _skill_md_exists(client, repo, branch, entry.name, timeout=timeout): + return _DiscoveredSkillDoc(skill_md_path=f"{entry.name}/SKILL.md", plugin_source_path=entry.name) + return None + + +async def _discover_root_skill_docs( + client: AsyncHTTPHandler, repo: str, branch: str, *, timeout: float +) -> tuple[_DiscoveredSkillDoc, ...]: + """Fallback for repos with no ``skills/`` wrapper folder: each top-level + directory that itself contains a ``SKILL.md`` is one skill (e.g. + ``investigate/SKILL.md`` sitting directly at repo root). No further + nesting - unlike the ``skills/`` convention, most top-level directories + in a repo like this are NOT skills at all, so this only checks one level + deep rather than also recursing into non-matching directories.""" + top_level = await _list_github_contents(client, repo, "", branch, timeout=timeout) + docs = await asyncio.gather( + *( + _check_root_skill_doc(client, repo, branch, entry, timeout=timeout) + for entry in top_level + if entry.type == "dir" + ) + ) + return tuple(doc for doc in docs if doc is not None) + + +def _plugin_manifest_skill_doc(raw_path: str) -> _DiscoveredSkillDoc | None: + normalized = _normalize_relative_path(raw_path) + if not normalized or not normalized.endswith("/SKILL.md"): + return None + return _DiscoveredSkillDoc( + skill_md_path=normalized, + plugin_source_path=normalized.removesuffix("/SKILL.md"), + ) + + +def _plugin_manifest_skill_docs(manifest: _ExternalPluginManifest) -> tuple[_DiscoveredSkillDoc, ...]: + """Turn a plugin.json's explicit ``skills: ["./guides/x/SKILL.md", ...]`` + path list into discovered docs, reusing the same traversal-safe path + normalization as marketplace.json's relative ``source`` field.""" + resolved = (_plugin_manifest_skill_doc(raw_path) for raw_path in manifest.skills) + return tuple(doc for doc in resolved if doc is not None) + + async def _fetch_skill_entry( client: AsyncHTTPHandler, repo: str, @@ -444,6 +516,54 @@ async def _fetch_skill_entry( # --- top-level fetch orchestration ----------------------------------------- +async def _fetch_entries_for_docs( + client: AsyncHTTPHandler, + repo: str, + branch: str, + marketplace_name: str, + docs: tuple[_DiscoveredSkillDoc, ...], + *, + timeout: float, +) -> tuple[ResolvedPluginEntry, ...]: + fetched = await asyncio.gather( + *( + _fetch_skill_entry(client, repo, branch, marketplace_name, doc, timeout=timeout) + for doc in docs + ) + ) + return tuple(entry for entry in fetched if entry is not None) + + +async def _fetch_github_fallback_entries( + client: AsyncHTTPHandler, resolved: ResolvedSource, branch: str, marketplace_name: str +) -> tuple[tuple[ResolvedPluginEntry, ...], MarketplaceSourceType]: + """Called once the repo has no ``.claude-plugin/marketplace.json``. Tries, + in order: an explicit single-plugin ``.claude-plugin/plugin.json`` skill + list, the ``skills/*/SKILL.md`` (or one level of category nesting) + convention, then a last-resort scan of top-level repo directories for a + directly-nested ``SKILL.md`` (e.g. ``investigate/SKILL.md`` with no + ``skills/`` wrapper at all).""" + repo, timeout = resolved.repo_or_url, DEFAULT_SYNC_TIMEOUT_SECONDS + + plugin_json_url = _build_plugin_json_url(resolved, branch) + plugin_json_response = await _http_get(client, plugin_json_url, timeout=timeout) + if plugin_json_response.status_code == 200: + plugin_manifest = _parse_plugin_manifest(plugin_json_response) + plugin_docs = _plugin_manifest_skill_docs(plugin_manifest) + if plugin_docs: + entries = await _fetch_entries_for_docs( + client, repo, branch, marketplace_name, plugin_docs, timeout=timeout + ) + if entries: + return entries, "claude_plugin_json" + + skills_dir_docs = await _discover_github_skill_docs(client, repo, branch, timeout=timeout) + if not skills_dir_docs: + skills_dir_docs = await _discover_root_skill_docs(client, repo, branch, timeout=timeout) + entries = await _fetch_entries_for_docs(client, repo, branch, marketplace_name, skills_dir_docs, timeout=timeout) + return entries, "skills_dir" + + async def _fetch_marketplace_entries( marketplace_row: MarketplaceRow, ) -> tuple[tuple[ResolvedPluginEntry, ...], MarketplaceSourceType]: @@ -466,24 +586,7 @@ async def _fetch_marketplace_entries( return entries, "claude_marketplace_json" if manifest_response.status_code == 404 and resolved.host == "github": - docs = await _discover_github_skill_docs( - client, resolved.repo_or_url, branch, timeout=DEFAULT_SYNC_TIMEOUT_SECONDS - ) - fetched = await asyncio.gather( - *( - _fetch_skill_entry( - client, - resolved.repo_or_url, - branch, - marketplace_row.name, - doc, - timeout=DEFAULT_SYNC_TIMEOUT_SECONDS, - ) - for doc in docs - ) - ) - entries = tuple(entry for entry in fetched if entry is not None) - return entries, "skills_dir" + return await _fetch_github_fallback_entries(client, resolved, branch, marketplace_row.name) if manifest_response.status_code == 404: raise MarketplaceSyncError( diff --git a/litellm/types/proxy/claude_code_endpoints.py b/litellm/types/proxy/claude_code_endpoints.py index 1b2fb4486f6..d79fe63c647 100644 --- a/litellm/types/proxy/claude_code_endpoints.py +++ b/litellm/types/proxy/claude_code_endpoints.py @@ -127,7 +127,7 @@ class MarketplaceResponse(BaseModel): # --- Multi-marketplace import (LiteLLM_SkillMarketplaceTable) --- -MarketplaceSourceType = Literal["claude_marketplace_json", "skills_dir", "managed"] +MarketplaceSourceType = Literal["claude_marketplace_json", "claude_plugin_json", "skills_dir", "managed"] class GithubSource(BaseModel): diff --git a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace_sync.py b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace_sync.py index 4c465000296..f217d5e2431 100644 --- a/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace_sync.py +++ b/tests/test_litellm/proxy/anthropic_endpoints/test_claude_code_marketplace_sync.py @@ -225,12 +225,15 @@ async def test_resolve_and_sync_falls_back_to_skills_directory_scan(monkeypatch) marketplace = await _create_marketplace(client, name="vercel-skills", source_ref="vercel-labs/skills") manifest_url = "https://raw.githubusercontent.com/vercel-labs/skills/main/.claude-plugin/marketplace.json" + plugin_json_url = "https://raw.githubusercontent.com/vercel-labs/skills/main/.claude-plugin/plugin.json" contents_url = "https://api.github.com/repos/vercel-labs/skills/contents/skills?ref=main" skill_md_url = "https://raw.githubusercontent.com/vercel-labs/skills/main/skills/find-skills/SKILL.md" async def _get(http_client, url, **kwargs): if url == manifest_url: return httpx.Response(404) + if url == plugin_json_url: + return httpx.Response(404) if url == contents_url: return httpx.Response( 200, @@ -260,6 +263,108 @@ async def test_resolve_and_sync_falls_back_to_skills_directory_scan(monkeypatch) } +@pytest.mark.asyncio +async def test_resolve_and_sync_reads_plugin_json_explicit_skill_list(monkeypatch): + """Regression test: a repo with no marketplace.json but a single-plugin + ``.claude-plugin/plugin.json`` (explicit ``skills: [...]`` path list, e.g. + inference-sh/skills) must resolve every listed SKILL.md - not fall through + to the skills/ directory-scan convention, which this repo doesn't use.""" + client = _make_fake_prisma_client() + marketplace = await _create_marketplace(client, name="inference-sh", source_ref="inference-sh/skills") + + manifest_url = "https://raw.githubusercontent.com/inference-sh/skills/main/.claude-plugin/marketplace.json" + plugin_json_url = "https://raw.githubusercontent.com/inference-sh/skills/main/.claude-plugin/plugin.json" + skill_md_url = "https://raw.githubusercontent.com/inference-sh/skills/main/guides/prompt-engineering/SKILL.md" + + async def _get(http_client, url, **kwargs): + if url == manifest_url: + return httpx.Response(404) + if url == plugin_json_url: + return httpx.Response( + 200, + json={ + "name": "inference-sh", + "description": "AI agent skills via inference.sh", + "skills": ["./guides/prompt-engineering/SKILL.md"], + }, + ) + if url == skill_md_url: + return httpx.Response( + 200, + text="---\nname: prompt-engineering\ndescription: Write better prompts.\n---\nBody.", + ) + raise AssertionError(f"unexpected url requested: {url}") + + monkeypatch.setattr(sync_module, "async_safe_get", _get) + + result = await resolve_and_sync(client, marketplace) + + assert result.status == "success" + assert result.plugin_count == 1 + + plugins = await client.db.litellm_claudecodeplugintable.find_many(where={"marketplace_id": marketplace.id}) + assert len(plugins) == 1 + assert plugins[0].name == "inference-sh--prompt-engineering" + source = json.loads(plugins[0].manifest_json)["source"] + assert source == { + "source": "git-subdir", + "url": "https://github.com/inference-sh/skills.git", + "path": "guides/prompt-engineering", + } + + +@pytest.mark.asyncio +async def test_resolve_and_sync_falls_back_to_root_directory_scan(monkeypatch): + """Regression test: a repo with no marketplace.json, no plugin.json, and no + skills/ folder at all (e.g. gstack, whose skills sit directly at repo root + as /SKILL.md) must still be discovered via a root-level scan.""" + client = _make_fake_prisma_client() + marketplace = await _create_marketplace(client, name="gstack", source_ref="garrytan/gstack") + + manifest_url = "https://raw.githubusercontent.com/garrytan/gstack/main/.claude-plugin/marketplace.json" + plugin_json_url = "https://raw.githubusercontent.com/garrytan/gstack/main/.claude-plugin/plugin.json" + skills_contents_url = "https://api.github.com/repos/garrytan/gstack/contents/skills?ref=main" + root_contents_url = "https://api.github.com/repos/garrytan/gstack/contents/?ref=main" + investigate_skill_md_url = "https://raw.githubusercontent.com/garrytan/gstack/main/investigate/SKILL.md" + bin_skill_md_url = "https://raw.githubusercontent.com/garrytan/gstack/main/bin/SKILL.md" + + async def _get(http_client, url, **kwargs): + if url in (manifest_url, plugin_json_url, skills_contents_url, bin_skill_md_url): + return httpx.Response(404) + if url == root_contents_url: + return httpx.Response( + 200, + json=[ + {"name": "investigate", "path": "investigate", "type": "dir"}, + {"name": "bin", "path": "bin", "type": "dir"}, + {"name": "README.md", "path": "README.md", "type": "file"}, + ], + ) + if url == investigate_skill_md_url: + return httpx.Response( + 200, + text="---\nname: investigate\ndescription: Systematic debugging.\n---\nBody.", + ) + raise AssertionError(f"unexpected url requested: {url}") + + monkeypatch.setattr(sync_module, "async_safe_get", _get) + + result = await resolve_and_sync(client, marketplace) + + assert result.status == "success" + assert result.plugin_count == 1 + + plugins = await client.db.litellm_claudecodeplugintable.find_many(where={"marketplace_id": marketplace.id}) + assert len(plugins) == 1 + assert plugins[0].name == "gstack--investigate" + source = json.loads(plugins[0].manifest_json)["source"] + assert source == { + "source": "git-subdir", + "url": "https://github.com/garrytan/gstack.git", + "path": "investigate", + } + + @pytest.mark.asyncio async def test_resolve_and_sync_connection_failure_is_unreachable(monkeypatch): client = _make_fake_prisma_client()