""" Tests for Skills API operations across providers """ import os import zipfile from abc import ABC, abstractmethod from contextlib import contextmanager from pathlib import Path from typing import Optional import pytest import litellm from litellm.types.llms.anthropic_skills import ( DeleteSkillResponse, ListSkillsResponse, Skill, ) @contextmanager def create_skill_zip(skill_name: str, unique_suffix: Optional[str] = None): """ Helper context manager to create a zip file for a skill. Args: skill_name: Name of the skill directory in test_skills_data/ unique_suffix: Optional suffix to make the skill name unique in the zip. When provided, the SKILL.md frontmatter name is rewritten to avoid duplicate-name conflicts on the API side. Yields: File handle to the zip file The zip file is automatically cleaned up after use. """ import time test_dir = Path(__file__).parent / "test_skills_data" skill_dir = test_dir / skill_name # Create a zip file containing the skill directory # When unique_suffix is set, folder name must match skill name in SKILL.md (Anthropic requirement) zip_folder_name = f"{skill_name}-{unique_suffix}" if unique_suffix else skill_name zip_path = test_dir / f"{skill_name}.zip" with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf: if unique_suffix is not None: # Rewrite SKILL.md with a unique name and use matching folder name skill_md = (skill_dir / "SKILL.md").read_text() skill_md = skill_md.replace( f"name: {skill_name}", f"name: {zip_folder_name}", ) zf.writestr(f"{zip_folder_name}/SKILL.md", skill_md) # Add any other files in the skill dir (e.g. subdirs) under the new folder name for f in skill_dir.rglob("*"): if f.is_file() and f.name != "SKILL.md": rel = f.relative_to(skill_dir) zf.write(f, arcname=f"{zip_folder_name}/{rel}") else: zf.write(skill_dir, arcname=skill_name) zf.write(skill_dir / "SKILL.md", arcname=f"{skill_name}/SKILL.md") try: with open(zip_path, "rb") as f: yield f finally: # Clean up zip file if zip_path.exists(): zip_path.unlink() class BaseSkillsAPITest(ABC): """ Base test class for Skills API operations. Tests create, list, get, and delete operations. """ @abstractmethod def get_custom_llm_provider(self) -> str: """Return the provider name (e.g., 'anthropic')""" pass @abstractmethod def get_api_key(self) -> Optional[str]: """Return the API key for the provider""" pass @abstractmethod def get_api_base(self) -> Optional[str]: """Return the API base URL for the provider""" pass def test_create_skill(self): """ Test creating a skill. Note: This test creates a skill but does not clean it up, as we want to verify it was created successfully. The test_delete_skill test will handle cleanup. """ import time custom_llm_provider = self.get_custom_llm_provider() api_key = self.get_api_key() api_base = self.get_api_base() if not api_key: pytest.skip(f"No API key provided for {custom_llm_provider}") litellm.set_verbose = True litellm._turn_on_debug() # Use helper to create skill zip skill_name = "test-skill-litellm" # Use unique title and unique skill name to avoid conflicts # with previous test runs (skills are never cleaned up in CI) ts = str(int(time.time())) unique_title = f"Test Skill {ts}" # Upload the skill with the zip file with create_skill_zip(skill_name, unique_suffix=ts) as zip_file: response = litellm.create_skill( display_title=unique_title, files=[zip_file], custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base, ) assert response is not None assert isinstance(response, Skill) assert response.id is not None print(f"Created skill: {response}") print(f"Skill ID: {response.id}") def test_list_skills(self): """ Test listing skills. """ custom_llm_provider = self.get_custom_llm_provider() api_key = self.get_api_key() api_base = self.get_api_base() if not api_key: pytest.skip(f"No API key provided for {custom_llm_provider}") # Enable debug logging os.environ["LITELLM_LOG"] = "DEBUG" litellm.set_verbose = True print(f"\n=== Testing list_skills ===") print("API Key: [REDACTED]") print(f"API Base: {api_base}") response = litellm.list_skills( limit=10, custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base, ) assert response is not None assert isinstance(response, ListSkillsResponse) assert hasattr(response, "data") print(f"Listed skills: {response}") def test_get_skill(self): """ Test getting a specific skill by ID. """ custom_llm_provider = self.get_custom_llm_provider() api_key = self.get_api_key() api_base = self.get_api_base() if not api_key: pytest.skip(f"No API key provided for {custom_llm_provider}") litellm.set_verbose = True # First list existing skills to see if any exist list_response = litellm.list_skills( limit=1, custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base, ) # Type assertion for linter assert isinstance(list_response, ListSkillsResponse) print(f"List response: {list_response}") # If there are existing skills, use the first one if list_response.data and len(list_response.data) > 0: skill_id = list_response.data[0].id should_cleanup = False print(f"Using existing skill: {skill_id}") # Now get the skill response = litellm.get_skill( skill_id=skill_id, custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base, ) assert response is not None assert isinstance(response, Skill) assert response.id == skill_id print(f"GET - Retrieved skill: {response}") def test_delete_skill(self): """ Test deleting a skill. Note: Anthropic requires deleting all skill versions before deleting the skill itself. This test is currently skipped as it would require additional API calls to delete versions. """ import time custom_llm_provider = self.get_custom_llm_provider() api_key = self.get_api_key() api_base = self.get_api_base() if not api_key: pytest.skip(f"No API key provided for {custom_llm_provider}") pytest.skip( "Anthropic requires deleting all skill versions first - skipping for now" ) litellm.set_verbose = True # Use helper to create skill zip skill_name = "test-delete-skill" # Use unique title and skill name to avoid conflicts ts = str(int(time.time())) unique_title = f"Test Delete Skill {ts}" # Create a skill specifically to delete with create_skill_zip(skill_name, unique_suffix=ts) as zip_file: created_skill = litellm.create_skill( display_title=unique_title, files=[zip_file], custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base, ) # Type assertion for linter assert isinstance(created_skill, Skill) skill_id = created_skill.id print(f"Created skill to delete: {skill_id}") # Now delete the skill response = litellm.delete_skill( skill_id=skill_id, custom_llm_provider=custom_llm_provider, api_key=api_key, api_base=api_base, ) assert response is not None assert isinstance(response, DeleteSkillResponse) assert response.type == "skill_deleted" print(f"Deleted skill response: {response}") # Live integration tests for the Anthropic Skills API are not run in CI because # the Skills API requires beta access (anthropic-beta: skills-2025-10-02) that # is not available on the standard API key used in CI. # # Transformation logic (URL construction, headers, request/response parsing) is # covered by unit tests in: # tests/test_litellm/test_anthropic_skills_transformation.py