diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md new file mode 100644 index 00000000000..12e8428caf0 --- /dev/null +++ b/litellm/proxy/client/cli/README.md @@ -0,0 +1,411 @@ +# LiteLLM Proxy CLI + +The LiteLLM Proxy CLI is a command-line tool for managing your LiteLLM proxy server. It provides commands for managing models, viewing server status, and interacting with the proxy server. + +## Installation + +```bash +pip install 'litellm[proxy]' +``` + +## Configuration + +The CLI can be configured using environment variables or command-line options: + +- `LITELLM_PROXY_URL`: Base URL of the LiteLLM proxy server (default: http://localhost:4000) +- `LITELLM_PROXY_API_KEY`: API key for authentication + +## Commands + +### Models Management + +The CLI provides several commands for managing models on your LiteLLM proxy server: + +#### List Models + +View all available models: + +```bash +litellm-proxy models list [--format table|json] +``` + +Options: + +- `--format`: Output format (table or json, default: table) + +#### Model Information + +Get detailed information about all models: + +```bash +litellm-proxy models info [options] +``` + +Options: + +- `--format`: Output format (table or json, default: table) +- `--columns`: Comma-separated list of columns to display. Valid columns: + - `public_model` + - `upstream_model` + - `credential_name` + - `created_at` + - `updated_at` + - `id` + - `input_cost` + - `output_cost` + +Default columns: `public_model`, `upstream_model`, `updated_at` + +#### Add Model + +Add a new model to the proxy: + +```bash +litellm-proxy models add [options] +``` + +Options: + +- `--param`, `-p`: Model parameters in key=value format (can be specified multiple times) +- `--info`, `-i`: Model info in key=value format (can be specified multiple times) + +Example: + +```bash +litellm-proxy models add gpt-4 -p api_key=sk-123 -p api_base=https://api.openai.com -i description="GPT-4 model" +``` + +#### Get Model Info + +Get information about a specific model: + +```bash +litellm-proxy models get [--id MODEL_ID] [--name MODEL_NAME] +``` + +Options: + +- `--id`: ID of the model to retrieve +- `--name`: Name of the model to retrieve + +#### Delete Model + +Delete a model from the proxy: + +```bash +litellm-proxy models delete +``` + +#### Update Model + +Update an existing model's configuration: + +```bash +litellm-proxy models update [options] +``` + +Options: + +- `--param`, `-p`: Model parameters in key=value format (can be specified multiple times) +- `--info`, `-i`: Model info in key=value format (can be specified multiple times) + +### Credentials Management + +The CLI provides commands for managing credentials on your LiteLLM proxy server: + +#### List Credentials + +View all available credentials: + +```bash +litellm-proxy credentials list [--format table|json] +``` + +Options: + +- `--format`: Output format (table or json, default: table) + +The table format displays: +- Credential Name +- Custom LLM Provider + +#### Create Credential + +Create a new credential: + +```bash +litellm-proxy credentials create --info --values +``` + +Options: + +- `--info`: JSON string containing credential info (e.g., custom_llm_provider) +- `--values`: JSON string containing credential values (e.g., api_key) + +Example: + +```bash +litellm-proxy credentials create azure-cred \ + --info '{"custom_llm_provider": "azure"}' \ + --values '{"api_key": "sk-123", "api_base": "https://example.azure.openai.com"}' +``` + +#### Get Credential + +Get information about a specific credential: + +```bash +litellm-proxy credentials get +``` + +#### Delete Credential + +Delete a credential: + +```bash +litellm-proxy credentials delete +``` + +### Keys Management + +The CLI provides commands for managing API keys on your LiteLLM proxy server: + +#### List Keys + +View all API keys: + +```bash +litellm-proxy keys list [--format table|json] [options] +``` + +Options: + +- `--format`: Output format (table or json, default: table) +- `--page`: Page number for pagination +- `--size`: Number of items per page +- `--user-id`: Filter keys by user ID +- `--team-id`: Filter keys by team ID +- `--organization-id`: Filter keys by organization ID +- `--key-hash`: Filter by specific key hash +- `--key-alias`: Filter by key alias +- `--return-full-object`: Return the full key object +- `--include-team-keys`: Include team keys in the response + +#### Generate Key + +Generate a new API key: + +```bash +litellm-proxy keys generate [options] +``` + +Options: + +- `--models`: Comma-separated list of allowed models +- `--aliases`: JSON string of model alias mappings +- `--spend`: Maximum spend limit for this key +- `--duration`: Duration for which the key is valid (e.g. '24h', '7d') +- `--key-alias`: Alias/name for the key +- `--team-id`: Team ID to associate the key with +- `--user-id`: User ID to associate the key with +- `--budget-id`: Budget ID to associate the key with +- `--config`: JSON string of additional configuration parameters + +Example: + +```bash +litellm-proxy keys generate --models gpt-4,gpt-3.5-turbo --spend 100 --duration 24h --key-alias my-key --team-id team123 +``` + +#### Delete Keys + +Delete API keys by key or alias: + +```bash +litellm-proxy keys delete [--keys ] [--key-aliases ] +``` + +Options: + +- `--keys`: Comma-separated list of API keys to delete +- `--key-aliases`: Comma-separated list of key aliases to delete + +Example: + +```bash +litellm-proxy keys delete --keys sk-key1,sk-key2 --key-aliases alias1,alias2 +``` + +#### Get Key Info + +Get information about a specific API key: + +```bash +litellm-proxy keys info --key +``` + +Options: + +- `--key`: The key hash to get information about + +Example: + +```bash +litellm-proxy keys info --key sk-key1 +``` + +### Chat Commands + +The CLI provides commands for interacting with chat models through your LiteLLM proxy server: + +#### Chat Completions + +Create a chat completion: + +```bash +litellm-proxy chat completions [options] +``` + +Arguments: +- `model`: The model to use (e.g., gpt-4, claude-2) + +Options: +- `--message`, `-m`: Messages in 'role:content' format. Can be specified multiple times to create a conversation. +- `--temperature`, `-t`: Sampling temperature between 0 and 2 +- `--top-p`: Nucleus sampling parameter between 0 and 1 +- `--n`: Number of completions to generate +- `--max-tokens`: Maximum number of tokens to generate +- `--presence-penalty`: Presence penalty between -2.0 and 2.0 +- `--frequency-penalty`: Frequency penalty between -2.0 and 2.0 +- `--user`: Unique identifier for the end user + +Examples: + +1. Simple completion: +```bash +litellm-proxy chat completions gpt-4 -m "user:Hello, how are you?" +``` + +2. Multi-message conversation: +```bash +litellm-proxy chat completions gpt-4 \ + -m "system:You are a helpful assistant" \ + -m "user:What's the capital of France?" \ + -m "assistant:The capital of France is Paris." \ + -m "user:What's its population?" +``` + +3. With generation parameters: +```bash +litellm-proxy chat completions gpt-4 \ + -m "user:Write a story" \ + --temperature 0.7 \ + --max-tokens 500 \ + --top-p 0.9 +``` + +### HTTP Commands + +The CLI provides commands for making direct HTTP requests to your LiteLLM proxy server: + +#### Make HTTP Request + +Make an HTTP request to any endpoint: + +```bash +litellm-proxy http request [options] +``` + +Arguments: +- `method`: HTTP method (GET, POST, PUT, DELETE, etc.) +- `uri`: URI path (will be appended to base_url) + +Options: +- `--data`, `-d`: Data to send in the request body (as JSON string) +- `--json`, `-j`: JSON data to send in the request body (as JSON string) +- `--header`, `-H`: HTTP headers in 'key:value' format. Can be specified multiple times. + +Examples: + +1. List models: +```bash +litellm-proxy http request GET /models +``` + +2. Create a chat completion: +```bash +litellm-proxy http request POST /chat/completions -j '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}' +``` + +3. Test connection with custom headers: +```bash +litellm-proxy http request GET /health/test_connection -H "X-Custom-Header:value" +``` + +## Environment Variables + +The CLI respects the following environment variables: + +- `LITELLM_PROXY_URL`: Base URL of the proxy server +- `LITELLM_PROXY_API_KEY`: API key for authentication + +## Examples + +1. List all models in table format: + +```bash +litellm-proxy models list +``` + +2. Add a new model with parameters: + +```bash +litellm-proxy models add gpt-4 -p api_key=sk-123 -p max_tokens=2048 +``` + +3. Get model information in JSON format: + +```bash +litellm-proxy models info --format json +``` + +4. Update model parameters: + +```bash +litellm-proxy models update model-123 -p temperature=0.7 -i description="Updated model" +``` + +5. List all credentials in table format: + +```bash +litellm-proxy credentials list +``` + +6. Create a new credential for Azure: + +```bash +litellm-proxy credentials create azure-prod \ + --info '{"custom_llm_provider": "azure"}' \ + --values '{"api_key": "sk-123", "api_base": "https://prod.azure.openai.com"}' +``` + +7. Make a custom HTTP request: + +```bash +litellm-proxy http request POST /chat/completions \ + -j '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}' \ + -H "X-Custom-Header:value" +``` + +## Error Handling + +The CLI will display appropriate error messages when: + +- The proxy server is not accessible +- Authentication fails +- Invalid parameters are provided +- The requested model or credential doesn't exist +- Invalid JSON is provided for credential creation +- Any other operation fails + +For detailed debugging, use the `--debug` flag with any command. diff --git a/litellm/proxy/client/cli/__init__.py b/litellm/proxy/client/cli/__init__.py new file mode 100644 index 00000000000..843a0095878 --- /dev/null +++ b/litellm/proxy/client/cli/__init__.py @@ -0,0 +1,5 @@ +"""CLI package for LiteLLM Proxy Client.""" + +from .main import cli + +__all__ = ["cli"] diff --git a/litellm/proxy/client/cli/commands/__init__.py b/litellm/proxy/client/cli/commands/__init__.py new file mode 100644 index 00000000000..e8309aef33a --- /dev/null +++ b/litellm/proxy/client/cli/commands/__init__.py @@ -0,0 +1 @@ +"""Command groups for the LiteLLM proxy CLI.""" diff --git a/litellm/proxy/client/cli/commands/chat.py b/litellm/proxy/client/cli/commands/chat.py new file mode 100644 index 00000000000..ea51f74652a --- /dev/null +++ b/litellm/proxy/client/cli/commands/chat.py @@ -0,0 +1,108 @@ +import json +from typing import Optional + +import click +import rich +import requests + +from ...chat import ChatClient + + +@click.group() +def chat(): + """Chat with models through the LiteLLM proxy server""" + pass + + +@chat.command() +@click.argument("model") +@click.option( + "--message", + "-m", + multiple=True, + help="Messages in 'role:content' format (e.g. 'user:Hello'). Can be specified multiple times.", +) +@click.option( + "--temperature", + "-t", + type=float, + help="Sampling temperature between 0 and 2", +) +@click.option( + "--top-p", + type=float, + help="Nucleus sampling parameter between 0 and 1", +) +@click.option( + "--n", + type=int, + help="Number of completions to generate", +) +@click.option( + "--max-tokens", + type=int, + help="Maximum number of tokens to generate", +) +@click.option( + "--presence-penalty", + type=float, + help="Presence penalty between -2.0 and 2.0", +) +@click.option( + "--frequency-penalty", + type=float, + help="Frequency penalty between -2.0 and 2.0", +) +@click.option( + "--user", + type=str, + help="Unique identifier for the end user", +) +@click.pass_context +def completions( + ctx: click.Context, + model: str, + message: tuple[str, ...], + temperature: Optional[float] = None, + top_p: Optional[float] = None, + n: Optional[int] = None, + max_tokens: Optional[int] = None, + presence_penalty: Optional[float] = None, + frequency_penalty: Optional[float] = None, + user: Optional[str] = None, +): + """Create a chat completion""" + if not message: + raise click.UsageError("At least one message is required") + + # Parse messages from role:content format + messages = [] + for msg in message: + try: + role, content = msg.split(":", 1) + messages.append({"role": role.strip(), "content": content.strip()}) + except ValueError: + raise click.BadParameter(f"Invalid message format: {msg}. Expected format: 'role:content'") + + client = ChatClient(ctx.obj["base_url"], ctx.obj["api_key"]) + try: + response = client.completions( + model=model, + messages=messages, + temperature=temperature, + top_p=top_p, + n=n, + max_tokens=max_tokens, + presence_penalty=presence_penalty, + frequency_penalty=frequency_penalty, + user=user, + ) + rich.print_json(data=response) + except requests.exceptions.HTTPError as e: + click.echo(f"Error: HTTP {e.response.status_code}", err=True) + try: + error_body = e.response.json() + rich.print_json(data=error_body) + except json.JSONDecodeError: + click.echo(e.response.text, err=True) + raise click.Abort() diff --git a/litellm/proxy/client/cli/commands/credentials.py b/litellm/proxy/client/cli/commands/credentials.py new file mode 100644 index 00000000000..44f4a112ce4 --- /dev/null +++ b/litellm/proxy/client/cli/commands/credentials.py @@ -0,0 +1,116 @@ +import json +from typing import Literal + +import click +import rich +import requests +from rich.table import Table + +from ...credentials import CredentialsManagementClient + + +@click.group() +def credentials(): + """Manage credentials for the LiteLLM proxy server""" + pass + + +@credentials.command() +@click.option( + "--format", + "output_format", + type=click.Choice(["table", "json"]), + default="table", + help="Output format (table or json)", +) +@click.pass_context +def list(ctx: click.Context, output_format: Literal["table", "json"]): + """List all credentials""" + client = CredentialsManagementClient(ctx.obj["base_url"], ctx.obj["api_key"]) + response = client.list() + assert isinstance(response, dict) + + if output_format == "json": + rich.print_json(data=response) + else: # table format + table = Table(title="Credentials") + + # Add columns + table.add_column("Credential Name", style="cyan") + table.add_column("Custom LLM Provider", style="green") + + # Add rows + for cred in response.get("credentials", []): + info = cred.get("credential_info", {}) + table.add_row( + str(cred.get("credential_name", "")), + str(info.get("custom_llm_provider", "")), + ) + + rich.print(table) + + +@credentials.command() +@click.argument("credential_name") +@click.option( + "--info", + type=str, + help="JSON string containing credential info", + required=True, +) +@click.option( + "--values", + type=str, + help="JSON string containing credential values", + required=True, +) +@click.pass_context +def create(ctx: click.Context, credential_name: str, info: str, values: str): + """Create a new credential""" + client = CredentialsManagementClient(ctx.obj["base_url"], ctx.obj["api_key"]) + try: + credential_info = json.loads(info) + credential_values = json.loads(values) + except json.JSONDecodeError as e: + raise click.BadParameter(f"Invalid JSON: {str(e)}") + + try: + response = client.create(credential_name, credential_info, credential_values) + rich.print_json(data=response) + except requests.exceptions.HTTPError as e: + click.echo(f"Error: HTTP {e.response.status_code}", err=True) + try: + error_body = e.response.json() + rich.print_json(data=error_body) + except json.JSONDecodeError: + click.echo(e.response.text, err=True) + raise click.Abort() + + +@credentials.command() +@click.argument("credential_name") +@click.pass_context +def delete(ctx: click.Context, credential_name: str): + """Delete a credential by name""" + client = CredentialsManagementClient(ctx.obj["base_url"], ctx.obj["api_key"]) + try: + response = client.delete(credential_name) + rich.print_json(data=response) + except requests.exceptions.HTTPError as e: + click.echo(f"Error: HTTP {e.response.status_code}", err=True) + try: + error_body = e.response.json() + rich.print_json(data=error_body) + except json.JSONDecodeError: + click.echo(e.response.text, err=True) + raise click.Abort() + + +@credentials.command() +@click.argument("credential_name") +@click.pass_context +def get(ctx: click.Context, credential_name: str): + """Get a credential by name""" + client = CredentialsManagementClient(ctx.obj["base_url"], ctx.obj["api_key"]) + response = client.get(credential_name) + rich.print_json(data=response) diff --git a/litellm/proxy/client/cli/commands/http.py b/litellm/proxy/client/cli/commands/http.py new file mode 100644 index 00000000000..dba36f9d92c --- /dev/null +++ b/litellm/proxy/client/cli/commands/http.py @@ -0,0 +1,100 @@ +import json as json_lib +from typing import Optional + +import click +import rich +import requests + +from ...http_client import HTTPClient + + +@click.group() +def http(): + """Make HTTP requests to the LiteLLM proxy server""" + pass + + +@http.command() +@click.argument("method") +@click.argument("uri") +@click.option( + "--data", + "-d", + type=str, + help="Data to send in the request body (as JSON string)", +) +@click.option( + "--json", + "-j", + type=str, + help="JSON data to send in the request body (as JSON string)", +) +@click.option( + "--header", + "-H", + multiple=True, + help="HTTP headers in 'key:value' format. Can be specified multiple times.", +) +@click.pass_context +def request( + ctx: click.Context, + method: str, + uri: str, + data: Optional[str] = None, + json: Optional[str] = None, + header: tuple[str, ...] = (), +): + """Make an HTTP request to the LiteLLM proxy server + + METHOD: HTTP method (GET, POST, PUT, DELETE, etc.) + URI: URI path (will be appended to base_url) + + Examples: + litellm http request GET /models + litellm http request POST /chat/completions -j '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}' + litellm http request GET /health/test_connection -H "X-Custom-Header:value" + """ + # Parse headers from key:value format + headers = {} + for h in header: + try: + key, value = h.split(":", 1) + headers[key.strip()] = value.strip() + except ValueError: + raise click.BadParameter(f"Invalid header format: {h}. Expected format: 'key:value'") + + # Parse JSON data if provided + json_data = None + if json: + try: + json_data = json_lib.loads(json) + except ValueError as e: + raise click.BadParameter(f"Invalid JSON format: {e}") + + # Parse data if provided + request_data = None + if data: + try: + request_data = json_lib.loads(data) + except ValueError: + # If not JSON, use as raw data + request_data = data + + client = HTTPClient(ctx.obj["base_url"], ctx.obj["api_key"]) + try: + response = client.request( + method=method, + uri=uri, + data=request_data, + json=json_data, + headers=headers, + ) + rich.print_json(data=response) + except requests.exceptions.HTTPError as e: + click.echo(f"Error: HTTP {e.response.status_code}", err=True) + try: + error_body = e.response.json() + rich.print_json(data=error_body) + except json_lib.JSONDecodeError: + click.echo(e.response.text, err=True) + raise click.Abort() diff --git a/litellm/proxy/client/cli/commands/keys.py b/litellm/proxy/client/cli/commands/keys.py new file mode 100644 index 00000000000..5ae81fb24d3 --- /dev/null +++ b/litellm/proxy/client/cli/commands/keys.py @@ -0,0 +1,158 @@ +import json +from typing import Literal, Optional + +import click +import rich +import requests +from rich.table import Table + +from ...keys import KeysManagementClient + + +@click.group() +def keys(): + """Manage API keys for the LiteLLM proxy server""" + pass + + +@keys.command() +@click.option("--page", type=int, help="Page number for pagination") +@click.option("--size", type=int, help="Number of items per page") +@click.option("--user-id", type=str, help="Filter keys by user ID") +@click.option("--team-id", type=str, help="Filter keys by team ID") +@click.option("--organization-id", type=str, help="Filter keys by organization ID") +@click.option("--key-hash", type=str, help="Filter by specific key hash") +@click.option("--key-alias", type=str, help="Filter by key alias") +@click.option("--return-full-object", is_flag=True, default=True, help="Return the full key object") +@click.option("--include-team-keys", is_flag=True, help="Include team keys in the response") +@click.option( + "--format", + "output_format", + type=click.Choice(["table", "json"]), + default="table", + help="Output format (table or json)", +) +@click.pass_context +def list( + ctx: click.Context, + page: Optional[int], + size: Optional[int], + user_id: Optional[str], + team_id: Optional[str], + organization_id: Optional[str], + key_hash: Optional[str], + key_alias: Optional[str], + include_team_keys: bool, + output_format: Literal["table", "json"], + return_full_object: bool, +): + """List all API keys""" + client = KeysManagementClient(ctx.obj["base_url"], ctx.obj["api_key"]) + response = client.list( + page=page, + size=size, + user_id=user_id, + team_id=team_id, + organization_id=organization_id, + key_hash=key_hash, + key_alias=key_alias, + return_full_object=return_full_object, + include_team_keys=include_team_keys, + ) + assert isinstance(response, dict) + + if output_format == "json": + rich.print_json(data=response) + else: + rich.print(f"Showing {len(response.get('keys', []))} keys out of {response.get('total_count', 0)}") + table = Table(title="API Keys") + table.add_column("Key Hash", style="cyan") + table.add_column("Alias", style="green") + table.add_column("User ID", style="magenta") + table.add_column("Team ID", style="yellow") + table.add_column("Spend", style="red") + for key in response.get("keys", []): + table.add_row( + str(key.get("token", "")), + str(key.get("key_alias", "")), + str(key.get("user_id", "")), + str(key.get("team_id", "")), + str(key.get("spend", "")), + ) + rich.print(table) + + +@keys.command() +@click.option("--models", type=str, help="Comma-separated list of allowed models") +@click.option("--aliases", type=str, help="JSON string of model alias mappings") +@click.option("--spend", type=float, help="Maximum spend limit for this key") +@click.option("--duration", type=str, help="Duration for which the key is valid (e.g. '24h', '7d')") +@click.option("--key-alias", type=str, help="Alias/name for the key") +@click.option("--team-id", type=str, help="Team ID to associate the key with") +@click.option("--user-id", type=str, help="User ID to associate the key with") +@click.option("--budget-id", type=str, help="Budget ID to associate the key with") +@click.option("--config", type=str, help="JSON string of additional configuration parameters") +@click.pass_context +def generate( + ctx: click.Context, + models: Optional[str], + aliases: Optional[str], + spend: Optional[float], + duration: Optional[str], + key_alias: Optional[str], + team_id: Optional[str], + user_id: Optional[str], + budget_id: Optional[str], + config: Optional[str], +): + """Generate a new API key""" + client = KeysManagementClient(ctx.obj["base_url"], ctx.obj["api_key"]) + try: + models_list = [m.strip() for m in models.split(",")] if models else None + aliases_dict = json.loads(aliases) if aliases else None + config_dict = json.loads(config) if config else None + except json.JSONDecodeError as e: + raise click.BadParameter(f"Invalid JSON: {str(e)}") + try: + response = client.generate( + models=models_list, + aliases=aliases_dict, + spend=spend, + duration=duration, + key_alias=key_alias, + team_id=team_id, + user_id=user_id, + budget_id=budget_id, + config=config_dict, + ) + rich.print_json(data=response) + except requests.exceptions.HTTPError as e: + click.echo(f"Error: HTTP {e.response.status_code}", err=True) + try: + error_body = e.response.json() + rich.print_json(data=error_body) + except json.JSONDecodeError: + click.echo(e.response.text, err=True) + raise click.Abort() + + +@keys.command() +@click.option("--keys", type=str, help="Comma-separated list of API keys to delete") +@click.option("--key-aliases", type=str, help="Comma-separated list of key aliases to delete") +@click.pass_context +def delete(ctx: click.Context, keys: Optional[str], key_aliases: Optional[str]): + """Delete API keys by key or alias""" + client = KeysManagementClient(ctx.obj["base_url"], ctx.obj["api_key"]) + keys_list = [k.strip() for k in keys.split(",")] if keys else None + aliases_list = [a.strip() for a in key_aliases.split(",")] if key_aliases else None + try: + response = client.delete(keys=keys_list, key_aliases=aliases_list) + rich.print_json(data=response) + except requests.exceptions.HTTPError as e: + click.echo(f"Error: HTTP {e.response.status_code}", err=True) + try: + error_body = e.response.json() + rich.print_json(data=error_body) + except json.JSONDecodeError: + click.echo(e.response.text, err=True) + raise click.Abort() diff --git a/litellm/proxy/client/cli/commands/models.py b/litellm/proxy/client/cli/commands/models.py new file mode 100644 index 00000000000..587eff993d7 --- /dev/null +++ b/litellm/proxy/client/cli/commands/models.py @@ -0,0 +1,277 @@ +# stdlib imports +from typing import Optional, Literal, Any +from datetime import datetime + +# third party imports +import click +import rich + +# local imports +from ... import Client + + +def format_iso_datetime_str(iso_datetime_str: Optional[str]) -> str: + """Format an ISO format datetime string to human-readable date with minute resolution.""" + if not iso_datetime_str: + return "" + try: + # Parse ISO format datetime string + dt = datetime.fromisoformat(iso_datetime_str.replace("Z", "+00:00")) + return dt.strftime("%Y-%m-%d %H:%M") + except (TypeError, ValueError): + return str(iso_datetime_str) + + +def format_timestamp(timestamp: Optional[int]) -> str: + """Format a Unix timestamp (integer) to human-readable date with minute resolution.""" + if timestamp is None: + return "" + try: + dt = datetime.fromtimestamp(timestamp) + return dt.strftime("%Y-%m-%d %H:%M") + except (TypeError, ValueError): + return str(timestamp) + + +def format_cost_per_1k_tokens(cost: Optional[float]) -> str: + """Format a per-token cost to cost per 1000 tokens.""" + if cost is None: + return "" + try: + # Convert string to float if needed + cost_float = float(cost) + # Multiply by 1000 and format to 4 decimal places + return f"${cost_float * 1000:.4f}" + except (TypeError, ValueError): + return str(cost) + + +def create_client(ctx: click.Context) -> Client: + """Helper function to create a client from context.""" + return Client(base_url=ctx.obj["base_url"], api_key=ctx.obj["api_key"]) + + +@click.group() +def models() -> None: + """Manage models on your LiteLLM proxy server""" + pass + + +@models.command("list") +@click.option( + "--format", + "output_format", + type=click.Choice(["table", "json"]), + default="table", + help="Output format (table or json)", +) +@click.pass_context +def list_models(ctx: click.Context, output_format: Literal["table", "json"]) -> None: + """List all available models""" + client = create_client(ctx) + models_list = client.models.list() + assert isinstance(models_list, list) + + if output_format == "json": + rich.print_json(data=models_list) + else: # table format + table = rich.table.Table(title="Available Models") + + # Add columns based on the data structure + table.add_column("ID", style="cyan") + table.add_column("Object", style="green") + table.add_column("Created", style="magenta") + table.add_column("Owned By", style="yellow") + + # Add rows + for model in models_list: + created = model.get("created") + # Convert string timestamp to integer if needed + if isinstance(created, str) and created.isdigit(): + created = int(created) + + table.add_row( + str(model.get("id", "")), + str(model.get("object", "model")), + format_timestamp(created) if isinstance(created, int) else format_iso_datetime_str(created), + str(model.get("owned_by", "")), + ) + + rich.print(table) + + +@models.command("add") +@click.argument("model-name") +@click.option( + "--param", + "-p", + multiple=True, + help="Model parameters in key=value format (can be specified multiple times)", +) +@click.option( + "--info", + "-i", + multiple=True, + help="Model info in key=value format (can be specified multiple times)", +) +@click.pass_context +def add_model(ctx: click.Context, model_name: str, param: tuple[str, ...], info: tuple[str, ...]) -> None: + """Add a new model to the proxy""" + # Convert parameters from key=value format to dict + model_params = dict(p.split("=", 1) for p in param) + model_info = dict(i.split("=", 1) for i in info) if info else None + + client = create_client(ctx) + result = client.models.new( + model_name=model_name, + model_params=model_params, + model_info=model_info, + ) + rich.print_json(data=result) + + +@models.command("delete") +@click.argument("model-id") +@click.pass_context +def delete_model(ctx: click.Context, model_id: str) -> None: + """Delete a model from the proxy""" + client = create_client(ctx) + result = client.models.delete(model_id=model_id) + rich.print_json(data=result) + + +@models.command("get") +@click.option("--id", "model_id", help="ID of the model to retrieve") +@click.option("--name", "model_name", help="Name of the model to retrieve") +@click.pass_context +def get_model(ctx: click.Context, model_id: Optional[str], model_name: Optional[str]) -> None: + """Get information about a specific model""" + if not model_id and not model_name: + raise click.UsageError("Either --id or --name must be provided") + + client = create_client(ctx) + result = client.models.get(model_id=model_id, model_name=model_name) + rich.print_json(data=result) + + +@models.command("info") +@click.option( + "--format", + "output_format", + type=click.Choice(["table", "json"]), + default="table", + help="Output format (table or json)", +) +@click.option( + "--columns", + "columns", + default="public_model,upstream_model,updated_at", + help="Comma-separated list of columns to display. Valid columns: public_model, upstream_model, credential_name, created_at, updated_at, id, input_cost, output_cost. Default: public_model,upstream_model,updated_at", +) +@click.pass_context +def get_models_info(ctx: click.Context, output_format: Literal["table", "json"], columns: str) -> None: + """Get detailed information about all models""" + client = create_client(ctx) + models_info = client.models.info() + assert isinstance(models_info, list) + + if output_format == "json": + rich.print_json(data=models_info) + else: # table format + table = rich.table.Table(title="Models Information") + + # Define all possible columns with their configurations + column_configs: dict[str, dict[str, Any]] = { + "public_model": { + "header": "Public Model", + "style": "cyan", + "get_value": lambda m: str(m.get("model_name", "")), + }, + "upstream_model": { + "header": "Upstream Model", + "style": "green", + "get_value": lambda m: str(m.get("litellm_params", {}).get("model", "")), + }, + "credential_name": { + "header": "Credential Name", + "style": "yellow", + "get_value": lambda m: str(m.get("litellm_params", {}).get("litellm_credential_name", "")), + }, + "created_at": { + "header": "Created At", + "style": "magenta", + "get_value": lambda m: format_iso_datetime_str(m.get("model_info", {}).get("created_at")), + }, + "updated_at": { + "header": "Updated At", + "style": "magenta", + "get_value": lambda m: format_iso_datetime_str(m.get("model_info", {}).get("updated_at")), + }, + "id": { + "header": "ID", + "style": "blue", + "get_value": lambda m: str(m.get("model_info", {}).get("id", "")), + }, + "input_cost": { + "header": "Input Cost", + "style": "green", + "justify": "right", + "get_value": lambda m: format_cost_per_1k_tokens(m.get("model_info", {}).get("input_cost_per_token")), + }, + "output_cost": { + "header": "Output Cost", + "style": "green", + "justify": "right", + "get_value": lambda m: format_cost_per_1k_tokens(m.get("model_info", {}).get("output_cost_per_token")), + }, + } + + # Add requested columns + requested_columns = [col.strip() for col in columns.split(",")] + for col_name in requested_columns: + if col_name in column_configs: + config = column_configs[col_name] + table.add_column(config["header"], style=config["style"], justify=config.get("justify", "left")) + else: + click.echo(f"Warning: Unknown column '{col_name}'", err=True) + + # Add rows with only the requested columns + for model in models_info: + row_values = [] + for col_name in requested_columns: + if col_name in column_configs: + row_values.append(column_configs[col_name]["get_value"](model)) + if row_values: + table.add_row(*row_values) + + rich.print(table) + + +@models.command("update") +@click.argument("model-id") +@click.option( + "--param", + "-p", + multiple=True, + help="Model parameters in key=value format (can be specified multiple times)", +) +@click.option( + "--info", + "-i", + multiple=True, + help="Model info in key=value format (can be specified multiple times)", +) +@click.pass_context +def update_model(ctx: click.Context, model_id: str, param: tuple[str, ...], info: tuple[str, ...]) -> None: + """Update an existing model's configuration""" + # Convert parameters from key=value format to dict + model_params = dict(p.split("=", 1) for p in param) + model_info = dict(i.split("=", 1) for i in info) if info else None + + client = create_client(ctx) + result = client.models.update( + model_id=model_id, + model_params=model_params, + model_info=model_info, + ) + rich.print_json(data=result) diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py new file mode 100644 index 00000000000..b4ddc900af6 --- /dev/null +++ b/litellm/proxy/client/cli/main.py @@ -0,0 +1,50 @@ +# stdlib imports +from typing import Optional + +# third party imports +import click + +# local imports +from .commands.models import models +from .commands.credentials import credentials +from .commands.chat import chat +from .commands.http import http +from .commands.keys import keys + + +@click.group() +@click.option( + "--base-url", + envvar="LITELLM_PROXY_URL", + show_envvar=True, + default="http://localhost:4000", + help="Base URL of the LiteLLM proxy server", +) +@click.option( + "--api-key", + envvar="LITELLM_PROXY_API_KEY", + show_envvar=True, + help="API key for authentication", +) +@click.pass_context +def cli(ctx: click.Context, base_url: str, api_key: Optional[str]) -> None: + """LiteLLM Proxy CLI - Manage your LiteLLM proxy server""" + ctx.ensure_object(dict) + ctx.obj["base_url"] = base_url + ctx.obj["api_key"] = api_key + + +# Add the models command group +cli.add_command(models) +# Add the credentials command group +cli.add_command(credentials) +# Add the chat command group +cli.add_command(chat) +# Add the http command group +cli.add_command(http) +# Add the keys command group +cli.add_command(keys) + + +if __name__ == "__main__": + cli() diff --git a/litellm/proxy/client/keys.py b/litellm/proxy/client/keys.py index 3628dfd59f7..24c6c2721a1 100644 --- a/litellm/proxy/client/keys.py +++ b/litellm/proxy/client/keys.py @@ -220,3 +220,34 @@ class KeysManagementClient: if e.response.status_code == 401: raise UnauthorizedError(e) raise + + def info(self, key: str, return_request: bool = False) -> Union[Dict[str, Any], requests.Request]: + """ + Get information about API keys. + + Args: + key (str): The key hash to get information about + return_request (bool): If True, returns the prepared request object instead of executing it + + Returns: + Union[Dict[str, Any], requests.Request]: Either the response from the server or a prepared request object if return_request is True + + Raises: + UnauthorizedError: If the request fails with a 401 status code + requests.exceptions.RequestException: If the request fails with any other error + """ + url = f"{self._base_url}/keys/info?key={key}" + request = requests.Request("GET", url, headers=self._get_headers()) + + if return_request: + return request + + session = requests.Session() + try: + response = session.send(request.prepare()) + response.raise_for_status() + return response.json() + except requests.exceptions.HTTPError as e: + if e.response.status_code == 401: + raise UnauthorizedError(e) + raise diff --git a/poetry.lock b/poetry.lock index 217b145a970..d8464199c94 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1721,6 +1721,32 @@ files = [ {file = "litellm_proxy_extras-0.1.15.tar.gz", hash = "sha256:0e9b9074023eea8954183746f196d4a62f5d9fda2838fac3cb94d6ee1899a308"}, ] +[[package]] +name = "markdown-it-py" +version = "3.0.0" +description = "Python port of markdown-it. Markdown parsing, done right!" +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" +files = [ + {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, + {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, +] + +[package.dependencies] +mdurl = ">=0.1,<1.0" + +[package.extras] +benchmarking = ["psutil", "pytest", "pytest-benchmark"] +code-style = ["pre-commit (>=3.0,<4.0)"] +compare = ["commonmark (>=0.9,<1.0)", "markdown (>=3.4,<4.0)", "mistletoe (>=1.0,<2.0)", "mistune (>=2.0,<3.0)", "panflute (>=2.3,<3.0)"] +linkify = ["linkify-it-py (>=1,<3)"] +plugins = ["mdit-py-plugins"] +profiling = ["gprof2dot"] +rtd = ["jupyter_sphinx", "mdit-py-plugins", "myst-parser", "pyyaml", "sphinx", "sphinx-copybutton", "sphinx-design", "sphinx_book_theme"] +testing = ["coverage", "pytest", "pytest-cov", "pytest-regressions"] + [[package]] name = "markupsafe" version = "2.1.5" @@ -1831,6 +1857,19 @@ cli = ["python-dotenv (>=1.0.0)", "typer (>=0.12.4)"] rich = ["rich (>=13.9.4)"] ws = ["websockets (>=15.0.1)"] +[[package]] +name = "mdurl" +version = "0.1.2" +description = "Markdown URL utilities" +optional = true +python-versions = ">=3.7" +groups = ["main"] +markers = "extra == \"proxy\"" +files = [ + {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, + {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, +] + [[package]] name = "ml-dtypes" version = "0.4.1" @@ -2981,6 +3020,22 @@ files = [ {file = "pyflakes-3.1.0.tar.gz", hash = "sha256:a0aae034c444db0071aa077972ba4768d40c830d9539fd45bf4cd3f8f6992efc"}, ] +[[package]] +name = "pygments" +version = "2.19.1" +description = "Pygments is a syntax highlighting package written in Python." +optional = true +python-versions = ">=3.8" +groups = ["main"] +markers = "extra == \"proxy\"" +files = [ + {file = "pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c"}, + {file = "pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f"}, +] + +[package.extras] +windows-terminal = ["colorama (>=0.4.6)"] + [[package]] name = "pyjwt" version = "2.9.0" @@ -3498,6 +3553,27 @@ files = [ [package.dependencies] httpx = ">=0.25.0" +[[package]] +name = "rich" +version = "13.7.1" +description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +optional = true +python-versions = ">=3.7.0" +groups = ["main"] +markers = "extra == \"proxy\"" +files = [ + {file = "rich-13.7.1-py3-none-any.whl", hash = "sha256:4edbae314f59eb482f54e9e30bf00d33350aaa94f4bfcd4e9e3110e64d0d7222"}, + {file = "rich-13.7.1.tar.gz", hash = "sha256:9be308cb1fe2f1f57d67ce99e95af38a1e2bc71ad9813b0e247cf7ffbcc3a432"}, +] + +[package.dependencies] +markdown-it-py = ">=2.2.0" +pygments = ">=2.13.0,<3.0.0" +typing-extensions = {version = ">=4.0.0,<5.0", markers = "python_version < \"3.9\""} + +[package.extras] +jupyter = ["ipywidgets (>=7.5.1,<9)"] + [[package]] name = "rpds-py" version = "0.20.1" @@ -4593,9 +4669,9 @@ type = ["pytest-mypy"] [extras] extra-proxy = ["azure-identity", "azure-keyvault-secrets", "google-cloud-kms", "prisma", "redisvl", "resend"] -proxy = ["PyJWT", "apscheduler", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-proxy-extras", "mcp", "orjson", "pynacl", "python-multipart", "pyyaml", "rq", "uvicorn", "uvloop", "websockets"] +proxy = ["PyJWT", "apscheduler", "backoff", "boto3", "cryptography", "fastapi", "fastapi-sso", "gunicorn", "litellm-proxy-extras", "mcp", "orjson", "pynacl", "python-multipart", "pyyaml", "rich", "rq", "uvicorn", "uvloop", "websockets"] [metadata] lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "4af55e1ec83871e3ff02ff6cf902e66128860dac7fe88bd57e18d7a4a99f790c" +content-hash = "ec95408547ecb69816b50eaffe96911a265781baeeb10681f9a2a12b2bed55eb" diff --git a/pyproject.toml b/pyproject.toml index de52bccd153..e03e8676dc7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ boto3 = {version = "1.34.34", optional = true} redisvl = {version = "^0.4.1", optional = true, markers = "python_version >= '3.9' and python_version < '3.14'"} mcp = {version = "1.5.0", optional = true, python = ">=3.10"} litellm-proxy-extras = {version = "0.1.15", optional = true} +rich = {version = "13.7.1", optional = true} [tool.poetry.extras] proxy = [ @@ -76,7 +77,8 @@ proxy = [ "websockets", "boto3", "mcp", - "litellm-proxy-extras" + "litellm-proxy-extras", + "rich", ] extra_proxy = [ @@ -93,6 +95,7 @@ profile = "black" [tool.poetry.scripts] litellm = 'litellm:run_server' +litellm-proxy = 'litellm.proxy.client.cli:cli' [tool.poetry.group.dev.dependencies] flake8 = "^6.1.0" diff --git a/requirements.txt b/requirements.txt index 56a2d6006b0..28056869e24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -44,6 +44,7 @@ tiktoken==0.8.0 # for calculating usage importlib-metadata==6.8.0 # for random utils tokenizers==0.20.2 # for calculating usage click==8.1.7 # for proxy cli +rich==13.7.1 # for litellm proxy cli jinja2==3.1.6 # for prompt templates aiohttp==3.10.2 # for network calls aioboto3==12.3.0 # for async sagemaker calls diff --git a/tests/litellm/proxy/client/cli/__init__.py b/tests/litellm/proxy/client/cli/__init__.py new file mode 100644 index 00000000000..352d47cd995 --- /dev/null +++ b/tests/litellm/proxy/client/cli/__init__.py @@ -0,0 +1 @@ +"""Tests for the LiteLLM Proxy Client CLI package.""" \ No newline at end of file diff --git a/tests/litellm/proxy/client/cli/test_chat_commands.py b/tests/litellm/proxy/client/cli/test_chat_commands.py new file mode 100644 index 00000000000..796229ccc45 --- /dev/null +++ b/tests/litellm/proxy/client/cli/test_chat_commands.py @@ -0,0 +1,241 @@ +import json +from unittest.mock import patch, MagicMock + +import pytest +import requests +from click.testing import CliRunner + +from litellm.proxy.client.cli.main import cli + + +@pytest.fixture +def mock_chat_client(): + with patch("litellm.proxy.client.cli.commands.chat.ChatClient") as mock: + yield mock + + +@pytest.fixture +def cli_runner(): + return CliRunner() + + +def test_chat_completions_success(cli_runner, mock_chat_client): + # Mock response data + mock_response = { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Hello! How can I help you today?", + }, + "finish_reason": "stop", + "index": 0, + } + ], + } + mock_instance = mock_chat_client.return_value + mock_instance.completions.return_value = mock_response + + # Run command + result = cli_runner.invoke( + cli, + [ + "chat", + "completions", + "gpt-4", + "-m", + "user:Hello!", + "--temperature", + "0.7", + "--max-tokens", + "100", + ], + ) + + # Verify + assert result.exit_code == 0 + output_data = json.loads(result.output) + assert output_data == mock_response + mock_instance.completions.assert_called_once_with( + model="gpt-4", + messages=[{"role": "user", "content": "Hello!"}], + temperature=0.7, + max_tokens=100, + top_p=None, + n=None, + presence_penalty=None, + frequency_penalty=None, + user=None, + ) + + +def test_chat_completions_multiple_messages(cli_runner, mock_chat_client): + # Mock response data + mock_response = { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Paris has a population of about 2.2 million.", + }, + "finish_reason": "stop", + "index": 0, + } + ], + } + mock_instance = mock_chat_client.return_value + mock_instance.completions.return_value = mock_response + + # Run command + result = cli_runner.invoke( + cli, + [ + "chat", + "completions", + "gpt-4", + "-m", + "system:You are a helpful assistant", + "-m", + "user:What's the population of Paris?", + ], + ) + + # Verify + assert result.exit_code == 0 + output_data = json.loads(result.output) + assert output_data == mock_response + mock_instance.completions.assert_called_once_with( + model="gpt-4", + messages=[ + {"role": "system", "content": "You are a helpful assistant"}, + {"role": "user", "content": "What's the population of Paris?"}, + ], + temperature=None, + max_tokens=None, + top_p=None, + n=None, + presence_penalty=None, + frequency_penalty=None, + user=None, + ) + + +def test_chat_completions_no_messages(cli_runner, mock_chat_client): + # Run command without any messages + result = cli_runner.invoke(cli, ["chat", "completions", "gpt-4"]) + + # Verify + assert result.exit_code == 2 + assert "At least one message is required" in result.output + mock_instance = mock_chat_client.return_value + mock_instance.completions.assert_not_called() + + +def test_chat_completions_invalid_message_format(cli_runner, mock_chat_client): + # Run command with invalid message format + result = cli_runner.invoke( + cli, ["chat", "completions", "gpt-4", "-m", "invalid-format"] + ) + + # Verify + assert result.exit_code == 2 + assert "Invalid message format" in result.output + mock_instance = mock_chat_client.return_value + mock_instance.completions.assert_not_called() + + +def test_chat_completions_http_error(cli_runner, mock_chat_client): + # Mock HTTP error + mock_instance = mock_chat_client.return_value + mock_error_response = MagicMock() + mock_error_response.status_code = 400 + mock_error_response.json.return_value = { + "error": "Invalid request", + "message": "Invalid model specified", + } + mock_instance.completions.side_effect = requests.exceptions.HTTPError( + response=mock_error_response + ) + + # Run command + result = cli_runner.invoke( + cli, ["chat", "completions", "invalid-model", "-m", "user:Hello"] + ) + + # Verify + assert result.exit_code == 1 + assert "Error: HTTP 400" in result.output + assert "Invalid request" in result.output + assert "Invalid model specified" in result.output + + +def test_chat_completions_all_parameters(cli_runner, mock_chat_client): + # Mock response data + mock_response = { + "id": "chatcmpl-123", + "object": "chat.completion", + "created": 1677858242, + "model": "gpt-4", + "choices": [ + { + "message": { + "role": "assistant", + "content": "Response with all parameters set", + }, + "finish_reason": "stop", + "index": 0, + } + ], + } + mock_instance = mock_chat_client.return_value + mock_instance.completions.return_value = mock_response + + # Run command with all available parameters + result = cli_runner.invoke( + cli, + [ + "chat", + "completions", + "gpt-4", + "-m", + "user:Test message", + "--temperature", + "0.7", + "--top-p", + "0.9", + "--n", + "1", + "--max-tokens", + "100", + "--presence-penalty", + "0.5", + "--frequency-penalty", + "0.5", + "--user", + "test-user", + ], + ) + + # Verify + assert result.exit_code == 0 + output_data = json.loads(result.output) + assert output_data == mock_response + mock_instance.completions.assert_called_once_with( + model="gpt-4", + messages=[{"role": "user", "content": "Test message"}], + temperature=0.7, + top_p=0.9, + n=1, + max_tokens=100, + presence_penalty=0.5, + frequency_penalty=0.5, + user="test-user", + ) \ No newline at end of file diff --git a/tests/litellm/proxy/client/cli/test_credentials_commands.py b/tests/litellm/proxy/client/cli/test_credentials_commands.py new file mode 100644 index 00000000000..de2bdec5ce4 --- /dev/null +++ b/tests/litellm/proxy/client/cli/test_credentials_commands.py @@ -0,0 +1,202 @@ +import json +from unittest.mock import patch, MagicMock + +import pytest +import requests +from click.testing import CliRunner + +from litellm.proxy.client.cli.main import cli + + +@pytest.fixture +def mock_credentials_client(): + with patch("litellm.proxy.client.cli.commands.credentials.CredentialsManagementClient") as mock: + yield mock + + +@pytest.fixture +def cli_runner(): + return CliRunner() + + +def test_list_credentials_table_format(cli_runner, mock_credentials_client): + # Mock response data + mock_response = { + "credentials": [ + { + "credential_name": "test-cred-1", + "credential_info": {"custom_llm_provider": "azure"}, + }, + { + "credential_name": "test-cred-2", + "credential_info": {"custom_llm_provider": "anthropic"}, + }, + ] + } + mock_instance = mock_credentials_client.return_value + mock_instance.list.return_value = mock_response + + # Run command + result = cli_runner.invoke(cli, ["credentials", "list"]) + + # Verify + assert result.exit_code == 0 + assert "test-cred-1" in result.output + assert "azure" in result.output + assert "test-cred-2" in result.output + assert "anthropic" in result.output + + +def test_list_credentials_json_format(cli_runner, mock_credentials_client): + # Mock response data + mock_response = { + "credentials": [ + { + "credential_name": "test-cred", + "credential_info": {"custom_llm_provider": "azure"}, + } + ] + } + mock_instance = mock_credentials_client.return_value + mock_instance.list.return_value = mock_response + + # Run command + result = cli_runner.invoke(cli, ["credentials", "list", "--format", "json"]) + + # Verify + assert result.exit_code == 0 + output_data = json.loads(result.output) + assert output_data == mock_response + + +def test_create_credential_success(cli_runner, mock_credentials_client): + # Mock response data + mock_response = {"status": "success", "credential_name": "test-cred"} + mock_instance = mock_credentials_client.return_value + mock_instance.create.return_value = mock_response + + # Run command + result = cli_runner.invoke( + cli, + [ + "credentials", + "create", + "test-cred", + "--info", + '{"custom_llm_provider": "azure"}', + "--values", + '{"api_key": "test-key"}', + ], + ) + + # Verify + assert result.exit_code == 0 + output_data = json.loads(result.output) + assert output_data == mock_response + mock_instance.create.assert_called_once_with( + "test-cred", + {"custom_llm_provider": "azure"}, + {"api_key": "test-key"}, + ) + + +def test_create_credential_invalid_json(cli_runner, mock_credentials_client): + # Run command with invalid JSON + result = cli_runner.invoke( + cli, + [ + "credentials", + "create", + "test-cred", + "--info", + "invalid-json", + "--values", + '{"api_key": "test-key"}', + ], + ) + + # Verify + assert result.exit_code == 2 + assert "Invalid JSON" in result.output + mock_instance = mock_credentials_client.return_value + mock_instance.create.assert_not_called() + + +def test_create_credential_http_error(cli_runner, mock_credentials_client): + # Mock HTTP error + mock_instance = mock_credentials_client.return_value + mock_error_response = MagicMock() + mock_error_response.status_code = 400 + mock_error_response.json.return_value = {"error": "Invalid request"} + mock_instance.create.side_effect = requests.exceptions.HTTPError(response=mock_error_response) + + # Run command + result = cli_runner.invoke( + cli, + [ + "credentials", + "create", + "test-cred", + "--info", + '{"custom_llm_provider": "azure"}', + "--values", + '{"api_key": "test-key"}', + ], + ) + + # Verify + assert result.exit_code == 1 + assert "Error: HTTP 400" in result.output + assert "Invalid request" in result.output + + +def test_delete_credential_success(cli_runner, mock_credentials_client): + # Mock response data + mock_response = {"status": "success", "message": "Credential deleted"} + mock_instance = mock_credentials_client.return_value + mock_instance.delete.return_value = mock_response + + # Run command + result = cli_runner.invoke(cli, ["credentials", "delete", "test-cred"]) + + # Verify + assert result.exit_code == 0 + output_data = json.loads(result.output) + assert output_data == mock_response + mock_instance.delete.assert_called_once_with("test-cred") + + +def test_delete_credential_http_error(cli_runner, mock_credentials_client): + # Mock HTTP error + mock_instance = mock_credentials_client.return_value + mock_error_response = MagicMock() + mock_error_response.status_code = 404 + mock_error_response.json.return_value = {"error": "Credential not found"} + mock_instance.delete.side_effect = requests.exceptions.HTTPError(response=mock_error_response) + + # Run command + result = cli_runner.invoke(cli, ["credentials", "delete", "test-cred"]) + + # Verify + assert result.exit_code == 1 + assert "Error: HTTP 404" in result.output + assert "Credential not found" in result.output + + +def test_get_credential_success(cli_runner, mock_credentials_client): + # Mock response data + mock_response = { + "credential_name": "test-cred", + "credential_info": {"custom_llm_provider": "azure"}, + } + mock_instance = mock_credentials_client.return_value + mock_instance.get.return_value = mock_response + + # Run command + result = cli_runner.invoke(cli, ["credentials", "get", "test-cred"]) + + # Verify + assert result.exit_code == 0 + output_data = json.loads(result.output) + assert output_data == mock_response + mock_instance.get.assert_called_once_with("test-cred") \ No newline at end of file diff --git a/tests/litellm/proxy/client/cli/test_keys_commands.py b/tests/litellm/proxy/client/cli/test_keys_commands.py new file mode 100644 index 00000000000..d3f1dcfb9a3 --- /dev/null +++ b/tests/litellm/proxy/client/cli/test_keys_commands.py @@ -0,0 +1,90 @@ +import json +import os +from unittest.mock import patch + +import pytest +from click.testing import CliRunner + +from litellm.proxy.client.cli import cli + + +@pytest.fixture +def cli_runner(): + return CliRunner() + + +@pytest.fixture(autouse=True) +def mock_env(): + with patch.dict(os.environ, {"LITELLM_PROXY_URL": "http://localhost:4000", "LITELLM_PROXY_API_KEY": "sk-test"}): + yield + + +@pytest.fixture +def mock_keys_client(): + with patch("litellm.proxy.client.cli.commands.keys.KeysManagementClient") as MockClient: + yield MockClient + + +def test_keys_list_json_format(mock_keys_client, cli_runner): + mock_keys_client.return_value.list.return_value = { + "keys": [{"token": "abc123", "key_alias": "alias1", "user_id": "u1", "team_id": "t1", "spend": 10.0}] + } + result = cli_runner.invoke(cli, ["keys", "list", "--format", "json"]) + assert result.exit_code == 0 + output_data = json.loads(result.output) + assert output_data == mock_keys_client.return_value.list.return_value + mock_keys_client.assert_called_once_with("http://localhost:4000", "sk-test") + mock_keys_client.return_value.list.assert_called_once() + + +def test_keys_list_table_format(mock_keys_client, cli_runner): + mock_keys_client.return_value.list.return_value = { + "keys": [{"token": "abc123", "key_alias": "alias1", "user_id": "u1", "team_id": "t1", "spend": 10.0}] + } + result = cli_runner.invoke(cli, ["keys", "list"]) + assert result.exit_code == 0 + assert "abc123" in result.output + assert "alias1" in result.output + assert "u1" in result.output + assert "t1" in result.output + assert "10.0" in result.output + mock_keys_client.assert_called_once_with("http://localhost:4000", "sk-test") + mock_keys_client.return_value.list.assert_called_once() + + +def test_keys_generate_success(mock_keys_client, cli_runner): + mock_keys_client.return_value.generate.return_value = {"key": "new-key", "spend": 100.0} + result = cli_runner.invoke(cli, ["keys", "generate", "--models", "gpt-4", "--spend", "100"]) + assert result.exit_code == 0 + assert "new-key" in result.output + mock_keys_client.return_value.generate.assert_called_once() + + +def test_keys_delete_success(mock_keys_client, cli_runner): + mock_keys_client.return_value.delete.return_value = {"status": "success", "deleted_keys": ["abc123"]} + result = cli_runner.invoke(cli, ["keys", "delete", "--keys", "abc123"]) + assert result.exit_code == 0 + assert "success" in result.output + assert "abc123" in result.output + mock_keys_client.return_value.delete.assert_called_once() + + +def test_keys_list_error_handling(mock_keys_client, cli_runner): + mock_keys_client.return_value.list.side_effect = Exception("API Error") + result = cli_runner.invoke(cli, ["keys", "list"]) + assert result.exit_code != 0 + assert "API Error" in str(result.exception) + + +def test_keys_generate_error_handling(mock_keys_client, cli_runner): + mock_keys_client.return_value.generate.side_effect = Exception("API Error") + result = cli_runner.invoke(cli, ["keys", "generate", "--models", "gpt-4"]) + assert result.exit_code != 0 + assert "API Error" in str(result.exception) + + +def test_keys_delete_error_handling(mock_keys_client, cli_runner): + mock_keys_client.return_value.delete.side_effect = Exception("API Error") + result = cli_runner.invoke(cli, ["keys", "delete", "--keys", "abc123"]) + assert result.exit_code != 0 + assert "API Error" in str(result.exception) diff --git a/tests/litellm/proxy/client/cli/test_models_commands.py b/tests/litellm/proxy/client/cli/test_models_commands.py new file mode 100644 index 00000000000..62b6c42c5de --- /dev/null +++ b/tests/litellm/proxy/client/cli/test_models_commands.py @@ -0,0 +1,232 @@ +# stdlib imports +import json +import os +from unittest.mock import patch + +# third party imports +from click.testing import CliRunner +import pytest + +# local imports +from litellm.proxy.client.cli import cli +from litellm.proxy.client.cli.commands.models import format_timestamp + + +@pytest.fixture +def mock_client(): + """Fixture to create a mock client with common setup""" + with patch("litellm.proxy.client.cli.commands.models.Client") as MockClient: + yield MockClient + + +@pytest.fixture +def cli_runner(): + """Fixture for Click CLI runner""" + return CliRunner() + + +@pytest.fixture(autouse=True) +def mock_env(): + """Fixture to set up environment variables for all tests""" + with patch.dict(os.environ, { + "LITELLM_PROXY_URL": "http://localhost:4000", + "LITELLM_PROXY_API_KEY": "sk-test" + }): + yield + + +@pytest.fixture +def mock_models_list(mock_client): + """Fixture to set up common mocking pattern for models list tests""" + mock_client.return_value.models.list.return_value = [ + { + "id": "model-123", + "object": "model", + "created": 1699848889, + "owned_by": "organization-123" + }, + { + "id": "model-456", + "object": "model", + "created": 1699848890, + "owned_by": "organization-456" + } + ] + + mock_client.assert_not_called() # Ensure clean slate + return mock_client + + +@pytest.fixture +def mock_models_info(mock_client): + """Fixture to set up models info mock""" + mock_client.return_value.models.info.return_value = [ + { + "model_name": "gpt-4", + "litellm_params": { + "model": "gpt-4", + "litellm_credential_name": "openai-1" + }, + "model_info": { + "id": "model-123", + "created_at": "2025-04-29T21:31:43.843000+00:00", + "updated_at": "2025-04-29T21:31:43.843000+00:00", + "input_cost_per_token": 0.00001, + "output_cost_per_token": 0.00002 + } + } + ] + + mock_client.assert_not_called() + return mock_client + + +def test_models_list_json_format(mock_models_list, cli_runner): + """Test the models list command with JSON output format""" + # Run the command + result = cli_runner.invoke(cli, ["models", "list", "--format", "json"]) + + # Check that the command succeeded + assert result.exit_code == 0 + + # Parse the output and verify it matches our mock data + output_data = json.loads(result.output) + assert output_data == mock_models_list.return_value.models.list.return_value + + # Verify the client was called correctly + mock_models_list.assert_called_once_with( + base_url="http://localhost:4000", + api_key="sk-test" + ) + mock_models_list.return_value.models.list.assert_called_once() + + +def test_models_list_table_format(mock_models_list, cli_runner): + """Test the models list command with table output format""" + # Run the command + result = cli_runner.invoke(cli, ["models", "list"]) + + # Check that the command succeeded + assert result.exit_code == 0 + + # Verify the output contains expected table elements + assert "ID" in result.output + assert "Object" in result.output + assert "Created" in result.output + assert "Owned By" in result.output + assert "model-123" in result.output + assert "organization-123" in result.output + assert format_timestamp(1699848889) in result.output + + # Verify the client was called correctly + mock_models_list.assert_called_once_with( + base_url="http://localhost:4000", + api_key="sk-test" + ) + mock_models_list.return_value.models.list.assert_called_once() + + +def test_models_list_with_base_url(mock_models_list, cli_runner): + """Test the models list command with custom base URL overriding env var""" + custom_base_url = "http://custom.server:8000" + + # Run the command with custom base URL + result = cli_runner.invoke(cli, [ + "--base-url", custom_base_url, + "models", "list" + ]) + + # Check that the command succeeded + assert result.exit_code == 0 + + # Verify the client was created with the custom base URL (overriding env var) + mock_models_list.assert_called_once_with( + base_url=custom_base_url, + api_key="sk-test" # Should still use env var for API key + ) + + +def test_models_list_with_api_key(mock_models_list, cli_runner): + """Test the models list command with API key overriding env var""" + custom_api_key = "custom-test-key" + + # Run the command with custom API key + result = cli_runner.invoke(cli, [ + "--api-key", custom_api_key, + "models", "list" + ]) + + # Check that the command succeeded + assert result.exit_code == 0 + + # Verify the client was created with the custom API key (overriding env var) + mock_models_list.assert_called_once_with( + base_url="http://localhost:4000", # Should still use env var for base URL + api_key=custom_api_key + ) + + +def test_models_list_error_handling(mock_client, cli_runner): + """Test error handling in the models list command""" + # Configure mock to raise an exception + mock_client.return_value.models.list.side_effect = Exception("API Error") + + # Run the command + result = cli_runner.invoke(cli, ["models", "list"]) + + # Check that the command failed + assert result.exit_code != 0 + assert "API Error" in str(result.exception) + + # Verify the client was created with env var values + mock_client.assert_called_once_with( + base_url="http://localhost:4000", + api_key="sk-test" + ) + + +def test_models_info_json_format(mock_models_info, cli_runner): + """Test the models info command with JSON output format""" + # Run the command + result = cli_runner.invoke(cli, ["models", "info", "--format", "json"]) + + # Check that the command succeeded + assert result.exit_code == 0 + + # Parse the output and verify it matches our mock data + output_data = json.loads(result.output) + assert output_data == mock_models_info.return_value.models.info.return_value + + # Verify the client was called correctly with env var values + mock_models_info.assert_called_once_with( + base_url="http://localhost:4000", + api_key="sk-test" + ) + mock_models_info.return_value.models.info.assert_called_once() + + +def test_models_info_table_format(mock_models_info, cli_runner): + """Test the models info command with table output format""" + # Run the command with default columns + result = cli_runner.invoke(cli, ["models", "info"]) + + # Check that the command succeeded + assert result.exit_code == 0 + + # Verify the output contains expected table elements + assert "Public Model" in result.output + assert "Upstream Model" in result.output + assert "Updated At" in result.output + assert "gpt-4" in result.output + assert "2025-04-29 21:31" in result.output + + # Verify seconds and microseconds are not shown + assert "21:31:43" not in result.output + assert "843000" not in result.output + + # Verify the client was called correctly with env var values + mock_models_info.assert_called_once_with( + base_url="http://localhost:4000", + api_key="sk-test" + ) + mock_models_info.return_value.models.info.assert_called_once() \ No newline at end of file diff --git a/tests/litellm/proxy/client/test_client.py b/tests/litellm/proxy/client/test_client.py index ceb872f87b7..9806237992e 100644 --- a/tests/litellm/proxy/client/test_client.py +++ b/tests/litellm/proxy/client/test_client.py @@ -73,7 +73,7 @@ def test_client_initialization(): api_key="test-key", timeout=60, ) - + # Check that http client is initialized correctly assert isinstance(client.http, HTTPClient) assert client.http._base_url == "http://localhost:4000" @@ -87,12 +87,12 @@ def test_client_default_timeout(): base_url="http://localhost:4000", api_key="test-key", ) - + assert client.http._timeout == 30 def test_client_without_api_key(): """Test that the client works without an API key.""" client = Client(base_url="http://localhost:4000") - + assert client.http._api_key is None diff --git a/tests/litellm/proxy/client/test_http_client.py b/tests/litellm/proxy/client/test_http_client.py index 4e7c16a5374..1a621959e03 100644 --- a/tests/litellm/proxy/client/test_http_client.py +++ b/tests/litellm/proxy/client/test_http_client.py @@ -26,13 +26,13 @@ def test_request_get(client): json={"models": []}, status=200, ) - + # Make request response = client.request("GET", "/models") - + # Check response assert response == {"models": []} - + # Check request assert len(responses.calls) == 1 assert responses.calls[0].request.url == "http://localhost:4000/models" @@ -49,23 +49,20 @@ def test_request_post_with_json(client): json={"id": "model-123"}, status=200, ) - + # Test data - json_data = { - "model": "gpt-4", - "params": {"temperature": 0.7} - } - + json_data = {"model": "gpt-4", "params": {"temperature": 0.7}} + # Make request response = client.request( "POST", "/models", json=json_data, ) - + # Check response assert response == {"id": "model-123"} - + # Check request assert len(responses.calls) == 1 assert responses.calls[0].request.url == "http://localhost:4000/models" @@ -82,7 +79,7 @@ def test_request_with_custom_headers(client): json={"models": []}, status=200, ) - + # Make request with custom headers custom_headers = { "X-Custom-Header": "test-value", @@ -93,7 +90,7 @@ def test_request_with_custom_headers(client): "/models", headers=custom_headers, ) - + # Check request headers assert len(responses.calls) == 1 request_headers = responses.calls[0].request.headers @@ -112,11 +109,11 @@ def test_request_http_error(client): json={"error": "Not authorized"}, status=401, ) - + # Check that request raises exception with pytest.raises(requests.exceptions.HTTPError) as exc_info: client.request("GET", "/models") - + assert exc_info.value.response.status_code == 401 @@ -130,7 +127,7 @@ def test_request_invalid_json(client): body="not json", status=200, ) - + # Check that request raises exception with pytest.raises(json.JSONDecodeError) as exc_info: client.request("GET", "/models") @@ -148,7 +145,7 @@ def test_base_url_trailing_slash(): def test_uri_leading_slash(): """Test that URIs with and without leading slashes work.""" client = HTTPClient(base_url="http://localhost:4000") - + with responses.RequestsMock() as rsps: # Mock endpoint rsps.add( @@ -156,12 +153,12 @@ def test_uri_leading_slash(): "http://localhost:4000/models", json={"models": []}, ) - + # Both of these should work and hit the same endpoint client.request("GET", "/models") client.request("GET", "models") - + # Check that both requests went to the same URL assert len(rsps.calls) == 2 assert rsps.calls[0].request.url == "http://localhost:4000/models" - assert rsps.calls[1].request.url == "http://localhost:4000/models" \ No newline at end of file + assert rsps.calls[1].request.url == "http://localhost:4000/models" diff --git a/tests/litellm/proxy/client/test_http_commands.py b/tests/litellm/proxy/client/test_http_commands.py new file mode 100644 index 00000000000..8a2228bc5c2 --- /dev/null +++ b/tests/litellm/proxy/client/test_http_commands.py @@ -0,0 +1,116 @@ +"""Tests for the HTTP command group.""" + +import json +import pytest +import responses +from click.testing import CliRunner + +from litellm.proxy.client.cli.commands.http import http + + +@pytest.fixture +def runner(): + """Create a CLI test runner.""" + return CliRunner() + + +@responses.activate +def test_request_get(runner): + """Test making a GET request.""" + responses.add( + responses.GET, + "http://localhost:4000/models", + json={"models": []}, + status=200, + ) + result = runner.invoke( + http, + ["request", "GET", "/models"], + obj={"base_url": "http://localhost:4000", "api_key": "sk-test-key"}, + ) + assert result.exit_code == 0 + assert "models" in result.output + + +@responses.activate +def test_request_post_with_json(runner): + """Test making a POST request with JSON data.""" + responses.add( + responses.POST, + "http://localhost:4000/chat/completions", + json={"choices": [{"message": {"content": "Hello!"}}]}, + status=200, + ) + result = runner.invoke( + http, + [ + "request", + "POST", + "/chat/completions", + "-j", + '{"model": "gpt-4", "messages": [{"role": "user", "content": "Hello"}]}', + ], + obj={"base_url": "http://localhost:4000", "api_key": "sk-test-key"}, + ) + assert result.exit_code == 0 + assert "choices" in result.output + + +@responses.activate +def test_request_with_headers(runner): + """Test making a request with custom headers.""" + responses.add( + responses.GET, + "http://localhost:4000/models", + json={"models": []}, + status=200, + ) + result = runner.invoke( + http, + [ + "request", + "GET", + "/models", + "-H", + "X-Custom-Header:value", + "-H", + "Accept:application/json", + ], + obj={"base_url": "http://localhost:4000", "api_key": "sk-test-key"}, + ) + assert result.exit_code == 0 + assert "models" in result.output + + +def test_request_invalid_json(runner): + """Test error handling for invalid JSON data.""" + result = runner.invoke( + http, + [ + "request", + "POST", + "/chat/completions", + "-j", + '{"invalid": json}', # Invalid JSON + ], + obj={"base_url": "http://localhost:4000", "api_key": "sk-test-key"}, + ) + assert result.exit_code == 2 # Click error code for invalid parameter + assert "Invalid JSON format" in result.output + + +def test_request_invalid_header(runner): + """Test error handling for invalid header format.""" + result = runner.invoke( + http, + [ + "request", + "GET", + "/models", + "-H", + "invalid-header", # Invalid header format + ], + obj={"base_url": "http://localhost:4000", "api_key": "sk-test-key"}, + ) + assert result.exit_code == 2 # Click error code for invalid parameter + assert "Invalid header format" in result.output \ No newline at end of file diff --git a/tests/litellm/proxy/client/test_keys.py b/tests/litellm/proxy/client/test_keys.py index 82dcafde3e6..85e4c371bb6 100644 --- a/tests/litellm/proxy/client/test_keys.py +++ b/tests/litellm/proxy/client/test_keys.py @@ -309,3 +309,42 @@ def test_delete_unauthorized_error(client, requests_mock): with pytest.raises(UnauthorizedError): client.delete(keys=["key-to-delete"]) + + +def test_info_request_minimal(client, base_url, api_key): + """Test info request with minimal parameters""" + request = client.info(key="test-key", return_request=True) + assert request.method == "GET" + assert request.url == f"{base_url}/keys/info?key=test-key" + assert request.headers["Content-Type"] == "application/json" + assert request.headers["Authorization"] == f"Bearer {api_key}" + + +def test_info_mock_response(client, requests_mock): + """Test info with a mocked successful response""" + mock_response = { + "key": "test-key", + "user_id": "user123", + "team_id": "team456", + "models": ["gpt-4"], + "spend": 100.0, + } + requests_mock.get(f"{client._base_url}/keys/info?key=test-key", json=mock_response) + response = client.info(key="test-key") + assert response == mock_response + + +def test_info_unauthorized_error(client, requests_mock): + """Test that info raises UnauthorizedError for 401 responses""" + requests_mock.get(f"{client._base_url}/keys/info?key=test-key", status_code=401, json={"error": "Unauthorized"}) + with pytest.raises(UnauthorizedError): + client.info(key="test-key") + + +def test_info_server_error(client, requests_mock): + """Test that info raises HTTPError for server errors""" + requests_mock.get( + f"{client._base_url}/keys/info?key=test-key", status_code=500, json={"error": "Internal Server Error"} + ) + with pytest.raises(requests.exceptions.HTTPError): + client.info(key="test-key")