From f2e5e822033356263d2549793ef0ab91733e0973 Mon Sep 17 00:00:00 2001 From: TensorNull Date: Wed, 3 Jun 2026 20:26:10 +0800 Subject: [PATCH] feat: enhance cometapi endpoint support --- README.md | 2 +- cookbook/LiteLLM_CometAPI.ipynb | 689 +++++++----------- litellm/images/main.py | 27 +- litellm/llms/cometapi/chat/transformation.py | 81 +- litellm/llms/cometapi/common_utils.py | 66 ++ litellm/llms/cometapi/embed/transformation.py | 18 +- .../image_generation/transformation.py | 113 ++- litellm/main.py | 159 ++-- litellm/utils.py | 8 + provider_endpoints_support.json | 8 +- .../chat/test_cometapi_chat_transformation.py | 195 ++--- .../llms/cometapi/test_cometapi_endpoints.py | 677 +++++++++++++++++ 12 files changed, 1324 insertions(+), 719 deletions(-) create mode 100644 tests/test_litellm/llms/cometapi/test_cometapi_endpoints.py diff --git a/README.md b/README.md index 9924aeb5829..cabb7b4ee1d 100644 --- a/README.md +++ b/README.md @@ -288,7 +288,7 @@ curl -X POST 'http://0.0.0.0:4000/v1/chat/completions' \ | [Codestral (`codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | | [Cohere (`cohere`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | ✅ | | | | | | ✅ | | [Cohere Chat (`cohere_chat`)](https://docs.litellm.ai/docs/providers/cohere) | ✅ | ✅ | ✅ | | | | | | | | -| [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ | | | | | | | +| [CometAPI (`cometapi`)](https://docs.litellm.ai/docs/providers/cometapi) | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | | | [CompactifAI (`compactifai`)](https://docs.litellm.ai/docs/providers/compactifai) | ✅ | ✅ | ✅ | | | | | | | | | [Custom (`custom`)](https://docs.litellm.ai/docs/providers/custom_llm_server) | ✅ | ✅ | ✅ | | | | | | | | | [Custom OpenAI (`custom_openai`)](https://docs.litellm.ai/docs/providers/openai_compatible) | ✅ | ✅ | ✅ | | | ✅ | ✅ | ✅ | ✅ | | diff --git a/cookbook/LiteLLM_CometAPI.ipynb b/cookbook/LiteLLM_CometAPI.ipynb index 0a7ab581ae3..ceab0f10683 100644 --- a/cookbook/LiteLLM_CometAPI.ipynb +++ b/cookbook/LiteLLM_CometAPI.ipynb @@ -2,74 +2,16 @@ "cells": [ { "cell_type": "markdown", - "metadata": { - "id": "iFEmsVJI_2BR" - }, + "metadata": {}, "source": [ "# LiteLLM CometAPI Cookbook" ] }, { "cell_type": "code", - "execution_count": 1, - "metadata": { - "id": "cBlUhCEP_xj4" - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: litellm in /Users/xmx/.miniforge3/lib/python3.12/site-packages (1.78.2)\n", - "Requirement already satisfied: aiohttp>=3.10 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.11.18)\n", - "Requirement already satisfied: click in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.3.0)\n", - "Requirement already satisfied: fastuuid>=0.13.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.13.3)\n", - "Requirement already satisfied: httpx>=0.23.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.28.1)\n", - "Requirement already satisfied: importlib-metadata>=6.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (8.6.1)\n", - "Requirement already satisfied: jinja2<4.0.0,>=3.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (3.1.6)\n", - "Requirement already satisfied: jsonschema<5.0.0,>=4.22.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (4.25.1)\n", - "Requirement already satisfied: openai>=2.8.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.109.1)\n", - "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (2.11.10)\n", - "Requirement already satisfied: python-dotenv>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (1.1.1)\n", - "Requirement already satisfied: tiktoken>=0.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.12.0)\n", - "Requirement already satisfied: tokenizers in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from litellm) (0.22.1)\n", - "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (2.6.1)\n", - "Requirement already satisfied: aiosignal>=1.1.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.4.0)\n", - "Requirement already satisfied: attrs>=17.3.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (25.3.0)\n", - "Requirement already satisfied: frozenlist>=1.1.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.6.0)\n", - "Requirement already satisfied: multidict<7.0,>=4.5 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (6.6.3)\n", - "Requirement already satisfied: propcache>=0.2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (0.3.1)\n", - "Requirement already satisfied: yarl<2.0,>=1.17.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from aiohttp>=3.10->litellm) (1.20.0)\n", - "Requirement already satisfied: anyio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (4.11.0)\n", - "Requirement already satisfied: certifi in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (2025.10.5)\n", - "Requirement already satisfied: httpcore==1.* in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (1.0.9)\n", - "Requirement already satisfied: idna in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpx>=0.23.0->litellm) (3.10)\n", - "Requirement already satisfied: h11>=0.16 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from httpcore==1.*->httpx>=0.23.0->litellm) (0.16.0)\n", - "Requirement already satisfied: zipp>=3.20 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from importlib-metadata>=6.8.0->litellm) (3.21.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jinja2<4.0.0,>=3.1.2->litellm) (3.0.3)\n", - "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (2025.9.1)\n", - "Requirement already satisfied: referencing>=0.28.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.36.2)\n", - "Requirement already satisfied: rpds-py>=0.7.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from jsonschema<5.0.0,>=4.22.0->litellm) (0.27.1)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.9.0)\n", - "Requirement already satisfied: jiter<1,>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (0.11.0)\n", - "Requirement already satisfied: sniffio in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (1.3.1)\n", - "Requirement already satisfied: tqdm>4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.67.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.11 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from openai>=2.8.0->litellm) (4.15.0)\n", - "Requirement already satisfied: annotated-types>=0.6.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.7.0)\n", - "Requirement already satisfied: pydantic-core==2.33.2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (2.33.2)\n", - "Requirement already satisfied: typing-inspection>=0.4.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from pydantic<3.0.0,>=2.5.0->litellm) (0.4.2)\n", - "Requirement already satisfied: regex>=2022.1.18 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2025.9.18)\n", - "Requirement already satisfied: requests>=2.26.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tiktoken>=0.7.0->litellm) (2.32.2)\n", - "Requirement already satisfied: huggingface-hub<2.0,>=0.16.4 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from tokenizers->litellm) (0.25.2)\n", - "Requirement already satisfied: filelock in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (3.15.4)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (2025.9.0)\n", - "Requirement already satisfied: packaging>=20.9 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (25.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from huggingface-hub<2.0,>=0.16.4->tokenizers->litellm) (6.0.3)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (3.4.0)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /Users/xmx/.miniforge3/lib/python3.12/site-packages (from requests>=2.26.0->tiktoken>=0.7.0->litellm) (1.26.20)\n" - ] - } - ], + "execution_count": null, + "metadata": {}, + "outputs": [], "source": [ "!pip install litellm" ] @@ -78,365 +20,28 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "# Completion" + "## Setup" ] }, { "cell_type": "code", "execution_count": null, - "metadata": { - "id": "p-MQqWOT_1a7" - }, + "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", - "os.environ['COMETAPI_KEY'] = \"Your_CometAPI_Key_Here\"\n", - "api_key = os.getenv('COMETAPI_KEY')" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "Ze8JqMqWAARO", - "outputId": "64f3e836-69fa-4f8e-fb35-088a913bbe98" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "ModelResponse(id='msg_017L3DDDit8AkEgHRe2DQBc9', created=1760589916, model='claude-sonnet-4-5-20250929', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='I\\'ll create a simple Python script that says hi.\\n\\n\\nhello.py\\n#!/usr/bin/env python3\\n\"\"\"\\nA simple script that says hi!\\n\"\"\"\\n\\ndef say_hi(name=None):\\n \"\"\"Say hi to someone, or just say hi generally.\"\"\"\\n if name:\\n print(f\"Hi, {name}!\")\\n else:\\n print(\"Hi!\")\\n\\nif __name__ == \"__main__\":\\n # Say hi generally\\n say_hi()\\n \\n # Say hi to someone specific\\n say_hi(\"World\")\\n\\n\\n\\nI\\'ve created a simple Python script called `hello.py` that:\\n\\n1. Defines a `say_hi()` function that can optionally take a name parameter\\n2. Prints \"Hi!\" if no name is provided\\n3. Prints \"Hi, [name]!\" if a name is provided\\n4. Demonstrates both usages when run\\n\\nYou can run it with:\\n```bash\\npython hello.py\\n```\\n\\nThis will output:\\n```\\nHi!\\nHi, World!\\n```\\n\\nWould you like me to create versions in other programming languages, or modify this in any way?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields=None), provider_specific_fields={})], usage=Usage(completion_tokens=290, prompt_tokens=26, total_tokens=316, completion_tokens_details=None, prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=None, text_tokens=None, image_tokens=None, cached_tokens_details={})))" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "from litellm import completion\n", - "response = completion(\n", - " model=\"cometapi/claude-sonnet-4-5-20250929\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "-LnhELrnAM_J", - "outputId": "d51c7ab7-d761-4bd1-f849-1534d9df4cd0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "ModelResponse(id='chatcmpl-CRA9Uo6nsQ9C7kMJv1J4kyNDFJym7', created=1760589916, model='gpt-5-chat-latest', object='chat.completion', system_fingerprint='fp_2da73a467a', choices=[Choices(finish_reason='stop', index=0, message=Message(content='Sure! I can help you write a simple code that prints out \"Hi\" in different programming languages. \\n\\nHere’s an example in **Python**:\\n\\n```python\\n# Simple Python program to say \"Hi\"\\nprint(\"Hi\")\\n```\\n\\nExample in **JavaScript**:\\n\\n```javascript\\n// Simple JavaScript program to say \"Hi\"\\nconsole.log(\"Hi\");\\n```\\n\\nExample in **C**:\\n\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi\\\\n\");\\n return 0;\\n}\\n```\\n\\nExample in **Java**:\\n\\n```java\\npublic class SayHi {\\n public static void main(String[] args) {\\n System.out.println(\"Hi\");\\n }\\n}\\n```\\n\\nWhich language would you like me to focus on, or do you want me to make it interactive so the program greets the user by name?', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={})], usage=Usage(completion_tokens=174, prompt_tokens=12, total_tokens=186, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)))" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"cometapi/gpt-5-chat-latest\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "id": "dJBOUYdwCEn1", - "outputId": "ffa18679-ec15-4dad-fe2b-68665cdf36b0" - }, - "outputs": [ - { - "data": { - "text/plain": [ - "ModelResponse(id='02176058998406949c23b3bf3d52941de23f13565a086747738f0', created=1760589991, model='deepseek-v3.2-exp', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content='Here are several ways to say \"hi\" in different programming languages:\\n\\n## Python\\n```python\\nprint(\"Hi!\")\\n```\\n\\n## JavaScript (Browser)\\n```javascript\\nconsole.log(\"Hi!\");\\n// or\\nalert(\"Hi!\");\\n```\\n\\n## JavaScript (Node.js)\\n```javascript\\nconsole.log(\"Hi!\");\\n```\\n\\n## Java\\n```java\\npublic class Hello {\\n public static void main(String[] args) {\\n System.out.println(\"Hi!\");\\n }\\n}\\n```\\n\\n## C\\n```c\\n#include \\n\\nint main() {\\n printf(\"Hi!\\\\n\");\\n return 0;\\n}\\n```\\n\\n## C++\\n```cpp\\n#include \\n\\nint main() {\\n std::cout << \"Hi!\" << std::endl;\\n return 0;\\n}\\n```\\n\\n## C#\\n```csharp\\nusing System;\\n\\nclass Program {\\n static void Main() {\\n Console.WriteLine(\"Hi!\");\\n }\\n}\\n```\\n\\n## PHP\\n```php\\n\\n```\\n\\n## Ruby\\n```ruby\\nputs \"Hi!\"\\n```\\n\\n## Go\\n```go\\npackage main\\n\\nimport \"fmt\"\\n\\nfunc main() {\\n fmt.Println(\"Hi!\")\\n}\\n```\\n\\n## Rust\\n```rust\\nfn main() {\\n println!(\"Hi!\");\\n}\\n```\\n\\n## Swift\\n```swift\\nprint(\"Hi!\")\\n```\\n\\n## Kotlin\\n```kotlin\\nfun main() {\\n println(\"Hi!\")\\n}\\n```\\n\\n## HTML (webpage)\\n```html\\n\\n\\n\\n Hi Page\\n\\n\\n

Hi!

\\n\\n\\n```\\n\\nThe Python version is probably the simplest if you\\'re just getting started!', role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}), provider_specific_fields={})], usage=Usage(completion_tokens=347, prompt_tokens=10, total_tokens=357, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=None, audio_tokens=None, reasoning_tokens=0, rejected_prediction_tokens=None, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=None, cached_tokens=0, text_tokens=None, image_tokens=None)), service_tier='default')" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "response = completion(\n", - " model=\"cometapi/deepseek-v3.2-exp\",\n", - " messages=[{\"role\": \"user\", \"content\": \"write code for saying hi\"}]\n", - ")\n", - "response" + "# CometAPI keys can be provided as COMETAPI_KEY or COMETAPI_API_KEY.\n", + "api_key = os.getenv(\"COMETAPI_KEY\") or os.getenv(\"COMETAPI_API_KEY\")\n", + "if api_key is None:\n", + " raise RuntimeError(\"Set COMETAPI_KEY or COMETAPI_API_KEY before running this notebook.\")\n" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "# Streaming" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Streaming Responses" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "I'm\n", - " doing\n", - " well\n", - " —\n", - " thanks\n", - " for\n", - " asking\n", - "!\n", - " How\n", - " can\n", - " I\n", - " help\n", - " you\n", - " today\n", - "?\n", - "\n" - ] - } - ], - "source": [ - "messages = [{\"role\": \"user\", \"content\": \"Hey, how's it going?\"}]\n", - "response = completion(model=\"cometapi/gpt-5-mini\", messages=messages, stream=True)\n", - "for part in response:\n", - " print(part.choices[0].delta.content or \"\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Async Completion" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ModelResponse(id='chatcmpl-CRAAkmfczlmCEnCM55D9CKexbRenn', created=1760589994, model='gpt-5-mini-2025-08-07', object='chat.completion', system_fingerprint=None, choices=[Choices(finish_reason='stop', index=0, message=Message(content=\"I'm doing well, thanks — how are you? How can I help today?\", role='assistant', tool_calls=None, function_call=None, provider_specific_fields={'refusal': None}, annotations=[]), provider_specific_fields={'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'protected_material_code': {'filtered': False, 'detected': False}, 'protected_material_text': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}})], usage=Usage(completion_tokens=26, prompt_tokens=12, total_tokens=38, completion_tokens_details=CompletionTokensDetailsWrapper(accepted_prediction_tokens=0, audio_tokens=0, reasoning_tokens=0, rejected_prediction_tokens=0, text_tokens=None), prompt_tokens_details=PromptTokensDetailsWrapper(audio_tokens=0, cached_tokens=0, text_tokens=None, image_tokens=None)), prompt_filter_results=[{'prompt_index': 0, 'content_filter_results': {'hate': {'filtered': False, 'severity': 'safe'}, 'jailbreak': {'filtered': False, 'detected': False}, 'self_harm': {'filtered': False, 'severity': 'safe'}, 'sexual': {'filtered': False, 'severity': 'safe'}, 'violence': {'filtered': False, 'severity': 'safe'}}}])\n" - ] - } - ], - "source": [ - "from litellm import acompletion\n", - "import asyncio\n", - "\n", - "async def test_get_response():\n", - " user_message = \"Hello, how are you?\"\n", - " messages = [{\"content\": user_message, \"role\": \"user\"}]\n", - " response = await acompletion(model=\"cometapi/gpt-5-mini\", messages=messages)\n", - " return response\n", - "\n", - "response = await test_get_response()\n", - "print(response)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Async Streaming" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "test acompletion + streaming\n", - "response: \n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='Hi', role='assistant', function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' —', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='’m', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' doing', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' well', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=',', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' thanks', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='!', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' How', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' are', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' What', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' can', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' I', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' help', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' you', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' with', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content=' today', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason=None, index=0, delta=Delta(provider_specific_fields=None, content='?', role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None, citations=None)\n", - "ModelResponseStream(id='chatcmpl-CRAAl9VMDBB5skZt638Qx86K9h1Hb', created=1760589996, model='gpt-5-mini', object='chat.completion.chunk', system_fingerprint=None, choices=[StreamingChoices(finish_reason='stop', index=0, delta=Delta(provider_specific_fields=None, content=None, role=None, function_call=None, tool_calls=None, audio=None), logprobs=None)], provider_specific_fields=None)\n" - ] - } - ], - "source": [ - "from litellm import acompletion\n", - "import asyncio, os, traceback\n", - "\n", - "async def completion_call():\n", - " try:\n", - " print(\"test acompletion + streaming\")\n", - " response = await acompletion(\n", - " model=\"cometapi/gpt-5-mini\", \n", - " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}], \n", - " stream=True\n", - " )\n", - " print(f\"response: {response}\")\n", - " async for chunk in response:\n", - " print(chunk)\n", - " except:\n", - " print(f\"error occurred: {traceback.format_exc()}\")\n", - " pass\n", - "\n", - "await completion_call()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Embedding" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n" - ] - } - ], - "source": [ - "import litellm\n", - "\n", - "\n", - "async def main():\n", - " response = await litellm.aembedding(\n", - " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"openai\" + the model name from ai/ml api\n", - " api_key=api_key, # your aiml api-key\n", - " api_base=\"https://api.cometapi.com/v1\", # 👈 the URL has changed from v2 to v1\n", - " input=\"Your text string\",\n", - " )\n", - " print(response)\n", - "\n", - "await main()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "EmbeddingResponse(model='text-embedding-3-small', data=[{'object': 'embedding', 'index': 0, 'embedding': [-0.018048199, 0.0047550877, -0.013976435, -0.021936804, -0.038773336, -0.03708264, 0.03854791, -0.0007172257, 0.026473511, -0.0027438616, -0.019823432, -0.011947598, -0.013426959, -0.0059914105, 0.020485623, 0.04269012, -0.028276922, -0.015216281, -0.03325039, 0.045057096, 0.0037477135, 0.015793936, -0.005188329, -0.0071713766, 0.008446445, 0.0070938864, 0.0027632343, 0.025656339, 0.022091785, -0.026797561, -0.029840818, -0.0542714, 0.017907308, -0.03454659, -0.014582269, 0.0429719, 0.03575826, 0.007939235, -0.010123054, -0.029587213, 0.018921727, 0.022556728, 0.019598005, 0.008819807, 0.01655475, 0.043310042, -0.034321167, 0.004441604, 0.032686826, 0.047226828, -0.0043253684, 0.006681779, 0.008995921, 0.06593721, 0.0066078105, -0.023190739, -0.00777721, 0.049875587, -0.028023317, -0.019386668, -0.0013252605, 0.009214303, -0.0055828253, -0.007432026, -0.01199691, 0.0054384116, -0.024247425, 0.047255006, -0.013743965, 0.012799991, 0.009883538, -0.005579303, -0.012616833, -0.014709071, -0.004473305, -0.022204498, 0.010454148, -0.02202134, 0.034631126, 0.0097567355, 0.041478455, -0.010750021, -0.0005340668, -0.041450277, -0.04516981, 0.014709071, -0.06142869, 0.021640932, -0.0008752884, -0.0012671428, -0.045254346, -0.004216178, -0.028220564, -0.011074071, 0.010693664, -0.0029481545, -0.039590508, -0.030178957, -0.019062618, -0.03539194, 0.019978413, 0.019161243, -0.025924034, 0.0058329077, -0.029305428, 0.028840488, 0.02731886, -0.0008048426, -0.056582022, 0.003043256, -0.08459125, -0.012560476, 0.00081100664, 0.0015771041, 0.015188103, -0.0049206354, 0.046888687, -0.027107522, 0.019245777, -0.011412211, 0.039336905, -0.018611765, 0.0429719, -0.04302826, -0.018132735, -0.0074883825, -0.02035882, -0.073038146, -0.042380158, 0.04485985, 0.03361671, -0.033842135, -0.0017259207, -0.0017091898, -0.049199305, -0.024148801, -0.044803493, 0.040943068, -0.03093977, 0.045057096, -0.0186963, -0.014053926, -0.009848315, -0.0070480965, -0.0060054995, -0.02813603, -0.022105874, 0.010235767, -0.004476827, 0.027220234, -0.026290352, 0.0216832, -0.05559578, 0.042464696, 0.0253182, -0.0031594916, 0.02266944, -0.030798879, -0.02896729, -0.0005891025, 0.004061197, -0.042295624, -0.008383043, 0.017146494, -0.025797231, -0.046522368, 0.044127215, 0.021105545, -0.04302826, 0.024430584, -0.014934498, -0.01556851, -0.075743265, -0.022415835, -0.028586883, 0.032743182, 0.03539194, -0.0034483192, -0.04669144, 0.051932603, 0.021626843, 0.03127791, 0.015610777, -0.016470214, -0.0056744046, -0.012743635, 0.060132485, 0.017428277, -0.0039942735, -0.017118316, 0.025106862, 0.008974788, 0.018513141, 0.0016035212, -0.0049593803, 0.0017514573, 0.031221554, -0.034856554, -0.011461522, 0.04621241, 0.044239927, 0.034715664, -0.0121941585, -0.012053267, -0.08526753, -0.011813751, -0.025769053, 0.0125182085, -0.0046670306, 0.038266126, 0.1187997, 0.005787118, -0.030038064, -0.054553185, -0.041506633, -0.024078354, 0.0071854657, -0.013391736, 0.03192601, -0.059625275, 0.0023458432, 0.027924692, 0.09163582, 0.030967949, 0.017639615, 0.01489223, 0.029559033, 0.042943723, 0.0003306547, 0.0047198646, 0.029897174, 0.00012603184, 0.0046811197, -0.0427183, 0.01789322, 0.018175002, -0.0081857955, 0.02581132, 0.009890582, 0.03840702, -0.094453655, -0.01076411, 0.06858598, 0.041647524, 0.033532172, 0.007467249, -0.008235107, -0.030967949, 0.0151317455, 0.027361127, 0.011834885, 0.008707094, 0.008178751, -0.022458103, 0.02844599, 0.003605061, -0.02399382, 0.05212985, 0.06041427, -0.023317542, -0.013335379, -0.044099037, -0.040802173, -0.0047656544, -0.023909286, 0.017315563, 0.017428277, 0.00736158, -0.0016070436, -0.055454887, -0.038012523, -0.020626513, 0.018273626, 0.03260229, -0.016991513, 0.038463376, -0.022458103, -0.0109613575, -0.021810003, 0.04846667, -0.042521052, 0.008601425, -0.019259866, -0.0040048403, -0.03308132, -0.02499415, 0.026783472, -0.032884073, 0.021824092, 0.013145176, -0.009186125, -0.01769597, 0.03240504, -0.015343083, -0.012539342, 0.03578644, -0.012299826, 0.011898286, 0.035730083, 0.058441788, 0.032010544, 0.048720278, 0.012926794, -0.0015207474, -0.03313768, 0.014540002, 0.020189751, 0.00029058868, 0.011531969, -0.022514459, 0.019752987, -0.037956167, 0.005272864, -0.042295624, -0.08521117, -0.03494109, 0.053313337, 0.029981708, -0.008150573, -0.053200625, 0.059681635, -0.035476476, 0.034828376, 0.00087881065, 0.025712697, 0.018668123, 0.03212326, 0.008474623, 0.017836861, 0.004910068, 0.016174342, -0.059681635, 0.04004136, 0.00753065, 0.008150573, -0.038012523, 0.0051178834, 0.012525253, 0.022119964, 0.030235313, 0.008242152, -0.01835816, -0.003150686, 0.010714797, -0.0033162334, -0.028882755, -0.06836055, 0.056159347, 0.013624207, 0.0077349427, 0.0066183778, 0.018583586, -0.008883208, -0.046550546, 0.046945043, -0.07393985, -0.017343743, 0.029530855, -0.010982491, 0.008129438, 0.009700378, -0.024613742, -0.0030097943, 0.0078053884, -0.006438741, 0.04770586, -0.008221018, 0.01654066, 0.02498006, -0.015793936, -0.010827511, 0.02399382, 0.03192601, 0.022923045, -0.029192716, 0.006724046, -0.04601516, 0.038519733, 0.031531516, 0.019443026, 0.000109466084, 0.03525105, -0.027248414, -0.038125236, 0.011771483, -0.007467249, 0.010285079, 0.01670973, -0.007861745, 0.026868006, 0.052327096, 0.026374886, -0.03905512, 0.031193376, -0.053566944, 0.04257741, -0.004670553, 0.0168788, 0.035166513, -0.057878222, 0.07095295, -0.009749691, -0.0137510095, -0.02151413, -0.02431787, 0.010073741, -0.05176353, -0.02083785, 0.003959051, -0.02682574, 0.062104966, -0.011461522, 0.04170388, 0.0076363184, 0.026177637, 0.0144413775, 0.014821785, -0.00046890447, 0.0050544823, 0.00032228927, -0.038970586, -0.011355854, -0.056300238, -0.04302826, -0.003545182, 0.04021043, 0.0051108385, -0.048438493, 0.00252548, -0.07692675, -0.0012433673, 0.0054278444, 0.029305428, -0.016188432, -0.003263399, -0.046156053, 6.031917e-05, 0.060977835, -0.016611107, -0.010637308, -0.012602744, 0.016442036, -0.051509928, -0.016991513, 0.0019407802, 0.019161243, 0.045282524, 0.031869654, -0.036941748, -0.035814617, -0.017850952, -0.027192056, -0.049734693, -0.020964652, 0.0228526, -0.025050506, 0.023472521, 0.025740875, -0.017738238, -0.009813092, -0.030883415, -0.012405495, -0.03277136, -0.029502677, 0.016780175, -0.04421175, -0.0020816717, 0.010341435, 0.059230782, -0.041901127, -0.04119667, 0.025924034, 0.02334572, -0.0008435878, 0.020654691, -0.022753974, 0.010700708, -0.013856677, -0.0121941585, -0.011391076, 0.006590199, 0.0050227814, -0.007960369, 0.0008418266, -0.0198657, 0.10781016, -0.0384352, -0.019147152, 0.0057237167, -0.0038181592, -0.047424074, -0.009341106, 0.018499052, -0.016906979, 0.005642704, -0.01837225, -0.038125236, -0.024895526, -0.010285079, -0.055708494, 0.014173684, -0.019724809, 0.00024215724, 0.04500074, 0.048804812, 0.009777869, -0.006572588, 0.008277375, 0.012328005, -0.012609788, 0.026079014, -0.012990195, 0.017963665, -0.007312268, -0.0015682983, 0.05446865, -0.01258161, 0.00035376972, -0.011299497, -0.036321826, -0.0071854657, 0.012969062, 0.026558045, -0.051819887, -0.0029146927, -0.044606246, -0.010383703, -0.03919601, 0.013624207, 0.0030978515, 0.0121941585, -0.0022225631, 0.0512845, -0.0029780937, -0.025191398, -0.015751667, -0.021006921, 0.0039520063, 0.04418357, 0.020570157, 0.00083390146, 0.020541979, -0.004807922, -0.0114263, -0.036152754, 0.018428607, -0.032658648, -0.002035882, 0.013828499, 0.03144698, -0.0003275727, -0.029756282, 0.008488712, 0.0041879993, 0.027826069, 0.0007273523, -0.018949905, -0.0029023646, 0.007861745, 0.011398122, 0.0125322975, 0.014976765, 0.006318983, -0.0066536004, -0.042915545, 0.025867676, -0.015272637, 0.034602948, 0.050241902, 0.014582269, 0.005987888, 0.015244459, -0.050213724, -0.003212326, 0.01315222, -0.022866689, -0.004772699, 0.035673723, -0.024796901, -0.00699174, -0.002072866, 0.022077696, 0.021147812, 0.005093227, -0.039618686, -0.0049241576, 0.012264604, -0.062104966, -0.0022613083, -0.004339458, 0.065486364, -0.0033320836, 0.02944632, 0.017498722, 0.0033039053, -0.020260196, -0.0154980635, -0.05460954, -0.03626547, 0.0072629564, 0.0028900367, 1.2954037e-05, 0.01769597, -0.0045930627, 0.022260854, 0.0027192058, -0.0010566862, -0.0005212985, 0.012158935, -0.0017312041, -0.035110157, -0.0036032998, -0.02317665, -0.01639977, 0.010327346, 0.018259536, -0.011095204, -0.00061904197, -0.023134382, -0.011989865, 0.0025924034, 0.0056708823, 0.03110884, -0.013462181, -0.021105545, 0.010376658, -0.010017385, -0.025106862, 0.026093103, 0.018456785, -0.02134506, 0.0066993902, 0.011891241, -0.010017385, -0.012687278, -0.017132405, 0.04717047, 0.012475941, -0.018752657, -0.008657781, 0.005276386, -0.02582541, 0.02913636, -0.0193444, -0.01101067, 0.029305428, 0.011736261, 0.043140974, 0.02135915, 0.00089422066, 0.009827181, 0.013638296, 0.013884856, -0.014004614, 0.010285079, 0.008108305, -0.04035132, -0.02978446, 0.008481667, -0.022289034, 0.01621661, -0.0057941624, -0.019090796, -0.01852723, -0.022923045, 0.0077208537, -0.039985005, -0.017428277, -0.009460864, 0.018301804, 0.0014397348, 0.04815671, -0.012187114, 0.018879458, 0.021739556, 0.018414518, -0.013462181, -0.06368295, 0.0057096276, 0.013088819, 0.0061640027, 0.031193376, -0.008728228, -0.019245777, 0.010735931, 0.012454808, 0.0397314, -0.017597347, 0.012278693, 0.0130465515, -0.025473181, -0.03215144, -0.0053292206, -0.0027068777, 0.014068015, -0.028079674, 0.016498392, 0.015159924, -0.009207259, -0.02334572, -0.0013710503, 0.008488712, 0.0012231142, 0.0020464489, -0.025149131, -0.021063277, -0.014427288, -0.035222873, 0.051030897, 0.016103897, 0.0063401167, -0.03093977, -0.004684642, -0.0070199184, -0.008495756, -0.0038674714, -0.012222337, -0.022556728, 0.0036015387, -0.040943068, -0.011362898, 0.016794264, 0.017766416, -0.014194817, 0.0011755633, -0.039759576, 0.011384032, -0.0006318103, 0.008298509, 0.04449353, 0.0004838742, 0.016935157, -0.011341765, 0.016864711, -0.00027892113, 0.0009140335, -0.031306088, -0.049452912, -0.0068367594, -0.00011216283, -0.005079138, -0.014420244, 0.01803411, 0.03984411, -0.026276262, -0.0011077593, -0.00063313113, -0.006301372, -0.019992502, 0.0064316965, -0.024289692, 0.0120039545, 0.0068649375, -0.017724149, -0.015667133, -0.0036490895, -0.007953324, 0.024627833, 0.024402406, 0.021810003, -0.015977094, 0.010524594, -0.0060054995, 0.0414221, -0.048551206, 0.01472316, 0.015427617, 0.0029217373, 0.012666144, 0.0048995013, 0.007326357, -0.04187295, -0.0064176074, -0.00674518, 0.0047762212, -0.053059734, -0.09541172, 0.022063607, 0.029530855, 0.01556851, 0.011292453, -0.0038709936, -0.0055370354, -0.016005272, -0.0035170037, -0.0572583, 0.038632445, 0.007981502, -0.005434889, -0.023895197, 0.0021380284, -0.0015084195, 0.016117986, 0.005434889, -0.014694982, -0.007104453, 0.011595369, -0.055229463, 0.0036455672, 0.0027104, -0.010052607, -0.023697948, -0.016315235, -0.002757951, 0.039505973, 0.011095204, 0.0002681341, 0.058948997, -0.0074883825, 0.0050122146, 0.040604927, 0.012912705, -0.025078684, 0.040464036, -0.008925476, -0.00876345, -0.040633105, -0.009024099, 0.024796901, 0.03592733, 0.03626547, -0.029474499, -0.00055431994, 0.0010839839, 0.016737908, 0.013286067, -0.005441934, 0.0059420983, -0.0121941585, 0.015089478, -0.010186454, -0.03477202, -0.0076363184, -0.0087141385, 0.0018439173, 0.028065585, -0.022331301, 0.0029516767, -0.045789734, 0.0010672531, 0.018287715, -0.015948916, 0.04849485, 0.0057589393, 0.0066219, 0.002196146, -0.047255006, 0.012116668, 0.02085194, 0.025924034, -0.0036737456, -0.02877004, 0.016906979, -0.037336245, -0.016258877, 0.010883868, -0.003765325, -0.0049523357, -0.002613537, -0.03263047, 0.023204828, 0.0049946033, -0.007692675, -0.034236632, 0.034095738, 0.020133393, 0.019259866, -0.014103238, 0.024599653, 0.005889264, 0.02430378, 0.0111233825, -0.018780835, -0.00040550332, 0.020232018, 0.03806888, 0.009890582, 0.032376863, 0.031052483, 0.01871039, 0.03891423, -0.0009739124, 0.002759712, 0.017498722, -0.01158128, -0.0045578396, 0.02744566, 0.06497915, 0.024853257, 0.004709298, 0.016667463, -0.00066263025, -0.018132735, -0.013138131, -0.01124314, -0.0125182085, -0.0038111147, 0.03361671, -0.007270001, 0.0012011, -0.01771006, -0.00039999973, 0.024021998, 0.0027896515, 0.0024744067, 0.0013965869, -0.05939985, 0.0014150789, -0.0052517303, 0.052524347, 0.015779847, -0.03327857, 0.042633764, 0.0059420983, -0.023387987, 0.0039097387, -0.028023317, -0.011863063, 0.004378203, 0.02052789, -0.063626595, -0.014864052, 0.014293442, -0.00015938349, -0.007932191, -0.0010954313, 0.023528878, -0.007467249, 0.0059667546, 0.017132405, 0.005730761, -0.00020495309, -0.032038722, 0.0036631785, 0.042915545, -0.029925352, 0.015667133, 0.018935816, -0.0072065997, 0.01556851, -0.025473181, 0.017625526, -0.0026698937, -0.007446115, -0.008622559, -0.043422755, -0.020133393, -0.0039801844, 0.01489223, -0.021655021, 0.015357172, -0.03640636, -0.005663838, -0.028530527, 0.0022648307, -0.00043015933, 0.043591827, -0.015526242, 0.011870108, -0.02530411, -0.016315235, -0.00032316984, -0.030150779, -0.0052552526, 0.020372909, 0.0075024716, 0.0104330145, -0.00055608107, -0.026248084, -0.015202192, -0.03341946, 0.031559695, -0.0012046222, 0.07185466, -0.039590508, 0.022979401, 0.05810365, 0.014025748, -0.029756282, -0.022866689, 0.0073897582, 0.037618026, -0.004180955, -0.0051566283, 0.009728557, -0.03604004, 0.040633105, 0.0026963109, -0.0054172776, 0.034095738, -0.00595971, 0.040943068, -0.031390622, 0.055962097, 0.02117599, -0.012912705, -0.019626183, 0.055877563, 0.017343743, -0.0035416598, 0.013257889, -0.0186963, 0.01656884, -0.06396473, -0.0055405577, 0.020767406, -0.0046564634, 0.045085277, -0.009221348, 0.013645341, 0.008777539, 0.004730432, -0.018625854, -0.011067026, 0.021500042, -0.015047211, 0.004600107, -0.0014344514, -0.0023740216, -0.016188432, 0.006209792, 0.0011993388, 0.004180955, -0.017160583, 0.014497734, 0.015371261, 0.018259536, -0.028333278, -0.008390088, 0.041929305, 0.003923828, 0.02550136, -0.003300383, -0.008058993, -0.010418925, 0.058216363, 0.01885128, -0.02020384, 0.002858336, -0.009806047, -0.022274945, 0.0070445742, 0.026670758, 0.008213974, -0.035307407, -0.027713355, 0.042915545, -0.039675042, -0.0029217373, 0.012053267, -0.003853382, 0.01133472, -0.010073741, 0.005878697, 0.0070938864, -0.035673723, 0.024205158, 0.005896309, 0.030573452, 0.02416289, -0.0072911344, 0.01738601, 0.017005602, -0.02846008, 0.0030344503, 0.018794924, -0.0148076955, -0.0344057, 0.025430914, 0.033503994, -0.0050580045, 0.0077138087, 0.03243322, 0.01372283, -0.005441934, 0.0073404466, -0.0007832686, -0.04767768, 0.0070480965, 0.015145835, 0.026233995, -0.01670973, -0.019513471, -0.014849963, 0.007953324, -0.0032176094, 0.006572588, -0.0012477703, 0.004230267, 0.004476827, -0.021810003, -0.030009886, -0.019273955, -0.0030414949, -0.002918215, 0.060639694, 0.024641922, 0.010327346, 0.026558045, 0.018921727, -0.025867676, -0.016117986, 0.023881108, 0.025360467, 0.009770825, 0.03792799, -0.022429924, 0.033363104, -0.0018914682, 0.04040768, 0.018484963, 0.0070199184, -0.017583257, 0.016258877, 0.010954313, -0.008939565, -0.024148801, -0.02498006, -0.007889924, 0.02748793, 0.0307707, 0.029756282, 0.0051425393, 0.0045719286, -0.03046074, 0.013596028, 0.025684519, -0.0033197557, 0.006967084, 0.03677268, 0.0120039545, -0.0032792494, -0.0032211316, -0.02399382, -0.026924362, -0.013920079, -0.0042197, 0.025346378, -0.0027015943, -0.016991513, 0.0031594916, -0.007579962, 0.018978084, 0.017681882, 0.0126591, 0.028939111, 0.008833896, 0.10183637, 0.0059632324, -0.05196078, -0.023697948, 0.011045893, -0.008777539, -0.013807366, 0.019273955, -0.025346378, 0.0074742935, 0.009961028, -0.010813422, 0.018597675, 0.009636978, 0.014948587, 0.024064265, -0.008693005, -0.020570157, 0.014194817, -0.026219906, -0.02299349, 0.011067026, 0.032066904, 0.013391736, -0.05148175, -0.009489042, -0.03062981, -0.0012847543, 0.07286908, 0.026529867, -0.00025008238, 0.013638296, 0.016089808, 0.018654034, -0.0020394044, -0.024543297, 0.0147795165, -0.009601755, 0.0018791402, -0.040520392, -0.003360262, 0.02216223, 0.0137650985, 0.0059914105, 0.0048361, -0.0009844792, 0.016977424, -0.00934815, 0.024233336, -0.013088819, -0.017555078, -0.0050263037, 0.010595039, -0.027516108, 0.0071537653, -0.023247095, -0.0017655464, -0.015948916, 0.058160007, -0.025966302, 0.0121941585, -0.012384362, -0.0015612538, 0.009946939, 0.00628376, 0.011327676, 0.0109613575, 0.008601425, -0.018329982, 0.055680316, -0.012778858, -0.0100807855, -0.011067026, -0.0036490895, -0.01356785, 0.0073193125, -0.014272308, -0.027403394, -0.030742522, 0.02862915, 0.03062981, -0.014596358, 0.021697288, 0.0042408337, 0.027572464, 0.0019601528, -0.037138995, -0.031306088, 0.041929305, 0.017738238, 0.004857234, 0.008256241, 0.0118278405, 0.021753646, -0.00160176, -0.0018333505, -0.0047374764, 0.042239267, 0.0058329077, -0.026459422, 0.015075389, 0.021147812, -0.005212985, 0.01281408, 0.017738238, 0.008242152, -0.020372909, -0.011081115, -0.011017715, 0.007706764, 0.01834407, 0.01954165, 0.037477136, -0.010278034, 0.015808025, 0.00031590514, -0.017681882, -0.008967743, -0.020612424, -0.025416825, -0.0037970257, -0.029868996, 0.01720285, -0.0144554665, 0.026727116, 0.00414221, 0.0040154075, 0.05838543, 0.0005622451, -0.025219576, 0.004180955, -0.002932304, -0.0090663675, 0.011574236, 0.02450103, -0.012553431, -0.020612424, -0.032095082, 0.015526242, 0.008974788, 0.0053151315, -0.0003112821, -0.017935487, -0.0076222294, 0.03358853, 0.029474499, -0.011496745, -0.012835215, -0.020739228, -0.012482986, -0.037871633, 0.0052517303, -0.012926794, -0.0025237189, 0.0020323596, 0.045113456, -0.04835396, -0.027755624, -0.0079955915, 0.007896968, 0.0072559114, 0.015047211, -0.0014573464, -0.014032792, 0.021091456, -0.0046071517, -0.0065232757, -0.02582541, -0.035870973, -0.015343083, 0.03254593, -0.028431902, -0.003286294, 0.014328664, 0.008840941, 0.015948916, 0.012835215, 0.019400757, -0.012342094, -0.010693664, 0.004772699, -0.03254593, 0.010707753, -0.016822444, -0.0032827717, 0.021246437, -0.04485985, -0.04384543, -0.015906649, -0.009707424, 0.02299349, 0.019513471, -0.010151232, 0.018963994, -0.0057976847, 0.05739919, -0.019922055, -0.029108182, -0.0106232185, 0.021077367, 0.0036455672, -0.026614401, 0.04497256, -0.04446535, -0.0004556959, -0.004578973, 0.003962573, -0.004910068, 0.015089478, -0.0301226, 0.007664497, 0.008375999, 0.031982366, 0.006135824, 0.02152822, -0.015469885, -0.007210122, 0.034715664, -0.01233505, 0.0004490916, -0.0144413775, -0.003150686, -0.02003477, -0.027924692, -0.0015850292, -0.009376328, -0.0035997776, -0.03240504, -0.010912046, 0.0031999978, 0.022303123, -0.008988877, 0.00024633997, -0.0035698381, 0.0070974086, -0.002599448, -0.042267445, -0.016935157, -0.0002481011, -0.041393917, 0.014483645, 0.019006262, -0.02813603, 0.0072030774, -7.3032425e-05, 0.01802002, -0.017188761, 0.015991183, 0.020401087, 0.03542012, 0.04469078, 0.04071764, 0.011095204, -0.031390622, -0.03254593, 0.014187773, 0.016272966, -0.009721513, -0.026388975, -0.014849963, -0.005642704, -0.022556728, 0.0064457855, -0.043450933, 0.010834555, -0.015977094, 0.020880118, -0.02385293, -0.054806788, 0.03789981, 0.0013516777, -0.026431242, -0.015540331, 0.016695641, -0.037167173, -0.021190079, 0.023881108, -0.0045860177, 0.0064105624, -0.007763121, -0.013053596, 0.024472851, -0.0004962022, -0.00976378, 0.060019772, -0.0057624616, -0.04384543, 0.010313257, 0.0076715415, 0.0025888812, -0.03589915, 0.008791628, -0.012785902, 0.01042597, 0.015653044, 0.04767768, -0.009869449, 0.0064457855, -0.010947268, -0.0077349427, -0.032715004, -0.023867019, -0.011327676, -0.00046274049, -0.036998104, 0.013913034, 0.012250515, -0.009996251, 0.021204168, 0.020091126, -0.003740669, -0.0049769916, -0.0140891485, 0.024064265, 0.0038815604, 0.025684519, 0.041788414, -0.013553761, 0.006681779, -0.0050826604, -0.018175002, 0.008228063, -0.006230926, -0.018907638, 0.0154839745, -0.028713685, -0.015047211, -0.019682541, 0.02516322, 0.040802173, 0.007213644, 0.011743305, -0.015963005, -0.03818159, 0.01191942, -0.031728763, -0.011863063, 0.023881108, 0.0053116092, -0.020992832, -0.017991843, -0.00405063, -0.017780505, -0.0057659843, 0.02978446, 0.031165197, 0.0014221234, 0.021316882, 0.026008569, -0.0018544842, -0.032658648, 0.028474169, 0.013109953, 0.018076377, 0.0007991189, -0.0042373114, 0.028910933, -0.0029358263, 0.021866359, 0.024472851, -0.002576553, -0.033532172, 0.01920351, -0.0095665315, -0.03093977, 0.0034817809, 0.018654034, -0.0074038478, 0.021443684, 0.0038604268, -0.02745975, 0.031587873, 0.0061146906, 0.022711707, -0.019795254, -0.016991513, -0.04471896, -0.007875834, -0.0034941088, -0.043789074, 0.021091456, 0.024909616, -0.013194487, -0.0042690123, 0.027896514, -0.018414518, -0.023303451, -0.025797231, -0.009524264]}], object='list', usage=Usage(completion_tokens=0, prompt_tokens=3, total_tokens=3, completion_tokens_details=None, prompt_tokens_details=None))\n" - ] - } - ], - "source": [ - "import litellm\n", - "\n", - "\n", - "async def main():\n", - " response = await litellm.aembedding(\n", - " model=\"cometapi/text-embedding-3-small\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n", - " api_key=api_key, # your CometAPI api-key\n", - " api_base=\"https://api.cometapi.com/v1\",\n", - " input=\"Your text string\",\n", - " )\n", - " print(response)\n", - "\n", - "\n", - "await main()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Async Image Generation" - ] - }, - { - "cell_type": "code", - "execution_count": 14, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "ImageResponse(created=1760591151, background=None, data=[ImageObject(b64_json=None, revised_prompt=\"Generate an image of an adorable baby sea otter. It should be floating on its back in a calm, clear ocean, playfully grasping a colorful shell in its small paws. The sun is setting in the background, casting a peaceful orange and purple hue across the sky and reflecting upon the ocean waves. The otter's fur is a deep, rich brown and appears silky and wet, with glints of sunlight catching on it. Its eyes are bright, expressing joy and curiosity as it examines its newfound treasure.\", url='https://oaidalleapiprodscus.blob.core.windows.net/private/org-OKnsK88id12jfvnKByup1O0l/user-3GxuMyEg9YMU8LFCPHi31prf/img-7PUEF8Wb6thGDAuZWLJjSnfP.png?st=2025-10-16T04%3A05%3A51Z&se=2025-10-16T06%3A05%3A51Z&sp=r&sv=2024-08-04&sr=b&rscd=inline&rsct=image/png&skoid=38e27a3b-6174-4d3e-90ac-d7d9ad49543f&sktid=a48cca56-e6da-484e-a814-9c849652bcb3&skt=2025-10-16T02%3A51%3A01Z&ske=2025-10-17T02%3A51%3A01Z&sks=b&skv=2024-08-04&sig=IZKG2VE%2B6VdOe5Tq0Zk/5bVyGK/oK/yO8g%2BDX4krpug%3D')], output_format=None, quality=None, size=None, usage=Usage(completion_tokens=0, prompt_tokens=0, total_tokens=0, completion_tokens_details=None, prompt_tokens_details=None, input_tokens=0, input_tokens_details={'image_tokens': 0, 'text_tokens': 0}, output_tokens=0))\n" - ] - } - ], - "source": [ - "import asyncio\n", - "\n", - "import litellm\n", - "\n", - "\n", - "async def main():\n", - " response = await litellm.aimage_generation(\n", - " model=\"cometapi/dall-e-3\", # The model name must include prefix \"cometapi/\" + the model name from CometAPI\n", - " api_key=api_key, # your cometapi api-key\n", - " api_base=\"https://api.cometapi.com/v1\",\n", - " prompt=\"A cute baby sea otter\",\n", - " )\n", - " print(response)\n", - "\n", - "\n", - "await main()" + "## Chat Completion" ] }, { @@ -444,7 +49,275 @@ "execution_count": null, "metadata": {}, "outputs": [], - "source": [] + "source": [ + "from litellm import completion\n", + "\n", + "response = completion(\n", + " model=\"cometapi/gpt-5.5\",\n", + " messages=[{\"role\": \"user\", \"content\": \"Write Python code that prints hi.\"}],\n", + " max_tokens=128,\n", + " api_key=api_key,\n", + ")\n", + "\n", + "print(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Streaming Chat Completion" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "messages = [{\"role\": \"user\", \"content\": \"Count to three.\"}]\n", + "response = completion(\n", + " model=\"cometapi/gpt-5.5\",\n", + " messages=messages,\n", + " max_tokens=64,\n", + " stream=True,\n", + " api_key=api_key,\n", + ")\n", + "\n", + "for part in response:\n", + " print(part.choices[0].delta.content or \"\", end=\"\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Async Chat Completion" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from litellm import acompletion\n", + "\n", + "async def run_async_chat():\n", + " response = await acompletion(\n", + " model=\"cometapi/gpt-5.5\",\n", + " messages=[{\"content\": \"Hello, how are you?\", \"role\": \"user\"}],\n", + " max_tokens=64,\n", + " api_key=api_key,\n", + " )\n", + " return response\n", + "\n", + "response = await run_async_chat()\n", + "print(response.choices[0].message.content)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Async Streaming Chat Completion" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "async def run_async_streaming_chat():\n", + " response = await acompletion(\n", + " model=\"cometapi/gpt-5.5\",\n", + " messages=[{\"content\": \"Stream one short sentence.\", \"role\": \"user\"}],\n", + " max_tokens=64,\n", + " stream=True,\n", + " api_key=api_key,\n", + " )\n", + " async for chunk in response:\n", + " print(chunk.choices[0].delta.content or \"\", end=\"\")\n", + "\n", + "await run_async_streaming_chat()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import litellm\n", + "\n", + "response = litellm.embedding(\n", + " model=\"cometapi/text-embedding-3-small\",\n", + " input=[\"LiteLLM routes this embedding request through CometAPI.\"],\n", + " api_key=api_key,\n", + ")\n", + "\n", + "print(len(response.data[0][\"embedding\"]))\n", + "print(response.usage)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Async Embeddings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "async def run_async_embedding():\n", + " response = await litellm.aembedding(\n", + " model=\"cometapi/text-embedding-3-small\",\n", + " input=\"Your text string\",\n", + " api_key=api_key,\n", + " )\n", + " return response\n", + "\n", + "response = await run_async_embedding()\n", + "print(len(response.data[0][\"embedding\"]))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Image Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "response = litellm.image_generation(\n", + " model=\"cometapi/gpt-image-2\",\n", + " prompt=\"A small comet over a clean API diagram\",\n", + " size=\"1024x1024\",\n", + " api_key=api_key,\n", + ")\n", + "\n", + "image = response.data[0]\n", + "print(image.url or image.b64_json[:64])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Async Image Generation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "async def run_async_image_generation():\n", + " response = await litellm.aimage_generation(\n", + " model=\"cometapi/gpt-image-2\",\n", + " prompt=\"A cute baby sea otter\",\n", + " size=\"1024x1024\",\n", + " api_key=api_key,\n", + " )\n", + " return response\n", + "\n", + "response = await run_async_image_generation()\n", + "image = response.data[0]\n", + "print(image.url or image.b64_json[:64])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Audio Speech" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import io\n", + "\n", + "speech_response = litellm.speech(\n", + " model=\"cometapi/tts-1\",\n", + " input=\"LiteLLM can route speech generation through CometAPI.\",\n", + " voice=\"alloy\",\n", + " api_key=api_key,\n", + ")\n", + "\n", + "speech_audio = io.BytesIO(speech_response.content)\n", + "speech_audio.name = \"speech.mp3\"\n", + "\n", + "print(len(speech_response.content))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Audio Transcription" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "speech_audio.seek(0)\n", + "transcription_response = litellm.transcription(\n", + " model=\"cometapi/whisper-1\",\n", + " file=speech_audio,\n", + " api_key=api_key,\n", + ")\n", + "\n", + "print(transcription_response.text)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Moderations" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "moderation_response = litellm.moderation(\n", + " model=\"omni-moderation-latest\",\n", + " custom_llm_provider=\"cometapi\",\n", + " input=\"I want to build a safe application.\",\n", + " api_key=api_key,\n", + ")\n", + "\n", + "print(moderation_response.results[0].categories)\n", + "print(moderation_response.results[0].category_scores)\n" + ] } ], "metadata": { @@ -452,7 +325,7 @@ "provenance": [] }, "kernelspec": { - "display_name": "base", + "display_name": "Python 3", "language": "python", "name": "python3" }, diff --git a/litellm/images/main.py b/litellm/images/main.py index d95b7287d20..1daa55a182a 100644 --- a/litellm/images/main.py +++ b/litellm/images/main.py @@ -152,6 +152,10 @@ def image_generation( model: Optional[str] = None, n: Optional[int] = None, quality: Optional[Union[str, ImageGenerationRequestQuality]] = None, + background: Optional[str] = None, + moderation: Optional[str] = None, + output_compression: Optional[int] = None, + output_format: Optional[str] = None, response_format: Optional[str] = None, size: Optional[str] = None, style: Optional[str] = None, @@ -176,6 +180,10 @@ def image_generation( model: Optional[str] = None, n: Optional[int] = None, quality: Optional[Union[str, ImageGenerationRequestQuality]] = None, + background: Optional[str] = None, + moderation: Optional[str] = None, + output_compression: Optional[int] = None, + output_format: Optional[str] = None, response_format: Optional[str] = None, size: Optional[str] = None, style: Optional[str] = None, @@ -200,6 +208,10 @@ def image_generation( # noqa: PLR0915 model: Optional[str] = None, n: Optional[int] = None, quality: Optional[Union[str, ImageGenerationRequestQuality]] = None, + background: Optional[str] = None, + moderation: Optional[str] = None, + output_compression: Optional[int] = None, + output_format: Optional[str] = None, response_format: Optional[str] = None, size: Optional[str] = None, style: Optional[str] = None, @@ -270,6 +282,14 @@ def image_generation( # noqa: PLR0915 non_default_params = { k: v for k, v in kwargs.items() if k not in default_params } # model-specific params - pass them straight to the model/provider + for param_name, param_value in { + "background": background, + "moderation": moderation, + "output_compression": output_compression, + "output_format": output_format, + }.items(): + if param_value is not None: + non_default_params[param_name] = param_value image_generation_config: Optional[BaseImageGenerationConfig] = None if ( @@ -404,6 +424,7 @@ def image_generation( # noqa: PLR0915 elif custom_llm_provider in ( litellm.LlmProviders.RECRAFT, litellm.LlmProviders.AIML, + litellm.LlmProviders.COMETAPI, litellm.LlmProviders.GEMINI, litellm.LlmProviders.FAL_AI, litellm.LlmProviders.STABILITY, @@ -417,9 +438,12 @@ def image_generation( # noqa: PLR0915 f"image generation config is not supported for {custom_llm_provider}" ) - # Resolve api_base from litellm.api_base if not explicitly provided _api_base = api_base or litellm.api_base + if custom_llm_provider == litellm.LlmProviders.COMETAPI: + _api_base = api_base + api_key = api_key or dynamic_api_key or litellm.cometapi_key litellm_params_dict["api_base"] = _api_base + litellm_params_dict["api_key"] = api_key return llm_http_handler.image_generation_handler( api_key=api_key, @@ -493,7 +517,6 @@ def image_generation( # noqa: PLR0915 ): if extra_headers is not None: optional_params["extra_headers"] = extra_headers - # Forward OpenAI organization if present (set by proxy pre-call utils) organization: Optional[str] = kwargs.get("organization", None) model_response = openai_chat_completions.image_generation( model=model, diff --git a/litellm/llms/cometapi/chat/transformation.py b/litellm/llms/cometapi/chat/transformation.py index 1e15ee188c6..45c74740dbb 100644 --- a/litellm/llms/cometapi/chat/transformation.py +++ b/litellm/llms/cometapi/chat/transformation.py @@ -5,17 +5,17 @@ Based on OpenAI-compatible API interface implementation Documentation: [CometAPI Documentation Link] """ -from typing import Any, AsyncIterator, Iterator, List, Optional, Tuple, Union +from typing import Any, AsyncIterator, Iterator, List, Optional, Union import httpx from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolParam +from litellm.types.llms.openai import AllMessageValues from litellm.types.utils import ModelResponse, ModelResponseStream from ...openai.chat.gpt_transformation import OpenAIGPTConfig -from ..common_utils import CometAPIException +from ..common_utils import CometAPIException, get_cometapi_complete_url class CometAPIConfig(OpenAIGPTConfig): @@ -26,47 +26,6 @@ class CometAPIConfig(OpenAIGPTConfig): and only need to override necessary methods to handle CometAPI-specific features """ - def map_openai_params( - self, - non_default_params: dict, - optional_params: dict, - model: str, - drop_params: bool, - ) -> dict: - """ - Map OpenAI format parameters to CometAPI format - """ - mapped_openai_params = super().map_openai_params( - non_default_params, optional_params, model, drop_params - ) - - # CometAPI-specific parameters (if any) - extra_body: dict[str, Any] = {} - # TODO: Add CometAPI-specific parameter handling here - # Example: - # custom_param = non_default_params.pop("custom_param", None) - # if custom_param is not None: - # extra_body["custom_param"] = custom_param - - if extra_body: - mapped_openai_params["extra_body"] = extra_body - - return mapped_openai_params - - def remove_cache_control_flag_from_messages_and_tools( - self, - model: str, - messages: List[AllMessageValues], - tools: Optional[List["ChatCompletionToolParam"]] = None, - ) -> Tuple[List[AllMessageValues], Optional[List["ChatCompletionToolParam"]]]: - """ - Remove cache control flags from messages and tools if not supported - """ - # For CometAPI, use default behavior (remove cache control) - return super().remove_cache_control_flag_from_messages_and_tools( - model, messages, tools - ) - def transform_request( self, model: str, @@ -81,10 +40,17 @@ class CometAPIConfig(OpenAIGPTConfig): Returns: dict: The transformed request. Sent as the body of the API call. """ - extra_body = optional_params.pop("extra_body", {}) + extra_body = optional_params.pop("extra_body", {}) or {} response = super().transform_request( model, messages, optional_params, litellm_params, headers ) + overlapping_keys = set(response).intersection(extra_body) + if overlapping_keys: + raise ValueError( + "CometAPI extra_body cannot override request fields: {}".format( + ", ".join(sorted(overlapping_keys)) + ) + ) response.update(extra_body) return response @@ -103,30 +69,7 @@ class CometAPIConfig(OpenAIGPTConfig): Returns: str: The complete URL for the API call. """ - # Default base - if api_base is None: - api_base = "https://api.cometapi.com/v1" - endpoint = "chat/completions" - - # Normalize - api_base = api_base.rstrip("/") - - # If endpoint already present, return as-is - if endpoint in api_base: - return api_base - - # Ensure we include /v1 prefix when missing - if api_base.endswith("/v1"): - return f"{api_base}/{endpoint}" - if api_base.endswith("/v1/"): - return f"{api_base}{endpoint}" - # If user provided https://api.cometapi.com, add /v1 - if api_base == "https://api.cometapi.com": - return f"{api_base}/v1/{endpoint}" - # Generic fallback: if '/v1' not in path, add it - if "/v1" not in api_base.split("//", 1)[-1]: - return f"{api_base}/v1/{endpoint}" - return f"{api_base}/{endpoint}" + return get_cometapi_complete_url(api_base, "chat/completions") def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] diff --git a/litellm/llms/cometapi/common_utils.py b/litellm/llms/cometapi/common_utils.py index 8cb0a304026..28bcc1307e6 100644 --- a/litellm/llms/cometapi/common_utils.py +++ b/litellm/llms/cometapi/common_utils.py @@ -1,4 +1,70 @@ +from typing import Optional +from urllib.parse import urlsplit, urlunsplit + from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str + +DEFAULT_COMETAPI_API_BASE = "https://api.cometapi.com/v1" + + +def get_cometapi_api_key(api_key: Optional[str] = None) -> Optional[str]: + return ( + api_key or get_secret_str("COMETAPI_KEY") or get_secret_str("COMETAPI_API_KEY") + ) + + +def get_cometapi_api_base(api_base: Optional[str] = None) -> str: + return ( + api_base + or get_secret_str("COMETAPI_BASE_URL") + or get_secret_str("COMETAPI_API_BASE") + or DEFAULT_COMETAPI_API_BASE + ) + + +def get_cometapi_complete_url(api_base: Optional[str], endpoint: str) -> str: + base_url = get_cometapi_api_base(api_base).rstrip("/") + normalized_endpoint = endpoint.strip("/") + parsed_base_url = urlsplit(base_url) + path_segments = [segment for segment in parsed_base_url.path.split("/") if segment] + endpoint_segments = normalized_endpoint.split("/") + invalid_version_segments = [ + segment + for segment in path_segments + if segment.startswith("v") and segment[1:2].isdigit() and segment != "v1" + ] + if "v1" not in path_segments and invalid_version_segments: + raise ValueError("CometAPI OpenAI-compatible endpoints require a /v1 api_base") + + if path_segments[-len(endpoint_segments) :] == endpoint_segments: + if "v1" not in path_segments: + raise ValueError( + "CometAPI OpenAI-compatible endpoints require a /v1 api_base" + ) + return base_url + + if "v1" in path_segments: + complete_path_segments = path_segments + endpoint_segments + else: + complete_path_segments = path_segments + ["v1"] + endpoint_segments + + complete_path = "/" + "/".join(complete_path_segments) + return urlunsplit( + ( + parsed_base_url.scheme, + parsed_base_url.netloc, + complete_path, + parsed_base_url.query, + parsed_base_url.fragment, + ) + ) + + +def require_cometapi_api_key(api_key: Optional[str] = None) -> str: + final_api_key = get_cometapi_api_key(api_key) + if not final_api_key: + raise ValueError("COMETAPI_KEY or COMETAPI_API_KEY is not set") + return final_api_key class CometAPIException(BaseLLMException): diff --git a/litellm/llms/cometapi/embed/transformation.py b/litellm/llms/cometapi/embed/transformation.py index d1972def8b7..23e1b6d440f 100644 --- a/litellm/llms/cometapi/embed/transformation.py +++ b/litellm/llms/cometapi/embed/transformation.py @@ -9,11 +9,14 @@ import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig -from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage -from ..common_utils import CometAPIException +from ..common_utils import ( + CometAPIException, + get_cometapi_complete_url, + require_cometapi_api_key, +) class CometAPIEmbeddingConfig(BaseEmbeddingConfig): @@ -39,11 +42,7 @@ class CometAPIEmbeddingConfig(BaseEmbeddingConfig): """ Get the complete URL for the CometAPI embedding endpoint. """ - api_base = ( - "https://api.cometapi.com/v1" if api_base is None else api_base.rstrip("/") - ) - complete_url = f"{api_base}/embeddings" - return complete_url + return get_cometapi_complete_url(api_base, "embeddings") def validate_environment( self, @@ -58,11 +57,10 @@ class CometAPIEmbeddingConfig(BaseEmbeddingConfig): """ Validate and set up authentication headers for CometAPI. """ - if api_key is None: - api_key = get_secret_str("COMETAPI_KEY") + final_api_key = require_cometapi_api_key(api_key) default_headers = { - "Authorization": f"Bearer {api_key}", + "Authorization": f"Bearer {final_api_key}", "accept": "application/json", "Content-Type": "application/json", } diff --git a/litellm/llms/cometapi/image_generation/transformation.py b/litellm/llms/cometapi/image_generation/transformation.py index bc6bd3f3ecc..53cf2107ab2 100644 --- a/litellm/llms/cometapi/image_generation/transformation.py +++ b/litellm/llms/cometapi/image_generation/transformation.py @@ -1,16 +1,23 @@ -from typing import TYPE_CHECKING, Any, List, Optional +from typing import TYPE_CHECKING, Any, List, Optional, Union import httpx +from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.image_generation.transformation import ( BaseImageGenerationConfig, ) -from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ( AllMessageValues, OpenAIImageGenerationOptionalParams, ) -from litellm.types.utils import ImageObject, ImageResponse +from litellm.types.utils import ImageResponse +from litellm.utils import convert_to_model_response_object + +from ..common_utils import ( + CometAPIException, + get_cometapi_complete_url, + require_cometapi_api_key, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj @@ -21,8 +28,28 @@ else: class CometAPIImageGenerationConfig(BaseImageGenerationConfig): - DEFAULT_BASE_URL: str = "https://api.cometapi.com" - IMAGE_GENERATION_ENDPOINT: str = "v1/images/generations" + @staticmethod + def _normalize_image_usage(response_data: dict) -> None: + usage = response_data.get("usage") + if not isinstance(usage, dict): + return + + if usage.get("input_tokens") is None: + usage["input_tokens"] = 0 + if usage.get("output_tokens") is None: + usage["output_tokens"] = 0 + if usage.get("total_tokens") is None: + usage["total_tokens"] = usage["input_tokens"] + usage["output_tokens"] + + input_tokens_details = usage.get("input_tokens_details") + if not isinstance(input_tokens_details, dict): + usage["input_tokens_details"] = {"image_tokens": 0, "text_tokens": 0} + return + + if input_tokens_details.get("image_tokens") is None: + input_tokens_details["image_tokens"] = 0 + if input_tokens_details.get("text_tokens") is None: + input_tokens_details["text_tokens"] = 0 def get_supported_openai_params( self, model: str @@ -31,11 +58,16 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): https://api.cometapi.com/v1/images/generations """ return [ + "background", + "moderation", "n", + "output_compression", + "output_format", "quality", "response_format", "size", "style", + "user", ] def map_openai_params( @@ -50,7 +82,6 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): for k in non_default_params.keys(): if k not in optional_params.keys(): if k in supported_params: - # CometAPI uses OpenAI-compatible parameters, so we can pass them directly optional_params[k] = non_default_params[k] elif drop_params: pass @@ -73,16 +104,7 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): """ Get the complete url for the request """ - complete_url: str = ( - api_base - or get_secret_str("COMETAPI_BASE_URL") - or get_secret_str("COMETAPI_API_BASE") - or self.DEFAULT_BASE_URL - ) - - complete_url = complete_url.rstrip("/") - complete_url = f"{complete_url}/{self.IMAGE_GENERATION_ENDPOINT}" - return complete_url + return get_cometapi_complete_url(api_base, "images/generations") def validate_environment( self, @@ -94,13 +116,7 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): api_key: Optional[str] = None, api_base: Optional[str] = None, ) -> dict: - final_api_key: Optional[str] = ( - api_key - or get_secret_str("COMETAPI_KEY") - or get_secret_str("COMETAPI_API_KEY") - ) - if not final_api_key: - raise ValueError("COMETAPI_KEY or COMETAPI_API_KEY is not set") + final_api_key = require_cometapi_api_key(api_key) headers["Authorization"] = f"Bearer {final_api_key}" headers["Content-Type"] = "application/json" @@ -119,7 +135,6 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): https://api.cometapi.com/v1/images/generations """ - # CometAPI uses OpenAI-compatible format request_body = { "prompt": prompt, "model": model, @@ -154,17 +169,43 @@ class CometAPIImageGenerationConfig(BaseImageGenerationConfig): headers=raw_response.headers, ) - if not model_response.data: - model_response.data = [] + if raw_response.status_code >= 400: + error_data = response_data.get("error", response_data) + error_message = ( + error_data.get("message") + if isinstance(error_data, dict) + else str(error_data) + ) + raise self.get_error_class( + error_message=error_message or str(response_data), + status_code=raw_response.status_code, + headers=raw_response.headers, + ) - # CometAPI returns OpenAI-compatible format - # Expected format: {"created": timestamp, "data": [{"url": "...", "b64_json": "..."}]} - if "data" in response_data: - for image_data in response_data["data"]: - image_obj = ImageObject( - b64_json=image_data.get("b64_json"), - url=image_data.get("url"), - ) - model_response.data.append(image_obj) + self._normalize_image_usage(response_data) + logging_obj.post_call( + input=request_data.get("prompt", ""), + api_key=api_key, + additional_args={"complete_input_dict": request_data}, + original_response=response_data, + ) + image_response: ImageResponse = convert_to_model_response_object( # type: ignore + response_object=response_data, + model_response_object=model_response, + response_type="image_generation", + ) - return model_response + image_response.size = optional_params.get("size") + image_response.quality = optional_params.get("quality") + image_response.output_format = optional_params.get( + "output_format", optional_params.get("response_format") + ) + + return image_response + + def get_error_class( + self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] + ) -> BaseLLMException: + return CometAPIException( + message=error_message, status_code=status_code, headers=headers + ) diff --git a/litellm/main.py b/litellm/main.py index 3ef094042e8..248a462d401 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -322,6 +322,20 @@ oci_transformation = OCIChatConfig() ovhcloud_transformation = OVHCloudChatConfig() lemonade_transformation = LemonadeChatConfig() + +def _get_cometapi_key_and_base( + api_key: Optional[str] = None, api_base: Optional[str] = None +) -> Tuple[str, str]: + from litellm.llms.cometapi.common_utils import ( + get_cometapi_api_base, + require_cometapi_api_key, + ) + + return require_cometapi_api_key( + api_key or litellm.cometapi_key + ), get_cometapi_api_base(api_base) + + MOCK_RESPONSE_TYPE = Union[str, Exception, dict, ModelResponse, ModelResponseStream] ####### COMPLETION ENDPOINTS ################ @@ -2538,18 +2552,8 @@ def completion( # type: ignore # noqa: PLR0915 stream=stream, ) elif custom_llm_provider == "cometapi": - api_key = ( - api_key - or litellm.cometapi_key - or get_secret_str("COMETAPI_KEY") - or litellm.api_key - ) - - api_base = ( - api_base - or litellm.api_base - or get_secret_str("COMETAPI_API_BASE") - or "https://api.cometapi.com/v1" + api_key, api_base = _get_cometapi_key_and_base( + api_key=api_key, api_base=api_base ) ## COMPLETION CALL @@ -5832,17 +5836,8 @@ def embedding( # noqa: PLR0915 litellm_params={}, ) elif custom_llm_provider == "cometapi": - api_key = ( - api_key - or litellm.cometapi_key - or get_secret_str("COMETAPI_KEY") - or litellm.api_key - ) - api_base = ( - api_base - or litellm.api_base - or get_secret_str("COMETAPI_API_BASE") - or "https://api.cometapi.com/v1" + api_key, api_base = _get_cometapi_key_and_base( + api_key=api_key, api_base=api_base ) response = base_llm_http_handler.embedding( model=model, @@ -6411,16 +6406,38 @@ def adapter_completion( def moderation( input: str, model: Optional[str] = None, api_key: Optional[str] = None, **kwargs ) -> OpenAIModerationResponse: - # only supports open ai for now - api_key = ( - api_key - or litellm.api_key - or litellm.openai_key - or get_secret_str("OPENAI_API_KEY") - ) - - # Extract api_base from kwargs api_base = kwargs.get("api_base", None) + custom_llm_provider = kwargs.get("custom_llm_provider", None) + _dynamic_api_key = None + _dynamic_api_base = None + try: + ( + model, + custom_llm_provider, + _dynamic_api_key, + _dynamic_api_base, + ) = litellm.get_llm_provider( + model=model or "", + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + except litellm.BadRequestError: + pass + + if custom_llm_provider == "cometapi": + api_key, api_base = _get_cometapi_key_and_base( + api_key=api_key or _dynamic_api_key, api_base=api_base or _dynamic_api_base + ) + else: + api_key = ( + api_key + or _dynamic_api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + ) + api_base = api_base or _dynamic_api_base openai_client = kwargs.get("client", None) if openai_client is None: @@ -6450,17 +6467,11 @@ async def amoderation( ) -> OpenAIModerationResponse: from openai import AsyncOpenAI - # only supports open ai for now - api_key = ( - api_key - or litellm.api_key - or litellm.openai_key - or get_secret_str("OPENAI_API_KEY") - ) optional_params = GenericLiteLLMParams(**kwargs) litellm_logging_obj: Optional[LiteLLMLoggingObj] = kwargs.get( "litellm_logging_obj", None ) + _dynamic_api_key = None _dynamic_api_base = None try: ( @@ -6478,6 +6489,21 @@ async def amoderation( # `model` is optional field for moderation - get_llm_provider will throw BadRequestError if model is not set / not recognized pass + if custom_llm_provider == "cometapi": + api_key, api_base = _get_cometapi_key_and_base( + api_key=api_key or _dynamic_api_key, + api_base=optional_params.api_base or _dynamic_api_base, + ) + else: + api_key = ( + api_key + or _dynamic_api_key + or litellm.api_key + or litellm.openai_key + or get_secret_str("OPENAI_API_KEY") + ) + api_base = optional_params.api_base or _dynamic_api_base + openai_client = kwargs.get("client", None) if openai_client is None or not isinstance(openai_client, AsyncOpenAI): # call helper to get OpenAI client @@ -6485,7 +6511,7 @@ async def amoderation( _openai_client: AsyncOpenAI = openai_chat_completions._get_openai_client( # type: ignore is_async=True, api_key=api_key, - api_base=optional_params.api_base or _dynamic_api_base, + api_base=api_base, ) else: _openai_client = openai_client @@ -6587,7 +6613,7 @@ async def atranscription(*args, **kwargs) -> TranscriptionResponse: @client -def transcription( +def transcription( # noqa: PLR0915 model: str, file: FileTypes, ## OPTIONAL OPENAI PARAMS ## @@ -6725,6 +6751,26 @@ def transcription( max_retries=max_retries, litellm_params=litellm_params_dict, ) + elif custom_llm_provider == "cometapi": + api_key, api_base = _get_cometapi_key_and_base( + api_key=api_key, api_base=api_base + ) + response = openai_audio_transcriptions.audio_transcriptions( + model=model, + audio_file=file, + optional_params=optional_params, + model_response=model_response, + atranscription=atranscription, + client=client, + timeout=timeout, + logging_obj=litellm_logging_obj, + max_retries=max_retries, + api_base=api_base, + api_key=api_key, + provider_config=provider_config, + litellm_params=litellm_params_dict, + shared_session=shared_session, + ) elif custom_llm_provider == "openai" or ( custom_llm_provider in litellm.openai_compatible_providers ): @@ -6740,8 +6786,6 @@ def transcription( or get_secret("OPENAI_ORGANIZATION") or None # default - https://github.com/openai/openai-python/blob/284c1799070c723c6a553337134148a7ab088dd8/openai/util.py#L105 ) - # set API KEY - api_key = api_key or litellm.api_key or litellm.openai_key or get_secret("OPENAI_API_KEY") # type: ignore response = openai_audio_transcriptions.audio_transcriptions( model=model, @@ -6953,7 +6997,33 @@ def speech( # noqa: PLR0915 Coroutine[Any, Any, HttpxBinaryResponseContent], None, ] = None - if ( + if custom_llm_provider == "cometapi": + if voice is None or not (isinstance(voice, str)): + raise litellm.BadRequestError( + message="'voice' is required to be passed as a string for OpenAI TTS", + model=model, + llm_provider=custom_llm_provider, + ) + api_key, api_base = _get_cometapi_key_and_base( + api_key=api_key or dynamic_api_key, api_base=api_base + ) + headers = headers or litellm.headers + response = openai_chat_completions.audio_speech( + model=model, + input=input, + voice=voice, + optional_params=optional_params, + api_key=api_key, + api_base=api_base, + organization=None, + project=None, + max_retries=max_retries, + timeout=timeout, + client=client, # pass AsyncOpenAI, OpenAI client + aspeech=aspeech, + shared_session=shared_session, + ) + elif ( custom_llm_provider == "openai" or custom_llm_provider in litellm.openai_compatible_providers ): @@ -6970,7 +7040,6 @@ def speech( # noqa: PLR0915 or get_secret("OPENAI_API_BASE") or "https://api.openai.com/v1" ) # type: ignore - # set API KEY api_key = ( api_key or litellm.api_key # for deepinfra/perplexity/anyscale we check in get_llm_provider and pass in the api key from there diff --git a/litellm/utils.py b/litellm/utils.py index 6188206148f..c9768a5d886 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -3143,6 +3143,10 @@ def get_optional_params_image_gen( model: Optional[str] = None, n: Optional[int] = None, quality: Optional[str] = None, + background: Optional[str] = None, + moderation: Optional[str] = None, + output_compression: Optional[int] = None, + output_format: Optional[str] = None, response_format: Optional[str] = None, size: Optional[str] = None, style: Optional[str] = None, @@ -3177,7 +3181,11 @@ def get_optional_params_image_gen( passed_params[k] = v default_params = { + "background": None, + "moderation": None, "n": None, + "output_compression": None, + "output_format": None, "quality": None, "response_format": None, "size": None, diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 3a01541060f..fc3fe65f1c2 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -607,10 +607,10 @@ "messages": true, "responses": true, "embeddings": true, - "image_generations": false, - "audio_transcriptions": false, - "audio_speech": false, - "moderations": false, + "image_generations": true, + "audio_transcriptions": true, + "audio_speech": true, + "moderations": true, "batches": false, "rerank": false, "a2a": true, diff --git a/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py b/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py index 0b3348c1b7f..8a237de25e5 100644 --- a/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py +++ b/tests/test_litellm/llms/cometapi/chat/test_cometapi_chat_transformation.py @@ -30,7 +30,7 @@ class TestCometAPIChatCompletionStreamingHandler: chunk = { "id": "test_id", "created": 1234567890, - "model": "gpt-3.5-turbo", + "model": "gpt-5.5", "usage": {"prompt_tokens": 10, "completion_tokens": 20, "total_tokens": 30}, "choices": [ {"delta": {"content": "test content", "reasoning": "test reasoning"}} @@ -44,7 +44,7 @@ class TestCometAPIChatCompletionStreamingHandler: assert result.id == "test_id" assert result.object == "chat.completion.chunk" assert result.created == 1234567890 - assert result.model == "gpt-3.5-turbo" + assert result.model == "gpt-5.5" assert result.usage.prompt_tokens == chunk["usage"]["prompt_tokens"] assert result.usage.completion_tokens == chunk["usage"]["completion_tokens"] assert result.usage.total_tokens == chunk["usage"]["total_tokens"] @@ -93,14 +93,14 @@ class TestCometAPIConfig: config = CometAPIConfig() transformed_request = config.transform_request( - model="cometapi/gpt-3.5-turbo", + model="cometapi/gpt-5.5", messages=[{"role": "user", "content": "Hello, world!"}], optional_params={}, litellm_params={}, headers={}, ) - assert transformed_request["model"] == "cometapi/gpt-3.5-turbo" + assert transformed_request["model"] == "cometapi/gpt-5.5" assert transformed_request["messages"] == [ {"role": "user", "content": "Hello, world!"} ] @@ -110,7 +110,7 @@ class TestCometAPIConfig: config = CometAPIConfig() transformed_request = config.transform_request( - model="cometapi/gpt-4", + model="cometapi/gpt-5.5", messages=[{"role": "user", "content": "Hello, world!"}], optional_params={"extra_body": {"custom_param": "custom_value"}}, litellm_params={}, @@ -123,12 +123,44 @@ class TestCometAPIConfig: {"role": "user", "content": "Hello, world!"} ] + def test_transform_request_allows_empty_extra_body(self): + config = CometAPIConfig() + + transformed_request = config.transform_request( + model="cometapi/gpt-5.5", + messages=[{"role": "user", "content": "Hello, world!"}], + optional_params={"extra_body": None}, + litellm_params={}, + headers={}, + ) + + assert transformed_request["model"] == "cometapi/gpt-5.5" + assert transformed_request["messages"] == [ + {"role": "user", "content": "Hello, world!"} + ] + + def test_transform_request_extra_body_cannot_override_core_fields(self): + """Test extra_body cannot override the generated request body""" + config = CometAPIConfig() + + with pytest.raises( + ValueError, + match="CometAPI extra_body cannot override request fields: model", + ): + config.transform_request( + model="cometapi/gpt-5.5", + messages=[{"role": "user", "content": "Hello, world!"}], + optional_params={"extra_body": {"model": "cometapi/gpt-5.5-all"}}, + litellm_params={}, + headers={}, + ) + def test_cache_control_flag_removal(self): """Test cache control flag removal from messages""" config = CometAPIConfig() transformed_request = config.transform_request( - model="cometapi/gpt-3.5-turbo", + model="cometapi/gpt-5.5", messages=[ { "role": "user", @@ -157,7 +189,7 @@ class TestCometAPIConfig: mapped_params = config.map_openai_params( non_default_params=non_default_params, optional_params={}, - model="cometapi/gpt-3.5-turbo", + model="cometapi/gpt-5.5", drop_params=False, ) @@ -179,142 +211,17 @@ class TestCometAPIConfig: assert error.message == "Test error" assert error.status_code == 400 + def test_get_complete_url(self): + """Test CometAPI chat endpoint URL normalization""" + config = CometAPIConfig() -# Integration test example (requires real API key) -@pytest.mark.skip(reason="Skipping integration test") -def test_cometapi_integration(): - """ - Integration test - requires real API key - Run with: pytest -k test_cometapi_integration -s - """ - import os - from litellm import completion - - # Try to get API key from multiple environment variables - api_key = ( - os.getenv("COMETAPI_API_KEY") - or os.getenv("COMETAPI_KEY") - or os.getenv("COMET_API_KEY") - ) - - if not api_key: - pytest.skip("COMETAPI_API_KEY not set - skipping integration test") - - response = completion( - model="cometapi/gpt-3.5-turbo", - messages=[{"role": "user", "content": "Say hello in one word"}], - api_key=api_key, - max_tokens=10, - temperature=0.7, - ) - - # Verify response structure - assert response.choices[0].message.content - assert len(response.choices[0].message.content.strip()) > 0 - assert response.model - assert response.usage - assert response.usage.total_tokens > 0 - - -def test_cometapi_streaming_integration(): - """ - Integration test for streaming - requires real API key - Run with: pytest -k test_cometapi_streaming_integration -s - """ - import os - from litellm import completion - - # Try to get API key from multiple environment variables - api_key = ( - os.getenv("COMETAPI_API_KEY") - or os.getenv("COMETAPI_KEY") - or os.getenv("COMET_API_KEY") - ) - - if not api_key: - pytest.skip("COMETAPI_API_KEY not set - skipping streaming integration test") - - try: - print( - f"🔍 Testing streaming with API key: {api_key[:6]}...{api_key[-4:]} (length: {len(api_key)})" + assert ( + config.get_complete_url( + api_base="https://api.cometapi.com/v1", + api_key=None, + model="gpt-5.5", + optional_params={}, + litellm_params={}, + ) + == "https://api.cometapi.com/v1/chat/completions" ) - print(f"🔍 API base URL: {os.getenv('COMETAPI_API_BASE', 'default')}") - - # test streaming API call - response = completion( - model="cometapi/gpt-3.5-turbo", - messages=[{"role": "user", "content": "Count from 1 to 5"}], - api_key=api_key, - max_tokens=50, - stream=True, - ) - - # collect streaming response - chunks = [] - content_parts = [] - - for chunk in response: - chunks.append(chunk) - if chunk.choices[0].delta.content: - content_parts.append(chunk.choices[0].delta.content) - - # Verify we received at least one chunk and content - assert len(chunks) > 0, "Should receive at least one chunk" - assert len(content_parts) > 0, "Should receive content in chunks" - - full_content = "".join(content_parts) - assert len(full_content.strip()) > 0, "Should have non-empty content" - - print(f"✅ Received {len(chunks)} chunks") - print(f"✅ Full content: {full_content}") - - except Exception as e: - print(f"❌ Streaming integration test error details:") - print(f" Error type: {type(e).__name__}") - print(f" Error message: {str(e)}") - if hasattr(e, "status_code"): - print(f" Status code: {e.status_code}") - if hasattr(e, "response"): - print(f" Response: {e.response}") - - # Re-raise with more context for pytest - pytest.fail(f"Streaming integration test failed: {type(e).__name__}: {str(e)}") - - -def test_cometapi_with_custom_base_url(): - """ - Test CometAPI with custom base URL - """ - import os - from litellm import completion - - api_key = ( - os.getenv("COMETAPI_API_KEY") - or os.getenv("COMETAPI_KEY") - or os.getenv("COMET_API_KEY") - ) - - custom_base_url = os.getenv("COMETAPI_API_BASE", "https://api.cometapi.com/v1") - - if not api_key: - pytest.skip("COMETAPI_API_KEY not set - skipping custom base URL test") - - try: - response = completion( - model="cometapi/gpt-3.5-turbo", - messages=[{"role": "user", "content": "Hello"}], - api_key=api_key, - api_base=custom_base_url, - max_tokens=5, - ) - - assert response.choices[0].message.content - print(f"✅ Custom base URL test passed: {response.choices[0].message.content}") - - except Exception as e: - pytest.fail(f"Custom base URL test failed: {str(e)}") - - -if __name__ == "__main__": - # Quick test runner - pytest.main([__file__, "-v"]) diff --git a/tests/test_litellm/llms/cometapi/test_cometapi_endpoints.py b/tests/test_litellm/llms/cometapi/test_cometapi_endpoints.py new file mode 100644 index 00000000000..ab7e681d476 --- /dev/null +++ b/tests/test_litellm/llms/cometapi/test_cometapi_endpoints.py @@ -0,0 +1,677 @@ +import io +import json +from pathlib import Path +from unittest.mock import MagicMock, patch + +import httpx +import pytest + +import litellm +from litellm import completion +from litellm.images.main import image_generation +from litellm.llms.cometapi.common_utils import ( + CometAPIException, + get_cometapi_api_base, + get_cometapi_api_key, + get_cometapi_complete_url, +) +from litellm.llms.cometapi.embed.transformation import CometAPIEmbeddingConfig +from litellm.llms.cometapi.image_generation.transformation import ( + CometAPIImageGenerationConfig, +) +from litellm.main import amoderation, embedding, moderation, speech, transcription +from litellm.types.utils import EmbeddingResponse, ImageResponse, TranscriptionResponse + + +def _clear_cometapi_env(monkeypatch): + monkeypatch.delenv("COMETAPI_KEY", raising=False) + monkeypatch.delenv("COMETAPI_API_KEY", raising=False) + monkeypatch.delenv("COMETAPI_BASE_URL", raising=False) + monkeypatch.delenv("COMETAPI_API_BASE", raising=False) + monkeypatch.setattr(litellm, "cometapi_key", None, raising=False) + + +def _pollute_openai_globals(monkeypatch): + monkeypatch.setattr(litellm, "api_key", "openai-global-key", raising=False) + monkeypatch.setattr(litellm, "openai_key", "openai-provider-key", raising=False) + monkeypatch.setattr(litellm, "api_base", "https://openai.invalid/v1", raising=False) + + +@pytest.mark.parametrize( + ("api_base", "expected"), + [ + (None, "https://api.cometapi.com/v1/embeddings"), + ("https://api.cometapi.com", "https://api.cometapi.com/v1/embeddings"), + ("https://api.cometapi.com/v1", "https://api.cometapi.com/v1/embeddings"), + ( + "https://api.cometapi.com/v1/embeddings", + "https://api.cometapi.com/v1/embeddings", + ), + ( + "https://proxy.example.com/openai/v1", + "https://proxy.example.com/openai/v1/embeddings", + ), + ( + "https://proxy.example.com/vertex/openai/v1", + "https://proxy.example.com/vertex/openai/v1/embeddings", + ), + ( + "https://proxy.example.com/api/v2/openai/v1", + "https://proxy.example.com/api/v2/openai/v1/embeddings", + ), + ( + "https://proxy.example.com/openai/v1/embeddings", + "https://proxy.example.com/openai/v1/embeddings", + ), + ], +) +def test_cometapi_embedding_url_normalization(api_base, expected, monkeypatch): + _clear_cometapi_env(monkeypatch) + + assert ( + CometAPIEmbeddingConfig().get_complete_url( + api_base=api_base, + api_key=None, + model="text-embedding-3-small", + optional_params={}, + litellm_params={}, + ) + == expected + ) + + +def test_cometapi_key_and_base_precedence(monkeypatch): + _clear_cometapi_env(monkeypatch) + monkeypatch.setenv("COMETAPI_KEY", "comet-env-key") + monkeypatch.setenv("COMETAPI_BASE_URL", "https://proxy.example.com/openai/v1") + + assert get_cometapi_api_key("explicit-key") == "explicit-key" + assert get_cometapi_api_key() == "comet-env-key" + assert get_cometapi_api_base("https://explicit.example.com/v1") == ( + "https://explicit.example.com/v1" + ) + assert get_cometapi_api_base() == "https://proxy.example.com/openai/v1" + + +def test_cometapi_complete_url_preserves_existing_v1_path(monkeypatch): + _clear_cometapi_env(monkeypatch) + + assert ( + get_cometapi_complete_url("https://proxy.example.com/openai/v1", "embeddings") + == "https://proxy.example.com/openai/v1/embeddings" + ) + + +@pytest.mark.parametrize( + "api_base", + [ + "https://proxy.example.com/openai/v1beta", + "https://proxy.example.com/openai/v1beta/embeddings", + "https://proxy.example.com/openai/v10", + "https://proxy.example.com/openai/embeddings", + ], +) +def test_cometapi_complete_url_rejects_non_v1_paths(api_base, monkeypatch): + _clear_cometapi_env(monkeypatch) + + with pytest.raises( + ValueError, + match="CometAPI OpenAI-compatible endpoints require a /v1 api_base", + ): + get_cometapi_complete_url(api_base, "embeddings") + + +def test_cometapi_embedding_uses_api_key_alias(monkeypatch): + _clear_cometapi_env(monkeypatch) + monkeypatch.setenv("COMETAPI_API_KEY", "comet-env-key") + + headers = CometAPIEmbeddingConfig().validate_environment( + headers={}, + model="text-embedding-3-small", + messages=[], + optional_params={}, + litellm_params={}, + ) + + assert headers["Authorization"] == "Bearer comet-env-key" + + +def test_cometapi_embedding_main_uses_cometapi_key_and_base(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + monkeypatch.setenv("COMETAPI_API_KEY", "comet-env-key") + monkeypatch.setenv("COMETAPI_BASE_URL", "https://proxy.example.com/openai/v1") + + with patch( + "litellm.main.base_llm_http_handler.embedding", + return_value=EmbeddingResponse(), + ) as mock_embedding: + embedding(model="cometapi/text-embedding-3-small", input=["hello"]) + + call_kwargs = mock_embedding.call_args.kwargs + assert call_kwargs["model"] == "text-embedding-3-small" + assert call_kwargs["api_key"] == "comet-env-key" + assert call_kwargs["api_base"] == "https://proxy.example.com/openai/v1" + + +def test_cometapi_completion_uses_cometapi_key_and_base(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + monkeypatch.setenv("COMETAPI_API_KEY", "comet-env-key") + monkeypatch.setenv("COMETAPI_BASE_URL", "https://proxy.example.com/openai/v1") + mock_response = MagicMock() + + with patch( + "litellm.main.base_llm_http_handler.completion", + return_value=mock_response, + ) as mock_completion: + response = completion( + model="cometapi/gpt-5.5", + messages=[{"role": "user", "content": "hello"}], + ) + + call_kwargs = mock_completion.call_args.kwargs + assert response is mock_response + assert call_kwargs["model"] == "gpt-5.5" + assert call_kwargs["api_key"] == "comet-env-key" + assert call_kwargs["api_base"] == "https://proxy.example.com/openai/v1" + assert call_kwargs["custom_llm_provider"] == "cometapi" + + +def test_cometapi_embedding_missing_key_fails(monkeypatch): + _clear_cometapi_env(monkeypatch) + + with pytest.raises(ValueError, match="COMETAPI_KEY or COMETAPI_API_KEY"): + CometAPIEmbeddingConfig().validate_environment( + headers={}, + model="text-embedding-3-small", + messages=[], + optional_params={}, + litellm_params={}, + ) + + +def test_cometapi_image_generation_url_normalization(): + config = CometAPIImageGenerationConfig() + + assert ( + config.get_complete_url( + api_base="https://api.cometapi.com/v1", + api_key=None, + model="gpt-image-2", + optional_params={}, + litellm_params={}, + ) + == "https://api.cometapi.com/v1/images/generations" + ) + assert ( + config.get_complete_url( + api_base="https://api.cometapi.com/v1/images/generations", + api_key=None, + model="gpt-image-2", + optional_params={}, + litellm_params={}, + ) + == "https://api.cometapi.com/v1/images/generations" + ) + + +def test_cometapi_image_generation_validate_environment_uses_api_key_alias(monkeypatch): + _clear_cometapi_env(monkeypatch) + monkeypatch.setenv("COMETAPI_API_KEY", "comet-env-key") + + headers = CometAPIImageGenerationConfig().validate_environment( + headers={}, + model="gpt-image-2", + messages=[], + optional_params={}, + litellm_params={}, + ) + + assert headers["Authorization"] == "Bearer comet-env-key" + assert headers["Content-Type"] == "application/json" + + +def test_cometapi_image_generation_maps_new_openai_params(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + with patch( + "litellm.images.main.llm_http_handler.image_generation_handler", + return_value=ImageResponse(data=[{"url": "https://example.com/image.png"}]), + ) as mock_image_handler: + image_generation( + model="cometapi/gpt-image-2", + prompt="A small comet over a clean API diagram", + api_key="comet-explicit-key", + output_compression=70, + output_format="png", + size="1024x1024", + ) + + call_kwargs = mock_image_handler.call_args.kwargs + assert call_kwargs["api_key"] == "comet-explicit-key" + assert call_kwargs["custom_llm_provider"] == "cometapi" + assert call_kwargs["litellm_params"]["api_base"] is None + assert ( + call_kwargs["image_generation_optional_request_params"]["output_compression"] + == 70 + ) + assert ( + call_kwargs["image_generation_optional_request_params"]["output_format"] + == "png" + ) + + +def test_cometapi_image_generation_normalizes_null_usage_fields(): + raw_response = httpx.Response( + 200, + json={ + "created": 123, + "data": [{"url": "https://example.com/image.png"}], + "usage": { + "input_tokens": 12, + "input_tokens_details": { + "image_tokens": None, + "text_tokens": None, + }, + "output_tokens": 100, + "total_tokens": 112, + }, + }, + request=httpx.Request("POST", "https://api.cometapi.com/v1/images/generations"), + ) + + response = CometAPIImageGenerationConfig().transform_image_generation_response( + model="gpt-image-2", + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={"prompt": "A small comet"}, + optional_params={"size": "1024x1024"}, + litellm_params={}, + encoding=None, + ) + + assert response.data[0].url == "https://example.com/image.png" + assert response.usage.input_tokens_details.image_tokens == 0 + assert response.usage.input_tokens_details.text_tokens == 0 + assert response.usage.input_tokens == 12 + assert response.usage.output_tokens == 100 + assert response.usage.total_tokens == 112 + + +def test_cometapi_image_generation_handles_missing_usage(): + raw_response = httpx.Response( + 200, + json={ + "created": 123, + "data": [{"url": "https://example.com/image.png"}], + }, + request=httpx.Request("POST", "https://api.cometapi.com/v1/images/generations"), + ) + + response = CometAPIImageGenerationConfig().transform_image_generation_response( + model="gpt-image-2", + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={"prompt": "A small comet"}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert response.data[0].url == "https://example.com/image.png" + assert response.usage.input_tokens == 0 + assert response.usage.input_tokens_details.image_tokens == 0 + assert response.usage.input_tokens_details.text_tokens == 0 + assert response.usage.output_tokens == 0 + assert response.usage.total_tokens == 0 + + +def test_cometapi_image_generation_normalizes_null_usage_totals(): + raw_response = httpx.Response( + 200, + json={ + "created": 123, + "data": [{"url": "https://example.com/image.png"}], + "usage": { + "input_tokens": None, + "input_tokens_details": None, + "output_tokens": None, + "total_tokens": None, + }, + }, + request=httpx.Request("POST", "https://api.cometapi.com/v1/images/generations"), + ) + + response = CometAPIImageGenerationConfig().transform_image_generation_response( + model="gpt-image-2", + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={"prompt": "A small comet"}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert response.usage.input_tokens == 0 + assert response.usage.input_tokens_details.image_tokens == 0 + assert response.usage.input_tokens_details.text_tokens == 0 + assert response.usage.output_tokens == 0 + assert response.usage.total_tokens == 0 + + +def test_cometapi_image_generation_raises_provider_error_on_error_response(): + raw_response = httpx.Response( + 500, + json={ + "error": { + "message": "Transparent background is not supported for this model.", + "type": "image_generation_user_error", + "param": "background", + "code": "invalid_value", + } + }, + request=httpx.Request("POST", "https://api.cometapi.com/v1/images/generations"), + ) + + with pytest.raises( + CometAPIException, + match="Transparent background is not supported for this model.", + ): + CometAPIImageGenerationConfig().transform_image_generation_response( + model="gpt-image-2", + raw_response=raw_response, + model_response=ImageResponse(), + logging_obj=MagicMock(), + request_data={"prompt": "A small comet"}, + optional_params={}, + litellm_params={}, + encoding=None, + ) + + +def test_cometapi_speech_uses_cometapi_key_and_base(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + monkeypatch.setenv("COMETAPI_API_KEY", "comet-env-key") + + with patch("litellm.main.openai_chat_completions.audio_speech") as mock_speech: + mock_speech.return_value = b"audio" + speech(model="cometapi/tts-1", input="hello", voice="alloy") + + call_kwargs = mock_speech.call_args.kwargs + assert call_kwargs["model"] == "tts-1" + assert call_kwargs["api_key"] == "comet-env-key" + assert call_kwargs["api_base"] == "https://api.cometapi.com/v1" + + +def test_cometapi_speech_uses_dynamic_api_key(monkeypatch): + _clear_cometapi_env(monkeypatch) + + with ( + patch( + "litellm.main.get_llm_provider", + return_value=("tts-1", "cometapi", "dynamic-comet-key", None), + ), + patch("litellm.main.openai_chat_completions.audio_speech") as mock_speech, + ): + mock_speech.return_value = b"audio" + speech(model="cometapi/tts-1", input="hello", voice="alloy") + + assert mock_speech.call_args.kwargs["api_key"] == "dynamic-comet-key" + + +def test_cometapi_speech_requires_voice(monkeypatch): + _clear_cometapi_env(monkeypatch) + + with pytest.raises(litellm.BadRequestError, match="'voice' is required"): + speech(model="cometapi/tts-1", input="hello", api_key="comet-explicit-key") + + +def test_cometapi_transcription_uses_cometapi_key_and_base(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + audio_file = io.BytesIO(b"not-real-audio") + audio_file.name = "sample.wav" + + with patch( + "litellm.main.openai_audio_transcriptions.audio_transcriptions", + return_value=TranscriptionResponse(text="hello"), + ) as mock_transcription: + transcription( + model="cometapi/whisper-1", + file=audio_file, + api_key="comet-explicit-key", + ) + + call_kwargs = mock_transcription.call_args.kwargs + assert call_kwargs["model"] == "whisper-1" + assert call_kwargs["api_key"] == "comet-explicit-key" + assert call_kwargs["api_base"] == "https://api.cometapi.com/v1" + + +def test_openai_transcription_fallback_is_unchanged(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + audio_file = io.BytesIO(b"not-real-audio") + audio_file.name = "sample.wav" + + with patch( + "litellm.main.openai_audio_transcriptions.audio_transcriptions", + return_value=TranscriptionResponse(text="hello"), + ) as mock_transcription: + transcription( + model="whisper-1", + file=audio_file, + api_key="openai-explicit-key", + ) + + call_kwargs = mock_transcription.call_args.kwargs + assert call_kwargs["model"] == "whisper-1" + assert call_kwargs["api_key"] == "openai-explicit-key" + assert call_kwargs["api_base"] == "https://openai.invalid/v1" + + +def test_openai_speech_fallback_is_unchanged(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + with patch("litellm.main.openai_chat_completions.audio_speech") as mock_speech: + mock_speech.return_value = b"audio" + speech( + model="tts-1", + input="hello", + voice="alloy", + api_key="openai-explicit-key", + ) + + call_kwargs = mock_speech.call_args.kwargs + assert call_kwargs["model"] == "tts-1" + assert call_kwargs["api_key"] == "openai-explicit-key" + assert call_kwargs["api_base"] == "https://openai.invalid/v1" + assert call_kwargs["organization"] is None + assert call_kwargs["project"] is None + + +def test_provider_endpoint_matrix_only_updates_cometapi(): + support_path = Path(__file__).parents[4] / "provider_endpoints_support.json" + providers = json.loads(support_path.read_text())["providers"] + + for endpoint in ( + "image_generations", + "audio_transcriptions", + "audio_speech", + "moderations", + ): + assert providers["cometapi"]["endpoints"][endpoint] is True + assert providers["a2a"]["endpoints"][endpoint] is False + assert providers["bedrock"]["endpoints"][endpoint] is False + + +class _ModerationResponse: + def model_dump(self): + return { + "id": "modr-test", + "model": "omni-moderation-latest", + "results": [ + { + "flagged": False, + "categories": {}, + "category_scores": {}, + "category_applied_input_types": {}, + } + ], + } + + +class _Moderations: + def __init__(self): + self.calls = [] + + def create(self, **kwargs): + self.calls.append(kwargs) + return _ModerationResponse() + + +class _OpenAIClient: + instances = [] + + def __init__(self, **kwargs): + self.kwargs = kwargs + self.moderations = _Moderations() + self.instances.append(self) + + +def test_cometapi_moderation_uses_cometapi_key_and_base(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + _OpenAIClient.instances = [] + with patch("litellm.main.openai.OpenAI", _OpenAIClient): + moderation( + model="cometapi/omni-moderation-latest", + input="hello", + api_key="comet-explicit-key", + ) + + assert _OpenAIClient.instances[0].kwargs == { + "api_key": "comet-explicit-key", + "base_url": "https://api.cometapi.com/v1", + } + assert _OpenAIClient.instances[0].moderations.calls[0]["model"] == ( + "omni-moderation-latest" + ) + + +def test_cometapi_moderation_accepts_bare_model_with_custom_provider(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + _OpenAIClient.instances = [] + with patch("litellm.main.openai.OpenAI", _OpenAIClient): + moderation( + model="omni-moderation-latest", + custom_llm_provider="cometapi", + input="hello", + api_key="comet-explicit-key", + ) + + assert _OpenAIClient.instances[0].kwargs == { + "api_key": "comet-explicit-key", + "base_url": "https://api.cometapi.com/v1", + } + assert _OpenAIClient.instances[0].moderations.calls[0]["model"] == ( + "omni-moderation-latest" + ) + + +def test_openai_moderation_fallback_is_unchanged(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + _OpenAIClient.instances = [] + with patch("litellm.main.openai.OpenAI", _OpenAIClient): + moderation(model="omni-moderation-latest", input="hello") + + assert _OpenAIClient.instances[0].kwargs == { + "api_key": "openai-global-key", + } + assert _OpenAIClient.instances[0].moderations.calls[0]["model"] == ( + "omni-moderation-latest" + ) + + +def test_openai_moderation_without_model_fallback_is_unchanged(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + _OpenAIClient.instances = [] + with patch("litellm.main.openai.OpenAI", _OpenAIClient): + moderation(input="hello") + + assert _OpenAIClient.instances[0].kwargs == { + "api_key": "openai-global-key", + } + assert "model" not in _OpenAIClient.instances[0].moderations.calls[0] + + +class _AsyncModerations: + def __init__(self): + self.calls = [] + + async def create(self, **kwargs): + self.calls.append(kwargs) + return _ModerationResponse() + + +class _AsyncOpenAIClient: + def __init__(self): + self.moderations = _AsyncModerations() + + +@pytest.mark.asyncio +async def test_cometapi_amoderation_uses_cometapi_key_and_base(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + fake_client = _AsyncOpenAIClient() + with patch( + "litellm.main.openai_chat_completions._get_openai_client", + return_value=fake_client, + ) as mock_get_client: + await amoderation( + model="cometapi/omni-moderation-latest", + input="hello", + api_key="comet-explicit-key", + ) + + assert mock_get_client.call_args.kwargs["api_key"] == "comet-explicit-key" + assert mock_get_client.call_args.kwargs["api_base"] == "https://api.cometapi.com/v1" + assert fake_client.moderations.calls[0]["model"] == "omni-moderation-latest" + + +@pytest.mark.asyncio +async def test_openai_amoderation_fallback_is_unchanged(monkeypatch): + _clear_cometapi_env(monkeypatch) + _pollute_openai_globals(monkeypatch) + + fake_client = _AsyncOpenAIClient() + with patch( + "litellm.main.openai_chat_completions._get_openai_client", + return_value=fake_client, + ) as mock_get_client: + await amoderation( + model="omni-moderation-latest", + input="hello", + api_key="openai-explicit-key", + ) + + assert mock_get_client.call_args.kwargs["api_key"] == "openai-explicit-key" + assert mock_get_client.call_args.kwargs["api_base"] is None + assert fake_client.moderations.calls[0]["model"] == "omni-moderation-latest"