From 159a880dcc5ba13cbd5c8384505d5985e7bd2ea3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:06:00 -0700 Subject: [PATCH 01/10] fix /v1/batches POST --- litellm/proxy/proxy_server.py | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1f35a06f0a2..1ec2b38149f 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4808,10 +4808,18 @@ async def create_batch( """ global proxy_logging_obj data: Dict = {} + try: - # Use orjson to parse JSON data, orjson speeds up requests significantly - form_data = await request.form() - data = {key: value for key, value in form_data.items() if key != "file"} + body = await request.body() + body_str = body.decode() + try: + data = ast.literal_eval(body_str) + except: + data = json.loads(body_str) + + verbose_proxy_logger.debug( + "Request received by LiteLLM:\n{}".format(json.dumps(data, indent=4)), + ) # Include original request and headers in the data data = await add_litellm_data_to_request( @@ -4915,10 +4923,18 @@ async def retrieve_batch( """ global proxy_logging_obj data: Dict = {} + data = {} try: - # Use orjson to parse JSON data, orjson speeds up requests significantly - form_data = await request.form() - data = {key: value for key, value in form_data.items() if key != "file"} + body = await request.body() + body_str = body.decode() + try: + data = ast.literal_eval(body_str) + except: + data = json.loads(body_str) + + verbose_proxy_logger.debug( + "Request received by LiteLLM:\n{}".format(json.dumps(data, indent=4)), + ) # Include original request and headers in the data data = await add_litellm_data_to_request( From 56ce7e892d3ed3966d7f477e1cd441ff490fefa0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:08:54 -0700 Subject: [PATCH 02/10] fix batches inserting metadata --- litellm/proxy/litellm_pre_call_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 7384dc30be5..ffea850a336 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -39,6 +39,8 @@ def _get_metadata_variable_name(request: Request) -> str: """ if "thread" in request.url.path or "assistant" in request.url.path: return "litellm_metadata" + if "batches" in request.url.path: + return "litellm_metadata" if "/v1/messages" in request.url.path: # anthropic API has a field called metadata return "litellm_metadata" From 12729ceece68db1888d67cf482511066e05cfb6d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:09:49 -0700 Subject: [PATCH 03/10] test - batches endpoint --- tests/test_openai_batches_endpoint.py | 41 +++++++++++++++++++++++++++ tests/test_openai_files_endpoints.py | 4 +-- 2 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 tests/test_openai_batches_endpoint.py diff --git a/tests/test_openai_batches_endpoint.py b/tests/test_openai_batches_endpoint.py new file mode 100644 index 00000000000..f996c7e8ba3 --- /dev/null +++ b/tests/test_openai_batches_endpoint.py @@ -0,0 +1,41 @@ +# What this tests ? +## Tests /batches endpoints +import pytest +import asyncio +import aiohttp, openai +from openai import OpenAI, AsyncOpenAI +from typing import Optional, List, Union +from test_openai_files_endpoints import upload_file, delete_file + + +BASE_URL = "http://localhost:4000" # Replace with your actual base URL +API_KEY = "sk-1234" # Replace with your actual API key + + +async def create_batch(session, input_file_id, endpoint, completion_window): + url = f"{BASE_URL}/v1/batches" + headers = {"Authorization": f"Bearer {API_KEY}", "Content-Type": "application/json"} + payload = { + "input_file_id": input_file_id, + "endpoint": endpoint, + "completion_window": completion_window, + } + + async with session.post(url, headers=headers, json=payload) as response: + assert response.status == 200, f"Expected status 200, got {response.status}" + result = await response.json() + print(f"Batch creation successful. Batch ID: {result.get('id', 'N/A')}") + return result + + +@pytest.mark.asyncio +async def test_file_operations(): + async with aiohttp.ClientSession() as session: + # Test file upload and get file_id + file_id = await upload_file(session, purpose="batch") + + batch_id = await create_batch(session, file_id, "/v1/chat/completions", "24h") + assert batch_id is not None + + # Test delete file + await delete_file(session, file_id) diff --git a/tests/test_openai_files_endpoints.py b/tests/test_openai_files_endpoints.py index d3922ab6994..1444b8a7064 100644 --- a/tests/test_openai_files_endpoints.py +++ b/tests/test_openai_files_endpoints.py @@ -30,11 +30,11 @@ async def test_file_operations(): await delete_file(session, file_id) -async def upload_file(session): +async def upload_file(session, purpose="fine-tune"): url = f"{BASE_URL}/v1/files" headers = {"Authorization": f"Bearer {API_KEY}"} data = aiohttp.FormData() - data.add_field("purpose", "fine-tune") + data.add_field("purpose", purpose) data.add_field( "file", b'{"prompt": "Hello", "completion": "Hi"}', filename="mydata.jsonl" ) From f627fa9b40c425377841539b5664d42c39d1a4a3 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:23:15 -0700 Subject: [PATCH 04/10] fix for GET /v1/batches{batch_id:path} --- litellm/proxy/proxy_server.py | 26 ++------------------------ 1 file changed, 2 insertions(+), 24 deletions(-) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1ec2b38149f..1bdbadd8301 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -4891,12 +4891,12 @@ async def create_batch( @router.get( - "/v1/batches{batch_id}", + "/v1/batches{batch_id:path}", dependencies=[Depends(user_api_key_auth)], tags=["batch"], ) @router.get( - "/batches{batch_id}", + "/batches{batch_id:path}", dependencies=[Depends(user_api_key_auth)], tags=["batch"], ) @@ -4923,29 +4923,7 @@ async def retrieve_batch( """ global proxy_logging_obj data: Dict = {} - data = {} try: - body = await request.body() - body_str = body.decode() - try: - data = ast.literal_eval(body_str) - except: - data = json.loads(body_str) - - verbose_proxy_logger.debug( - "Request received by LiteLLM:\n{}".format(json.dumps(data, indent=4)), - ) - - # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, - request=request, - general_settings=general_settings, - user_api_key_dict=user_api_key_dict, - version=version, - proxy_config=proxy_config, - ) - _retrieve_batch_request = RetrieveBatchRequest( batch_id=batch_id, ) From 2541d5f6259514a9f40a41368cab152907849ff9 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:26:39 -0700 Subject: [PATCH 05/10] add verbose_logger.debug to retrieve batch --- litellm/llms/openai.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/litellm/llms/openai.py b/litellm/llms/openai.py index fae8a448ad8..94000233cac 100644 --- a/litellm/llms/openai.py +++ b/litellm/llms/openai.py @@ -24,6 +24,7 @@ from pydantic import BaseModel from typing_extensions import overload, override import litellm +from litellm._logging import verbose_logger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.types.utils import ProviderField from litellm.utils import ( @@ -2534,6 +2535,7 @@ class OpenAIBatchesAPI(BaseLLM): retrieve_batch_data: RetrieveBatchRequest, openai_client: AsyncOpenAI, ) -> Batch: + verbose_logger.debug("retrieving batch, args= %s", retrieve_batch_data) response = await openai_client.batches.retrieve(**retrieve_batch_data) return response From 812dd5e162ddcc5ee00793a48446241382b818e6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:40:10 -0700 Subject: [PATCH 06/10] test get batches by id --- tests/test_openai_batches_endpoint.py | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/tests/test_openai_batches_endpoint.py b/tests/test_openai_batches_endpoint.py index f996c7e8ba3..75e3c3f881e 100644 --- a/tests/test_openai_batches_endpoint.py +++ b/tests/test_openai_batches_endpoint.py @@ -28,14 +28,37 @@ async def create_batch(session, input_file_id, endpoint, completion_window): return result +async def get_batch_by_id(session, batch_id): + url = f"{BASE_URL}/v1/batches/{batch_id}" + headers = {"Authorization": f"Bearer {API_KEY}"} + + async with session.get(url, headers=headers) as response: + if response.status == 200: + result = await response.json() + return result + else: + print(f"Error: Failed to get batch. Status code: {response.status}") + return None + + @pytest.mark.asyncio -async def test_file_operations(): +async def test_batches_operations(): async with aiohttp.ClientSession() as session: # Test file upload and get file_id file_id = await upload_file(session, purpose="batch") - batch_id = await create_batch(session, file_id, "/v1/chat/completions", "24h") + create_batch_response = await create_batch( + session, file_id, "/v1/chat/completions", "24h" + ) + batch_id = create_batch_response.get("id") assert batch_id is not None + # Test get batch + get_batch_response = await get_batch_by_id(session, batch_id) + print("response from get batch", get_batch_response) + + assert get_batch_response["id"] == batch_id + assert get_batch_response["input_file_id"] == file_id + # Test delete file await delete_file(session, file_id) From f4048bc89055c18c8ce73856f8f6258d9d05c012 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:41:53 -0700 Subject: [PATCH 07/10] docs batches api --- docs/my-website/docs/batches.md | 81 +++++++++++++++++---------------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/docs/my-website/docs/batches.md b/docs/my-website/docs/batches.md index 51f3bb5cad2..6956a47bee7 100644 --- a/docs/my-website/docs/batches.md +++ b/docs/my-website/docs/batches.md @@ -18,6 +18,47 @@ Call an existing Assistant. + + +```bash +$ export OPENAI_API_KEY="sk-..." + +$ litellm + +# RUNNING on http://0.0.0.0:4000 +``` + +**Create File for Batch Completion** + +```shell +curl https://api.openai.com/v1/files \ + -H "Authorization: Bearer sk-1234" \ + -F purpose="batch" \ + -F file="@mydata.jsonl" +``` + +**Create Batch Request** + +```bash +curl http://localhost:4000/v1/batches \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{ + "input_file_id": "file-abc123", + "endpoint": "/v1/chat/completions", + "completion_window": "24h" + }' +``` + +**Retrieve the Specific Batch** + +```bash +curl http://localhost:4000/v1/batches/batch_abc123 \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ +``` + + **Create File for Batch Completion** @@ -78,47 +119,7 @@ print("file content = ", file_content) ``` - -```bash -$ export OPENAI_API_KEY="sk-..." - -$ litellm - -# RUNNING on http://0.0.0.0:4000 -``` - -**Create File for Batch Completion** - -```shell -curl https://api.openai.com/v1/files \ - -H "Authorization: Bearer sk-1234" \ - -F purpose="batch" \ - -F file="@mydata.jsonl" -``` - -**Create Batch Request** - -```bash -curl http://localhost:4000/v1/batches \ - -H "Authorization: Bearer sk-1234" \ - -H "Content-Type: application/json" \ - -d '{ - "input_file_id": "file-abc123", - "endpoint": "/v1/chat/completions", - "completion_window": "24h" - }' -``` - -**Retrieve the Specific Batch** - -```bash -curl http://localhost:4000/v1/batches/batch_abc123 \ - -H "Authorization: Bearer sk-1234" \ - -H "Content-Type: application/json" \ -``` - - ## [👉 Proxy API Reference](https://litellm-api.up.railway.app/#/batch) From dd37d1d032cc1a6091cae73c6e9a3af11ee3db09 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:42:45 -0700 Subject: [PATCH 08/10] use correct link on http://localhost:4000 --- docs/my-website/docs/batches.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/batches.md b/docs/my-website/docs/batches.md index 6956a47bee7..91cc86bb45f 100644 --- a/docs/my-website/docs/batches.md +++ b/docs/my-website/docs/batches.md @@ -31,7 +31,7 @@ $ litellm **Create File for Batch Completion** ```shell -curl https://api.openai.com/v1/files \ +curl http://localhost:4000/v1/files \ -H "Authorization: Bearer sk-1234" \ -F purpose="batch" \ -F file="@mydata.jsonl" From 90648bee6082682932b02441d7942a3402755377 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:50:44 -0700 Subject: [PATCH 09/10] docs batches API --- docs/my-website/docs/batches.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/batches.md b/docs/my-website/docs/batches.md index 91cc86bb45f..b5386a900d1 100644 --- a/docs/my-website/docs/batches.md +++ b/docs/my-website/docs/batches.md @@ -1,7 +1,7 @@ import Tabs from '@theme/Tabs'; import TabItem from '@theme/TabItem'; -# Batches API +# [BETA] Batches API Covers Batches, Files From f8b9c7128e48536468415cb9ae991acc10ced6db Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Fri, 26 Jul 2024 18:51:13 -0700 Subject: [PATCH 10/10] docs batches --- docs/my-website/docs/batches.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/docs/my-website/docs/batches.md b/docs/my-website/docs/batches.md index b5386a900d1..2199e318fdd 100644 --- a/docs/my-website/docs/batches.md +++ b/docs/my-website/docs/batches.md @@ -8,8 +8,6 @@ Covers Batches, Files ## Quick Start -Call an existing Assistant. - - Create File for Batch Completion - Create Batch Request