From 03cda2468158907f2839c4d4f243a30fcd86421a Mon Sep 17 00:00:00 2001 From: "Jugal D. Bhatt" <55304795+jugaldb@users.noreply.github.com> Date: Wed, 21 May 2025 21:30:18 -0500 Subject: [PATCH 01/36] Verbose error on admin add (#10978) --- .../management_endpoints/team_endpoints.py | 9 +++- .../litellm/proxy/test_team_member_update.py | 39 ++++++++++++++ .../src/components/networking.tsx | 51 +++++++++++++------ .../src/components/team/edit_membership.tsx | 4 +- .../src/components/team/team_info.tsx | 46 ++++++++++++----- 5 files changed, 118 insertions(+), 31 deletions(-) create mode 100644 tests/litellm/proxy/test_team_member_update.py diff --git a/litellm/proxy/management_endpoints/team_endpoints.py b/litellm/proxy/management_endpoints/team_endpoints.py index f2ea89dd318..60f05902522 100644 --- a/litellm/proxy/management_endpoints/team_endpoints.py +++ b/litellm/proxy/management_endpoints/team_endpoints.py @@ -1110,7 +1110,7 @@ async def team_member_update( Update team member budgets and team member role """ - from litellm.proxy.proxy_server import prisma_client + from litellm.proxy.proxy_server import prisma_client, premium_user if prisma_client is None: raise HTTPException(status_code=500, detail={"error": "No db connected"}) @@ -1118,6 +1118,13 @@ async def team_member_update( if data.team_id is None: raise HTTPException(status_code=400, detail={"error": "No team id passed in"}) + + if data.role == "admin" and not premium_user: + # exactly the same text your proxy throws for add: + raise HTTPException( + status_code=400, + detail="Assigning team admins is a premium feature. You must be a LiteLLM Enterprise user to use this feature. If you have a license please set `LITELLM_LICENSE` in your env. Get a 7 day trial key here: https://www.litellm.ai/#trial. Pricing: https://www.litellm.ai/#pricing" + ) if data.user_id is None and data.user_email is None: raise HTTPException( status_code=400, diff --git a/tests/litellm/proxy/test_team_member_update.py b/tests/litellm/proxy/test_team_member_update.py new file mode 100644 index 00000000000..7f534b0b618 --- /dev/null +++ b/tests/litellm/proxy/test_team_member_update.py @@ -0,0 +1,39 @@ +import pytest +from fastapi import HTTPException +from starlette.requests import Request + +from litellm.proxy.management_endpoints.team_endpoints import team_member_update +from litellm.proxy._types import TeamMemberUpdateRequest +import litellm.proxy.proxy_server as proxy_server + +@pytest.mark.asyncio +async def test_team_member_update_admin_requires_premium(monkeypatch): + # Arrange: patch prisma_client and premium_user + monkeypatch.setattr(proxy_server, 'prisma_client', object()) + monkeypatch.setattr(proxy_server, 'premium_user', False) + + # Create a request body that tries to set role=admin + data = TeamMemberUpdateRequest( + team_id="team-1234", + user_id="user-1", + user_email=None, + role="admin", + max_budget_in_team=None, + ) + scope = {"type": "http", "method": "POST", "path": "/team/member_update"} + request = Request(scope) + + # We don't need a full auth object since premium check happens before auth is used + auth = object() + + # Act & Assert: expect HTTPException 400 with the exact premium feature message + with pytest.raises(HTTPException) as exc_info: + await team_member_update(data, request, auth) + + assert exc_info.value.status_code == 400 + expected_msg = ( + "Assigning team admins is a premium feature. You must be a LiteLLM Enterprise user to use this feature. " + "If you have a license please set `LITELLM_LICENSE` in your env. Get a 7 day trial key here: https://www.litellm.ai/#trial. " + "Pricing: https://www.litellm.ai/#pricing" + ) + assert exc_info.value.detail == expected_msg diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index 86d63cfc17b..ba9d1cb34a5 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -3358,14 +3358,15 @@ export interface Member { export const teamMemberAddCall = async ( accessToken: string, teamId: string, - formValues: Member // Assuming formValues is an object + formValues: Member ) => { try { - console.log("Form Values in teamMemberAddCall:", formValues); // Log the form values before making the API call + console.log("Form Values in teamMemberAddCall:", formValues); const url = proxyBaseUrl ? `${proxyBaseUrl}/team/member_add` : `/team/member_add`; + const response = await fetch(url, { method: "POST", headers: { @@ -3374,21 +3375,30 @@ export const teamMemberAddCall = async ( }, body: JSON.stringify({ team_id: teamId, - member: formValues, // Include formValues in the request body + member: formValues, }), }); if (!response.ok) { - const errorData = await response.text(); - handleError(errorData); - console.error("Error response from the server:", errorData); - throw new Error("Network response was not ok"); + // Read and parse JSON error body + const errorText = await response.text(); + let parsedError: any = {}; + + try { + parsedError = JSON.parse(errorText); + } catch (e) { + console.warn("Failed to parse error body as JSON:", errorText); + } + + const rawMessage = parsedError?.detail?.error || "Failed to add team member"; + const err = new Error(rawMessage); + (err as any).raw = parsedError; + throw err; } const data = await response.json(); console.log("API Response:", data); return data; - // Handle success - you might want to update some state or UI based on the created key } catch (error) { console.error("Failed to create key:", error); throw error; @@ -3401,7 +3411,7 @@ export const teamMemberUpdateCall = async ( formValues: Member // Assuming formValues is an object ) => { try { - console.log("Form Values in teamMemberAddCall:", formValues); // Log the form values before making the API call + console.log("Form Values in teamMemberUpdateCall:", formValues); const url = proxyBaseUrl ? `${proxyBaseUrl}/team/member_update` @@ -3420,21 +3430,30 @@ export const teamMemberUpdateCall = async ( }); if (!response.ok) { - const errorData = await response.text(); - handleError(errorData); - console.error("Error response from the server:", errorData); - throw new Error("Network response was not ok"); + // Read and parse JSON error body + const errorText = await response.text(); + let parsedError: any = {}; + + try { + parsedError = JSON.parse(errorText); + } catch (e) { + console.warn("Failed to parse error body as JSON:", errorText); + } + + const rawMessage = parsedError?.detail?.error || "Failed to add team member"; + const err = new Error(rawMessage); + (err as any).raw = parsedError; + throw err; } const data = await response.json(); console.log("API Response:", data); return data; - // Handle success - you might want to update some state or UI based on the created key } catch (error) { - console.error("Failed to create key:", error); + console.error("Failed to update team member:", error); throw error; } -} +}; export const teamMemberDeleteCall = async ( accessToken: string, diff --git a/ui/litellm-dashboard/src/components/team/edit_membership.tsx b/ui/litellm-dashboard/src/components/team/edit_membership.tsx index 27e64193e35..5ffc1dd5945 100644 --- a/ui/litellm-dashboard/src/components/team/edit_membership.tsx +++ b/ui/litellm-dashboard/src/components/team/edit_membership.tsx @@ -78,9 +78,9 @@ const MemberModal = ({ onSubmit(formData); form.resetFields(); - message.success(`Successfully ${mode === 'add' ? 'added' : 'updated'} member`); + // message.success(`Successfully ${mode === 'add' ? 'added' : 'updated'} member`); } catch (error) { - message.error('Failed to submit form'); + // message.error('Failed to submit form'); console.error('Form submission error:', error); } }; diff --git a/ui/litellm-dashboard/src/components/team/team_info.tsx b/ui/litellm-dashboard/src/components/team/team_info.tsx index f195e0845df..d502097b8f5 100644 --- a/ui/litellm-dashboard/src/components/team/team_info.tsx +++ b/ui/litellm-dashboard/src/components/team/team_info.tsx @@ -115,26 +115,36 @@ const TeamInfoView: React.FC = ({ const handleMemberCreate = async (values: any) => { try { - if (accessToken == null) { - return; - } - + if (accessToken == null) return; + const member: Member = { user_email: values.user_email, user_id: values.user_id, role: values.role, - } - const response = await teamMemberAddCall(accessToken, teamId, member); - + }; + + await teamMemberAddCall(accessToken, teamId, member); + message.success("Team member added successfully"); setIsAddMemberModalVisible(false); form.resetFields(); fetchTeamInfo(); - } catch (error) { - message.error("Failed to add team member"); + } catch (error: any) { + let errMsg = "Failed to add team member"; + + if (error?.raw?.detail?.error?.includes("Assigning team admins is a premium feature")) { + errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; + } else if (error?.message) { + errMsg = error.message; + } + + message.error(errMsg); console.error("Error adding team member:", error); } }; + + + const handleMemberUpdate = async (values: any) => { try { @@ -147,17 +157,29 @@ const TeamInfoView: React.FC = ({ user_id: values.user_id, role: values.role, } + message.destroy(); // Remove all existing toasts - const response = await teamMemberUpdateCall(accessToken, teamId, member); + await teamMemberUpdateCall(accessToken, teamId, member); message.success("Team member updated successfully"); setIsEditMemberModalVisible(false); fetchTeamInfo(); - } catch (error) { - message.error("Failed to update team member"); + } catch (error: any) { + let errMsg = "Failed to update team member"; + if (error?.raw?.detail?.includes("Assigning team admins is a premium feature")) { + errMsg = "Assigning admins is an enterprise-only feature. Please upgrade your LiteLLM plan to enable this."; + } else if (error?.message) { + errMsg = error.message; + } + setIsEditMemberModalVisible(false); + + message.destroy(); // Remove all existing toasts + + message.error(errMsg); console.error("Error updating team member:", error); } }; + const handleMemberDelete = async (member: Member) => { try { From 0cde73ffb7a671b3ec5df7f738e9c352cea2e059 Mon Sep 17 00:00:00 2001 From: tanjiro <56165694+NANDINI-star@users.noreply.github.com> Date: Thu, 22 May 2025 12:41:54 +0900 Subject: [PATCH 02/36] Spend rounded to 4 for Organizations and Users page (#11023) * spend rounded to 4 * fixed for organization and users table --- .../src/components/organizations.tsx | 261 +++++++++++------- .../src/components/view_users/columns.tsx | 2 +- 2 files changed, 167 insertions(+), 96 deletions(-) diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/components/organizations.tsx index f9ee248b82d..66eccdcc38e 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/components/organizations.tsx @@ -264,101 +264,172 @@ const OrganizationsTable: React.FC = ({ - - {organizations && organizations.length > 0 - ? organizations - .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()) - .map((org: Organization) => ( - - -
- - - -
- - {org.organization_alias} - - {org.created_at ? new Date(org.created_at).toLocaleDateString() : "N/A"} - - {org.spend} - - {org.litellm_budget_table?.max_budget !== null && org.litellm_budget_table?.max_budget !== undefined ? org.litellm_budget_table?.max_budget : "No limit"} - - - {Array.isArray(org.models) && ( -
- {org.models.length === 0 ? ( - - All Proxy Models - - ) : ( - org.models.map((model, index) => - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - - ) - ) - )} -
- )} - - - - TPM: {org.litellm_budget_table?.tpm_limit ? org.litellm_budget_table?.tpm_limit : "Unlimited"} -
- RPM: {org.litellm_budget_table?.rpm_limit ? org.litellm_budget_table?.rpm_limit : "Unlimited"} - - - - {org.members?.length || 0} Members - - - {userRole === "Admin" && ( - <> - { - setSelectedOrgId(org.organization_id); - setEditOrg(true); - }} - /> - handleDelete(org.organization_id)} - icon={TrashIcon} - size="sm" - /> - - )} - - - )) - : null} - - - - - - - - - - + + {organizations && organizations.length > 0 + ? organizations + .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()) + .map((org: Organization) => ( + + +
+ + + +
+ + {org.organization_alias} + + {org.created_at ? new Date(org.created_at).toLocaleDateString() : "N/A"} + + {org.spend.toFixed(4)} + + {org.litellm_budget_table?.max_budget !== null && org.litellm_budget_table?.max_budget !== undefined ? org.litellm_budget_table?.max_budget : "No limit"} + + + {Array.isArray(org.models) && ( +
+ {org.models.length === 0 ? ( + + All Proxy Models + + ) : ( + org.models.map((model, index) => + model === "all-proxy-models" ? ( + + All Proxy Models + + ) : ( + + {model.length > 30 + ? `${getModelDisplayName(model).slice(0, 30)}...` + : getModelDisplayName(model)} + + ) + ) + )} +
+ )} + + + + TPM: {org.litellm_budget_table?.tpm_limit ? org.litellm_budget_table?.tpm_limit : "Unlimited"} +
+ RPM: {org.litellm_budget_table?.rpm_limit ? org.litellm_budget_table?.rpm_limit : "Unlimited"} + + + + {org.members?.length || 0} Members + + + {userRole === "Admin" && ( + <> + { + setSelectedOrgId(org.organization_id); + setEditOrg(true); + }} + /> + handleDelete(org.organization_id)} + icon={TrashIcon} + size="sm" + /> + + )} + + + )) + : null} + + + + + {(userRole === "Admin" || userRole === "Org Admin") && ( + + + +
+ + + + + + + All Proxy Models + + {userModels && userModels.length > 0 && userModels.map((model) => ( + + {getModelDisplayName(model)} + + ))} + + + + + + + + + daily + weekly + monthly + + + + + + + + + + + + + +
+ +
+
+ + + )} + + + {isDeleteModalOpen ? (
diff --git a/ui/litellm-dashboard/src/components/view_users/columns.tsx b/ui/litellm-dashboard/src/components/view_users/columns.tsx index 6abf7680344..3a5f9c3cbb6 100644 --- a/ui/litellm-dashboard/src/components/view_users/columns.tsx +++ b/ui/litellm-dashboard/src/components/view_users/columns.tsx @@ -39,7 +39,7 @@ export const columns = ( accessorKey: "spend", cell: ({ row }) => ( - {row.original.spend ? row.original.spend.toFixed(2) : "-"} + {row.original.spend ? row.original.spend.toFixed(4) : "-"} ), }, From 85d577c8e674ec771f0636e89b27623b41c28b7e Mon Sep 17 00:00:00 2001 From: Jay Gowdy <130084966+jgowdy-godaddy@users.noreply.github.com> Date: Wed, 21 May 2025 20:58:11 -0700 Subject: [PATCH 03/36] Fix: Handle dict objects in Anthropic streaming response (#11032) * fix: handle dict objects in Anthropic streaming response Fix issue where dictionary objects in Anthropic streaming responses were not properly converted to SSE format strings before being yielded, causing AttributeError: 'dict' object has no attribute 'encode' * fix: refactor Anthropic streaming response handling - Added STREAM_SSE_DATA_PREFIX constant in constants.py - Created return_anthropic_chunk helper function for better maintainability - Using safe_dumps from safe_json_dumps.py for improved JSON serialization - Added unit test for dictionary object handling in streaming response * fix: correct patch path in anthropic_endpoints test --- litellm/constants.py | 1 + .../proxy/anthropic_endpoints/endpoints.py | 25 +++++++- .../proxy/anthropic_endpoints/__init__.py | 0 .../anthropic_endpoints/test_endpoints.py | 61 +++++++++++++++++++ 4 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 tests/litellm/proxy/anthropic_endpoints/__init__.py create mode 100644 tests/litellm/proxy/anthropic_endpoints/test_endpoints.py diff --git a/litellm/constants.py b/litellm/constants.py index 37d44f64c54..148cb9847c8 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -141,6 +141,7 @@ DEFAULT_MAX_TOKENS_FOR_TRITON = int(os.getenv("DEFAULT_MAX_TOKENS_FOR_TRITON", 2 #### Networking settings #### request_timeout: float = float(os.getenv("REQUEST_TIMEOUT", 6000)) # time in seconds STREAM_SSE_DONE_STRING: str = "[DONE]" +STREAM_SSE_DATA_PREFIX: str = "data: " ### SPEND TRACKING ### DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND = float( os.getenv("DEFAULT_REPLICATE_GPU_PRICE_PER_SECOND", 0.001400) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 78078b93f81..2395df3faf4 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -12,6 +12,8 @@ from fastapi.responses import StreamingResponse import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import STREAM_SSE_DATA_PREFIX +from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing @@ -22,6 +24,24 @@ from litellm.proxy.utils import ProxyLogging router = APIRouter() +def return_anthropic_chunk(chunk: str | dict) -> str: + """ + Helper function to format streaming chunks for Anthropic API format + + Args: + chunk: A string or dictionary to be returned in SSE format + + Returns: + str: A properly formatted SSE chunk string + """ + if isinstance(chunk, dict): + # Use safe_dumps for proper JSON serialization with circular reference detection + chunk_str = safe_dumps(chunk) + return f"{STREAM_SSE_DATA_PREFIX}{chunk_str}\n\n" + else: + return chunk + + async def async_data_generator_anthropic( response, user_api_key_dict: UserAPIKeyAuth, @@ -40,7 +60,8 @@ async def async_data_generator_anthropic( user_api_key_dict=user_api_key_dict, response=chunk ) - yield chunk + # Format chunk using helper function + yield return_anthropic_chunk(chunk) except Exception as e: verbose_proxy_logger.exception( "litellm.proxy.proxy_server.async_data_generator(): Exception occured - {}".format( @@ -69,7 +90,7 @@ async def async_data_generator_anthropic( code=getattr(e, "status_code", 500), ) error_returned = json.dumps({"error": proxy_exception.to_dict()}) - yield f"data: {error_returned}\n\n" + yield f"{STREAM_SSE_DATA_PREFIX}{error_returned}\n\n" @router.post( diff --git a/tests/litellm/proxy/anthropic_endpoints/__init__.py b/tests/litellm/proxy/anthropic_endpoints/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/litellm/proxy/anthropic_endpoints/test_endpoints.py b/tests/litellm/proxy/anthropic_endpoints/test_endpoints.py new file mode 100644 index 00000000000..2dbdf345042 --- /dev/null +++ b/tests/litellm/proxy/anthropic_endpoints/test_endpoints.py @@ -0,0 +1,61 @@ +""" +Test for anthropic_endpoints/endpoints.py, focusing on handling dictionary objects in streaming responses +""" + +import json +import unittest +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from litellm.proxy.anthropic_endpoints.endpoints import async_data_generator_anthropic + + +class TestAnthropicEndpoints(unittest.TestCase): + @patch("litellm.litellm_core_utils.safe_json_dumps.safe_dumps") + @pytest.mark.asyncio + async def test_async_data_generator_anthropic_dict_handling(self, mock_safe_dumps): + """Test async_data_generator_anthropic handles dictionary chunks properly""" + # Setup + mock_response = AsyncMock() + mock_response.__aiter__.return_value = [ + {"type": "message_start", "message": {"id": "msg_123"}}, + "text chunk data", + {"type": "content_block_delta", "delta": {"text": "more data"}}, + "text chunk data again", + ] + + mock_user_api_key_dict = MagicMock() + mock_request_data = {} + mock_proxy_logging_obj = MagicMock() + mock_proxy_logging_obj.async_post_call_streaming_hook = AsyncMock(side_effect=lambda **kwargs: kwargs["response"]) + + # Configure safe_dumps to return a properly formatted JSON string + mock_safe_dumps.side_effect = lambda chunk: json.dumps(chunk) + + # Execute + result = [chunk async for chunk in async_data_generator_anthropic( + response=mock_response, + user_api_key_dict=mock_user_api_key_dict, + request_data=mock_request_data, + proxy_logging_obj=mock_proxy_logging_obj, + )] + + # Verify + expected_result = [ + 'data: {"type": "message_start", "message": {"id": "msg_123"}}\n\n', + 'text chunk data', + 'data: {"type": "content_block_delta", "delta": {"text": "more data"}}\n\n', + 'text chunk data again', + ] + + self.assertEqual(result, expected_result) + + # Assert safe_dumps was called for dictionary objects + mock_safe_dumps.assert_any_call({"type": "message_start", "message": {"id": "msg_123"}}) + mock_safe_dumps.assert_any_call({"type": "content_block_delta", "delta": {"text": "more data"}}) + assert mock_safe_dumps.call_count == 2 # Called twice, once for each dict object + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file From e487f1e17ddcc5f23883ecf15b914b56b66c2b90 Mon Sep 17 00:00:00 2001 From: bepotp Date: Thu, 22 May 2025 05:59:18 +0200 Subject: [PATCH 04/36] feat: add Databricks Llama 4 Maverick model cost (#11008) Co-authored-by: Tommy PLANEL --- .../model_prices_and_context_window_backup.json | 16 +++++++++++++++- model_prices_and_context_window.json | 16 +++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 8d7d062b818..4bea9baee3e 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12350,6 +12350,20 @@ "metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."}, "supports_tool_choice": true }, + "databricks/databricks-llama-4-maverick": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.000005, + "input_dbu_cost_per_token": 0.00007143, + "output_cost_per_token": 0.000015, + "output_dbu_cost_per_token": 0.00021429, + "litellm_provider": "databricks", + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "metadata": {"notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."}, + "supports_tool_choice": true + }, "databricks/databricks-dbrx-instruct": { "max_tokens": 32768, "max_input_tokens": 32768, @@ -12976,4 +12990,4 @@ "litellm_provider": "featherless_ai", "mode": "chat" } -} \ No newline at end of file +} diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 8d7d062b818..4bea9baee3e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12350,6 +12350,20 @@ "metadata": {"notes": "Input/output cost per token is dbu cost * $0.070, based on databricks Llama 3.1 70B conversion. Number provided for reference, '*_dbu_cost_per_token' used in actual calculation."}, "supports_tool_choice": true }, + "databricks/databricks-llama-4-maverick": { + "max_tokens": 128000, + "max_input_tokens": 128000, + "max_output_tokens": 128000, + "input_cost_per_token": 0.000005, + "input_dbu_cost_per_token": 0.00007143, + "output_cost_per_token": 0.000015, + "output_dbu_cost_per_token": 0.00021429, + "litellm_provider": "databricks", + "mode": "chat", + "source": "https://www.databricks.com/product/pricing/foundation-model-serving", + "metadata": {"notes": "Databricks documentation now provides both DBU costs (_dbu_cost_per_token) and dollar costs(_cost_per_token)."}, + "supports_tool_choice": true + }, "databricks/databricks-dbrx-instruct": { "max_tokens": 32768, "max_input_tokens": 32768, @@ -12976,4 +12990,4 @@ "litellm_provider": "featherless_ai", "mode": "chat" } -} \ No newline at end of file +} From 546a508c8c7f39bdee920310a37e2267e80bd931 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 21 May 2025 21:36:38 -0700 Subject: [PATCH 05/36] test: mark flaky test --- tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py b/tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py index f0a3497a12d..d97e83b7dbb 100644 --- a/tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py +++ b/tests/litellm/proxy/hooks/test_parallel_request_limiter_v2.py @@ -19,6 +19,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v2 import ( from litellm.proxy.utils import InternalUsageCache, ProxyLogging, hash_token +@pytest.mark.flaky(reruns=3) @pytest.mark.asyncio async def test_normal_router_call_v2(monkeypatch): """ @@ -340,6 +341,7 @@ async def test_normal_router_call_rpm(monkeypatch, rate_limit_object): ) +@pytest.mark.flaky(reruns=3) @pytest.mark.asyncio async def test_streaming_router_call_v2(monkeypatch): """ From 58f958f30a56a704a844c9ccdbbcc7b134185ffd Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Wed, 21 May 2025 21:40:53 -0700 Subject: [PATCH 06/36] Litellm dev 05 21 2025 p2 (#11039) * feat: initial commit adding managed file support to fine tuning endpoints * feat(fine_tuning/endpoints.py): working call to openai finetuning route Uses litellm managed files for finetuning api support * feat(fine-tuning/main.py): refactor to use LiteLLMFineTuningJob pydantic object includes 'hidden_params' * fix: initial commit adding unified finetuning id support return a unified finetuning id we can use to understand which deployment to route the ft request to * test: fix test * feat(managed_files.py): return unified finetuning job id on create finetuning job enables retrieve, delete to work with litellm managed files * test: update test * fix: fix linting error * fix: fix ruff linting error * test: fix check --- enterprise/enterprise_hooks/managed_files.py | 67 +++++++++++++++- litellm/fine_tuning/main.py | 5 +- litellm/llms/ollama/common_utils.py | 4 +- litellm/llms/openai/fine_tuning/handler.py | 16 ++-- litellm/llms/vertex_ai/fine_tuning/handler.py | 10 +-- litellm/proxy/_new_secret_config.yaml | 4 +- litellm/proxy/common_request_processing.py | 1 + .../proxy/fine_tuning_endpoints/endpoints.py | 78 +++++++++++++++---- litellm/router.py | 5 ++ litellm/types/llms/openai.py | 2 +- litellm/types/utils.py | 22 +++++- .../enterprise_hooks/test_managed_files.py | 45 +++++++++++ .../llms/azure/test_azure_common_utils.py | 1 + 13 files changed, 221 insertions(+), 39 deletions(-) diff --git a/enterprise/enterprise_hooks/managed_files.py b/enterprise/enterprise_hooks/managed_files.py index 0dc86294d36..78e1cdfd98b 100644 --- a/enterprise/enterprise_hooks/managed_files.py +++ b/enterprise/enterprise_hooks/managed_files.py @@ -23,7 +23,12 @@ from litellm.types.llms.openai import ( OpenAIFileObject, OpenAIFilesPurpose, ) -from litellm.types.utils import LiteLLMBatch, LLMResponseTypes, SpecialEnums +from litellm.types.utils import ( + LiteLLMBatch, + LiteLLMFineTuningJob, + LLMResponseTypes, + SpecialEnums, +) if TYPE_CHECKING: from opentelemetry.trace import Span as _Span @@ -138,12 +143,16 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "acreate_batch", "aretrieve_batch", "afile_content", + "acreate_fine_tuning_job", ], ) -> Union[Exception, str, Dict, None]: """ - Detect litellm_proxy/ file_id - add dictionary of mappings of litellm_proxy/ file_id -> provider_file_id => {litellm_proxy/file_id: {"model_id": id, "file_id": provider_file_id}} """ + print( + "CALLS ASYNC PRE CALL HOOK - DATA={}, CALL_TYPE={}".format(data, call_type) + ) if call_type == CallTypes.completion.value: messages = data.get("messages") if messages: @@ -196,7 +205,15 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): data["batch_id"] = self.get_batch_id_from_unified_batch_id( potential_batch_id ) + elif call_type == CallTypes.acreate_fine_tuning_job.value: + input_file_id = cast(Optional[str], data.get("training_file")) + if input_file_id: + model_file_id_mapping = await self.get_model_file_id_mapping( + [input_file_id], user_api_key_dict.parent_otel_span + ) + data["model_file_id_mapping"] = model_file_id_mapping + print("DATA={}".format(data)) return data async def async_pre_call_deployment_hook( @@ -205,8 +222,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ Allow modifying the request just before it's sent to the deployment. """ + print( + "CALLS ASYNC PRE CALL DEPLOYMENT HOOK - KWARGS={}, CALL_TYPE={}".format( + kwargs, call_type + ) + ) + accessor_key: Optional[str] = None if call_type and call_type == CallTypes.acreate_batch: - input_file_id = cast(Optional[str], kwargs.get("input_file_id")) + accessor_key = "input_file_id" + elif call_type and call_type == CallTypes.acreate_fine_tuning_job: + accessor_key = "training_file" + else: + return kwargs + + if accessor_key: + input_file_id = cast(Optional[str], kwargs.get(accessor_key)) model_file_id_mapping = cast( Optional[Dict[str, Dict[str, str]]], kwargs.get("model_file_id_mapping") ) @@ -217,7 +247,8 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_id, None ) if mapped_file_id: - kwargs["input_file_id"] = mapped_file_id + kwargs[accessor_key] = mapped_file_id + return kwargs def get_file_ids_from_messages(self, messages: List[AllMessageValues]) -> List[str]: @@ -383,6 +414,20 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return response + def get_unified_generic_response_id( + self, model_id: str, generic_response_id: str + ) -> str: + unified_generic_response_id = ( + SpecialEnums.LITELLM_MANAGED_GENERIC_RESPONSE_COMPLETE_STR.value.format( + model_id, generic_response_id + ) + ) + return ( + base64.urlsafe_b64encode(unified_generic_response_id.encode()) + .decode() + .rstrip("=") + ) + def get_unified_batch_id(self, batch_id: str, model_id: str) -> str: unified_batch_id = SpecialEnums.LITELLM_MANAGED_BATCH_COMPLETE_STR.value.format( model_id, batch_id @@ -455,7 +500,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_id=model_id, model_name=model_name, ) - + return response + elif isinstance(response, LiteLLMFineTuningJob): + ## Check if unified_file_id is in the response + print(f"hidden params={response._hidden_params}") + unified_file_id = response._hidden_params.get( + "unified_file_id" + ) # managed file id + model_id = cast(Optional[str], response._hidden_params.get("model_id")) + print("MODEL_ID={}".format(model_id)) + model_name = cast(Optional[str], response._hidden_params.get("model_name")) + if unified_file_id and model_id: + response.id = self.get_unified_generic_response_id( + model_id=model_id, generic_response_id=response.id + ) + return response return await super().async_post_call_success_hook( data, user_api_key_dict, response ) diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index b7efcb40d42..55e45f75012 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -28,6 +28,7 @@ from litellm.types.llms.openai import ( Hyperparameters, ) from litellm.types.router import * +from litellm.types.utils import LiteLLMFineTuningJob from litellm.utils import client, supports_httpx_timeout ####### ENVIRONMENT VARIABLES ################### @@ -50,7 +51,7 @@ async def acreate_fine_tuning_job( extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> FineTuningJob: +) -> LiteLLMFineTuningJob: """ Async: Creates and executes a batch from an uploaded file of request @@ -104,7 +105,7 @@ def create_fine_tuning_job( extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: +) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: """ Creates a fine-tuning job which begins the process of creating a new model from a given dataset. diff --git a/litellm/llms/ollama/common_utils.py b/litellm/llms/ollama/common_utils.py index 4bee5e358c1..daff7a12065 100644 --- a/litellm/llms/ollama/common_utils.py +++ b/litellm/llms/ollama/common_utils.py @@ -1,4 +1,4 @@ -from typing import Optional, Union +from typing import List, Optional, Union import httpx @@ -67,7 +67,7 @@ class OllamaModelInfo(BaseLLMModelInfo): # env var OLLAMA_API_BASE or default return api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434" - def get_models(self, api_key=None, api_base: Optional[str] = None) -> list[str]: + def get_models(self, api_key=None, api_base: Optional[str] = None) -> List[str]: """ List all models available on the Ollama server via /api/tags endpoint. """ diff --git a/litellm/llms/openai/fine_tuning/handler.py b/litellm/llms/openai/fine_tuning/handler.py index 2b697f85d2d..aa4b7e20319 100644 --- a/litellm/llms/openai/fine_tuning/handler.py +++ b/litellm/llms/openai/fine_tuning/handler.py @@ -1,10 +1,11 @@ -from typing import Any, Coroutine, Optional, Union +from typing import Any, Coroutine, Optional, Union, cast import httpx from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI from openai.types.fine_tuning import FineTuningJob from litellm._logging import verbose_logger +from litellm.types.utils import LiteLLMFineTuningJob class OpenAIFineTuningAPI: @@ -55,11 +56,12 @@ class OpenAIFineTuningAPI: self, create_fine_tuning_job_data: dict, openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI], - ) -> FineTuningJob: + ) -> LiteLLMFineTuningJob: response = await openai_client.fine_tuning.jobs.create( **create_fine_tuning_job_data ) - return response + + return LiteLLMFineTuningJob(**response.model_dump()) def create_fine_tuning_job( self, @@ -74,7 +76,7 @@ class OpenAIFineTuningAPI: client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = None, - ) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: + ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: openai_client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = self.get_openai_client( @@ -104,8 +106,10 @@ class OpenAIFineTuningAPI: verbose_logger.debug( "creating fine tuning job, args= %s", create_fine_tuning_job_data ) - response = openai_client.fine_tuning.jobs.create(**create_fine_tuning_job_data) - return response + response = cast(OpenAI, openai_client).fine_tuning.jobs.create( + **create_fine_tuning_job_data + ) + return LiteLLMFineTuningJob(**response.model_dump()) async def acancel_fine_tuning_job( self, diff --git a/litellm/llms/vertex_ai/fine_tuning/handler.py b/litellm/llms/vertex_ai/fine_tuning/handler.py index 7ea8527fd41..4d7f8cec02d 100644 --- a/litellm/llms/vertex_ai/fine_tuning/handler.py +++ b/litellm/llms/vertex_ai/fine_tuning/handler.py @@ -1,10 +1,9 @@ import json import traceback from datetime import datetime -from typing import Literal, Optional, Union +from typing import Any, Coroutine, Literal, Optional, Union import httpx -from openai.types.fine_tuning.fine_tuning_job import FineTuningJob import litellm from litellm._logging import verbose_logger @@ -20,6 +19,7 @@ from litellm.types.llms.vertex_ai import ( ResponseSupervisedTuningSpec, ResponseTuningJob, ) +from litellm.types.utils import LiteLLMFineTuningJob class VertexFineTuningAPI(VertexLLM): @@ -113,7 +113,7 @@ class VertexFineTuningAPI(VertexLLM): def convert_vertex_response_to_open_ai_response( self, response: ResponseTuningJob - ) -> FineTuningJob: + ) -> LiteLLMFineTuningJob: status: Literal[ "validating_files", "queued", "running", "succeeded", "failed", "cancelled" ] = "queued" @@ -134,7 +134,7 @@ class VertexFineTuningAPI(VertexLLM): response.get("supervisedTuningSpec", None) or {} ) training_uri: str = _supervisedTuningSpec.get("trainingDatasetUri", "") or "" - return FineTuningJob( + return LiteLLMFineTuningJob( id=response.get("name", "") or "", created_at=created_at, fine_tuned_model=response.get("tunedModelDisplayName", ""), @@ -226,7 +226,7 @@ class VertexFineTuningAPI(VertexLLM): timeout: Union[float, httpx.Timeout], kwargs: Optional[dict] = None, original_hyperparameters: Optional[dict] = {}, - ): + ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: verbose_logger.debug( "creating fine tuning job, args= %s", create_fine_tuning_job_data ) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 74f81459b88..78880ba55cf 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -2,9 +2,9 @@ model_list: - model_name: "gemini-2.0-flash" litellm_params: model: gemini/gemini-2.0-flash-live-001 - - model_name: "gpt-4o-mini-openai" + - model_name: "gpt-4.1-openai" litellm_params: - model: gpt-4o-mini + model: gpt-4.1-mini-2025-04-14 api_key: os.environ/OPENAI_API_KEY model_info: access_groups: ["default-openai-models"] diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 2fd56af0b91..325e812409a 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -117,6 +117,7 @@ class ProxyBaseLLMRequestProcessing: "acreate_batch", "aretrieve_batch", "afile_content", + "acreate_fine_tuning_job", ], version: Optional[str] = None, user_model: Optional[str] = None, diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index d4c4250b37f..04d76646cff 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -7,16 +7,20 @@ import asyncio import traceback -from typing import Optional +from typing import Optional, cast -from fastapi import APIRouter, Depends, Request, Response +from fastapi import APIRouter, Depends, HTTPException, Request, Response import litellm from litellm._logging import verbose_proxy_logger from litellm.proxy._types import * from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, +) from litellm.proxy.utils import handle_exception_on_proxy +from litellm.types.utils import LiteLLMFineTuningJob router = APIRouter() @@ -96,8 +100,8 @@ async def create_fine_tuning_job( ``` """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, premium_user, proxy_config, proxy_logging_obj, @@ -117,25 +121,68 @@ async def create_fine_tuning_job( ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type="acreate_fine_tuning_job", ) - # get configs for custom_llm_provider - llm_provider_config = get_fine_tuning_provider_config( - custom_llm_provider=fine_tuning_request.custom_llm_provider, + ## CHECK IF MANAGED FILE ID + unified_file_id: Union[str, Literal[False]] = False + training_file = fine_tuning_request.training_file + response: Optional[LiteLLMFineTuningJob] = None + if training_file: + unified_file_id = _is_base64_encoded_unified_file_id(training_file) + ## IF SO, Route based on that + if unified_file_id: + """ """ + if llm_router is None: + raise HTTPException( + status_code=500, + detail={ + "error": "LLM Router not initialized. Ensure models added to proxy." + }, + ) + + response = cast( + LiteLLMFineTuningJob, await llm_router.acreate_fine_tuning_job(**data) + ) + response.training_file = unified_file_id + response._hidden_params["unified_file_id"] = unified_file_id + ## ELSE, Route based on custom_llm_provider + elif fine_tuning_request.custom_llm_provider: + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=fine_tuning_request.custom_llm_provider, + ) + + # add llm_provider_config to data + if llm_provider_config is not None: + data.update(llm_provider_config) + + response = await litellm.acreate_fine_tuning_job(**data) + + if response is None: + raise ValueError( + "Invalid request, No litellm managed file id or custom_llm_provider provided." + ) + + ### CALL HOOKS ### - modify outgoing data + _response = await proxy_logging_obj.post_call_success_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, ) - - # add llm_provider_config to data - if llm_provider_config is not None: - data.update(llm_provider_config) - - response = await litellm.acreate_fine_tuning_job(**data) + if _response is not None and isinstance(_response, LiteLLMFineTuningJob): + response = _response ### ALERTING ### asyncio.create_task( @@ -166,12 +213,11 @@ async def create_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( + verbose_proxy_logger.exception( "litellm.proxy.proxy_server.create_fine_tuning_job(): Exception occurred - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) raise handle_exception_on_proxy(e) diff --git a/litellm/router.py b/litellm/router.py index 38f187d9a8b..4b562d669a6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -752,6 +752,9 @@ class Router: self._arealtime = self.factory_function( litellm._arealtime, call_type="_arealtime" ) + self.acreate_fine_tuning_job = self.factory_function( + litellm.acreate_fine_tuning_job, call_type="acreate_fine_tuning_job" + ) def validate_fallbacks(self, fallback_param: Optional[List]): """ @@ -3159,6 +3162,7 @@ class Router: "afile_delete", "afile_content", "_arealtime", + "acreate_fine_tuning_job", ] = "assistants", ): """ @@ -3207,6 +3211,7 @@ class Router: "anthropic_messages", "aresponses", "_arealtime", + "acreate_fine_tuning_job", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 0d880a4b1cf..f5aa27a9eaf 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -878,7 +878,7 @@ class FineTuningJobCreate(BaseModel): class LiteLLMFineTuningJobCreate(FineTuningJobCreate): - custom_llm_provider: Literal["openai", "azure", "vertex_ai"] + custom_llm_provider: Optional[Literal["openai", "azure", "vertex_ai"]] = None model_config = { "extra": "allow" diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 2b230d089c3..612b03adcbb 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -44,6 +44,7 @@ from .llms.openai import ( ChatCompletionToolCallChunk, ChatCompletionUsageBlock, FileSearchTool, + FineTuningJob, OpenAIChatCompletionChunk, OpenAIFileObject, OpenAIRealtimeStreamList, @@ -2256,6 +2257,18 @@ class SelectTokenizerResponse(TypedDict): tokenizer: Any +class LiteLLMFineTuningJob(FineTuningJob): + _hidden_params: dict = {} + + def __init__(self, **kwargs): + if "error" in kwargs and kwargs["error"] is not None: + # check if error is all None - if so, set error to None + if all(value is None for value in kwargs["error"].values()): + kwargs["error"] = None + super().__init__(**kwargs) + self._hidden_params = kwargs.get("_hidden_params", {}) + + class LiteLLMBatch(Batch): _hidden_params: dict = {} usage: Optional[Usage] = None @@ -2360,9 +2373,16 @@ class SpecialEnums(Enum): LITELLM_MANAGED_BATCH_COMPLETE_STR = "litellm_proxy;model_id:{};llm_batch_id:{}" + LITELLM_MANAGED_GENERIC_RESPONSE_COMPLETE_STR = "litellm_proxy;model_id:{};generic_response_id:{}" # generic implementation of 'managed batches' - used for finetuning and any future work. + LLMResponseTypes = Union[ - ModelResponse, EmbeddingResponse, ImageResponse, OpenAIFileObject, LiteLLMBatch + ModelResponse, + EmbeddingResponse, + ImageResponse, + OpenAIFileObject, + LiteLLMBatch, + LiteLLMFineTuningJob, ] diff --git a/tests/enterprise/enterprise_hooks/test_managed_files.py b/tests/enterprise/enterprise_hooks/test_managed_files.py index e84034ed099..19f5a7c3404 100644 --- a/tests/enterprise/enterprise_hooks/test_managed_files.py +++ b/tests/enterprise/enterprise_hooks/test_managed_files.py @@ -13,6 +13,9 @@ from unittest.mock import MagicMock from enterprise.enterprise_hooks.managed_files import _PROXY_LiteLLMManagedFiles from litellm.caching import DualCache +from litellm.proxy.openai_files_endpoints.common_utils import ( + _is_base64_encoded_unified_file_id, +) from litellm.types.utils import SpecialEnums @@ -161,3 +164,45 @@ async def test_async_pre_call_hook_batch_retrieve(): # assert len(batch_files) == 1 # assert assistant_files[0].id == file1.id # assert batch_files[0].id == file2.id + + +@pytest.mark.asyncio +async def test_async_post_call_success_hook_for_unified_finetuning_job(): + from litellm.types.utils import LiteLLMFineTuningJob + + unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxiZTQ0ZDVlYi1mNDU3LTRiNzktOWM4My01N2QxMTMxYWM0YzY7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00LjEtb3BlbmFpO2xsbV9vdXRwdXRfZmlsZV9pZCxmaWxlLURKMnQ0OWZlQ2NTQk5vNG9oekZ6NGc7bGxtX291dHB1dF9maWxlX21vZGVsX2lkLGRiNjY5ODcwNzdkZTdmYzZjNzAzY2Y1MDczMGU2MmNkOWQ3YTU1N2NlNjVmMDUzNTFkYTM4YTA3ZjBlZDEyNzQ" + provider_ft_job = LiteLLMFineTuningJob( + object="fine_tuning.job", + id="ftjob-0kEBV5b4sPrFcMnuzmYSzU1G", + model="gpt-3.5-turbo-0613", + created_at=1692779769, + finished_at=None, + fine_tuned_model=None, + organization_id="org-dUVLhaAQ37YCGwVC2QVY8sdB", + result_files=[], + status="validating_files", + validation_file=None, + training_file="file-azQuKMLAmiFdEjxpCcbI11zF", + hyperparameters={"n_epochs": 8}, + trained_tokens=None, + seed=0, + ) + provider_ft_job._hidden_params = { + "unified_file_id": unified_file_id, + "model_id": "gpt-3.5-turbo-0613", + } + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=MagicMock() + ) + data = { + "user_api_key_dict": {"parent_otel_span": MagicMock()}, + } + + response = await proxy_managed_files.async_post_call_success_hook( + data=data, + user_api_key_dict=MagicMock(), + response=provider_ft_job, + ) + + assert isinstance(response, LiteLLMFineTuningJob) + assert _is_base64_encoded_unified_file_id(response.id) diff --git a/tests/litellm/llms/azure/test_azure_common_utils.py b/tests/litellm/llms/azure/test_azure_common_utils.py index abdd1cb9294..54916e4daba 100644 --- a/tests/litellm/llms/azure/test_azure_common_utils.py +++ b/tests/litellm/llms/azure/test_azure_common_utils.py @@ -391,6 +391,7 @@ def test_select_azure_base_url_called(setup_mocks): "add_message", "arun_thread_stream", "aresponses", + "acreate_fine_tuning_job", ] ], ) From 1cd25950062b5359b6ab725b6b208b7088c30f20 Mon Sep 17 00:00:00 2001 From: jmorenoc-o <52289208+jmorenoc-o@users.noreply.github.com> Date: Thu, 22 May 2025 06:42:57 +0200 Subject: [PATCH 07/36] Fixes the InvitationLink Prisma find_many query (#11031) Related: https://github.com/BerriAI/litellm/commit/3b6c6d05dd8f8bcd83f776cdc1c8fc64d3d85d13#r157675103 We should use "order", according to the prisma python docs https://prisma-client-py.readthedocs.io/en/stable/reference/limitations/#order-argument Also we are using "order" in other files of the project: https://github.com/search?q=repo%3ABerriAI%2Flitellm%20order%3D%7B&type=code --- .../enterprise_callbacks/send_emails/base_email.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py index 9ea22074c01..779d4f2eb37 100644 --- a/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py +++ b/enterprise/litellm_enterprise/enterprise_callbacks/send_emails/base_email.py @@ -189,7 +189,7 @@ class BaseEmailLogger(CustomLogger): # get the latest invitation link for the user invitation_rows = await prisma_client.db.litellm_invitationlink.find_many( where={"user_id": user_id}, - orderBy={"created_at": "desc"}, + order={"created_at": "desc"}, ) if len(invitation_rows) > 0: invitation_row = invitation_rows[0] From cd496fee2e3e7a3838e385b731614d39bbe9514e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 21 May 2025 22:04:41 -0700 Subject: [PATCH 08/36] fix: fix linting error --- litellm/proxy/anthropic_endpoints/endpoints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/anthropic_endpoints/endpoints.py b/litellm/proxy/anthropic_endpoints/endpoints.py index 2395df3faf4..a84c1e84ab0 100644 --- a/litellm/proxy/anthropic_endpoints/endpoints.py +++ b/litellm/proxy/anthropic_endpoints/endpoints.py @@ -24,13 +24,13 @@ from litellm.proxy.utils import ProxyLogging router = APIRouter() -def return_anthropic_chunk(chunk: str | dict) -> str: +def return_anthropic_chunk(chunk: Any) -> str: """ Helper function to format streaming chunks for Anthropic API format - + Args: chunk: A string or dictionary to be returned in SSE format - + Returns: str: A properly formatted SSE chunk string """ From 2b50b43ae2b7790a85a2f37daaf69ac1725749df Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Wed, 21 May 2025 22:27:36 -0700 Subject: [PATCH 09/36] Support passing `prompt_label` to langfuse (#11018) * fix: add prompt label support to prompt management hook * feat: support 'prompt_label' parameter for langfuse prompt management Closes https://github.com/BerriAI/litellm/discussions/9003#discussioncomment-13221555 * fix(litellm_logging.py): deep copy optional params to avoid mutation while logging * fix(log-consistent-optional-param-values-across-providers): ensures params can be used for finetuning from providers * fix: fix linting error * test: update test * test: update langfuse tests * fix(litellm_logging.py): avoid deepcopying optional params might contain thread object --- .../anthropic_cache_control_hook.py | 9 ++++---- litellm/integrations/custom_logger.py | 2 ++ .../integrations/custom_prompt_management.py | 2 ++ litellm/integrations/humanloop.py | 7 ++---- .../langfuse/langfuse_prompt_management.py | 20 ++++++++++------- .../integrations/prompt_management_base.py | 5 +++++ .../vector_stores/bedrock_vector_store.py | 22 ++++++++++--------- litellm/litellm_core_utils/litellm_logging.py | 4 ++++ litellm/main.py | 11 ++++++++-- litellm/proxy/_new_secret_config.yaml | 18 +++++++++++---- litellm/proxy/auth/auth_checks.py | 17 +++++++++----- litellm/proxy/custom_prompt_management.py | 1 + litellm/router.py | 15 ++++++++++--- litellm/types/utils.py | 1 + litellm/utils.py | 8 +++++++ .../test_custom_prompt_management.py | 1 + .../completion.json | 4 +--- .../completion_with_complex_metadata.json | 4 +--- .../completion_with_langfuse_metadata.json | 4 +--- .../completion_with_no_choices.json | 4 +--- .../completion_with_tags.json | 4 +--- .../completion_with_tags_stream.json | 4 +--- .../complex_metadata.json | 4 +--- .../complex_metadata_2.json | 4 +--- .../empty_metadata.json | 4 +--- .../metadata_with_function.json | 4 +--- .../metadata_with_lock.json | 4 +--- .../nested_metadata.json | 4 +--- .../simple_metadata.json | 4 +--- .../simple_metadata2.json | 4 +--- .../simple_metadata3.json | 4 +--- 31 files changed, 116 insertions(+), 87 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index c138b3cc254..5c75e452ab7 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -28,6 +28,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Apply cache control directives based on specified injection points. @@ -79,10 +80,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): # Case 1: Target by specific index if targetted_index is not None: if 0 <= targetted_index < len(messages): - messages[targetted_index] = ( - AnthropicCacheControlHook._safe_insert_cache_control_in_message( - messages[targetted_index], control - ) + messages[ + targetted_index + ] = AnthropicCacheControlHook._safe_insert_cache_control_in_message( + messages[targetted_index], control ) # Case 2: Target by role elif targetted_role is not None: diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 960dc715e7e..ce97b9a292d 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -87,6 +87,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: @@ -104,6 +105,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: diff --git a/litellm/integrations/custom_prompt_management.py b/litellm/integrations/custom_prompt_management.py index 9d05e7b2426..061aadc3c05 100644 --- a/litellm/integrations/custom_prompt_management.py +++ b/litellm/integrations/custom_prompt_management.py @@ -18,6 +18,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: @@ -43,6 +44,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: raise NotImplementedError( "Custom prompt management does not support compile prompt helper" diff --git a/litellm/integrations/humanloop.py b/litellm/integrations/humanloop.py index 853fbe148cc..c62ab1110ff 100644 --- a/litellm/integrations/humanloop.py +++ b/litellm/integrations/humanloop.py @@ -155,11 +155,8 @@ class HumanloopLogger(CustomLogger): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - ) -> Tuple[ - str, - List[AllMessageValues], - dict, - ]: + prompt_label: Optional[str] = None, + ) -> Tuple[str, List[AllMessageValues], dict,]: humanloop_api_key = dynamic_callback_params.get( "humanloop_api_key" ) or get_secret_str("HUMANLOOP_API_KEY") diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index b4149d7ad97..8fe9cb63dea 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -130,9 +130,12 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge return "langfuse" def _get_prompt_from_id( - self, langfuse_prompt_id: str, langfuse_client: LangfuseClass + self, + langfuse_prompt_id: str, + langfuse_client: LangfuseClass, + prompt_label: Optional[str] = None, ) -> PROMPT_CLIENT: - return langfuse_client.get_prompt(langfuse_prompt_id) + return langfuse_client.get_prompt(langfuse_prompt_id, label=prompt_label) def _compile_prompt( self, @@ -176,11 +179,8 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, - ) -> Tuple[ - str, - List[AllMessageValues], - dict, - ]: + prompt_label: Optional[str] = None, + ) -> Tuple[str, List[AllMessageValues], dict,]: return self.get_chat_completion_prompt( model, messages, @@ -188,6 +188,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_id, prompt_variables, dynamic_callback_params, + prompt_label=prompt_label, ) def should_run_prompt_management( @@ -211,6 +212,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: langfuse_client = langfuse_client_init( langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"), @@ -219,7 +221,9 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge langfuse_host=dynamic_callback_params.get("langfuse_host"), ) langfuse_prompt_client = self._get_prompt_from_id( - langfuse_prompt_id=prompt_id, langfuse_client=langfuse_client + langfuse_prompt_id=prompt_id, + langfuse_client=langfuse_client, + prompt_label=prompt_label, ) ## SET PROMPT diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py index 270c34be8a6..c9e7adbccbd 100644 --- a/litellm/integrations/prompt_management_base.py +++ b/litellm/integrations/prompt_management_base.py @@ -33,6 +33,7 @@ class PromptManagementBase(ABC): prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: pass @@ -49,11 +50,13 @@ class PromptManagementBase(ABC): prompt_variables: Optional[dict], client_messages: List[AllMessageValues], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: compiled_prompt_client = self._compile_prompt_helper( prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, ) try: @@ -82,6 +85,7 @@ class PromptManagementBase(ABC): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: if prompt_id is None: raise ValueError("prompt_id is required for Prompt Management Base class") @@ -95,6 +99,7 @@ class PromptManagementBase(ABC): prompt_variables=prompt_variables, client_messages=messages, dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, ) completed_messages = prompt_template["completed_messages"] or messages diff --git a/litellm/integrations/vector_stores/bedrock_vector_store.py b/litellm/integrations/vector_stores/bedrock_vector_store.py index e0af1a66364..9015757000b 100644 --- a/litellm/integrations/vector_stores/bedrock_vector_store.py +++ b/litellm/integrations/vector_stores/bedrock_vector_store.py @@ -75,6 +75,7 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Retrieves the context from the Bedrock Knowledge Base and appends it to the messages. @@ -99,10 +100,11 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): f"Bedrock Knowledge Base Response: {bedrock_kb_response}" ) - context_message, context_string = ( - self.get_chat_completion_message_from_bedrock_kb_response( - bedrock_kb_response - ) + ( + context_message, + context_string, + ) = self.get_chat_completion_message_from_bedrock_kb_response( + bedrock_kb_response ) if context_message is not None: messages.append(context_message) @@ -126,9 +128,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): ) ) - litellm_logging_obj.model_call_details["vector_store_request_metadata"] = ( - vector_store_request_metadata - ) + litellm_logging_obj.model_call_details[ + "vector_store_request_metadata" + ] = vector_store_request_metadata return model, messages, non_default_params @@ -140,9 +142,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): """ Transform a BedrockKBResponse to a VectorStoreSearchResponse """ - retrieval_results: Optional[List[BedrockKBRetrievalResult]] = ( - bedrock_kb_response.get("retrievalResults", None) - ) + retrieval_results: Optional[ + List[BedrockKBRetrievalResult] + ] = bedrock_kb_response.get("retrievalResults", None) vector_store_search_response: VectorStoreSearchResponse = ( VectorStoreSearchResponse(search_query=query, data=[]) ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 88ce34245a6..dc5cffa2290 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -539,6 +539,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_id: Optional[str], prompt_variables: Optional[dict], prompt_management_logger: Optional[CustomLogger] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: custom_logger = ( prompt_management_logger @@ -559,6 +560,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=self.standard_callback_dynamic_params, + prompt_label=prompt_label, ) self.messages = messages return model, messages, non_default_params @@ -572,6 +574,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_variables: Optional[dict], prompt_management_logger: Optional[CustomLogger] = None, tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: custom_logger = ( prompt_management_logger @@ -594,6 +597,7 @@ class Logging(LiteLLMLoggingBaseClass): dynamic_callback_params=self.standard_callback_dynamic_params, litellm_logging_obj=self, tools=tools, + prompt_label=prompt_label, ) self.messages = messages return model, messages, non_default_params diff --git a/litellm/main.py b/litellm/main.py index 7cae5acd97b..1c1f4879cc8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -97,6 +97,7 @@ from litellm.utils import ( get_optional_params_image_gen, get_optional_params_transcription, get_secret, + get_standard_openai_params, mock_completion_streaming_obj, read_config_args, supports_httpx_timeout, @@ -428,6 +429,7 @@ async def acompletion( prompt_id=kwargs.get("prompt_id", None), prompt_variables=kwargs.get("prompt_variables", None), tools=tools, + prompt_label=kwargs.get("prompt_label", None), ) ######################################################### @@ -983,6 +985,7 @@ def completion( # type: ignore # noqa: PLR0915 assistant_continue_message=assistant_continue_message, ) ######## end of unpacking kwargs ########### + standard_openai_params = get_standard_openai_params(params=args) non_default_params = get_non_default_completion_params(kwargs=kwargs) litellm_params = {} # used to prevent unbound var errors ## PROMPT MANAGEMENT HOOKS ## @@ -1001,6 +1004,7 @@ def completion( # type: ignore # noqa: PLR0915 non_default_params=non_default_params, prompt_id=prompt_id, prompt_variables=prompt_variables, + prompt_label=kwargs.get("prompt_label", None), ) try: @@ -1234,10 +1238,13 @@ def completion( # type: ignore # noqa: PLR0915 max_retries=max_retries, timeout=timeout, ) - logging.update_environment_variables( + cast(LiteLLMLoggingObj, logging).update_environment_variables( model=model, user=user, - optional_params=optional_params, + optional_params={ + **standard_openai_params, + **non_default_params, + }, # [IMPORTANT] - using standard_openai_params ensures consistent params logged to langfuse for finetuning / eval datasets. litellm_params=litellm_params, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 78880ba55cf..a67ce254685 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,8 +1,8 @@ model_list: - - model_name: "gemini-2.0-flash" + - model_name: "gemini-2.0-flash-gemini" litellm_params: - model: gemini/gemini-2.0-flash-live-001 - - model_name: "gpt-4.1-openai" + model: gemini/gemini-2.0-flash + - model_name: "gpt-4o-mini-openai" litellm_params: model: gpt-4.1-mini-2025-04-14 api_key: os.environ/OPENAI_API_KEY @@ -71,6 +71,16 @@ model_list: model: mistral/* api_key: os.environ/MISTRAL_API_KEY access_groups: ["beta-models"] + - model_name: my-langfuse-model + litellm_params: + model: langfuse/gpt-3.5-turbo + prompt_id: "jokes" + prompt_label: "latest" + api_key: os.environ/OPENAI_API_KEY litellm_settings: - cache: true \ No newline at end of file + callbacks: ["langfuse"] + +general_settings: + store_model_in_db: true + store_prompts_in_spend_logs: true diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 3c759e839ec..1ac694f9475 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -670,15 +670,20 @@ class UserObjectCache: - update user object in cache """ if isinstance(user_object, LiteLLM_UserTable): - user_object = user_object.model_dump() - for k, v in user_object.items(): - if isinstance(v, datetime): - user_object[k] = v.isoformat() - await self.user_api_key_cache.async_set_cache(key=user_id, value=user_object) + user_object_dict = user_object.model_dump() + else: + user_object_dict = user_object + + for k, v in user_object_dict.items(): + if isinstance(v, datetime): + user_object_dict[k] = v.isoformat() + await self.user_api_key_cache.async_set_cache( + key=user_id, value=user_object_dict + ) if self.internal_usage_cache is not None: await self.internal_usage_cache.async_set_cache( key=user_id, - value=user_object, + value=user_object_dict, litellm_parent_otel_span=litellm_parent_otel_span, ) diff --git a/litellm/proxy/custom_prompt_management.py b/litellm/proxy/custom_prompt_management.py index fc16f4a4903..8cf20da5e92 100644 --- a/litellm/proxy/custom_prompt_management.py +++ b/litellm/proxy/custom_prompt_management.py @@ -15,6 +15,7 @@ class X42PromptManagement(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: diff --git a/litellm/router.py b/litellm/router.py index 4b562d669a6..f5fa1886024 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1700,9 +1700,13 @@ class Router: specific_deployment=kwargs.pop("specific_deployment", None), ) - litellm_model = prompt_management_deployment["litellm_params"].get( - "model", None + self._update_kwargs_with_deployment( + deployment=prompt_management_deployment, kwargs=kwargs ) + data = prompt_management_deployment["litellm_params"].copy() + + litellm_model = data.get("model", None) + prompt_id = kwargs.get("prompt_id") or prompt_management_deployment[ "litellm_params" ].get("prompt_id", None) @@ -1711,6 +1715,9 @@ class Router: ) or prompt_management_deployment["litellm_params"].get( "prompt_variables", None ) + prompt_label = kwargs.get("prompt_label", None) or prompt_management_deployment[ + "litellm_params" + ].get("prompt_label", None) if prompt_id is None or not isinstance(prompt_id, str): raise ValueError( @@ -1731,14 +1738,16 @@ class Router: non_default_params=get_non_default_completion_params(kwargs=kwargs), prompt_id=prompt_id, prompt_variables=prompt_variables, + prompt_label=prompt_label, ) - kwargs = {**kwargs, **optional_params} + kwargs = {**data, **kwargs, **optional_params} kwargs["model"] = model kwargs["messages"] = messages kwargs["litellm_logging_obj"] = litellm_logging_object kwargs["prompt_id"] = prompt_id kwargs["prompt_variables"] = prompt_variables + kwargs["prompt_label"] = prompt_label _model_list = self.get_model_list(model_name=model) if _model_list is None or len(_model_list) == 0: # if direct call to model diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 612b03adcbb..4d23ec0f395 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2084,6 +2084,7 @@ all_litellm_params = [ "allowed_openai_params", "litellm_session_id", "use_litellm_proxy", + "prompt_label", ] + list(StandardCallbackDynamicParams.__annotations__.keys()) diff --git a/litellm/utils.py b/litellm/utils.py index 773196077d1..65d825c979c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6835,6 +6835,14 @@ def _add_path_to_api_base(api_base: str, ending_path: str) -> str: return str(modified_url.copy_with(params=original_url.params)) +def get_standard_openai_params(params: dict) -> dict: + return { + k: v + for k, v in params.items() + if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS and v is not None + } + + def get_non_default_completion_params(kwargs: dict) -> dict: openai_params = litellm.OPENAI_CHAT_COMPLETION_PARAMS default_params = openai_params + all_litellm_params diff --git a/tests/litellm/integrations/test_custom_prompt_management.py b/tests/litellm/integrations/test_custom_prompt_management.py index 09ba32b2033..f5855abf71e 100644 --- a/tests/litellm/integrations/test_custom_prompt_management.py +++ b/tests/litellm/integrations/test_custom_prompt_management.py @@ -33,6 +33,7 @@ class TestCustomPromptManagement(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str], ) -> Tuple[str, List[AllMessageValues], dict]: print( "TestCustomPromptManagement: running get_chat_completion_prompt for prompt_id: ", diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json index 4dfe9630ff9..b2a2c83b51a 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json @@ -62,9 +62,7 @@ "endTime": "2025-01-16T11:28:55.124353-08:00", "completionStartTime": "2025-01-16T11:28:55.124353-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json index 4c5f345eaa5..9d30a82b8d2 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json @@ -103,9 +103,7 @@ "endTime": "2025-01-22T09:27:51.702048-08:00", "completionStartTime": "2025-01-22T09:27:51.702048-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json index d4882c962d8..7c2fc6c5f35 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json @@ -81,9 +81,7 @@ "endTime": "2025-01-22T09:19:11.234200-08:00", "completionStartTime": "2025-01-22T09:19:11.234200-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json index 0683ff9ba9f..cb9f007c2d5 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json @@ -52,9 +52,7 @@ "endTime": "2025-02-06T16:23:27.644253-08:00", "completionStartTime": "2025-02-06T16:23:27.644253-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 10, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json index 3a87c0ad739..c4cbe1e68af 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json @@ -71,9 +71,7 @@ "endTime": "2025-01-22T07:31:28.962389-08:00", "completionStartTime": "2025-01-22T07:31:28.962389-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json index 6495ed947d6..cd882af614d 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json @@ -71,9 +71,7 @@ "endTime": "2025-01-22T08:38:26.015666-08:00", "completionStartTime": "2025-01-22T08:38:26.015666-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json index 01dcd264883..5c8d5c5b88d 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json @@ -78,9 +78,7 @@ "endTime": "2025-01-22T09:59:39.365756-08:00", "completionStartTime": "2025-01-22T09:59:39.365756-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json index 1b7b91930e9..4533262ef42 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json @@ -70,9 +70,7 @@ "endTime": "2025-01-22T10:06:50.958374-08:00", "completionStartTime": "2025-01-22T10:06:50.958374-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json index 8c1711ee98e..39a88320bbf 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json @@ -64,9 +64,7 @@ "endTime": "2025-01-22T09:59:32.880691-08:00", "completionStartTime": "2025-01-22T09:59:32.880691-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json index 0b1309425e3..e73ef0d9ed6 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json @@ -64,9 +64,7 @@ "endTime": "2025-01-22T09:59:36.161959-08:00", "completionStartTime": "2025-01-22T09:59:36.161959-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json index 8c1711ee98e..39a88320bbf 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json @@ -64,9 +64,7 @@ "endTime": "2025-01-22T09:59:32.880691-08:00", "completionStartTime": "2025-01-22T09:59:32.880691-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json index bb24688aa5c..efd3bbae323 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json @@ -70,9 +70,7 @@ "endTime": "2025-01-22T09:55:28.853979-08:00", "completionStartTime": "2025-01-22T09:55:28.853979-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json index d40ec6bafca..8cb1cced89d 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json @@ -70,9 +70,7 @@ "endTime": "2025-01-22T09:53:53.753431-08:00", "completionStartTime": "2025-01-22T09:53:53.753431-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json index 610bc461a13..0de688644b9 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json @@ -74,9 +74,7 @@ "endTime": "2025-01-22T09:56:35.476236-08:00", "completionStartTime": "2025-01-22T09:56:35.476236-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json index d21c58fdee4..f0ad3e9e712 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json @@ -78,9 +78,7 @@ "endTime": "2025-01-22T09:56:38.785762-08:00", "completionStartTime": "2025-01-22T09:56:38.785762-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, From bfb04d3e7583f91efbf9bdece0d18ca6f2a24c00 Mon Sep 17 00:00:00 2001 From: tanjiro <56165694+NANDINI-star@users.noreply.github.com> Date: Thu, 22 May 2025 23:23:26 +0900 Subject: [PATCH 10/36] added cloding tags for + indentation changes (#11046) --- .../src/components/organizations.tsx | 329 +++++++++--------- 1 file changed, 166 insertions(+), 163 deletions(-) diff --git a/ui/litellm-dashboard/src/components/organizations.tsx b/ui/litellm-dashboard/src/components/organizations.tsx index 66eccdcc38e..86a62ac42c5 100644 --- a/ui/litellm-dashboard/src/components/organizations.tsx +++ b/ui/litellm-dashboard/src/components/organizations.tsx @@ -264,173 +264,175 @@ const OrganizationsTable: React.FC = ({ - - {organizations && organizations.length > 0 - ? organizations - .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()) - .map((org: Organization) => ( - - -
- - - -
- - {org.organization_alias} - - {org.created_at ? new Date(org.created_at).toLocaleDateString() : "N/A"} - - {org.spend.toFixed(4)} - - {org.litellm_budget_table?.max_budget !== null && org.litellm_budget_table?.max_budget !== undefined ? org.litellm_budget_table?.max_budget : "No limit"} - - - {Array.isArray(org.models) && ( -
- {org.models.length === 0 ? ( - - All Proxy Models - - ) : ( - org.models.map((model, index) => - model === "all-proxy-models" ? ( - - All Proxy Models - - ) : ( - - {model.length > 30 - ? `${getModelDisplayName(model).slice(0, 30)}...` - : getModelDisplayName(model)} - - ) - ) - )} -
- )} - - - - TPM: {org.litellm_budget_table?.tpm_limit ? org.litellm_budget_table?.tpm_limit : "Unlimited"} -
- RPM: {org.litellm_budget_table?.rpm_limit ? org.litellm_budget_table?.rpm_limit : "Unlimited"} - - - - {org.members?.length || 0} Members - - - {userRole === "Admin" && ( - <> - { - setSelectedOrgId(org.organization_id); - setEditOrg(true); - }} - /> - handleDelete(org.organization_id)} - icon={TrashIcon} - size="sm" - /> - - )} - - - )) - : null} - - - - - {(userRole === "Admin" || userRole === "Org Admin") && ( - - - -
- - - - - + {organizations && organizations.length > 0 + ? organizations + .sort((a, b) => new Date(b.created_at).getTime() - new Date(a.created_at).getTime()) + .map((org: Organization) => ( + + +
+ + + +
+ + {org.organization_alias} + + {org.created_at ? new Date(org.created_at).toLocaleDateString() : "N/A"} + + {org.spend.toFixed(4)} + + {org.litellm_budget_table?.max_budget !== null && org.litellm_budget_table?.max_budget !== undefined ? org.litellm_budget_table?.max_budget : "No limit"} + + + {Array.isArray(org.models) && ( +
+ {org.models.length === 0 ? ( + + All Proxy Models + + ) : ( + org.models.map((model, index) => + model === "all-proxy-models" ? ( + + All Proxy Models + + ) : ( + + {model.length > 30 + ? `${getModelDisplayName(model).slice(0, 30)}...` + : getModelDisplayName(model)} + + ) + ) + )} +
+ )} + + + + TPM: {org.litellm_budget_table?.tpm_limit ? org.litellm_budget_table?.tpm_limit : "Unlimited"} +
+ RPM: {org.litellm_budget_table?.rpm_limit ? org.litellm_budget_table?.rpm_limit : "Unlimited"} + + + + {org.members?.length || 0} Members + + + {userRole === "Admin" && ( + <> + { + setSelectedOrgId(org.organization_id); + setEditOrg(true); + }} + /> + handleDelete(org.organization_id)} + icon={TrashIcon} + size="sm" + /> + + )} + + + )) + : null} + + + + + {(userRole === "Admin" || userRole === "Org Admin") && ( + + + + + + + + + + + All Proxy Models + + {userModels && userModels.length > 0 && userModels.map((model) => ( + + {getModelDisplayName(model)} + + ))} + + - - - - - - daily - weekly - monthly - - - - - - - - + + + + + + daily + weekly + monthly + + + + + + + + - - - - -
- -
- - - - )} - - - + + + +
+ +
+ +
+ + )} + + + + + + {isDeleteModalOpen ? (
@@ -467,6 +469,7 @@ const OrganizationsTable: React.FC = ({
) : <>} +
); }; From dd4a65b83a7c66b6deb4a4059e616445721d448c Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 07:24:10 -0700 Subject: [PATCH 11/36] Feat: add MCP to Responses API and bump openai python sdk (#11029) * feat: add MCP to responses API * feat: bump openai version to 1.75.0 * docs MCP + responses API * fixes: type checking * fixes: type checking * build: use latest openai 1.81.0 * fix: linting error * fix: linting error * fix: test * fix: linting errors * fix: test * fix: test * fix: linting * Revert "fix: linting" This reverts commit ebb19ff8cb1f8fcc3e224390e351676daccb33de. * fix: linting --- .circleci/config.yml | 18 +-- .circleci/requirements.txt | 2 +- .github/workflows/test-linting.yml | 4 +- .../docs/providers/openai/responses_api.md | 130 ++++++++++++++++++ litellm/llms/bedrock/image/cost_calculator.py | 4 +- .../image_generation/cost_calculator.py | 4 +- .../transformation.py | 4 +- litellm/responses/streaming_iterator.py | 5 +- litellm/types/llms/openai.py | 2 +- litellm/types/utils.py | 45 +++++- poetry.lock | 2 +- pyproject.toml | 2 +- requirements.txt | 2 +- .../base_image_generation_test.py | 11 +- .../test_openai_responses_api.py | 51 ++++++- 15 files changed, 254 insertions(+), 32 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4306fa5cf05..c523b1d191e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -95,7 +95,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.68.2 + pip install openai==1.81.0 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -218,7 +218,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.68.2 + pip install openai==1.81.0 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -325,7 +325,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.68.2 + pip install openai==1.81.0 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -581,7 +581,7 @@ jobs: pip install opentelemetry-api==1.25.0 pip install opentelemetry-sdk==1.25.0 pip install opentelemetry-exporter-otlp==1.25.0 - pip install openai==1.68.2 + pip install openai==1.81.0 pip install prisma==0.11.0 pip install "detect_secrets==1.5.0" pip install "httpx==0.24.1" @@ -1472,7 +1472,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.68.2" + pip install "openai==1.81.0" - run: name: Install Grype command: | @@ -1610,7 +1610,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.68.2" + pip install "openai==1.81.0" # Run pytest and generate JUnit XML report - run: name: Build Docker image @@ -1733,7 +1733,7 @@ jobs: pip install "aiodynamo==23.10.1" pip install "asyncio==3.4.3" pip install "PyGithub==1.59.1" - pip install "openai==1.68.2" + pip install "openai==1.81.0" - run: name: Build Docker image command: docker build -t my-app:latest -f ./docker/Dockerfile.database . @@ -2256,7 +2256,7 @@ jobs: pip install "pytest-asyncio==0.21.1" pip install "google-cloud-aiplatform==1.43.0" pip install aiohttp - pip install "openai==1.68.2" + pip install "openai==1.81.0" pip install "assemblyai==0.37.0" python -m pip install --upgrade pip pip install "pydantic==2.10.2" @@ -2644,7 +2644,7 @@ jobs: pip install "pytest-retry==1.6.3" pip install "pytest-asyncio==0.21.1" pip install aiohttp - pip install "openai==1.68.2" + pip install "openai==1.81.0" python -m pip install --upgrade pip pip install "pydantic==2.10.2" pip install "pytest==7.3.1" diff --git a/.circleci/requirements.txt b/.circleci/requirements.txt index 0e2362c4e3d..b720d15a7fd 100644 --- a/.circleci/requirements.txt +++ b/.circleci/requirements.txt @@ -1,5 +1,5 @@ # used by CI/CD testing -openai==1.68.2 +openai==1.81.0 python-dotenv tiktoken importlib_metadata diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index 0e1c895c3a4..ceeedbe7e13 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -22,9 +22,9 @@ jobs: - name: Install dependencies run: | - pip install openai==1.68.2 + pip install openai==1.81.0 poetry install --with dev - pip install openai==1.68.2 + pip install openai==1.81.0 diff --git a/docs/my-website/docs/providers/openai/responses_api.md b/docs/my-website/docs/providers/openai/responses_api.md index 578ce038f37..e88512ecfd4 100644 --- a/docs/my-website/docs/providers/openai/responses_api.md +++ b/docs/my-website/docs/providers/openai/responses_api.md @@ -318,3 +318,133 @@ print(response) + + +## MCP Tools + + + + +```python showLineNumbers title="MCP Tools with LiteLLM SDK" +import litellm +from typing import Optional + +# Configure MCP Tools +MCP_TOOLS = [ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "allowed_tools": ["ask_question"] + } +] + +# Step 1: Make initial request - OpenAI will use MCP LIST and return MCP calls for approval +response = litellm.responses( + model="openai/gpt-4.1", + tools=MCP_TOOLS, + input="What transport protocols does the 2025-03-26 version of the MCP spec support?" +) + +# Get the MCP approval ID +mcp_approval_id = None +for output in response.output: + if output.type == "mcp_approval_request": + mcp_approval_id = output.id + break + +# Step 2: Send followup with approval for the MCP call +response_with_mcp_call = litellm.responses( + model="openai/gpt-4.1", + tools=MCP_TOOLS, + input=[ + { + "type": "mcp_approval_response", + "approve": True, + "approval_request_id": mcp_approval_id + } + ], + previous_response_id=response.id, +) + +print(response_with_mcp_call) +``` + + + + +1. Set up config.yaml + +```yaml showLineNumbers title="OpenAI Proxy Configuration" +model_list: + - model_name: openai/gpt-4.1 + litellm_params: + model: openai/gpt-4.1 + api_key: os.environ/OPENAI_API_KEY +``` + +2. Start LiteLLM Proxy Server + +```bash title="Start LiteLLM Proxy Server" +litellm --config /path/to/config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +3. Test it! + +```python showLineNumbers title="MCP Tools with OpenAI SDK via LiteLLM Proxy" +from openai import OpenAI +from typing import Optional + +# Initialize client with your proxy URL +client = OpenAI( + base_url="http://localhost:4000", # Your proxy URL + api_key="your-api-key" # Your proxy API key +) + +# Configure MCP Tools +MCP_TOOLS = [ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "allowed_tools": ["ask_question"] + } +] + +# Step 1: Make initial request - OpenAI will use MCP LIST and return MCP calls for approval +response = client.responses.create( + model="openai/gpt-4.1", + tools=MCP_TOOLS, + input="What transport protocols does the 2025-03-26 version of the MCP spec support?" +) + +# Get the MCP approval ID +mcp_approval_id = None +for output in response.output: + if output.type == "mcp_approval_request": + mcp_approval_id = output.id + break + +# Step 2: Send followup with approval for the MCP call +response_with_mcp_call = client.responses.create( + model="openai/gpt-4.1", + tools=MCP_TOOLS, + input=[ + { + "type": "mcp_approval_response", + "approve": True, + "approval_request_id": mcp_approval_id + } + ], + previous_response_id=response.id, +) + +print(response_with_mcp_call) +``` + + + + + diff --git a/litellm/llms/bedrock/image/cost_calculator.py b/litellm/llms/bedrock/image/cost_calculator.py index 0a20b44cb38..a0dc91d7119 100644 --- a/litellm/llms/bedrock/image/cost_calculator.py +++ b/litellm/llms/bedrock/image/cost_calculator.py @@ -37,5 +37,7 @@ def cost_calculator( ) output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 - num_images: int = len(image_response.data) + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) return output_cost_per_image * num_images diff --git a/litellm/llms/vertex_ai/image_generation/cost_calculator.py b/litellm/llms/vertex_ai/image_generation/cost_calculator.py index 2ba18c095bd..646c6080a2e 100644 --- a/litellm/llms/vertex_ai/image_generation/cost_calculator.py +++ b/litellm/llms/vertex_ai/image_generation/cost_calculator.py @@ -19,5 +19,7 @@ def cost_calculator( ) output_cost_per_image: float = _model_info.get("output_cost_per_image") or 0.0 - num_images: int = len(image_response.data) + num_images: int = 0 + if image_response.data: + num_images = len(image_response.data) return output_cost_per_image * num_images diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index 5812daad462..baffca3ac68 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -457,8 +457,8 @@ class LiteLLMCompletionResponsesConfig: function=ChatCompletionToolParamFunctionChunk( name=tool["name"], description=tool.get("description") or "", - parameters=tool.get("parameters", {}), - strict=tool.get("strict", False), + parameters=dict(tool.get("parameters", {}) or {}), + strict=tool.get("strict", False) or False, ), ) ) diff --git a/litellm/responses/streaming_iterator.py b/litellm/responses/streaming_iterator.py index a111fbec094..e9e41789f09 100644 --- a/litellm/responses/streaming_iterator.py +++ b/litellm/responses/streaming_iterator.py @@ -44,7 +44,7 @@ class BaseResponsesAPIStreamingIterator: self.responses_api_provider_config = responses_api_provider_config self.completed_response: Optional[ResponsesAPIStreamingResponse] = None self.start_time = datetime.now() - + # set request kwargs self.litellm_metadata = litellm_metadata self.custom_llm_provider = custom_llm_provider @@ -330,7 +330,8 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator): def _collect_text(self, resp: ResponsesAPIResponse) -> str: out = "" for out_item in resp.output: - if out_item.type == "message": + item_type = getattr(out_item, "type", None) + if item_type == "message": for c in getattr(out_item, "content", []): out += c.text return out diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index f5aa27a9eaf..367f86fd3f4 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -1000,7 +1000,7 @@ class ResponsesAPIResponse(BaseLiteLLMOpenAIResponseObject): model: Optional[str] object: Optional[str] output: Union[ - List[ResponseOutputItem], + List[Union[ResponseOutputItem, Dict]], List[Union[GenericResponseOutputItem, OutputFunctionToolCall]], ] parallel_tool_calls: bool diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4d23ec0f395..a9acce9a797 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -33,6 +33,7 @@ from pydantic import BaseModel, ConfigDict, Field, PrivateAttr, model_validator from typing_extensions import Callable, Dict, Required, TypedDict, override import litellm +from litellm.types.llms.base import BaseLiteLLMOpenAIResponseObject from ..litellm_core_utils.core_helpers import map_finish_reason from .guardrails import GuardrailEventHooks @@ -1542,19 +1543,46 @@ class ImageObject(OpenAIImage): return self.dict() +class ImageUsageInputTokensDetails(BaseLiteLLMOpenAIResponseObject): + image_tokens: int + """The number of image tokens in the input prompt.""" + + text_tokens: int + """The number of text tokens in the input prompt.""" + + +class ImageUsage(BaseLiteLLMOpenAIResponseObject): + input_tokens: int + """The number of tokens (images and text) in the input prompt.""" + + input_tokens_details: ImageUsageInputTokensDetails + """The input tokens detailed information for the image generation.""" + + output_tokens: int + """The number of image tokens in the output image.""" + + total_tokens: int + """The total number of tokens (images and text) used for the image generation.""" + + from openai.types.images_response import ImagesResponse as OpenAIImageResponse -class ImageResponse(OpenAIImageResponse): +class ImageResponse(OpenAIImageResponse, BaseLiteLLMOpenAIResponseObject): _hidden_params: dict = {} - usage: Usage + + usage: Optional[ImageUsage] = None # type: ignore + """ + Users might use litellm with older python versions, we don't want this to break for them. + Happens when their OpenAIImageResponse has the old OpenAI usage class. + """ def __init__( self, created: Optional[int] = None, data: Optional[List[ImageObject]] = None, response_ms=None, - usage: Optional[Usage] = None, + usage: Optional[ImageUsage] = None, hidden_params: Optional[dict] = None, ): if response_ms: @@ -1577,9 +1605,14 @@ class ImageResponse(OpenAIImageResponse): _data.append(ImageObject(**d)) elif isinstance(d, BaseModel): _data.append(ImageObject(**d.model_dump())) - _usage = usage or Usage( - prompt_tokens=0, - completion_tokens=0, + + _usage = usage or ImageUsage( + input_tokens=0, + input_tokens_details=ImageUsageInputTokensDetails( + image_tokens=0, + text_tokens=0, + ), + output_tokens=0, total_tokens=0, ) super().__init__(created=created, data=_data, usage=_usage) # type: ignore diff --git a/poetry.lock b/poetry.lock index 9c361b597d5..2bd91505cb2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -4935,4 +4935,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "4c385d4e27013d9cacf9573f5532c5e13d22fc27af6344e87ba2417340c9be93" +content-hash = "15bad8ae37c1e7cf21555b0023150bfd4bd7d6d548828f6c65a66283d14a189b" diff --git a/pyproject.toml b/pyproject.toml index 2607937e7b1..6a5ed32e6e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ Documentation = "https://docs.litellm.ai" [tool.poetry.dependencies] python = ">=3.8.1,<4.0, !=3.9.7" httpx = ">=0.23.0" -openai = ">=1.68.2, <1.76.0" +openai = ">=1.68.2" python-dotenv = ">=0.2.0" tiktoken = ">=0.7.0" importlib-metadata = ">=6.8.0" diff --git a/requirements.txt b/requirements.txt index db040686f1e..e3aae6f90fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ # LITELLM PROXY DEPENDENCIES # anyio==4.5.0 # openai + http req. httpx==0.27.0 # Pin Httpx dependency -openai==1.68.2 # openai req. +openai==1.81.0 # openai req. fastapi==0.115.5 # server dep backoff==2.2.1 # server dep pyyaml==6.0.2 # server dep diff --git a/tests/image_gen_tests/base_image_generation_test.py b/tests/image_gen_tests/base_image_generation_test.py index df64d353c71..c3a5cfb2251 100644 --- a/tests/image_gen_tests/base_image_generation_test.py +++ b/tests/image_gen_tests/base_image_generation_test.py @@ -68,10 +68,17 @@ class BaseImageGenTest(ABC): assert logged_standard_logging_payload is not None assert logged_standard_logging_payload["response_cost"] is not None assert logged_standard_logging_payload["response_cost"] > 0 - + import openai from openai.types.images_response import ImagesResponse - ImagesResponse.model_validate(response.model_dump()) + # print openai version + print("openai version=", openai.__version__) + + response_dict = dict(response) + if "usage" in response_dict: + response_dict["usage"] = dict(response_dict["usage"]) + print("response usage=", response_dict.get("usage")) + ImagesResponse.model_validate(response_dict) for d in response.data: assert isinstance(d, Image) diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 333f99f9d46..4c3998e0c31 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -2,7 +2,7 @@ import os import sys import pytest import asyncio -from typing import Optional +from typing import Optional, cast from unittest.mock import patch, AsyncMock sys.path.insert(0, os.path.abspath("../..")) @@ -1032,4 +1032,51 @@ def test_basic_computer_use_preview_tool_call(): # Validate the input format assert isinstance(request_body["input"], str) assert request_body["input"] == "Check the latest OpenAI news on bing.com." - \ No newline at end of file + + + +def test_mcp_tools_with_responses_api(): + litellm._turn_on_debug() + MCP_TOOLS = [ + { + "type": "mcp", + "server_label": "deepwiki", + "server_url": "https://mcp.deepwiki.com/mcp", + "allowed_tools": ["ask_question"] + } + ] + MODEL = "openai/gpt-4.1" + USER_QUERY = "What transport protocols does the 2025-03-26 version of the MCP spec (modelcontextprotocol/modelcontextprotocol) support?" + ######################################################### + # Step 1: OpenAI will use MCP LIST, and return a list of MCP calls for our approval + response = litellm.responses( + model=MODEL, + tools=MCP_TOOLS, + input=USER_QUERY + ) + print(response) + + response = cast(ResponsesAPIResponse, response) + + mcp_approval_id: Optional[str] + for output in response.output: + if output.type == "mcp_approval_request": + mcp_approval_id = output.id + break + + # Step 2: Send followup with approval for the MCP call + response_with_mcp_call = litellm.responses( + model=MODEL, + tools=MCP_TOOLS, + input=[ + { + "type": "mcp_approval_response", + "approve": True, + "approval_request_id": mcp_approval_id + } + ], + previous_response_id=response.id, + ) + print(response_with_mcp_call) + + From 1c652b67b656d168fdcbe31332905e99b00df67a Mon Sep 17 00:00:00 2001 From: tanjiro <56165694+NANDINI-star@users.noreply.github.com> Date: Thu, 22 May 2025 23:31:43 +0900 Subject: [PATCH 12/36] Model filter on logs (#11048) * add model filter * remove calling all models --- .../components/common_components/filter.tsx | 1 + .../src/components/view_logs/index.tsx | 15 ++------------- .../components/view_logs/log_filter_logic.tsx | 19 ------------------- 3 files changed, 3 insertions(+), 32 deletions(-) diff --git a/ui/litellm-dashboard/src/components/common_components/filter.tsx b/ui/litellm-dashboard/src/components/common_components/filter.tsx index 067fadfc6ba..8351ea791ad 100644 --- a/ui/litellm-dashboard/src/components/common_components/filter.tsx +++ b/ui/litellm-dashboard/src/components/common_components/filter.tsx @@ -88,6 +88,7 @@ const FilterComponent: React.FC = ({ "Key Alias", "User ID", "Key Hash", + "Model" ]; return ( diff --git a/ui/litellm-dashboard/src/components/view_logs/index.tsx b/ui/litellm-dashboard/src/components/view_logs/index.tsx index 2c05c3138d7..b6be331aba5 100644 --- a/ui/litellm-dashboard/src/components/view_logs/index.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/index.tsx @@ -226,7 +226,6 @@ export default function SpendLogsTable({ filteredLogs, allTeams: hookAllTeams, allKeyAliases, - allModels, handleFilterChange, handleFilterReset } = useLogFilterLogic({ @@ -408,17 +407,7 @@ export default function SpendLogsTable({ { name: 'Model', label: 'Model', - isSearchable: true, - searchFn: async (searchText: string) => { - if (!allModels || allModels.length === 0) return []; - const filtered = allModels.filter((model: string) => { - return model.toLowerCase().includes(searchText.toLowerCase()); - }); - return filtered.map((model: string) => ({ - label: model, - value: model - })); - } + isSearchable: false, }, { name: 'Key Alias', @@ -440,7 +429,7 @@ export default function SpendLogsTable({ name: 'Key Hash', label: 'Key Hash', isSearchable: false, - } + }, ] // When a session is selected, render the SessionView component diff --git a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx index 1f2b765ded7..e61bfbf109b 100644 --- a/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx +++ b/ui/litellm-dashboard/src/components/view_logs/log_filter_logic.tsx @@ -198,24 +198,6 @@ export function useLogFilterLogic({ enabled: !!accessToken, }); - const { data: allModels = [] } = useQuery({ - queryKey: ['allModels', accessToken, userID, userRole], - queryFn: async () => { - if (!accessToken || !userID || !userRole) return []; - - const response = await modelAvailableCall( - accessToken, - userID, - userRole, - false, // return_wildcard_routes - null // teamID - ); - - return response.data.map((model: { id: string }) => model.id); - }, - enabled: !!accessToken && !!userID && !!userRole, - }); - // Update filters state const handleFilterChange = (newFilters: Partial) => { setFilters(prev => { @@ -252,7 +234,6 @@ export function useLogFilterLogic({ filteredLogs, allKeyAliases, allTeams, - allModels, handleFilterChange, handleFilterReset, }; From d595c4ef2d077267f9246e3d96b212bec42115a4 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 07:32:35 -0700 Subject: [PATCH 13/36] =?UTF-8?q?bump:=20version=201.70.3=20=E2=86=92=201.?= =?UTF-8?q?70.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 6a5ed32e6e9..d3fd3ecf0cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "litellm" -version = "1.70.3" +version = "1.70.4" description = "Library to easily interface with LLM API providers" authors = ["BerriAI"] license = "MIT" @@ -136,7 +136,7 @@ requires = ["poetry-core", "wheel"] build-backend = "poetry.core.masonry.api" [tool.commitizen] -version = "1.70.3" +version = "1.70.4" version_files = [ "pyproject.toml:^version" ] From d95c3a16f3e76464745768e0a9be37596c27e713 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 08:55:07 -0700 Subject: [PATCH 14/36] docs fix ad hoc recognizer --- docs/my-website/docs/proxy/guardrails/pii_masking_v2.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md index 427308cf221..c93eb52a2a7 100644 --- a/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md +++ b/docs/my-website/docs/proxy/guardrails/pii_masking_v2.md @@ -418,7 +418,7 @@ guardrails: Send ad-hoc recognizers to presidio `/analyze` by passing a json file to the proxy -[**Example** ad-hoc recognizer](../../../../litellm/proxy/hooks/example_presidio_ad_hoc_recognize) +[**Example** ad-hoc recognizer](https://github.com/BerriAI/litellm/blob/b69b7503db5aa039a49b7ca96ae5b34db0d25a3d/litellm/proxy/hooks/example_presidio_ad_hoc_recognizer.json) #### Define ad-hoc recognizer on your LiteLLM config.yaml From 2c90ca0189557ec0054c5204e6c86a2aeda4ecc8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 09:33:22 -0700 Subject: [PATCH 15/36] docs fix example --- docs/my-website/docs/pass_through/vertex_ai.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/my-website/docs/pass_through/vertex_ai.md b/docs/my-website/docs/pass_through/vertex_ai.md index b99f0fcf982..d3f4e75e31d 100644 --- a/docs/my-website/docs/pass_through/vertex_ai.md +++ b/docs/my-website/docs/pass_through/vertex_ai.md @@ -116,7 +116,7 @@ curl \ ```bash -curl http://localhost:4000/vertex_ai/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \ +curl http://localhost:4000/vertex_ai/v1/projects/${PROJECT_ID}/locations/us-central1/publishers/google/models/${MODEL_ID}:generateContent \ -H "Content-Type: application/json" \ -H "x-litellm-api-key: Bearer sk-1234" \ -d '{ From 197c608078ee44f367bb7359837d0d7a69a992a2 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 13:33:54 -0700 Subject: [PATCH 16/36] [Feat] Add claude-4 model family (#11060) * add new claude-sonnet-4-2025051 * feat: add bedrock claude-4 models * add bedrock claude-4 models * add vertx_ai/claude-sonnet-4 * fix provider=bedrock_converse * feat: ensure thinking is supported for claude-4 model family --- litellm/__init__.py | 2 + litellm/llms/anthropic/chat/transformation.py | 6 +- .../bedrock/chat/converse_transformation.py | 9 +- ...odel_prices_and_context_window_backup.json | 260 ++++++++++++++++++ model_prices_and_context_window.json | 260 ++++++++++++++++++ 5 files changed, 532 insertions(+), 5 deletions(-) diff --git a/litellm/__init__.py b/litellm/__init__.py index dd09810d46f..5061c8e7e5b 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -354,6 +354,8 @@ project = None config_path = None vertex_ai_safety_settings: Optional[dict] = None BEDROCK_CONVERSE_MODELS = [ + "anthropic.claude-opus-4-20250514-v1:0", + "anthropic.claude-sonnet-4-20250514-v1:0", "anthropic.claude-3-7-sonnet-20250219-v1:0", "anthropic.claude-3-5-haiku-20241022-v1:0", "anthropic.claude-3-5-sonnet-20241022-v2:0", diff --git a/litellm/llms/anthropic/chat/transformation.py b/litellm/llms/anthropic/chat/transformation.py index 9052cec97cf..d7756bead07 100644 --- a/litellm/llms/anthropic/chat/transformation.py +++ b/litellm/llms/anthropic/chat/transformation.py @@ -49,6 +49,7 @@ from litellm.utils import ( Usage, add_dummy_tool, has_tool_call_blocks, + supports_reasoning, token_counter, ) @@ -121,7 +122,10 @@ class AnthropicConfig(AnthropicModelInfo, BaseConfig): "web_search_options", ] - if "claude-3-7-sonnet" in model: + if "claude-3-7-sonnet" in model or supports_reasoning( + model=model, + custom_llm_provider=self.custom_llm_provider, + ): params.append("thinking") return params diff --git a/litellm/llms/bedrock/chat/converse_transformation.py b/litellm/llms/bedrock/chat/converse_transformation.py index 7dad73d871d..f857f47af32 100644 --- a/litellm/llms/bedrock/chat/converse_transformation.py +++ b/litellm/llms/bedrock/chat/converse_transformation.py @@ -45,7 +45,7 @@ from litellm.types.utils import ( PromptTokensDetailsWrapper, Usage, ) -from litellm.utils import add_dummy_tool, has_tool_call_blocks +from litellm.utils import add_dummy_tool, has_tool_call_blocks, supports_reasoning from ..common_utils import BedrockError, BedrockModelInfo, get_bedrock_tool_name @@ -146,9 +146,10 @@ class AmazonConverseConfig(BaseConfig): # only anthropic and mistral support tool choice config. otherwise (E.g. cohere) will fail the call - https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_ToolChoice.html supported_params.append("tool_choice") - if ( - "claude-3-7" in model - ): # [TODO]: move to a 'supports_reasoning_content' param from model cost map + if "claude-3-7" in model or supports_reasoning( + model=model, + custom_llm_provider=self.custom_llm_provider, + ): supported_params.append("thinking") supported_params.append("reasoning_effort") return supported_params diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 4bea9baee3e..6e2f66c466b 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4684,6 +4684,58 @@ "deprecation_date": "2025-06-01", "supports_tool_choice": true }, + "claude-opus-4-20250514": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "claude-sonnet-4-20250514": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "claude-3-7-sonnet-latest": { "supports_computer_use": true, "max_tokens": 128000, @@ -6753,6 +6805,58 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/claude-opus-4@20250514": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "vertex_ai/claude-sonnet-4@20250514": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "vertex_ai/claude-3-haiku": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9332,6 +9436,58 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "anthropic.claude-3-7-sonnet-20250219-v1:0": { "supports_computer_use": true, "max_tokens": 8192, @@ -9482,6 +9638,58 @@ "supports_tool_choice": true, "supports_reasoning": true }, + "us.anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "us.anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "us.anthropic.claude-3-haiku-20240307-v1:0": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9603,6 +9811,58 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "eu.anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "eu.anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { "max_tokens": 8192, "max_input_tokens": 200000, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 4bea9baee3e..6e2f66c466b 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4684,6 +4684,58 @@ "deprecation_date": "2025-06-01", "supports_tool_choice": true }, + "claude-opus-4-20250514": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "claude-sonnet-4-20250514": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "anthropic", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "claude-3-7-sonnet-latest": { "supports_computer_use": true, "max_tokens": 128000, @@ -6753,6 +6805,58 @@ "supports_reasoning": true, "supports_tool_choice": true }, + "vertex_ai/claude-opus-4@20250514": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "vertex_ai/claude-sonnet-4@20250514": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "vertex_ai-anthropic_models", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "vertex_ai/claude-3-haiku": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9332,6 +9436,58 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "anthropic.claude-3-7-sonnet-20250219-v1:0": { "supports_computer_use": true, "max_tokens": 8192, @@ -9482,6 +9638,58 @@ "supports_tool_choice": true, "supports_reasoning": true }, + "us.anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "us.anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "us.anthropic.claude-3-haiku-20240307-v1:0": { "max_tokens": 4096, "max_input_tokens": 200000, @@ -9603,6 +9811,58 @@ "supports_pdf_input": true, "supports_tool_choice": true }, + "eu.anthropic.claude-opus-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 15e-6, + "output_cost_per_token": 75e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 18.75e-6, + "cache_read_input_token_cost": 1.5e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, + "eu.anthropic.claude-sonnet-4-20250514-v1:0": { + "max_tokens": 128000, + "max_input_tokens": 200000, + "max_output_tokens": 128000, + "input_cost_per_token": 3e-6, + "output_cost_per_token": 15e-6, + "search_context_cost_per_query": { + "search_context_size_low": 1e-2, + "search_context_size_medium": 1e-2, + "search_context_size_high": 1e-2 + }, + "cache_creation_input_token_cost": 3.75e-6, + "cache_read_input_token_cost": 0.3e-6, + "litellm_provider": "bedrock_converse", + "mode": "chat", + "supports_function_calling": true, + "supports_vision": true, + "tool_use_system_prompt_tokens": 159, + "supports_assistant_prefill": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_reasoning": true, + "supports_computer_use": true + }, "eu.anthropic.claude-3-5-haiku-20241022-v1:0": { "max_tokens": 8192, "max_input_tokens": 200000, From 89daa1dbad2b50a2c1197a446764ec75a3ad042d Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 13:41:09 -0700 Subject: [PATCH 17/36] docs add claude-4 models --- docs/my-website/docs/providers/anthropic.md | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/providers/anthropic.md b/docs/my-website/docs/providers/anthropic.md index 990a9b5122e..4ab4eb06086 100644 --- a/docs/my-website/docs/providers/anthropic.md +++ b/docs/my-website/docs/providers/anthropic.md @@ -4,6 +4,8 @@ import TabItem from '@theme/TabItem'; # Anthropic LiteLLM supports all anthropic models. +- `claude-4` (`claude-opus-4-20250514`, `claude-sonnet-4-20250514`) +- `claude-3.7` (`claude-3-7-sonnet-20250219`) - `claude-3.5` (`claude-3-5-sonnet-20240620`) - `claude-3` (`claude-3-haiku-20240307`, `claude-3-opus-20240229`, `claude-3-sonnet-20240229`) - `claude-2` @@ -64,7 +66,7 @@ from litellm import completion os.environ["ANTHROPIC_API_KEY"] = "your-api-key" messages = [{"role": "user", "content": "Hey! how's it going?"}] -response = completion(model="claude-3-opus-20240229", messages=messages) +response = completion(model="claude-opus-4-20250514", messages=messages) print(response) ``` @@ -80,7 +82,7 @@ from litellm import completion os.environ["ANTHROPIC_API_KEY"] = "your-api-key" messages = [{"role": "user", "content": "Hey! how's it going?"}] -response = completion(model="claude-3-opus-20240229", messages=messages, stream=True) +response = completion(model="claude-opus-4-20250514", messages=messages, stream=True) for chunk in response: print(chunk["choices"][0]["delta"]["content"]) # same as openai format ``` @@ -102,9 +104,9 @@ export ANTHROPIC_API_KEY="your-api-key" ```yaml model_list: - - model_name: claude-3 ### RECEIVED MODEL NAME ### + - model_name: claude-4 ### RECEIVED MODEL NAME ### litellm_params: # all params accepted by litellm.completion() - https://docs.litellm.ai/docs/completion/input - model: claude-3-opus-20240229 ### MODEL NAME sent to `litellm.completion()` ### + model: claude-opus-4-20250514 ### MODEL NAME sent to `litellm.completion()` ### api_key: "os.environ/ANTHROPIC_API_KEY" # does os.getenv("AZURE_API_KEY_EU") ``` @@ -156,7 +158,7 @@ curl --location 'http://0.0.0.0:4000/chat/completions' \ ```bash -$ litellm --model claude-3-opus-20240229 +$ litellm --model claude-opus-4-20250514 # Server running on http://0.0.0.0:4000 ``` @@ -244,6 +246,9 @@ print(response) | Model Name | Function Call | |------------------|--------------------------------------------| +| claude-opus-4 | `completion('claude-opus-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` | +| claude-sonnet-4 | `completion('claude-sonnet-4-20250514', messages)` | `os.environ['ANTHROPIC_API_KEY']` | +| claude-3.7 | `completion('claude-3-7-sonnet-20250219', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-3-5-sonnet | `completion('claude-3-5-sonnet-20240620', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-3-haiku | `completion('claude-3-haiku-20240307', messages)` | `os.environ['ANTHROPIC_API_KEY']` | | claude-3-opus | `completion('claude-3-opus-20240229', messages)` | `os.environ['ANTHROPIC_API_KEY']` | From 0be7e7d0888a0b7ee5e10d1dbc8dfc0fda992155 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 14:11:19 -0700 Subject: [PATCH 18/36] Revert "Support passing `prompt_label` to langfuse (#11018)" This reverts commit 2b50b43ae2b7790a85a2f37daaf69ac1725749df. --- .../anthropic_cache_control_hook.py | 9 ++++---- litellm/integrations/custom_logger.py | 2 -- .../integrations/custom_prompt_management.py | 2 -- litellm/integrations/humanloop.py | 7 ++++-- .../langfuse/langfuse_prompt_management.py | 20 +++++++---------- .../integrations/prompt_management_base.py | 5 ----- .../vector_stores/bedrock_vector_store.py | 22 +++++++++---------- litellm/litellm_core_utils/litellm_logging.py | 4 ---- litellm/main.py | 11 ++-------- litellm/proxy/_new_secret_config.yaml | 18 ++++----------- litellm/proxy/auth/auth_checks.py | 17 +++++--------- litellm/proxy/custom_prompt_management.py | 1 - litellm/router.py | 15 +++---------- litellm/types/utils.py | 1 - litellm/utils.py | 8 ------- .../test_custom_prompt_management.py | 1 - .../completion.json | 4 +++- .../completion_with_complex_metadata.json | 4 +++- .../completion_with_langfuse_metadata.json | 4 +++- .../completion_with_no_choices.json | 4 +++- .../completion_with_tags.json | 4 +++- .../completion_with_tags_stream.json | 4 +++- .../complex_metadata.json | 4 +++- .../complex_metadata_2.json | 4 +++- .../empty_metadata.json | 4 +++- .../metadata_with_function.json | 4 +++- .../metadata_with_lock.json | 4 +++- .../nested_metadata.json | 4 +++- .../simple_metadata.json | 4 +++- .../simple_metadata2.json | 4 +++- .../simple_metadata3.json | 4 +++- 31 files changed, 87 insertions(+), 116 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 5c75e452ab7..c138b3cc254 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -28,7 +28,6 @@ class AnthropicCacheControlHook(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Apply cache control directives based on specified injection points. @@ -80,10 +79,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): # Case 1: Target by specific index if targetted_index is not None: if 0 <= targetted_index < len(messages): - messages[ - targetted_index - ] = AnthropicCacheControlHook._safe_insert_cache_control_in_message( - messages[targetted_index], control + messages[targetted_index] = ( + AnthropicCacheControlHook._safe_insert_cache_control_in_message( + messages[targetted_index], control + ) ) # Case 2: Target by role elif targetted_role is not None: diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index ce97b9a292d..960dc715e7e 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -87,7 +87,6 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, - prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: @@ -105,7 +104,6 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: diff --git a/litellm/integrations/custom_prompt_management.py b/litellm/integrations/custom_prompt_management.py index 061aadc3c05..9d05e7b2426 100644 --- a/litellm/integrations/custom_prompt_management.py +++ b/litellm/integrations/custom_prompt_management.py @@ -18,7 +18,6 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: @@ -44,7 +43,6 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, ) -> PromptManagementClient: raise NotImplementedError( "Custom prompt management does not support compile prompt helper" diff --git a/litellm/integrations/humanloop.py b/litellm/integrations/humanloop.py index c62ab1110ff..853fbe148cc 100644 --- a/litellm/integrations/humanloop.py +++ b/litellm/integrations/humanloop.py @@ -155,8 +155,11 @@ class HumanloopLogger(CustomLogger): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, - ) -> Tuple[str, List[AllMessageValues], dict,]: + ) -> Tuple[ + str, + List[AllMessageValues], + dict, + ]: humanloop_api_key = dynamic_callback_params.get( "humanloop_api_key" ) or get_secret_str("HUMANLOOP_API_KEY") diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index 8fe9cb63dea..b4149d7ad97 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -130,12 +130,9 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge return "langfuse" def _get_prompt_from_id( - self, - langfuse_prompt_id: str, - langfuse_client: LangfuseClass, - prompt_label: Optional[str] = None, + self, langfuse_prompt_id: str, langfuse_client: LangfuseClass ) -> PROMPT_CLIENT: - return langfuse_client.get_prompt(langfuse_prompt_id, label=prompt_label) + return langfuse_client.get_prompt(langfuse_prompt_id) def _compile_prompt( self, @@ -179,8 +176,11 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, - prompt_label: Optional[str] = None, - ) -> Tuple[str, List[AllMessageValues], dict,]: + ) -> Tuple[ + str, + List[AllMessageValues], + dict, + ]: return self.get_chat_completion_prompt( model, messages, @@ -188,7 +188,6 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_id, prompt_variables, dynamic_callback_params, - prompt_label=prompt_label, ) def should_run_prompt_management( @@ -212,7 +211,6 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, ) -> PromptManagementClient: langfuse_client = langfuse_client_init( langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"), @@ -221,9 +219,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge langfuse_host=dynamic_callback_params.get("langfuse_host"), ) langfuse_prompt_client = self._get_prompt_from_id( - langfuse_prompt_id=prompt_id, - langfuse_client=langfuse_client, - prompt_label=prompt_label, + langfuse_prompt_id=prompt_id, langfuse_client=langfuse_client ) ## SET PROMPT diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py index c9e7adbccbd..270c34be8a6 100644 --- a/litellm/integrations/prompt_management_base.py +++ b/litellm/integrations/prompt_management_base.py @@ -33,7 +33,6 @@ class PromptManagementBase(ABC): prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, ) -> PromptManagementClient: pass @@ -50,13 +49,11 @@ class PromptManagementBase(ABC): prompt_variables: Optional[dict], client_messages: List[AllMessageValues], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, ) -> PromptManagementClient: compiled_prompt_client = self._compile_prompt_helper( prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=dynamic_callback_params, - prompt_label=prompt_label, ) try: @@ -85,7 +82,6 @@ class PromptManagementBase(ABC): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: if prompt_id is None: raise ValueError("prompt_id is required for Prompt Management Base class") @@ -99,7 +95,6 @@ class PromptManagementBase(ABC): prompt_variables=prompt_variables, client_messages=messages, dynamic_callback_params=dynamic_callback_params, - prompt_label=prompt_label, ) completed_messages = prompt_template["completed_messages"] or messages diff --git a/litellm/integrations/vector_stores/bedrock_vector_store.py b/litellm/integrations/vector_stores/bedrock_vector_store.py index 9015757000b..e0af1a66364 100644 --- a/litellm/integrations/vector_stores/bedrock_vector_store.py +++ b/litellm/integrations/vector_stores/bedrock_vector_store.py @@ -75,7 +75,6 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, - prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Retrieves the context from the Bedrock Knowledge Base and appends it to the messages. @@ -100,11 +99,10 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): f"Bedrock Knowledge Base Response: {bedrock_kb_response}" ) - ( - context_message, - context_string, - ) = self.get_chat_completion_message_from_bedrock_kb_response( - bedrock_kb_response + context_message, context_string = ( + self.get_chat_completion_message_from_bedrock_kb_response( + bedrock_kb_response + ) ) if context_message is not None: messages.append(context_message) @@ -128,9 +126,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): ) ) - litellm_logging_obj.model_call_details[ - "vector_store_request_metadata" - ] = vector_store_request_metadata + litellm_logging_obj.model_call_details["vector_store_request_metadata"] = ( + vector_store_request_metadata + ) return model, messages, non_default_params @@ -142,9 +140,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): """ Transform a BedrockKBResponse to a VectorStoreSearchResponse """ - retrieval_results: Optional[ - List[BedrockKBRetrievalResult] - ] = bedrock_kb_response.get("retrievalResults", None) + retrieval_results: Optional[List[BedrockKBRetrievalResult]] = ( + bedrock_kb_response.get("retrievalResults", None) + ) vector_store_search_response: VectorStoreSearchResponse = ( VectorStoreSearchResponse(search_query=query, data=[]) ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index dc5cffa2290..88ce34245a6 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -539,7 +539,6 @@ class Logging(LiteLLMLoggingBaseClass): prompt_id: Optional[str], prompt_variables: Optional[dict], prompt_management_logger: Optional[CustomLogger] = None, - prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: custom_logger = ( prompt_management_logger @@ -560,7 +559,6 @@ class Logging(LiteLLMLoggingBaseClass): prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=self.standard_callback_dynamic_params, - prompt_label=prompt_label, ) self.messages = messages return model, messages, non_default_params @@ -574,7 +572,6 @@ class Logging(LiteLLMLoggingBaseClass): prompt_variables: Optional[dict], prompt_management_logger: Optional[CustomLogger] = None, tools: Optional[List[Dict]] = None, - prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: custom_logger = ( prompt_management_logger @@ -597,7 +594,6 @@ class Logging(LiteLLMLoggingBaseClass): dynamic_callback_params=self.standard_callback_dynamic_params, litellm_logging_obj=self, tools=tools, - prompt_label=prompt_label, ) self.messages = messages return model, messages, non_default_params diff --git a/litellm/main.py b/litellm/main.py index 1c1f4879cc8..7cae5acd97b 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -97,7 +97,6 @@ from litellm.utils import ( get_optional_params_image_gen, get_optional_params_transcription, get_secret, - get_standard_openai_params, mock_completion_streaming_obj, read_config_args, supports_httpx_timeout, @@ -429,7 +428,6 @@ async def acompletion( prompt_id=kwargs.get("prompt_id", None), prompt_variables=kwargs.get("prompt_variables", None), tools=tools, - prompt_label=kwargs.get("prompt_label", None), ) ######################################################### @@ -985,7 +983,6 @@ def completion( # type: ignore # noqa: PLR0915 assistant_continue_message=assistant_continue_message, ) ######## end of unpacking kwargs ########### - standard_openai_params = get_standard_openai_params(params=args) non_default_params = get_non_default_completion_params(kwargs=kwargs) litellm_params = {} # used to prevent unbound var errors ## PROMPT MANAGEMENT HOOKS ## @@ -1004,7 +1001,6 @@ def completion( # type: ignore # noqa: PLR0915 non_default_params=non_default_params, prompt_id=prompt_id, prompt_variables=prompt_variables, - prompt_label=kwargs.get("prompt_label", None), ) try: @@ -1238,13 +1234,10 @@ def completion( # type: ignore # noqa: PLR0915 max_retries=max_retries, timeout=timeout, ) - cast(LiteLLMLoggingObj, logging).update_environment_variables( + logging.update_environment_variables( model=model, user=user, - optional_params={ - **standard_openai_params, - **non_default_params, - }, # [IMPORTANT] - using standard_openai_params ensures consistent params logged to langfuse for finetuning / eval datasets. + optional_params=optional_params, litellm_params=litellm_params, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index a67ce254685..78880ba55cf 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,8 +1,8 @@ model_list: - - model_name: "gemini-2.0-flash-gemini" + - model_name: "gemini-2.0-flash" litellm_params: - model: gemini/gemini-2.0-flash - - model_name: "gpt-4o-mini-openai" + model: gemini/gemini-2.0-flash-live-001 + - model_name: "gpt-4.1-openai" litellm_params: model: gpt-4.1-mini-2025-04-14 api_key: os.environ/OPENAI_API_KEY @@ -71,16 +71,6 @@ model_list: model: mistral/* api_key: os.environ/MISTRAL_API_KEY access_groups: ["beta-models"] - - model_name: my-langfuse-model - litellm_params: - model: langfuse/gpt-3.5-turbo - prompt_id: "jokes" - prompt_label: "latest" - api_key: os.environ/OPENAI_API_KEY litellm_settings: - callbacks: ["langfuse"] - -general_settings: - store_model_in_db: true - store_prompts_in_spend_logs: true + cache: true \ No newline at end of file diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 1ac694f9475..3c759e839ec 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -670,20 +670,15 @@ class UserObjectCache: - update user object in cache """ if isinstance(user_object, LiteLLM_UserTable): - user_object_dict = user_object.model_dump() - else: - user_object_dict = user_object - - for k, v in user_object_dict.items(): - if isinstance(v, datetime): - user_object_dict[k] = v.isoformat() - await self.user_api_key_cache.async_set_cache( - key=user_id, value=user_object_dict - ) + user_object = user_object.model_dump() + for k, v in user_object.items(): + if isinstance(v, datetime): + user_object[k] = v.isoformat() + await self.user_api_key_cache.async_set_cache(key=user_id, value=user_object) if self.internal_usage_cache is not None: await self.internal_usage_cache.async_set_cache( key=user_id, - value=user_object_dict, + value=user_object, litellm_parent_otel_span=litellm_parent_otel_span, ) diff --git a/litellm/proxy/custom_prompt_management.py b/litellm/proxy/custom_prompt_management.py index 8cf20da5e92..fc16f4a4903 100644 --- a/litellm/proxy/custom_prompt_management.py +++ b/litellm/proxy/custom_prompt_management.py @@ -15,7 +15,6 @@ class X42PromptManagement(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: diff --git a/litellm/router.py b/litellm/router.py index f5fa1886024..4b562d669a6 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1700,13 +1700,9 @@ class Router: specific_deployment=kwargs.pop("specific_deployment", None), ) - self._update_kwargs_with_deployment( - deployment=prompt_management_deployment, kwargs=kwargs + litellm_model = prompt_management_deployment["litellm_params"].get( + "model", None ) - data = prompt_management_deployment["litellm_params"].copy() - - litellm_model = data.get("model", None) - prompt_id = kwargs.get("prompt_id") or prompt_management_deployment[ "litellm_params" ].get("prompt_id", None) @@ -1715,9 +1711,6 @@ class Router: ) or prompt_management_deployment["litellm_params"].get( "prompt_variables", None ) - prompt_label = kwargs.get("prompt_label", None) or prompt_management_deployment[ - "litellm_params" - ].get("prompt_label", None) if prompt_id is None or not isinstance(prompt_id, str): raise ValueError( @@ -1738,16 +1731,14 @@ class Router: non_default_params=get_non_default_completion_params(kwargs=kwargs), prompt_id=prompt_id, prompt_variables=prompt_variables, - prompt_label=prompt_label, ) - kwargs = {**data, **kwargs, **optional_params} + kwargs = {**kwargs, **optional_params} kwargs["model"] = model kwargs["messages"] = messages kwargs["litellm_logging_obj"] = litellm_logging_object kwargs["prompt_id"] = prompt_id kwargs["prompt_variables"] = prompt_variables - kwargs["prompt_label"] = prompt_label _model_list = self.get_model_list(model_name=model) if _model_list is None or len(_model_list) == 0: # if direct call to model diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a9acce9a797..9bde73b786c 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2117,7 +2117,6 @@ all_litellm_params = [ "allowed_openai_params", "litellm_session_id", "use_litellm_proxy", - "prompt_label", ] + list(StandardCallbackDynamicParams.__annotations__.keys()) diff --git a/litellm/utils.py b/litellm/utils.py index 65d825c979c..773196077d1 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6835,14 +6835,6 @@ def _add_path_to_api_base(api_base: str, ending_path: str) -> str: return str(modified_url.copy_with(params=original_url.params)) -def get_standard_openai_params(params: dict) -> dict: - return { - k: v - for k, v in params.items() - if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS and v is not None - } - - def get_non_default_completion_params(kwargs: dict) -> dict: openai_params = litellm.OPENAI_CHAT_COMPLETION_PARAMS default_params = openai_params + all_litellm_params diff --git a/tests/litellm/integrations/test_custom_prompt_management.py b/tests/litellm/integrations/test_custom_prompt_management.py index f5855abf71e..09ba32b2033 100644 --- a/tests/litellm/integrations/test_custom_prompt_management.py +++ b/tests/litellm/integrations/test_custom_prompt_management.py @@ -33,7 +33,6 @@ class TestCustomPromptManagement(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - prompt_label: Optional[str], ) -> Tuple[str, List[AllMessageValues], dict]: print( "TestCustomPromptManagement: running get_chat_completion_prompt for prompt_id: ", diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json index b2a2c83b51a..4dfe9630ff9 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json @@ -62,7 +62,9 @@ "endTime": "2025-01-16T11:28:55.124353-08:00", "completionStartTime": "2025-01-16T11:28:55.124353-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json index 9d30a82b8d2..4c5f345eaa5 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json @@ -103,7 +103,9 @@ "endTime": "2025-01-22T09:27:51.702048-08:00", "completionStartTime": "2025-01-22T09:27:51.702048-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json index 7c2fc6c5f35..d4882c962d8 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json @@ -81,7 +81,9 @@ "endTime": "2025-01-22T09:19:11.234200-08:00", "completionStartTime": "2025-01-22T09:19:11.234200-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json index cb9f007c2d5..0683ff9ba9f 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json @@ -52,7 +52,9 @@ "endTime": "2025-02-06T16:23:27.644253-08:00", "completionStartTime": "2025-02-06T16:23:27.644253-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 10, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json index c4cbe1e68af..3a87c0ad739 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json @@ -71,7 +71,9 @@ "endTime": "2025-01-22T07:31:28.962389-08:00", "completionStartTime": "2025-01-22T07:31:28.962389-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json index cd882af614d..6495ed947d6 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json @@ -71,7 +71,9 @@ "endTime": "2025-01-22T08:38:26.015666-08:00", "completionStartTime": "2025-01-22T08:38:26.015666-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json index 5c8d5c5b88d..01dcd264883 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json @@ -78,7 +78,9 @@ "endTime": "2025-01-22T09:59:39.365756-08:00", "completionStartTime": "2025-01-22T09:59:39.365756-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json index 4533262ef42..1b7b91930e9 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json @@ -70,7 +70,9 @@ "endTime": "2025-01-22T10:06:50.958374-08:00", "completionStartTime": "2025-01-22T10:06:50.958374-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json index 39a88320bbf..8c1711ee98e 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json @@ -64,7 +64,9 @@ "endTime": "2025-01-22T09:59:32.880691-08:00", "completionStartTime": "2025-01-22T09:59:32.880691-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json index e73ef0d9ed6..0b1309425e3 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json @@ -64,7 +64,9 @@ "endTime": "2025-01-22T09:59:36.161959-08:00", "completionStartTime": "2025-01-22T09:59:36.161959-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json index 39a88320bbf..8c1711ee98e 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json @@ -64,7 +64,9 @@ "endTime": "2025-01-22T09:59:32.880691-08:00", "completionStartTime": "2025-01-22T09:59:32.880691-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json index efd3bbae323..bb24688aa5c 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json @@ -70,7 +70,9 @@ "endTime": "2025-01-22T09:55:28.853979-08:00", "completionStartTime": "2025-01-22T09:55:28.853979-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json index 8cb1cced89d..d40ec6bafca 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json @@ -70,7 +70,9 @@ "endTime": "2025-01-22T09:53:53.753431-08:00", "completionStartTime": "2025-01-22T09:53:53.753431-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json index 0de688644b9..610bc461a13 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json @@ -74,7 +74,9 @@ "endTime": "2025-01-22T09:56:35.476236-08:00", "completionStartTime": "2025-01-22T09:56:35.476236-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json index f0ad3e9e712..d21c58fdee4 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json @@ -78,7 +78,9 @@ "endTime": "2025-01-22T09:56:38.785762-08:00", "completionStartTime": "2025-01-22T09:56:38.785762-08:00", "model": "gpt-3.5-turbo", - "modelParameters": {}, + "modelParameters": { + "extra_body": "{}" + }, "usage": { "input": 10, "output": 20, From 754a94db97ec256f9d999097abf4eb0ebea0f7cd Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 14:14:39 -0700 Subject: [PATCH 19/36] Revert "Revert "Support passing `prompt_label` to langfuse (#11018)"" This reverts commit 0be7e7d0888a0b7ee5e10d1dbc8dfc0fda992155. --- .../anthropic_cache_control_hook.py | 9 ++++---- litellm/integrations/custom_logger.py | 2 ++ .../integrations/custom_prompt_management.py | 2 ++ litellm/integrations/humanloop.py | 7 ++---- .../langfuse/langfuse_prompt_management.py | 20 ++++++++++------- .../integrations/prompt_management_base.py | 5 +++++ .../vector_stores/bedrock_vector_store.py | 22 ++++++++++--------- litellm/litellm_core_utils/litellm_logging.py | 4 ++++ litellm/main.py | 11 ++++++++-- litellm/proxy/_new_secret_config.yaml | 18 +++++++++++---- litellm/proxy/auth/auth_checks.py | 17 +++++++++----- litellm/proxy/custom_prompt_management.py | 1 + litellm/router.py | 15 ++++++++++--- litellm/types/utils.py | 1 + litellm/utils.py | 8 +++++++ .../test_custom_prompt_management.py | 1 + .../completion.json | 4 +--- .../completion_with_complex_metadata.json | 4 +--- .../completion_with_langfuse_metadata.json | 4 +--- .../completion_with_no_choices.json | 4 +--- .../completion_with_tags.json | 4 +--- .../completion_with_tags_stream.json | 4 +--- .../complex_metadata.json | 4 +--- .../complex_metadata_2.json | 4 +--- .../empty_metadata.json | 4 +--- .../metadata_with_function.json | 4 +--- .../metadata_with_lock.json | 4 +--- .../nested_metadata.json | 4 +--- .../simple_metadata.json | 4 +--- .../simple_metadata2.json | 4 +--- .../simple_metadata3.json | 4 +--- 31 files changed, 116 insertions(+), 87 deletions(-) diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index c138b3cc254..5c75e452ab7 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -28,6 +28,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Apply cache control directives based on specified injection points. @@ -79,10 +80,10 @@ class AnthropicCacheControlHook(CustomPromptManagement): # Case 1: Target by specific index if targetted_index is not None: if 0 <= targetted_index < len(messages): - messages[targetted_index] = ( - AnthropicCacheControlHook._safe_insert_cache_control_in_message( - messages[targetted_index], control - ) + messages[ + targetted_index + ] = AnthropicCacheControlHook._safe_insert_cache_control_in_message( + messages[targetted_index], control ) # Case 2: Target by role elif targetted_role is not None: diff --git a/litellm/integrations/custom_logger.py b/litellm/integrations/custom_logger.py index 960dc715e7e..ce97b9a292d 100644 --- a/litellm/integrations/custom_logger.py +++ b/litellm/integrations/custom_logger.py @@ -87,6 +87,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: @@ -104,6 +105,7 @@ class CustomLogger: # https://docs.litellm.ai/docs/observability/custom_callbac prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: diff --git a/litellm/integrations/custom_prompt_management.py b/litellm/integrations/custom_prompt_management.py index 9d05e7b2426..061aadc3c05 100644 --- a/litellm/integrations/custom_prompt_management.py +++ b/litellm/integrations/custom_prompt_management.py @@ -18,6 +18,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: @@ -43,6 +44,7 @@ class CustomPromptManagement(CustomLogger, PromptManagementBase): prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: raise NotImplementedError( "Custom prompt management does not support compile prompt helper" diff --git a/litellm/integrations/humanloop.py b/litellm/integrations/humanloop.py index 853fbe148cc..c62ab1110ff 100644 --- a/litellm/integrations/humanloop.py +++ b/litellm/integrations/humanloop.py @@ -155,11 +155,8 @@ class HumanloopLogger(CustomLogger): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, - ) -> Tuple[ - str, - List[AllMessageValues], - dict, - ]: + prompt_label: Optional[str] = None, + ) -> Tuple[str, List[AllMessageValues], dict,]: humanloop_api_key = dynamic_callback_params.get( "humanloop_api_key" ) or get_secret_str("HUMANLOOP_API_KEY") diff --git a/litellm/integrations/langfuse/langfuse_prompt_management.py b/litellm/integrations/langfuse/langfuse_prompt_management.py index b4149d7ad97..8fe9cb63dea 100644 --- a/litellm/integrations/langfuse/langfuse_prompt_management.py +++ b/litellm/integrations/langfuse/langfuse_prompt_management.py @@ -130,9 +130,12 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge return "langfuse" def _get_prompt_from_id( - self, langfuse_prompt_id: str, langfuse_client: LangfuseClass + self, + langfuse_prompt_id: str, + langfuse_client: LangfuseClass, + prompt_label: Optional[str] = None, ) -> PROMPT_CLIENT: - return langfuse_client.get_prompt(langfuse_prompt_id) + return langfuse_client.get_prompt(langfuse_prompt_id, label=prompt_label) def _compile_prompt( self, @@ -176,11 +179,8 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, - ) -> Tuple[ - str, - List[AllMessageValues], - dict, - ]: + prompt_label: Optional[str] = None, + ) -> Tuple[str, List[AllMessageValues], dict,]: return self.get_chat_completion_prompt( model, messages, @@ -188,6 +188,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_id, prompt_variables, dynamic_callback_params, + prompt_label=prompt_label, ) def should_run_prompt_management( @@ -211,6 +212,7 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: langfuse_client = langfuse_client_init( langfuse_public_key=dynamic_callback_params.get("langfuse_public_key"), @@ -219,7 +221,9 @@ class LangfusePromptManagement(LangFuseLogger, PromptManagementBase, CustomLogge langfuse_host=dynamic_callback_params.get("langfuse_host"), ) langfuse_prompt_client = self._get_prompt_from_id( - langfuse_prompt_id=prompt_id, langfuse_client=langfuse_client + langfuse_prompt_id=prompt_id, + langfuse_client=langfuse_client, + prompt_label=prompt_label, ) ## SET PROMPT diff --git a/litellm/integrations/prompt_management_base.py b/litellm/integrations/prompt_management_base.py index 270c34be8a6..c9e7adbccbd 100644 --- a/litellm/integrations/prompt_management_base.py +++ b/litellm/integrations/prompt_management_base.py @@ -33,6 +33,7 @@ class PromptManagementBase(ABC): prompt_id: str, prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: pass @@ -49,11 +50,13 @@ class PromptManagementBase(ABC): prompt_variables: Optional[dict], client_messages: List[AllMessageValues], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> PromptManagementClient: compiled_prompt_client = self._compile_prompt_helper( prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, ) try: @@ -82,6 +85,7 @@ class PromptManagementBase(ABC): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: if prompt_id is None: raise ValueError("prompt_id is required for Prompt Management Base class") @@ -95,6 +99,7 @@ class PromptManagementBase(ABC): prompt_variables=prompt_variables, client_messages=messages, dynamic_callback_params=dynamic_callback_params, + prompt_label=prompt_label, ) completed_messages = prompt_template["completed_messages"] or messages diff --git a/litellm/integrations/vector_stores/bedrock_vector_store.py b/litellm/integrations/vector_stores/bedrock_vector_store.py index e0af1a66364..9015757000b 100644 --- a/litellm/integrations/vector_stores/bedrock_vector_store.py +++ b/litellm/integrations/vector_stores/bedrock_vector_store.py @@ -75,6 +75,7 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): dynamic_callback_params: StandardCallbackDynamicParams, litellm_logging_obj: LiteLLMLoggingObj, tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Retrieves the context from the Bedrock Knowledge Base and appends it to the messages. @@ -99,10 +100,11 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): f"Bedrock Knowledge Base Response: {bedrock_kb_response}" ) - context_message, context_string = ( - self.get_chat_completion_message_from_bedrock_kb_response( - bedrock_kb_response - ) + ( + context_message, + context_string, + ) = self.get_chat_completion_message_from_bedrock_kb_response( + bedrock_kb_response ) if context_message is not None: messages.append(context_message) @@ -126,9 +128,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): ) ) - litellm_logging_obj.model_call_details["vector_store_request_metadata"] = ( - vector_store_request_metadata - ) + litellm_logging_obj.model_call_details[ + "vector_store_request_metadata" + ] = vector_store_request_metadata return model, messages, non_default_params @@ -140,9 +142,9 @@ class BedrockVectorStore(BaseVectorStore, BaseAWSLLM): """ Transform a BedrockKBResponse to a VectorStoreSearchResponse """ - retrieval_results: Optional[List[BedrockKBRetrievalResult]] = ( - bedrock_kb_response.get("retrievalResults", None) - ) + retrieval_results: Optional[ + List[BedrockKBRetrievalResult] + ] = bedrock_kb_response.get("retrievalResults", None) vector_store_search_response: VectorStoreSearchResponse = ( VectorStoreSearchResponse(search_query=query, data=[]) ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 88ce34245a6..dc5cffa2290 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -539,6 +539,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_id: Optional[str], prompt_variables: Optional[dict], prompt_management_logger: Optional[CustomLogger] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: custom_logger = ( prompt_management_logger @@ -559,6 +560,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_id=prompt_id, prompt_variables=prompt_variables, dynamic_callback_params=self.standard_callback_dynamic_params, + prompt_label=prompt_label, ) self.messages = messages return model, messages, non_default_params @@ -572,6 +574,7 @@ class Logging(LiteLLMLoggingBaseClass): prompt_variables: Optional[dict], prompt_management_logger: Optional[CustomLogger] = None, tools: Optional[List[Dict]] = None, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: custom_logger = ( prompt_management_logger @@ -594,6 +597,7 @@ class Logging(LiteLLMLoggingBaseClass): dynamic_callback_params=self.standard_callback_dynamic_params, litellm_logging_obj=self, tools=tools, + prompt_label=prompt_label, ) self.messages = messages return model, messages, non_default_params diff --git a/litellm/main.py b/litellm/main.py index 7cae5acd97b..1c1f4879cc8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -97,6 +97,7 @@ from litellm.utils import ( get_optional_params_image_gen, get_optional_params_transcription, get_secret, + get_standard_openai_params, mock_completion_streaming_obj, read_config_args, supports_httpx_timeout, @@ -428,6 +429,7 @@ async def acompletion( prompt_id=kwargs.get("prompt_id", None), prompt_variables=kwargs.get("prompt_variables", None), tools=tools, + prompt_label=kwargs.get("prompt_label", None), ) ######################################################### @@ -983,6 +985,7 @@ def completion( # type: ignore # noqa: PLR0915 assistant_continue_message=assistant_continue_message, ) ######## end of unpacking kwargs ########### + standard_openai_params = get_standard_openai_params(params=args) non_default_params = get_non_default_completion_params(kwargs=kwargs) litellm_params = {} # used to prevent unbound var errors ## PROMPT MANAGEMENT HOOKS ## @@ -1001,6 +1004,7 @@ def completion( # type: ignore # noqa: PLR0915 non_default_params=non_default_params, prompt_id=prompt_id, prompt_variables=prompt_variables, + prompt_label=kwargs.get("prompt_label", None), ) try: @@ -1234,10 +1238,13 @@ def completion( # type: ignore # noqa: PLR0915 max_retries=max_retries, timeout=timeout, ) - logging.update_environment_variables( + cast(LiteLLMLoggingObj, logging).update_environment_variables( model=model, user=user, - optional_params=optional_params, + optional_params={ + **standard_openai_params, + **non_default_params, + }, # [IMPORTANT] - using standard_openai_params ensures consistent params logged to langfuse for finetuning / eval datasets. litellm_params=litellm_params, custom_llm_provider=custom_llm_provider, ) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index 78880ba55cf..a67ce254685 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -1,8 +1,8 @@ model_list: - - model_name: "gemini-2.0-flash" + - model_name: "gemini-2.0-flash-gemini" litellm_params: - model: gemini/gemini-2.0-flash-live-001 - - model_name: "gpt-4.1-openai" + model: gemini/gemini-2.0-flash + - model_name: "gpt-4o-mini-openai" litellm_params: model: gpt-4.1-mini-2025-04-14 api_key: os.environ/OPENAI_API_KEY @@ -71,6 +71,16 @@ model_list: model: mistral/* api_key: os.environ/MISTRAL_API_KEY access_groups: ["beta-models"] + - model_name: my-langfuse-model + litellm_params: + model: langfuse/gpt-3.5-turbo + prompt_id: "jokes" + prompt_label: "latest" + api_key: os.environ/OPENAI_API_KEY litellm_settings: - cache: true \ No newline at end of file + callbacks: ["langfuse"] + +general_settings: + store_model_in_db: true + store_prompts_in_spend_logs: true diff --git a/litellm/proxy/auth/auth_checks.py b/litellm/proxy/auth/auth_checks.py index 3c759e839ec..1ac694f9475 100644 --- a/litellm/proxy/auth/auth_checks.py +++ b/litellm/proxy/auth/auth_checks.py @@ -670,15 +670,20 @@ class UserObjectCache: - update user object in cache """ if isinstance(user_object, LiteLLM_UserTable): - user_object = user_object.model_dump() - for k, v in user_object.items(): - if isinstance(v, datetime): - user_object[k] = v.isoformat() - await self.user_api_key_cache.async_set_cache(key=user_id, value=user_object) + user_object_dict = user_object.model_dump() + else: + user_object_dict = user_object + + for k, v in user_object_dict.items(): + if isinstance(v, datetime): + user_object_dict[k] = v.isoformat() + await self.user_api_key_cache.async_set_cache( + key=user_id, value=user_object_dict + ) if self.internal_usage_cache is not None: await self.internal_usage_cache.async_set_cache( key=user_id, - value=user_object, + value=user_object_dict, litellm_parent_otel_span=litellm_parent_otel_span, ) diff --git a/litellm/proxy/custom_prompt_management.py b/litellm/proxy/custom_prompt_management.py index fc16f4a4903..8cf20da5e92 100644 --- a/litellm/proxy/custom_prompt_management.py +++ b/litellm/proxy/custom_prompt_management.py @@ -15,6 +15,7 @@ class X42PromptManagement(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str] = None, ) -> Tuple[str, List[AllMessageValues], dict]: """ Returns: diff --git a/litellm/router.py b/litellm/router.py index 4b562d669a6..f5fa1886024 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1700,9 +1700,13 @@ class Router: specific_deployment=kwargs.pop("specific_deployment", None), ) - litellm_model = prompt_management_deployment["litellm_params"].get( - "model", None + self._update_kwargs_with_deployment( + deployment=prompt_management_deployment, kwargs=kwargs ) + data = prompt_management_deployment["litellm_params"].copy() + + litellm_model = data.get("model", None) + prompt_id = kwargs.get("prompt_id") or prompt_management_deployment[ "litellm_params" ].get("prompt_id", None) @@ -1711,6 +1715,9 @@ class Router: ) or prompt_management_deployment["litellm_params"].get( "prompt_variables", None ) + prompt_label = kwargs.get("prompt_label", None) or prompt_management_deployment[ + "litellm_params" + ].get("prompt_label", None) if prompt_id is None or not isinstance(prompt_id, str): raise ValueError( @@ -1731,14 +1738,16 @@ class Router: non_default_params=get_non_default_completion_params(kwargs=kwargs), prompt_id=prompt_id, prompt_variables=prompt_variables, + prompt_label=prompt_label, ) - kwargs = {**kwargs, **optional_params} + kwargs = {**data, **kwargs, **optional_params} kwargs["model"] = model kwargs["messages"] = messages kwargs["litellm_logging_obj"] = litellm_logging_object kwargs["prompt_id"] = prompt_id kwargs["prompt_variables"] = prompt_variables + kwargs["prompt_label"] = prompt_label _model_list = self.get_model_list(model_name=model) if _model_list is None or len(_model_list) == 0: # if direct call to model diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 9bde73b786c..a9acce9a797 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2117,6 +2117,7 @@ all_litellm_params = [ "allowed_openai_params", "litellm_session_id", "use_litellm_proxy", + "prompt_label", ] + list(StandardCallbackDynamicParams.__annotations__.keys()) diff --git a/litellm/utils.py b/litellm/utils.py index 773196077d1..65d825c979c 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -6835,6 +6835,14 @@ def _add_path_to_api_base(api_base: str, ending_path: str) -> str: return str(modified_url.copy_with(params=original_url.params)) +def get_standard_openai_params(params: dict) -> dict: + return { + k: v + for k, v in params.items() + if k in litellm.OPENAI_CHAT_COMPLETION_PARAMS and v is not None + } + + def get_non_default_completion_params(kwargs: dict) -> dict: openai_params = litellm.OPENAI_CHAT_COMPLETION_PARAMS default_params = openai_params + all_litellm_params diff --git a/tests/litellm/integrations/test_custom_prompt_management.py b/tests/litellm/integrations/test_custom_prompt_management.py index 09ba32b2033..f5855abf71e 100644 --- a/tests/litellm/integrations/test_custom_prompt_management.py +++ b/tests/litellm/integrations/test_custom_prompt_management.py @@ -33,6 +33,7 @@ class TestCustomPromptManagement(CustomPromptManagement): prompt_id: Optional[str], prompt_variables: Optional[dict], dynamic_callback_params: StandardCallbackDynamicParams, + prompt_label: Optional[str], ) -> Tuple[str, List[AllMessageValues], dict]: print( "TestCustomPromptManagement: running get_chat_completion_prompt for prompt_id: ", diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json index 4dfe9630ff9..b2a2c83b51a 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion.json @@ -62,9 +62,7 @@ "endTime": "2025-01-16T11:28:55.124353-08:00", "completionStartTime": "2025-01-16T11:28:55.124353-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json index 4c5f345eaa5..9d30a82b8d2 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_complex_metadata.json @@ -103,9 +103,7 @@ "endTime": "2025-01-22T09:27:51.702048-08:00", "completionStartTime": "2025-01-22T09:27:51.702048-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json index d4882c962d8..7c2fc6c5f35 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_langfuse_metadata.json @@ -81,9 +81,7 @@ "endTime": "2025-01-22T09:19:11.234200-08:00", "completionStartTime": "2025-01-22T09:19:11.234200-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json index 0683ff9ba9f..cb9f007c2d5 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_no_choices.json @@ -52,9 +52,7 @@ "endTime": "2025-02-06T16:23:27.644253-08:00", "completionStartTime": "2025-02-06T16:23:27.644253-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 10, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json index 3a87c0ad739..c4cbe1e68af 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags.json @@ -71,9 +71,7 @@ "endTime": "2025-01-22T07:31:28.962389-08:00", "completionStartTime": "2025-01-22T07:31:28.962389-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json index 6495ed947d6..cd882af614d 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/completion_with_tags_stream.json @@ -71,9 +71,7 @@ "endTime": "2025-01-22T08:38:26.015666-08:00", "completionStartTime": "2025-01-22T08:38:26.015666-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json index 01dcd264883..5c8d5c5b88d 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata.json @@ -78,9 +78,7 @@ "endTime": "2025-01-22T09:59:39.365756-08:00", "completionStartTime": "2025-01-22T09:59:39.365756-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json index 1b7b91930e9..4533262ef42 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/complex_metadata_2.json @@ -70,9 +70,7 @@ "endTime": "2025-01-22T10:06:50.958374-08:00", "completionStartTime": "2025-01-22T10:06:50.958374-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json index 8c1711ee98e..39a88320bbf 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/empty_metadata.json @@ -64,9 +64,7 @@ "endTime": "2025-01-22T09:59:32.880691-08:00", "completionStartTime": "2025-01-22T09:59:32.880691-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json index 0b1309425e3..e73ef0d9ed6 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_function.json @@ -64,9 +64,7 @@ "endTime": "2025-01-22T09:59:36.161959-08:00", "completionStartTime": "2025-01-22T09:59:36.161959-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json index 8c1711ee98e..39a88320bbf 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/metadata_with_lock.json @@ -64,9 +64,7 @@ "endTime": "2025-01-22T09:59:32.880691-08:00", "completionStartTime": "2025-01-22T09:59:32.880691-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json index bb24688aa5c..efd3bbae323 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/nested_metadata.json @@ -70,9 +70,7 @@ "endTime": "2025-01-22T09:55:28.853979-08:00", "completionStartTime": "2025-01-22T09:55:28.853979-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json index d40ec6bafca..8cb1cced89d 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata.json @@ -70,9 +70,7 @@ "endTime": "2025-01-22T09:53:53.753431-08:00", "completionStartTime": "2025-01-22T09:53:53.753431-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json index 610bc461a13..0de688644b9 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata2.json @@ -74,9 +74,7 @@ "endTime": "2025-01-22T09:56:35.476236-08:00", "completionStartTime": "2025-01-22T09:56:35.476236-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, diff --git a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json index d21c58fdee4..f0ad3e9e712 100644 --- a/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json +++ b/tests/logging_callback_tests/langfuse_expected_request_body/simple_metadata3.json @@ -78,9 +78,7 @@ "endTime": "2025-01-22T09:56:38.785762-08:00", "completionStartTime": "2025-01-22T09:56:38.785762-08:00", "model": "gpt-3.5-turbo", - "modelParameters": { - "extra_body": "{}" - }, + "modelParameters": {}, "usage": { "input": 10, "output": 20, From 889f0093e07483419fd259555e39508898b7b131 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 22 May 2025 14:46:42 -0700 Subject: [PATCH 20/36] fix: fix checking optional params from logging object for function call --- .../prompt_templates/common_utils.py | 9 +++++++++ litellm/litellm_core_utils/streaming_handler.py | 10 +++++----- .../vertex_and_google_ai_studio_gemini.py | 17 +++++++++-------- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 387c072ffd7..a99a2677e8f 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -573,3 +573,12 @@ def get_tool_call_names(tools: List[ChatCompletionToolParam]) -> List[str]: if tool_call_name: tool_call_names.append(tool_call_name) return tool_call_names + + +def is_function_call(optional_params: dict) -> bool: + """ + Checks if the optional params contain the function call + """ + if "functions" in optional_params and optional_params.get("functions"): + return True + return False diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index dcc6ea36a30..5ae1dcf9889 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -149,14 +149,14 @@ class CustomStreamWrapper: ) def check_is_function_call(self, logging_obj) -> bool: + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + is_function_call, + ) + if hasattr(logging_obj, "optional_params") and isinstance( logging_obj.optional_params, dict ): - if ( - "litellm_param_is_function_call" in logging_obj.optional_params - and logging_obj.optional_params["litellm_param_is_function_call"] - is True - ): + if is_function_call(logging_obj.optional_params): return True return False diff --git a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py index cd67be3545a..902f8257248 100644 --- a/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py +++ b/litellm/llms/vertex_ai/gemini/vertex_and_google_ai_studio_gemini.py @@ -455,9 +455,6 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): and value ): optional_params["tools"] = self._map_function(value=value) - optional_params["litellm_param_is_function_call"] = ( - True if param == "functions" else False - ) elif param == "tool_choice" and ( isinstance(value, str) or isinstance(value, dict) ): @@ -880,8 +877,14 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): else: return "stop" - def _process_candidates(self, _candidates, model_response, litellm_params): + def _process_candidates( + self, _candidates, model_response, standard_optional_params: dict + ): """Helper method to process candidates and extract metadata""" + from litellm.litellm_core_utils.prompt_templates.common_utils import ( + is_function_call, + ) + grounding_metadata: List[dict] = [] safety_ratings: List = [] citation_metadata: List = [] @@ -918,9 +921,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): functions, tools = self._transform_parts( parts=candidate["content"]["parts"], index=candidate.get("index", idx), - is_function_call=litellm_params.get( - "litellm_param_is_function_call" - ), + is_function_call=is_function_call(standard_optional_params), ) if "logprobsResult" in candidate: @@ -1019,7 +1020,7 @@ class VertexGeminiConfig(VertexAIBaseConfig, BaseConfig): safety_ratings, citation_metadata, ) = self._process_candidates( - _candidates, model_response, litellm_params + _candidates, model_response, logging_obj.optional_params ) usage = self._calculate_usage(completion_response=completion_response) From 469d3951770e0080eb7a495f539834c179cb5476 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Thu, 22 May 2025 15:02:01 -0700 Subject: [PATCH 21/36] test: update groq test - change on their end --- tests/local_testing/test_stream_chunk_builder.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/local_testing/test_stream_chunk_builder.py b/tests/local_testing/test_stream_chunk_builder.py index 14ff8278505..63907eb7d5e 100644 --- a/tests/local_testing/test_stream_chunk_builder.py +++ b/tests/local_testing/test_stream_chunk_builder.py @@ -931,9 +931,6 @@ def execute_completion(opts: dict): print("\n\n") assembly = litellm.stream_chunk_builder(partial_streaming_chunks) print(f"assembly.choices[0].message.tool_calls: {assembly.choices[0].message.tool_calls}") - assert len(assembly.choices[0].message.tool_calls) == 3, ( - assembly.choices[0].message.tool_calls[0].function.arguments[0] - ) print(assembly.choices[0].message.tool_calls) for tool_call in assembly.choices[0].message.tool_calls: json.loads(tool_call.function.arguments) # assert valid json - https://github.com/BerriAI/litellm/issues/10034 From 70f32154c5aebd9901b14ce1a985b217d553b082 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 22 May 2025 17:20:41 -0700 Subject: [PATCH 22/36] Litellm managed file updates combined (#11040) * Add LiteLLM Managed file support for `retrieve`, `list` and `cancel` finetuning jobs (#11033) * feat: initial commit adding managed file support to fine tuning endpoints * feat(fine_tuning/endpoints.py): working call to openai finetuning route Uses litellm managed files for finetuning api support * feat(fine-tuning/main.py): refactor to use LiteLLMFineTuningJob pydantic object includes 'hidden_params' * fix: initial commit adding unified finetuning id support return a unified finetuning id we can use to understand which deployment to route the ft request to * test: fix test * feat(managed_files.py): return unified finetuning job id on create finetuning job enables retrieve, delete to work with litellm managed files * feat(managed_files.py): support managed files for cancel ft job endpoint * feat(managed_files.py): support managed files for cancel ft job endpoint * feat(fine_tuning_endpoints/endpoints.py): add managed files support to list finetuning jobs * feat(finetuning_endpoints/main): add managed files support for retrieving ft job Makes it easier to control permissions for ft endpoint * LiteLLM Managed Files - Enforce validation check if user can access finetuning job (#11034) * feat: initial commit adding managed file support to fine tuning endpoints * feat(fine_tuning/endpoints.py): working call to openai finetuning route Uses litellm managed files for finetuning api support * feat(fine-tuning/main.py): refactor to use LiteLLMFineTuningJob pydantic object includes 'hidden_params' * fix: initial commit adding unified finetuning id support return a unified finetuning id we can use to understand which deployment to route the ft request to * test: fix test * feat(managed_files.py): return unified finetuning job id on create finetuning job enables retrieve, delete to work with litellm managed files * feat(managed_files.py): support managed files for cancel ft job endpoint * feat(managed_files.py): support managed files for cancel ft job endpoint * feat(fine_tuning_endpoints/endpoints.py): add managed files support to list finetuning jobs * feat(finetuning_endpoints/main): add managed files support for retrieving ft job Makes it easier to control permissions for ft endpoint * feat(managed_files.py): store create fine-tune / batch response object in db storing this allows us to filter files returned on list based on what user created * feat(managed_files.py): Ensures users can't retrieve / modify each others jobs * fix: fix check * fix: fix ruff check errors * test: update to handle testing * fix: suppress linting warning - openai 'seed' is none on azure * test: update tests * test: update test --- enterprise/enterprise_hooks/__init__.py | 1 - enterprise/enterprise_hooks/managed_files.py | 154 +++++++++-- litellm/fine_tuning/main.py | 18 +- litellm/llms/openai/fine_tuning/handler.py | 21 +- litellm/proxy/_new_secret_config.yaml | 4 +- litellm/proxy/_types.py | 9 + litellm/proxy/common_request_processing.py | 3 + .../proxy/fine_tuning_endpoints/endpoints.py | 251 ++++++++++++++---- litellm/proxy/schema.prisma | 16 +- litellm/router.py | 16 ++ litellm/router_strategy/lowest_tpm_rpm.py | 2 + litellm/types/utils.py | 1 + tests/batches_tests/test_fine_tuning_api.py | 18 +- .../enterprise_hooks/test_managed_files.py | 40 ++- .../llms/azure/test_azure_common_utils.py | 3 + 15 files changed, 444 insertions(+), 113 deletions(-) diff --git a/enterprise/enterprise_hooks/__init__.py b/enterprise/enterprise_hooks/__init__.py index 830d97886a6..9cfe9218f00 100644 --- a/enterprise/enterprise_hooks/__init__.py +++ b/enterprise/enterprise_hooks/__init__.py @@ -1,4 +1,3 @@ -import os from typing import Dict, Literal, Type, Union from litellm.integrations.custom_logger import CustomLogger diff --git a/enterprise/enterprise_hooks/managed_files.py b/enterprise/enterprise_hooks/managed_files.py index 78e1cdfd98b..480ead78386 100644 --- a/enterprise/enterprise_hooks/managed_files.py +++ b/enterprise/enterprise_hooks/managed_files.py @@ -1,17 +1,25 @@ # What is this? ## This hook is used to check for LiteLLM managed files in the request body, and replace them with model-specific file id +import asyncio import base64 import json import uuid from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast +from fastapi import HTTPException + from litellm import Router, verbose_logger from litellm.caching.caching import DualCache from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.prompt_templates.common_utils import extract_file_data from litellm.llms.base_llm.files.transformation import BaseFileEndpoints -from litellm.proxy._types import CallTypes, LiteLLM_ManagedFileTable, UserAPIKeyAuth +from litellm.proxy._types import ( + CallTypes, + LiteLLM_ManagedFileTable, + LiteLLM_ManagedObjectTable, + UserAPIKeyAuth, +) from litellm.proxy.openai_files_endpoints.common_utils import ( _is_base64_encoded_unified_file_id, convert_b64_uid_to_unified_uid, @@ -82,6 +90,41 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): } ) + async def store_unified_object_id( + self, + unified_object_id: str, + file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob], + litellm_parent_otel_span: Optional[Span], + model_object_id: str, + file_purpose: Literal["batch", "fine-tune"], + user_api_key_dict: UserAPIKeyAuth, + ) -> None: + verbose_logger.info( + f"Storing LiteLLM Managed {file_purpose} object with id={unified_object_id} in cache" + ) + litellm_managed_object = LiteLLM_ManagedObjectTable( + unified_object_id=unified_object_id, + model_object_id=model_object_id, + file_purpose=file_purpose, + file_object=file_object, + ) + await self.internal_usage_cache.async_set_cache( + key=unified_object_id, + value=litellm_managed_object.model_dump(), + litellm_parent_otel_span=litellm_parent_otel_span, + ) + + await self.prisma_client.db.litellm_managedobjecttable.create( + data={ + "unified_object_id": unified_object_id, + "file_object": file_object.model_dump_json(), + "model_object_id": model_object_id, + "file_purpose": file_purpose, + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, + } + ) + async def get_unified_file_id( self, file_id: str, litellm_parent_otel_span: Optional[Span] = None ) -> Optional[LiteLLM_ManagedFileTable]: @@ -126,6 +169,21 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) return initial_value.file_object + async def can_user_call_unified_object_id( + self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + ## check if the user has access to the unified object id + ## check if the user has access to the unified object id + user_id = user_api_key_dict.user_id + managed_object = ( + await self.prisma_client.db.litellm_managedobjecttable.find_first( + where={"unified_object_id": unified_object_id} + ) + ) + if managed_object: + return managed_object.created_by == user_id + return False + async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -144,6 +202,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "aretrieve_batch", "afile_content", "acreate_fine_tuning_job", + "aretrieve_fine_tuning_job", + "alist_fine_tuning_jobs", + "acancel_fine_tuning_job", ], ) -> Union[Exception, str, Dict, None]: """ @@ -185,25 +246,50 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) data["model_file_id_mapping"] = model_file_id_mapping - elif call_type == CallTypes.aretrieve_batch.value: - retrieve_batch_id = cast(Optional[str], data.get("batch_id")) - potential_batch_id = ( - _is_base64_encoded_unified_file_id(retrieve_batch_id) - if retrieve_batch_id + elif ( + call_type == CallTypes.aretrieve_batch.value + or call_type == CallTypes.acancel_fine_tuning_job.value + or call_type == CallTypes.aretrieve_fine_tuning_job.value + ): + accessor_key: Optional[str] = None + retrieve_object_id: Optional[str] = None + if call_type == CallTypes.aretrieve_batch.value: + accessor_key = "batch_id" + elif ( + call_type == CallTypes.acancel_fine_tuning_job.value + or call_type == CallTypes.aretrieve_fine_tuning_job.value + ): + accessor_key = "fine_tuning_job_id" + + if accessor_key: + retrieve_object_id = cast(Optional[str], data.get(accessor_key)) + + potential_llm_object_id = ( + _is_base64_encoded_unified_file_id(retrieve_object_id) + if retrieve_object_id else False ) - if potential_batch_id: + if potential_llm_object_id and retrieve_object_id: + ## VALIDATE USER HAS ACCESS TO THE OBJECT ## + if not await self.can_user_call_unified_object_id( + retrieve_object_id, user_api_key_dict + ): + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to the object {retrieve_object_id}", + ) + ## for managed batch id - get the model id potential_model_id = self.get_model_id_from_unified_batch_id( - potential_batch_id + potential_llm_object_id ) if potential_model_id is None: raise Exception( - f"LiteLLM Managed Batch ID with id={retrieve_batch_id} is invalid - does not contain encoded model_id." + f"LiteLLM Managed {accessor_key} with id={retrieve_object_id} is invalid - does not contain encoded model_id." ) data["model"] = potential_model_id - data["batch_id"] = self.get_batch_id_from_unified_batch_id( - potential_batch_id + data[accessor_key] = self.get_batch_id_from_unified_batch_id( + potential_llm_object_id ) elif call_type == CallTypes.acreate_fine_tuning_job.value: input_file_id = cast(Optional[str], data.get("training_file")) @@ -211,7 +297,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_file_id_mapping = await self.get_model_file_id_mapping( [input_file_id], user_api_key_dict.parent_otel_span ) - data["model_file_id_mapping"] = model_file_id_mapping print("DATA={}".format(data)) return data @@ -222,11 +307,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): """ Allow modifying the request just before it's sent to the deployment. """ - print( - "CALLS ASYNC PRE CALL DEPLOYMENT HOOK - KWARGS={}, CALL_TYPE={}".format( - kwargs, call_type - ) - ) accessor_key: Optional[str] = None if call_type and call_type == CallTypes.acreate_batch: accessor_key = "input_file_id" @@ -472,7 +552,10 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): def get_batch_id_from_unified_batch_id(self, file_id: str) -> str: ## use regex to get the batch_id from the file_id - return file_id.split("llm_batch_id:")[1].split(",")[0] + if "llm_batch_id" in file_id: + return file_id.split("llm_batch_id:")[1].split(",")[0] + else: + return file_id.split("generic_response_id:")[1].split(",")[0] async def async_post_call_success_hook( self, data: Dict, user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes @@ -487,6 +570,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) # managed batch id model_id = cast(Optional[str], response._hidden_params.get("model_id")) model_name = cast(Optional[str], response._hidden_params.get("model_name")) + original_response_id = response.id if (unified_batch_id or unified_file_id) and model_id: response.id = self.get_unified_batch_id( batch_id=response.id, model_id=model_id @@ -500,24 +584,42 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): model_id=model_id, model_name=model_name, ) - return response + asyncio.create_task( + self.store_unified_object_id( + unified_object_id=response.id, + file_object=response, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + model_object_id=original_response_id, + file_purpose="batch", + user_api_key_dict=user_api_key_dict, + ) + ) elif isinstance(response, LiteLLMFineTuningJob): ## Check if unified_file_id is in the response - print(f"hidden params={response._hidden_params}") unified_file_id = response._hidden_params.get( "unified_file_id" ) # managed file id + unified_finetuning_job_id = response._hidden_params.get( + "unified_finetuning_job_id" + ) # managed finetuning job id model_id = cast(Optional[str], response._hidden_params.get("model_id")) - print("MODEL_ID={}".format(model_id)) model_name = cast(Optional[str], response._hidden_params.get("model_name")) - if unified_file_id and model_id: + original_response_id = response.id + if (unified_file_id or unified_finetuning_job_id) and model_id: response.id = self.get_unified_generic_response_id( model_id=model_id, generic_response_id=response.id ) - return response - return await super().async_post_call_success_hook( - data, user_api_key_dict, response - ) + asyncio.create_task( + self.store_unified_object_id( + unified_object_id=response.id, + file_object=response, + litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + model_object_id=original_response_id, + file_purpose="fine-tune", + user_api_key_dict=user_api_key_dict, + ) + ) + return response async def afile_retrieve( self, file_id: str, litellm_parent_otel_span: Optional[Span] diff --git a/litellm/fine_tuning/main.py b/litellm/fine_tuning/main.py index 55e45f75012..f5b8b097026 100644 --- a/litellm/fine_tuning/main.py +++ b/litellm/fine_tuning/main.py @@ -22,11 +22,7 @@ from litellm.llms.azure.fine_tuning.handler import AzureOpenAIFineTuningAPI from litellm.llms.openai.fine_tuning.handler import OpenAIFineTuningAPI from litellm.llms.vertex_ai.fine_tuning.handler import VertexFineTuningAPI from litellm.secret_managers.main import get_secret_str -from litellm.types.llms.openai import ( - FineTuningJob, - FineTuningJobCreate, - Hyperparameters, -) +from litellm.types.llms.openai import FineTuningJobCreate, Hyperparameters from litellm.types.router import * from litellm.types.utils import LiteLLMFineTuningJob from litellm.utils import client, supports_httpx_timeout @@ -289,13 +285,14 @@ def create_fine_tuning_job( raise e +@client async def acancel_fine_tuning_job( fine_tuning_job_id: str, custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> FineTuningJob: +) -> LiteLLMFineTuningJob: """ Async: Immediately cancel a fine-tune job. """ @@ -326,13 +323,14 @@ async def acancel_fine_tuning_job( raise e +@client def cancel_fine_tuning_job( fine_tuning_job_id: str, custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: +) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: """ Immediately cancel a fine-tune job. @@ -610,13 +608,14 @@ def list_fine_tuning_jobs( raise e +@client async def aretrieve_fine_tuning_job( fine_tuning_job_id: str, custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> FineTuningJob: +) -> LiteLLMFineTuningJob: """ Async: Get info about a fine-tuning job. """ @@ -647,13 +646,14 @@ async def aretrieve_fine_tuning_job( raise e +@client def retrieve_fine_tuning_job( fine_tuning_job_id: str, custom_llm_provider: Literal["openai", "azure", "vertex_ai"] = "openai", extra_headers: Optional[Dict[str, str]] = None, extra_body: Optional[Dict[str, str]] = None, **kwargs, -) -> Union[FineTuningJob, Coroutine[Any, Any, FineTuningJob]]: +) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: """ Get info about a fine-tuning job. """ diff --git a/litellm/llms/openai/fine_tuning/handler.py b/litellm/llms/openai/fine_tuning/handler.py index aa4b7e20319..9804ff3539e 100644 --- a/litellm/llms/openai/fine_tuning/handler.py +++ b/litellm/llms/openai/fine_tuning/handler.py @@ -2,7 +2,6 @@ from typing import Any, Coroutine, Optional, Union, cast import httpx from openai import AsyncAzureOpenAI, AsyncOpenAI, AzureOpenAI, OpenAI -from openai.types.fine_tuning import FineTuningJob from litellm._logging import verbose_logger from litellm.types.utils import LiteLLMFineTuningJob @@ -115,11 +114,11 @@ class OpenAIFineTuningAPI: self, fine_tuning_job_id: str, openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI], - ) -> FineTuningJob: + ) -> LiteLLMFineTuningJob: response = await openai_client.fine_tuning.jobs.cancel( fine_tuning_job_id=fine_tuning_job_id ) - return response + return LiteLLMFineTuningJob(**response.model_dump()) def cancel_fine_tuning_job( self, @@ -134,7 +133,7 @@ class OpenAIFineTuningAPI: client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = None, - ): + ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: openai_client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = self.get_openai_client( @@ -162,10 +161,10 @@ class OpenAIFineTuningAPI: openai_client=openai_client, ) verbose_logger.debug("canceling fine tuning job, args= %s", fine_tuning_job_id) - response = openai_client.fine_tuning.jobs.cancel( + response = cast(OpenAI, openai_client).fine_tuning.jobs.cancel( fine_tuning_job_id=fine_tuning_job_id ) - return response + return LiteLLMFineTuningJob(**response.model_dump()) async def alist_fine_tuning_jobs( self, @@ -226,11 +225,11 @@ class OpenAIFineTuningAPI: self, fine_tuning_job_id: str, openai_client: Union[AsyncOpenAI, AsyncAzureOpenAI], - ) -> FineTuningJob: + ) -> LiteLLMFineTuningJob: response = await openai_client.fine_tuning.jobs.retrieve( fine_tuning_job_id=fine_tuning_job_id ) - return response + return LiteLLMFineTuningJob(**response.model_dump()) def retrieve_fine_tuning_job( self, @@ -245,7 +244,7 @@ class OpenAIFineTuningAPI: client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = None, - ): + ) -> Union[LiteLLMFineTuningJob, Coroutine[Any, Any, LiteLLMFineTuningJob]]: openai_client: Optional[ Union[OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI] ] = self.get_openai_client( @@ -273,7 +272,7 @@ class OpenAIFineTuningAPI: openai_client=openai_client, ) verbose_logger.debug("retrieving fine tuning job, id= %s", fine_tuning_job_id) - response = openai_client.fine_tuning.jobs.retrieve( + response = cast(OpenAI, openai_client).fine_tuning.jobs.retrieve( fine_tuning_job_id=fine_tuning_job_id ) - return response + return LiteLLMFineTuningJob(**response.model_dump()) diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index a67ce254685..c995567ed13 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -5,13 +5,13 @@ model_list: - model_name: "gpt-4o-mini-openai" litellm_params: model: gpt-4.1-mini-2025-04-14 - api_key: os.environ/OPENAI_API_KEY + api_key: os.environ/OPENAI_API_KEY_2 model_info: access_groups: ["default-openai-models"] - model_name: "gpt-4o-realtime-preview" litellm_params: model: gpt-4o-realtime-preview-2024-10-01 - api_key: os.environ/OPENAI_API_KEY + api_key: os.environ/OPENAI_API_KEY_2 - model_name: "bedrock-nova" litellm_params: model: us.amazon.nova-pro-v1:0 diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 088d1dd7d4c..1dcf417623a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -23,6 +23,8 @@ from litellm.types.utils import ( EmbeddingResponse, GenericBudgetConfigType, ImageResponse, + LiteLLMBatch, + LiteLLMFineTuningJob, LiteLLMPydanticObjectBase, ModelResponse, ProviderField, @@ -2879,3 +2881,10 @@ class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase): unified_file_id: str file_object: OpenAIFileObject model_mappings: Dict[str, str] + + +class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase): + unified_object_id: str + model_object_id: str + file_purpose: Literal["batch", "fine-tune"] + file_object: Union[LiteLLMBatch, LiteLLMFineTuningJob] diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index 325e812409a..678dd2693a8 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -118,6 +118,9 @@ class ProxyBaseLLMRequestProcessing: "aretrieve_batch", "afile_content", "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", ], version: Optional[str] = None, user_model: Optional[str] = None, diff --git a/litellm/proxy/fine_tuning_endpoints/endpoints.py b/litellm/proxy/fine_tuning_endpoints/endpoints.py index 04d76646cff..be7f83c65ef 100644 --- a/litellm/proxy/fine_tuning_endpoints/endpoints.py +++ b/litellm/proxy/fine_tuning_endpoints/endpoints.py @@ -6,10 +6,9 @@ ########################################################################## import asyncio -import traceback from typing import Optional, cast -from fastapi import APIRouter, Depends, HTTPException, Request, Response +from fastapi import APIRouter, Depends, HTTPException, Query, Request, Response import litellm from litellm._logging import verbose_proxy_logger @@ -163,7 +162,6 @@ async def create_fine_tuning_job( llm_provider_config = get_fine_tuning_provider_config( custom_llm_provider=fine_tuning_request.custom_llm_provider, ) - # add llm_provider_config to data if llm_provider_config is not None: data.update(llm_provider_config) @@ -237,7 +235,7 @@ async def retrieve_fine_tuning_job( request: Request, fastapi_response: Response, fine_tuning_job_id: str, - custom_llm_provider: Literal["openai", "azure"], + custom_llm_provider: Optional[Literal["openai", "azure"]] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ @@ -249,41 +247,99 @@ async def retrieve_fine_tuning_job( - `fine_tuning_job_id`: The ID of the fine-tuning job to retrieve. """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, premium_user, proxy_config, proxy_logging_obj, version, ) - data: dict = {} + data: dict = {"fine_tuning_job_id": fine_tuning_job_id} try: if premium_user is not True: raise ValueError( f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}" ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type=CallTypes.aretrieve_fine_tuning_job.value, ) - # get configs for custom_llm_provider - llm_provider_config = get_fine_tuning_provider_config( - custom_llm_provider=custom_llm_provider + try: + request_body = await request.json() + except Exception: + request_body = {} + + custom_llm_provider = request_body.get("custom_llm_provider", None) + + ## CHECK IF MANAGED FILE ID + unified_finetuning_job_id: Union[str, Literal[False]] = False + response: Optional[LiteLLMFineTuningJob] = None + if fine_tuning_job_id: + unified_finetuning_job_id = _is_base64_encoded_unified_file_id( + fine_tuning_job_id + ) + if unified_finetuning_job_id: + if llm_router is None: + raise HTTPException( + status_code=500, + detail={ + "error": "LLM Router not initialized. Ensure models added to proxy." + }, + ) + response = cast( + LiteLLMFineTuningJob, + await llm_router.aretrieve_fine_tuning_job( + **data, + ), + ) + response._hidden_params[ + "unified_finetuning_job_id" + ] = unified_finetuning_job_id + elif custom_llm_provider: + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=custom_llm_provider + ) + + if llm_provider_config is not None: + data.update(llm_provider_config) + + response = await litellm.aretrieve_fine_tuning_job( + **data, + ) + + if response is None: + raise HTTPException( + status_code=400, + detail="Invalid request, No litellm managed file id or custom_llm_provider provided.", + ) + + ### CALL HOOKS ### - modify outgoing data + _response = await proxy_logging_obj.post_call_success_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, ) + if _response is not None and isinstance(_response, LiteLLMFineTuningJob): + response = _response - if llm_provider_config is not None: - data.update(llm_provider_config) - - response = await litellm.aretrieve_fine_tuning_job( - **data, - fine_tuning_job_id=fine_tuning_job_id, + ### ALERTING ### + asyncio.create_task( + proxy_logging_obj.update_request_status( + litellm_call_id=data.get("litellm_call_id", ""), status="success" + ) ) ### RESPONSE HEADERS ### @@ -309,12 +365,11 @@ async def retrieve_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format( + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.retrieve_fine_tuning_job(): Exception occurred - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) raise handle_exception_on_proxy(e) @@ -333,7 +388,11 @@ async def retrieve_fine_tuning_job( async def list_fine_tuning_jobs( request: Request, fastapi_response: Response, - custom_llm_provider: Literal["openai", "azure"], + custom_llm_provider: Optional[Literal["openai", "azure"]] = None, + target_model_names: Optional[str] = Query( + default=None, + description="Comma separated list of model names to filter by. Example: 'gpt-4o,gpt-4o-mini'", + ), after: Optional[str] = None, limit: Optional[int] = None, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), @@ -348,8 +407,8 @@ async def list_fine_tuning_jobs( - `limit`: Number of fine-tuning jobs to retrieve (default is 20). """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, premium_user, proxy_config, proxy_logging_obj, @@ -363,28 +422,60 @@ async def list_fine_tuning_jobs( f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}" ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type=CallTypes.alist_fine_tuning_jobs.value, ) - # get configs for custom_llm_provider - llm_provider_config = get_fine_tuning_provider_config( - custom_llm_provider=custom_llm_provider - ) + response: Optional[Any] = None + if target_model_names and isinstance(target_model_names, str): + target_model_names_list = target_model_names.split(",") + if len(target_model_names_list) != 1: + raise HTTPException( + status_code=400, + detail="target_model_names on list fine-tuning jobs must be a list of one model name. Example: ['gpt-4o']", + ) + ## Use router to list fine-tuning jobs for that model + if llm_router is None: + raise HTTPException( + status_code=500, + detail="LLM Router not initialized. Ensure models added to proxy.", + ) + data["model"] = target_model_names_list[0] + response = await llm_router.alist_fine_tuning_jobs( + **data, + after=after, + limit=limit, + ) + return response + elif custom_llm_provider: + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=custom_llm_provider + ) - if llm_provider_config is not None: - data.update(llm_provider_config) + if llm_provider_config is not None: + data.update(llm_provider_config) - response = await litellm.alist_fine_tuning_jobs( - **data, - after=after, - limit=limit, - ) + response = await litellm.alist_fine_tuning_jobs( + **data, + after=after, + limit=limit, + ) + if response is None: + raise HTTPException( + status_code=400, + detail="Invalid request, No litellm managed file id or custom_llm_provider provided.", + ) ### RESPONSE HEADERS ### hidden_params = getattr(response, "_hidden_params", {}) or {} @@ -409,12 +500,11 @@ async def list_fine_tuning_jobs( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( + verbose_proxy_logger.exception( "litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) raise handle_exception_on_proxy(e) @@ -446,45 +536,99 @@ async def cancel_fine_tuning_job( - `fine_tuning_job_id`: The ID of the fine-tuning job to cancel. """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, premium_user, proxy_config, proxy_logging_obj, version, ) - data: dict = {} + data: dict = {"fine_tuning_job_id": fine_tuning_job_id} try: if premium_user is not True: raise ValueError( f"Only premium users can use this endpoint + {CommonProxyErrors.not_premium_user.value}" ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type=CallTypes.acancel_fine_tuning_job.value, ) - request_body = await request.json() + try: + request_body = await request.json() + except Exception: + request_body = {} custom_llm_provider = request_body.get("custom_llm_provider", None) - # get configs for custom_llm_provider - llm_provider_config = get_fine_tuning_provider_config( - custom_llm_provider=custom_llm_provider + ## CHECK IF MANAGED FILE ID + unified_finetuning_job_id: Union[str, Literal[False]] = False + response: Optional[LiteLLMFineTuningJob] = None + if fine_tuning_job_id: + unified_finetuning_job_id = _is_base64_encoded_unified_file_id( + fine_tuning_job_id + ) + if unified_finetuning_job_id: + if llm_router is None: + raise HTTPException( + status_code=500, + detail={ + "error": "LLM Router not initialized. Ensure models added to proxy." + }, + ) + response = cast( + LiteLLMFineTuningJob, + await llm_router.acancel_fine_tuning_job( + **data, + ), + ) + response._hidden_params[ + "unified_finetuning_job_id" + ] = unified_finetuning_job_id + else: + # get configs for custom_llm_provider + llm_provider_config = get_fine_tuning_provider_config( + custom_llm_provider=custom_llm_provider + ) + + if llm_provider_config is not None: + data.update(llm_provider_config) + + response = await litellm.acancel_fine_tuning_job( + **data, + ) + + if response is None: + raise HTTPException( + status_code=400, + detail="Invalid request, No litellm managed file id or custom_llm_provider provided.", + ) + + ### CALL HOOKS ### - modify outgoing data + _response = await proxy_logging_obj.post_call_success_hook( + data=data, + user_api_key_dict=user_api_key_dict, + response=response, ) + if _response is not None and isinstance(_response, LiteLLMFineTuningJob): + response = _response - if llm_provider_config is not None: - data.update(llm_provider_config) - - response = await litellm.acancel_fine_tuning_job( - **data, - fine_tuning_job_id=fine_tuning_job_id, + ### ALERTING ### + asyncio.create_task( + proxy_logging_obj.update_request_status( + litellm_call_id=data.get("litellm_call_id", ""), status="success" + ) ) ### RESPONSE HEADERS ### @@ -510,10 +654,9 @@ async def cancel_fine_tuning_job( await proxy_logging_obj.post_call_failure_hook( user_api_key_dict=user_api_key_dict, original_exception=e, request_data=data ) - verbose_proxy_logger.error( - "litellm.proxy.proxy_server.list_fine_tuning_jobs(): Exception occurred - {}".format( + verbose_proxy_logger.exception( + "litellm.proxy.proxy_server.cancel_fine_tuning_job(): Exception occurred - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) raise handle_exception_on_proxy(e) diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index 1d6f3b52118..e97dc7d2ae1 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -453,13 +453,27 @@ model LiteLLM_ManagedFileTable { id String @id @default(uuid()) unified_file_id String @unique // The base64 encoded unified file ID file_object Json // Stores the OpenAIFileObject - model_mappings Json // Stores the mapping of model_id -> provider_file_id + model_mappings Json created_at DateTime @default(now()) updated_at DateTime @updatedAt @@index([unified_file_id]) } +model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the + id String @id @default(uuid()) + unified_object_id String @unique // The base64 encoded unified file ID + model_object_id String @unique // the id returned by the backend API provider + file_object Json // Stores the OpenAIFileObject + file_purpose String // either 'batch' or 'fine-tune' + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @updatedAt + updated_by String? + + @@index([unified_object_id]) + @@index([model_object_id]) +} model LiteLLM_ManagedVectorStoresTable { vector_store_id String @id diff --git a/litellm/router.py b/litellm/router.py index f5fa1886024..6556791a84c 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -755,6 +755,15 @@ class Router: self.acreate_fine_tuning_job = self.factory_function( litellm.acreate_fine_tuning_job, call_type="acreate_fine_tuning_job" ) + self.acancel_fine_tuning_job = self.factory_function( + litellm.acancel_fine_tuning_job, call_type="acancel_fine_tuning_job" + ) + self.alist_fine_tuning_jobs = self.factory_function( + litellm.alist_fine_tuning_jobs, call_type="alist_fine_tuning_jobs" + ) + self.aretrieve_fine_tuning_job = self.factory_function( + litellm.aretrieve_fine_tuning_job, call_type="aretrieve_fine_tuning_job" + ) def validate_fallbacks(self, fallback_param: Optional[List]): """ @@ -2439,6 +2448,7 @@ class Router: messages=kwargs.get("messages", None), specific_deployment=kwargs.pop("specific_deployment", None), ) + self._update_kwargs_with_deployment( deployment=deployment, kwargs=kwargs, function_name="generic_api_call" ) @@ -3172,6 +3182,9 @@ class Router: "afile_content", "_arealtime", "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", ] = "assistants", ): """ @@ -3221,6 +3234,9 @@ class Router: "aresponses", "_arealtime", "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, diff --git a/litellm/router_strategy/lowest_tpm_rpm.py b/litellm/router_strategy/lowest_tpm_rpm.py index 121df00d305..735ddb3f802 100644 --- a/litellm/router_strategy/lowest_tpm_rpm.py +++ b/litellm/router_strategy/lowest_tpm_rpm.py @@ -96,6 +96,8 @@ class LowestTPMLoggingHandler(CustomLogger): if kwargs["litellm_params"].get("metadata") is None: pass else: + if "litellm_params" not in kwargs: + return model_group = kwargs["litellm_params"]["metadata"].get( "model_group", None ) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index a9acce9a797..310a4332f08 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -2293,6 +2293,7 @@ class SelectTokenizerResponse(TypedDict): class LiteLLMFineTuningJob(FineTuningJob): _hidden_params: dict = {} + seed: Optional[int] = None # type: ignore def __init__(self, **kwargs): if "error" in kwargs and kwargs["error"] is not None: diff --git a/tests/batches_tests/test_fine_tuning_api.py b/tests/batches_tests/test_fine_tuning_api.py index e6c20e129e7..3561d99d0f6 100644 --- a/tests/batches_tests/test_fine_tuning_api.py +++ b/tests/batches_tests/test_fine_tuning_api.py @@ -525,9 +525,12 @@ async def test_mock_openai_cancel_fine_tune_job(): client = AsyncOpenAI(api_key="fake-api-key") with patch.object(client.fine_tuning.jobs, "cancel") as mock_cancel: - await litellm.acancel_fine_tuning_job( - fine_tuning_job_id="ft-123", client=client - ) + try: + await litellm.acancel_fine_tuning_job( + fine_tuning_job_id="ft-123", client=client + ) + except Exception as e: + print("error=", e) # Only verify that the client was called with correct parameters mock_cancel.assert_called_once_with(fine_tuning_job_id="ft-123") @@ -541,10 +544,13 @@ async def test_mock_openai_retrieve_fine_tune_job(): client = AsyncOpenAI(api_key="fake-api-key") with patch.object(client.fine_tuning.jobs, "retrieve") as mock_retrieve: + try: + response = await litellm.aretrieve_fine_tuning_job( + fine_tuning_job_id="ft-123", client=client + ) + except Exception as e: + print("error=", e) - response = await litellm.aretrieve_fine_tuning_job( - fine_tuning_job_id="ft-123", client=client - ) # Verify the request mock_retrieve.assert_called_once_with(fine_tuning_job_id="ft-123") diff --git a/tests/enterprise/enterprise_hooks/test_managed_files.py b/tests/enterprise/enterprise_hooks/test_managed_files.py index 19f5a7c3404..04a2717f788 100644 --- a/tests/enterprise/enterprise_hooks/test_managed_files.py +++ b/tests/enterprise/enterprise_hooks/test_managed_files.py @@ -9,7 +9,7 @@ sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path -from unittest.mock import MagicMock +from unittest.mock import AsyncMock, MagicMock from enterprise.enterprise_hooks.managed_files import _PROXY_LiteLLMManagedFiles from litellm.caching import DualCache @@ -45,11 +45,19 @@ def test_get_file_ids_from_messages(): @pytest.mark.asyncio async def test_async_pre_call_hook_batch_retrieve(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + return_value = MagicMock() + return_value.created_by = "123" + prisma_client.db.litellm_managedobjecttable.find_first.return_value = return_value proxy_managed_files = _PROXY_LiteLLMManagedFiles( - DualCache(), prisma_client=MagicMock() + DualCache(), prisma_client=prisma_client ) data = { - "user_api_key_dict": {"parent_otel_span": MagicMock()}, + "user_api_key_dict": UserAPIKeyAuth( + user_id="123", parent_otel_span=MagicMock() + ), "data": { "batch_id": "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDpteS1nZW5lcmFsLWF6dXJlLWRlcGxveW1lbnQ7bGxtX2JhdGNoX2lkOmJhdGNoX2EzMjJiNmJhLWFjN2UtNDg4OC05MjljLTFhZDM0NDJmMDZlZA", }, @@ -206,3 +214,29 @@ async def test_async_post_call_success_hook_for_unified_finetuning_job(): assert isinstance(response, LiteLLMFineTuningJob) assert _is_base64_encoded_unified_file_id(response.id) + + +@pytest.mark.asyncio +async def test_async_pre_call_hook_for_unified_finetuning_job(): + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + return_value = MagicMock() + return_value.created_by = "123" + prisma_client.db.litellm_managedobjecttable.find_first.return_value = return_value + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=prisma_client + ) + data = { + "user_api_key_dict": UserAPIKeyAuth( + user_id="123", parent_otel_span=MagicMock() + ), + "data": { + "fine_tuning_job_id": "bGl0ZWxsbV9wcm94eTttb2RlbF9pZDo0OTIxODU4MWY3OGViZTllZjE4NDE0ZmE0ZjdmYjlmYTc0YzA5NWVkMTEyY2E4NDBkZDU2ZGZmZTliZDMwZGQxO2dlbmVyaWNfcmVzcG9uc2VfaWQ6ZnRqb2ItalRCeXM3YlZzYnlaRE93TDlHbHBZcVhS", + }, + "call_type": "acancel_fine_tuning_job", + "cache": MagicMock(), + } + + response = await proxy_managed_files.async_pre_call_hook(**data) + assert response["fine_tuning_job_id"] == "ftjob-jTBys7bVsbyZDOwL9GlpYqXR" diff --git a/tests/litellm/llms/azure/test_azure_common_utils.py b/tests/litellm/llms/azure/test_azure_common_utils.py index 54916e4daba..03d9d252198 100644 --- a/tests/litellm/llms/azure/test_azure_common_utils.py +++ b/tests/litellm/llms/azure/test_azure_common_utils.py @@ -392,6 +392,9 @@ def test_select_azure_base_url_called(setup_mocks): "arun_thread_stream", "aresponses", "acreate_fine_tuning_job", + "acancel_fine_tuning_job", + "alist_fine_tuning_jobs", + "aretrieve_fine_tuning_job", ] ], ) From 5c90e51ad48b9998959167e4d9b41a4c934a7bfb Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 17:55:29 -0700 Subject: [PATCH 23/36] (build) fix context window for claude 4 model family --- ...odel_prices_and_context_window_backup.json | 40 +++++++++---------- model_prices_and_context_window.json | 40 +++++++++---------- 2 files changed, 40 insertions(+), 40 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 6e2f66c466b..eca0bbcfc89 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -4685,9 +4685,9 @@ "supports_tool_choice": true }, "claude-opus-4-20250514": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -4711,9 +4711,9 @@ "supports_computer_use": true }, "claude-sonnet-4-20250514": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { @@ -6806,9 +6806,9 @@ "supports_tool_choice": true }, "vertex_ai/claude-opus-4@20250514": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -6832,9 +6832,9 @@ "supports_computer_use": true }, "vertex_ai/claude-sonnet-4@20250514": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { @@ -9437,9 +9437,9 @@ "supports_tool_choice": true }, "anthropic.claude-opus-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -9463,9 +9463,9 @@ "supports_computer_use": true }, "anthropic.claude-sonnet-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { @@ -9639,9 +9639,9 @@ "supports_reasoning": true }, "us.anthropic.claude-opus-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -9665,9 +9665,9 @@ "supports_computer_use": true }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { @@ -9812,9 +9812,9 @@ "supports_tool_choice": true }, "eu.anthropic.claude-opus-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -9838,9 +9838,9 @@ "supports_computer_use": true }, "eu.anthropic.claude-sonnet-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 6e2f66c466b..eca0bbcfc89 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -4685,9 +4685,9 @@ "supports_tool_choice": true }, "claude-opus-4-20250514": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -4711,9 +4711,9 @@ "supports_computer_use": true }, "claude-sonnet-4-20250514": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { @@ -6806,9 +6806,9 @@ "supports_tool_choice": true }, "vertex_ai/claude-opus-4@20250514": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -6832,9 +6832,9 @@ "supports_computer_use": true }, "vertex_ai/claude-sonnet-4@20250514": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { @@ -9437,9 +9437,9 @@ "supports_tool_choice": true }, "anthropic.claude-opus-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -9463,9 +9463,9 @@ "supports_computer_use": true }, "anthropic.claude-sonnet-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { @@ -9639,9 +9639,9 @@ "supports_reasoning": true }, "us.anthropic.claude-opus-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -9665,9 +9665,9 @@ "supports_computer_use": true }, "us.anthropic.claude-sonnet-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { @@ -9812,9 +9812,9 @@ "supports_tool_choice": true }, "eu.anthropic.claude-opus-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 32000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 32000, "input_cost_per_token": 15e-6, "output_cost_per_token": 75e-6, "search_context_cost_per_query": { @@ -9838,9 +9838,9 @@ "supports_computer_use": true }, "eu.anthropic.claude-sonnet-4-20250514-v1:0": { - "max_tokens": 128000, + "max_tokens": 64000, "max_input_tokens": 200000, - "max_output_tokens": 128000, + "max_output_tokens": 64000, "input_cost_per_token": 3e-6, "output_cost_per_token": 15e-6, "search_context_cost_per_query": { From c8a00889700100d4469158122ea85f45e29c8751 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 18:04:15 -0700 Subject: [PATCH 24/36] [Fix] Reliability Fix - Removing code that was creating threads on errors (#11066) * fix: only init langfuse if active * fix: only init langfuse if active * fix: add initialized_langfuse_clients count * fix: add MAX_LANGFUSE_INITIALIZED_CLIENTS * fix: use safe init langfuse * test: init langfuse clients * test: test_langfuse_not_initialized_returns_none_early * docs MAX_LANGFUSE_INITIALIZED_CLIENTS * fix: use correct langfuse callback * fix: code qa --- docs/my-website/docs/proxy/config_settings.md | 1 + litellm/__init__.py | 1 + litellm/constants.py | 3 + litellm/integrations/SlackAlerting/utils.py | 17 ++++-- litellm/integrations/langfuse/langfuse.py | 25 +++++++- .../logging_callback_manager.py | 9 +++ litellm/proxy/proxy_config.yaml | 21 +------ .../test_slack_alerting_utils.py | 39 +++++++++++++ tests/litellm/integrations/test_langfuse.py | 57 +++++++++++++++++++ 9 files changed, 147 insertions(+), 26 deletions(-) create mode 100644 tests/litellm/integrations/SlackAlerting/test_slack_alerting_utils.py create mode 100644 tests/litellm/integrations/test_langfuse.py diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index fdd68c953f6..f8d219c6a02 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -528,6 +528,7 @@ router_settings: | MAX_TOKEN_TRIMMING_ATTEMPTS | Maximum number of attempts to trim a token message. Default is 10 | MAXIMUM_TRACEBACK_LINES_TO_LOG | Maximum number of lines to log in traceback in LiteLLM Logs UI. Default is 100 | MAX_RETRY_DELAY | Maximum delay in seconds for retrying requests. Default is 8.0 +| MAX_LANGFUSE_INITIALIZED_CLIENTS | Maximum number of Langfuse clients to initialize on proxy. Default is 20. This is set since langfuse initializes 1 thread everytime a client is initialized. We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times. | MIN_NON_ZERO_TEMPERATURE | Minimum non-zero temperature value. Default is 0.0001 | MINIMUM_PROMPT_CACHE_TOKEN_COUNT | Minimum token count for caching a prompt. Default is 1024 | MISTRAL_API_BASE | Base URL for Mistral API diff --git a/litellm/__init__.py b/litellm/__init__.py index 5061c8e7e5b..5e40d3e3d8d 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -126,6 +126,7 @@ _known_custom_logger_compatible_callbacks: List = list( callbacks: List[ Union[Callable, _custom_logger_compatible_callbacks_literal, CustomLogger] ] = [] +initialized_langfuse_clients: int = 0 langfuse_default_tags: Optional[List[str]] = None langsmith_batch_size: Optional[int] = None prometheus_initialize_budget_metrics: Optional[bool] = False diff --git a/litellm/constants.py b/litellm/constants.py index 148cb9847c8..bb5d56978be 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -153,6 +153,9 @@ FIREWORKS_AI_16_B = int(os.getenv("FIREWORKS_AI_16_B", 16)) FIREWORKS_AI_80_B = int(os.getenv("FIREWORKS_AI_80_B", 80)) #### Logging callback constants #### REDACTED_BY_LITELM_STRING = "REDACTED_BY_LITELM" +MAX_LANGFUSE_INITIALIZED_CLIENTS = int( + os.getenv("MAX_LANGFUSE_INITIALIZED_CLIENTS", 20) +) ############### LLM Provider Constants ############### ### ANTHROPIC CONSTANTS ### diff --git a/litellm/integrations/SlackAlerting/utils.py b/litellm/integrations/SlackAlerting/utils.py index 0dc8bae5a6a..e695266c88b 100644 --- a/litellm/integrations/SlackAlerting/utils.py +++ b/litellm/integrations/SlackAlerting/utils.py @@ -5,6 +5,7 @@ Utils used for slack alerting import asyncio from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +import litellm from litellm.proxy._types import AlertType from litellm.secret_managers.main import get_secret @@ -69,7 +70,12 @@ async def _add_langfuse_trace_id_to_alert( -> trace_id -> litellm_call_id """ - # do nothing for now + if "langfuse" not in litellm.logging_callback_manager._get_all_callbacks(): + return None + ######################################################### + # Only run if langfuse is added as a callback + ######################################################### + if ( request_data is not None and request_data.get("litellm_logging_obj", None) is not None @@ -82,11 +88,12 @@ async def _add_langfuse_trace_id_to_alert( if trace_id is not None: break await asyncio.sleep(3) # wait 3s before retrying for trace id - - _langfuse_object = litellm_logging_obj._get_callback_object( + ######################################################### + langfuse_object = litellm_logging_obj._get_callback_object( service_name="langfuse" ) - if _langfuse_object is not None: - base_url = _langfuse_object.Langfuse.base_url + if langfuse_object is not None: + base_url = langfuse_object.Langfuse.base_url return f"{base_url}/trace/{trace_id}" + return None diff --git a/litellm/integrations/langfuse/langfuse.py b/litellm/integrations/langfuse/langfuse.py index 02862c52c5d..2674f2ace0a 100644 --- a/litellm/integrations/langfuse/langfuse.py +++ b/litellm/integrations/langfuse/langfuse.py @@ -10,6 +10,7 @@ from packaging.version import Version import litellm from litellm._logging import verbose_logger +from litellm.constants import MAX_LANGFUSE_INITIALIZED_CLIENTS from litellm.litellm_core_utils.redact_messages import redact_user_api_key_info from litellm.llms.custom_httpx.http_handler import _get_httpx_client from litellm.secret_managers.main import str_to_bool @@ -27,12 +28,13 @@ from litellm.types.utils import ( ) if TYPE_CHECKING: - from langfuse.client import StatefulTraceClient + from langfuse.client import Langfuse, StatefulTraceClient from litellm.litellm_core_utils.litellm_logging import DynamicLoggingCache else: DynamicLoggingCache = Any StatefulTraceClient = Any + Langfuse = Any class LangFuseLogger: @@ -84,8 +86,7 @@ class LangFuseLogger: if Version(self.langfuse_sdk_version) >= Version("2.6.0"): parameters["sdk_integration"] = "litellm" - - self.Langfuse = Langfuse(**parameters) + self.Langfuse: Langfuse = self.safe_init_langfuse_client(parameters) # set the current langfuse project id in the environ # this is used by Alerting to link to the correct project @@ -124,6 +125,24 @@ class LangFuseLogger: else: self.upstream_langfuse = None + def safe_init_langfuse_client(self, parameters: dict) -> Langfuse: + """ + Safely init a langfuse client if the number of initialized clients is less than the max + + Note: + - Langfuse initializes 1 thread everytime a client is initialized. + - We've had an incident in the past where we reached 100% cpu utilization because Langfuse was initialized several times. + """ + from langfuse import Langfuse + + if litellm.initialized_langfuse_clients >= MAX_LANGFUSE_INITIALIZED_CLIENTS: + raise Exception( + f"Max langfuse clients reached: {litellm.initialized_langfuse_clients} is greater than {MAX_LANGFUSE_INITIALIZED_CLIENTS}" + ) + langfuse_client = Langfuse(**parameters) + litellm.initialized_langfuse_clients += 1 + return langfuse_client + @staticmethod def add_metadata_from_header(litellm_params: dict, metadata: dict) -> dict: """ diff --git a/litellm/litellm_core_utils/logging_callback_manager.py b/litellm/litellm_core_utils/logging_callback_manager.py index dec3add4e1b..e1bddc65497 100644 --- a/litellm/litellm_core_utils/logging_callback_manager.py +++ b/litellm/litellm_core_utils/logging_callback_manager.py @@ -266,3 +266,12 @@ class LoggingCallbackManager: if isinstance(callback, callback_type) and callback not in all_callbacks: all_callbacks.append(callback) return all_callbacks + + def callback_is_active(self, callback_type: Type[CustomLogger]) -> bool: + """ + Returns True if any of the active callbacks are of the given type + """ + return any( + isinstance(callback, callback_type) + for callback in self._get_all_callbacks() + ) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index ae0c4b2de48..2e37fbe9776 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -2,25 +2,10 @@ model_list: - model_name: openai/gpt-4o litellm_params: model: openai/gpt-4o - api_key: any_key - api_base: https://exampleopenaiendpoint-production.up.railway.app/ + api_key: a + api_base: hi general_settings: store_prompts_in_spend_logs: true - - - -guardrails: - - guardrail_name: "custom-pre-guard" - litellm_params: - guardrail: custom_guardrail.myCustomGuardrail # 👈 Key change - mode: "pre_call" # runs async_pre_call_hook - - guardrail_name: "custom-during-guard" - litellm_params: - guardrail: custom_guardrail.myCustomGuardrail - mode: "during_call" # runs async_moderation_hook - - guardrail_name: "custom-post-guard" - litellm_params: - guardrail: custom_guardrail.myCustomGuardrail - mode: "post_call" # runs async_post_call_success_hook \ No newline at end of file + alerting: ["slack"] \ No newline at end of file diff --git a/tests/litellm/integrations/SlackAlerting/test_slack_alerting_utils.py b/tests/litellm/integrations/SlackAlerting/test_slack_alerting_utils.py new file mode 100644 index 00000000000..027fed1b5ff --- /dev/null +++ b/tests/litellm/integrations/SlackAlerting/test_slack_alerting_utils.py @@ -0,0 +1,39 @@ +import json +import os +import sys +from typing import Optional +from unittest.mock import MagicMock + +import pytest + +# Adds the grandparent directory to sys.path to allow importing project modules +sys.path.insert(0, os.path.abspath("../..")) + +import litellm +from litellm.integrations.langfuse.langfuse_prompt_management import ( + LangfusePromptManagement, +) +from litellm.integrations.SlackAlerting.utils import _add_langfuse_trace_id_to_alert +from litellm.litellm_core_utils.logging_callback_manager import LoggingCallbackManager + + +@pytest.mark.asyncio +async def test_langfuse_not_initialized_returns_none_early(): + """ + Test that when no LangfusePromptManagement is initialized, + the function returns None immediately without executing further logic + """ + # Ensure no Langfuse logger is in the callback manager + litellm.logging_callback_manager = LoggingCallbackManager() + + # Create request data that would normally trigger processing + request_data = {"litellm_logging_obj": MagicMock(), "trace_id": "test-trace-id"} + + # Call the function + result = await _add_langfuse_trace_id_to_alert(request_data) + + # Should return None early without processing request_data + assert result is None + + # Verify the litellm_logging_obj was never accessed (early return) + request_data["litellm_logging_obj"].assert_not_called() diff --git a/tests/litellm/integrations/test_langfuse.py b/tests/litellm/integrations/test_langfuse.py new file mode 100644 index 00000000000..7e47f76b995 --- /dev/null +++ b/tests/litellm/integrations/test_langfuse.py @@ -0,0 +1,57 @@ +import json +import os +import sys +from typing import Optional + +# Adds the grandparent directory to sys.path to allow importing project modules +sys.path.insert(0, os.path.abspath("../..")) + +import asyncio +from unittest.mock import patch + +import pytest + +import litellm +from litellm.integrations.langfuse.langfuse import LangFuseLogger + + +def test_max_langfuse_clients_limit(): + """ + Test that the max langfuse clients limit is respected when initializing multiple clients + """ + # Set max clients to 2 for testing + with patch( + "litellm.integrations.langfuse.langfuse.MAX_LANGFUSE_INITIALIZED_CLIENTS", 2 + ): + # Reset the counter + litellm.initialized_langfuse_clients = 0 + + # First client should succeed + logger1 = LangFuseLogger( + langfuse_public_key="test_key_1", + langfuse_secret="test_secret_1", + langfuse_host="https://test1.langfuse.com", + ) + assert litellm.initialized_langfuse_clients == 1 + + # Second client should succeed + logger2 = LangFuseLogger( + langfuse_public_key="test_key_2", + langfuse_secret="test_secret_2", + langfuse_host="https://test2.langfuse.com", + ) + assert litellm.initialized_langfuse_clients == 2 + + # Third client should fail with exception + with pytest.raises(Exception) as exc_info: + logger3 = LangFuseLogger( + langfuse_public_key="test_key_3", + langfuse_secret="test_secret_3", + langfuse_host="https://test3.langfuse.com", + ) + + # Verify the error message contains the expected text + assert "Max langfuse clients reached" in str(exc_info.value) + + # Counter should still be 2 (third client failed to initialize) + assert litellm.initialized_langfuse_clients == 2 From a7a5b22393ebd8278b96e07117d5e8d3d3da80e0 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 18:05:28 -0700 Subject: [PATCH 25/36] [Feat] Add Azure AD certificate-based authentication (#11069) * feat: add cert based auth for Azure get_azure_ad_token_provider * test: tests azure cert auth * fix update poetry * fix: fix linting --- .../get_azure_ad_token_provider.py | 45 ++++-- .../get_azure_ad_token_provider.py | 7 + poetry.lock | 43 +++--- pyproject.toml | 1 + .../test_get_azure_ad_token_provider.py | 138 ++++++++++++++++++ 5 files changed, 199 insertions(+), 35 deletions(-) create mode 100644 litellm/types/secret_managers/get_azure_ad_token_provider.py create mode 100644 tests/litellm/secret_managers/test_get_azure_ad_token_provider.py diff --git a/litellm/secret_managers/get_azure_ad_token_provider.py b/litellm/secret_managers/get_azure_ad_token_provider.py index 5403675b979..c982856b5ed 100644 --- a/litellm/secret_managers/get_azure_ad_token_provider.py +++ b/litellm/secret_managers/get_azure_ad_token_provider.py @@ -1,5 +1,9 @@ import os -from typing import Callable +from typing import Any, Callable, Optional, Union + +from litellm.types.secret_managers.get_azure_ad_token_provider import ( + AzureCredentialType, +) def get_azure_ad_token_provider() -> Callable[[], str]: @@ -15,24 +19,45 @@ def get_azure_ad_token_provider() -> Callable[[], str]: Callable that returns a temporary authentication token. """ import azure.identity as identity - from azure.identity import get_bearer_token_provider + from azure.identity import ( + CertificateCredential, + ClientSecretCredential, + ManagedIdentityCredential, + get_bearer_token_provider, + ) azure_scope = os.environ.get( "AZURE_SCOPE", "https://cognitiveservices.azure.com/.default" ) - cred = os.environ.get("AZURE_CREDENTIAL", "ClientSecretCredential") - - cred_cls = getattr(identity, cred) - # ClientSecretCredential, DefaultAzureCredential, AzureCliCredential - if cred == "ClientSecretCredential": - credential = cred_cls( + cred: Union[AzureCredentialType, str] = AzureCredentialType( + os.environ.get("AZURE_CREDENTIAL", AzureCredentialType.ClientSecretCredential) + ) + credential: Optional[ + Union[ + ClientSecretCredential, + ManagedIdentityCredential, + CertificateCredential, + Any, + ] + ] = None + if cred == AzureCredentialType.ClientSecretCredential: + credential = ClientSecretCredential( client_id=os.environ["AZURE_CLIENT_ID"], client_secret=os.environ["AZURE_CLIENT_SECRET"], tenant_id=os.environ["AZURE_TENANT_ID"], ) - elif cred == "ManagedIdentityCredential": - credential = cred_cls(client_id=os.environ["AZURE_CLIENT_ID"]) + elif cred == AzureCredentialType.ManagedIdentityCredential: + credential = ManagedIdentityCredential(client_id=os.environ["AZURE_CLIENT_ID"]) + elif cred == AzureCredentialType.CertificateCredential: + credential = CertificateCredential( + client_id=os.environ["AZURE_CLIENT_ID"], + tenant_id=os.environ["AZURE_TENANT_ID"], + certificate_path=os.environ["AZURE_CERTIFICATE_PATH"], + ) else: + cred_cls = getattr(identity, cred) credential = cred_cls() + if credential is None: + raise ValueError("No credential provided") return get_bearer_token_provider(credential, azure_scope) diff --git a/litellm/types/secret_managers/get_azure_ad_token_provider.py b/litellm/types/secret_managers/get_azure_ad_token_provider.py new file mode 100644 index 00000000000..f318b4333bd --- /dev/null +++ b/litellm/types/secret_managers/get_azure_ad_token_provider.py @@ -0,0 +1,7 @@ +from enum import Enum + + +class AzureCredentialType(str, Enum): + ClientSecretCredential = "ClientSecretCredential" + ManagedIdentityCredential = "ManagedIdentityCredential" + CertificateCredential = "CertificateCredential" diff --git a/poetry.lock b/poetry.lock index 2bd91505cb2..cadc7cd6d76 100644 --- a/poetry.lock +++ b/poetry.lock @@ -258,10 +258,9 @@ tests-mypy = ["mypy (>=1.11.1) ; platform_python_implementation == \"CPython\" a name = "azure-core" version = "1.33.0" description = "Microsoft Azure Core Library for Python" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"extra-proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "azure_core-1.33.0-py3-none-any.whl", hash = "sha256:9b5b6d0223a1d38c37500e6971118c1e0f13f54951e6893968b38910bc9cda8f"}, {file = "azure_core-1.33.0.tar.gz", hash = "sha256:f367aa07b5e3005fec2c1e184b882b0b039910733907d001c20fb08ebb8c0eb9"}, @@ -280,10 +279,9 @@ tracing = ["opentelemetry-api (>=1.26,<2.0)"] name = "azure-identity" version = "1.21.0" description = "Microsoft Azure Identity Library for Python" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"extra-proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "azure_identity-1.21.0-py3-none-any.whl", hash = "sha256:258ea6325537352440f71b35c3dffe9d240eae4a5126c1b7ce5efd5766bd9fd9"}, {file = "azure_identity-1.21.0.tar.gz", hash = "sha256:ea22ce6e6b0f429bc1b8d9212d5b9f9877bd4c82f1724bfa910760612c07a9a6"}, @@ -499,7 +497,7 @@ version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "proxy-dev"] files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -569,7 +567,7 @@ files = [ {file = "cffi-1.17.1-cp39-cp39-win_amd64.whl", hash = "sha256:d016c76bdd850f3c626af19b0542c9677ba156e4ee4fccfdd7848803533ef662"}, {file = "cffi-1.17.1.tar.gz", hash = "sha256:1c39c6016c32bc48dd54561950ebd6836e1670f2ae46128f67cf49e789c52824"}, ] -markers = {main = "(extra == \"proxy\" or extra == \"extra-proxy\") and (platform_python_implementation != \"PyPy\" or extra == \"proxy\")", dev = "platform_python_implementation != \"PyPy\""} +markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} [package.dependencies] pycparser = "*" @@ -729,7 +727,7 @@ version = "43.0.3" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." optional = false python-versions = ">=3.7" -groups = ["main", "dev"] +groups = ["main", "dev", "proxy-dev"] files = [ {file = "cryptography-43.0.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:bf7a1932ac4176486eab36a19ed4c0492da5d97123f1406cf15e41b05e787d2e"}, {file = "cryptography-43.0.3-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63efa177ff54aec6e1c0aefaa1a241232dcd37413835a9b674b6e3f0ae2bfd3e"}, @@ -759,7 +757,6 @@ files = [ {file = "cryptography-43.0.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:2ce6fae5bdad59577b44e4dfed356944fbf1d925269114c28be377692643b4ff"}, {file = "cryptography-43.0.3.tar.gz", hash = "sha256:315b9001266a492a6ff443b61238f956b214dbec9910a081ba5b6646a055a805"}, ] -markers = {main = "extra == \"proxy\" or extra == \"extra-proxy\""} [package.dependencies] cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} @@ -1986,10 +1983,9 @@ dev = ["absl-py", "pyink", "pylint (>=2.6.0)", "pytest", "pytest-xdist"] name = "msal" version = "1.32.3" description = "The Microsoft Authentication Library (MSAL) for Python library enables your app to access the Microsoft Cloud by supporting authentication of users with Microsoft Azure Active Directory accounts (AAD) and Microsoft Accounts (MSA) using industry standard OAuth2 and OpenID Connect." -optional = true +optional = false python-versions = ">=3.7" -groups = ["main"] -markers = "extra == \"extra-proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "msal-1.32.3-py3-none-any.whl", hash = "sha256:b2798db57760b1961b142f027ffb7c8169536bf77316e99a0df5c4aaebb11569"}, {file = "msal-1.32.3.tar.gz", hash = "sha256:5eea038689c78a5a70ca8ecbe1245458b55a857bd096efb6989c69ba15985d35"}, @@ -2007,10 +2003,9 @@ broker = ["pymsalruntime (>=0.14,<0.18) ; python_version >= \"3.6\" and platform name = "msal-extensions" version = "1.3.0" description = "Microsoft Authentication Library extensions (MSAL EX) provides a persistence API that can save your data on disk, encrypted on Windows, macOS and Linux. Concurrent data access will be coordinated by a file lock mechanism." -optional = true +optional = false python-versions = ">=3.7" -groups = ["main"] -markers = "extra == \"extra-proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "msal_extensions-1.3.0-py3-none-any.whl", hash = "sha256:105328ddcbdd342016c9949d8f89e3917554740c8ab26669c0fa0e069e730a0e"}, {file = "msal_extensions-1.3.0.tar.gz", hash = "sha256:96918996642b38c78cd59b55efa0f06fd1373c90e0949be8615697c048fba62c"}, @@ -2937,12 +2932,12 @@ version = "2.22" description = "C parser in Python" optional = false python-versions = ">=3.8" -groups = ["main", "dev"] +groups = ["main", "dev", "proxy-dev"] files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, ] -markers = {main = "(extra == \"proxy\" or extra == \"extra-proxy\") and (platform_python_implementation != \"PyPy\" or extra == \"proxy\")", dev = "platform_python_implementation != \"PyPy\""} +markers = {main = "platform_python_implementation != \"PyPy\" or extra == \"proxy\"", dev = "platform_python_implementation != \"PyPy\"", proxy-dev = "platform_python_implementation != \"PyPy\""} [[package]] name = "pydantic" @@ -3136,10 +3131,9 @@ windows-terminal = ["colorama (>=0.4.6)"] name = "pyjwt" version = "2.9.0" description = "JSON Web Token implementation in Python" -optional = true +optional = false python-versions = ">=3.8" -groups = ["main"] -markers = "extra == \"extra-proxy\" or extra == \"proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "PyJWT-2.9.0-py3-none-any.whl", hash = "sha256:3b02fb0f44517787776cf48f2ae25d8e14f300e6d7545a4315cee571a415e850"}, {file = "pyjwt-2.9.0.tar.gz", hash = "sha256:7e1e5b56cc735432a7369cbfa0efe50fa113ebecdc04ae6922deba8b84582d0c"}, @@ -3879,10 +3873,9 @@ crt = ["botocore[crt] (>=1.33.2,<2.0a.0)"] name = "six" version = "1.17.0" description = "Python 2 and 3 compatibility utilities" -optional = true +optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" -groups = ["main"] -markers = "extra == \"extra-proxy\" or extra == \"proxy\"" +groups = ["main", "proxy-dev"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -4935,4 +4928,4 @@ utils = ["numpydoc"] [metadata] lock-version = "2.1" python-versions = ">=3.8.1,<4.0, !=3.9.7" -content-hash = "15bad8ae37c1e7cf21555b0023150bfd4bd7d6d548828f6c65a66283d14a189b" +content-hash = "fe1bc122aaeae89043f8099bfe7fda082110b06ddee5147cbfcdd085f072ef72" diff --git a/pyproject.toml b/pyproject.toml index d3fd3ecf0cc..0cc0c4a826e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,6 +130,7 @@ prometheus-client = "0.20.0" opentelemetry-api = "1.25.0" opentelemetry-sdk = "1.25.0" opentelemetry-exporter-otlp = "1.25.0" +azure-identity = "^1.15.0" [build-system] requires = ["poetry-core", "wheel"] diff --git a/tests/litellm/secret_managers/test_get_azure_ad_token_provider.py b/tests/litellm/secret_managers/test_get_azure_ad_token_provider.py new file mode 100644 index 00000000000..7fd427dae48 --- /dev/null +++ b/tests/litellm/secret_managers/test_get_azure_ad_token_provider.py @@ -0,0 +1,138 @@ +import json +import os +import sys +from typing import Optional +from unittest.mock import MagicMock, patch + +# Adds the grandparent directory to sys.path to allow importing project modules +sys.path.insert(0, os.path.abspath("../..")) + +import pytest + +from litellm.secret_managers.get_azure_ad_token_provider import ( + get_azure_ad_token_provider, +) + + +class TestGetAzureAdTokenProvider: + @patch.dict( + os.environ, + { + "AZURE_CLIENT_ID": "test-client-id", + "AZURE_CLIENT_SECRET": "test-client-secret", + "AZURE_TENANT_ID": "test-tenant-id", + "AZURE_SCOPE": "https://cognitiveservices.azure.com/.default", + "AZURE_CREDENTIAL": "ClientSecretCredential", + }, + ) + @patch("azure.identity.get_bearer_token_provider") + @patch("azure.identity.ClientSecretCredential") + def test_get_azure_ad_token_provider_client_secret_credential( + self, mock_client_secret_credential, mock_get_bearer_token_provider + ): + """Test get_azure_ad_token_provider with ClientSecretCredential.""" + # Mock the Azure identity credential instance + mock_credential_instance = MagicMock() + mock_client_secret_credential.return_value = mock_credential_instance + + # Mock the bearer token provider + mock_token_provider = MagicMock(return_value="mock-token") + mock_get_bearer_token_provider.return_value = mock_token_provider + + # Call the function + result = get_azure_ad_token_provider() + + # Assertions + assert callable(result) + mock_client_secret_credential.assert_called_once_with( + client_id="test-client-id", + client_secret="test-client-secret", + tenant_id="test-tenant-id", + ) + mock_get_bearer_token_provider.assert_called_once_with( + mock_credential_instance, "https://cognitiveservices.azure.com/.default" + ) + + # Test that the returned callable works + token = result() + assert token == "mock-token" + + @patch.dict( + os.environ, + { + "AZURE_CLIENT_ID": "test-client-id", + "AZURE_SCOPE": "https://cognitiveservices.azure.com/.default", + "AZURE_CREDENTIAL": "ManagedIdentityCredential", + }, + ) + @patch("azure.identity.get_bearer_token_provider") + @patch("azure.identity.ManagedIdentityCredential") + def test_get_azure_ad_token_provider_managed_identity_credential( + self, mock_managed_identity_credential, mock_get_bearer_token_provider + ): + """Test get_azure_ad_token_provider with ManagedIdentityCredential.""" + # Mock the Azure identity credential instance + mock_credential_instance = MagicMock() + mock_managed_identity_credential.return_value = mock_credential_instance + + # Mock the bearer token provider + mock_token_provider = MagicMock(return_value="mock-managed-identity-token") + mock_get_bearer_token_provider.return_value = mock_token_provider + + # Call the function + result = get_azure_ad_token_provider() + + # Assertions + assert callable(result) + mock_managed_identity_credential.assert_called_once_with( + client_id="test-client-id" + ) + mock_get_bearer_token_provider.assert_called_once_with( + mock_credential_instance, "https://cognitiveservices.azure.com/.default" + ) + + # Test that the returned callable works + token = result() + assert token == "mock-managed-identity-token" + + @patch.dict( + os.environ, + { + "AZURE_CLIENT_ID": "test-client-id", + "AZURE_TENANT_ID": "test-tenant-id", + "AZURE_CERTIFICATE_PATH": "/path/to/cert.pem", + "AZURE_SCOPE": "https://cognitiveservices.azure.com/.default", + "AZURE_CREDENTIAL": "CertificateCredential", + }, + ) + @patch("azure.identity.get_bearer_token_provider") + @patch("azure.identity.CertificateCredential") + def test_get_azure_ad_token_provider_certificate_credential( + self, mock_certificate_credential, mock_get_bearer_token_provider + ): + """Test get_azure_ad_token_provider with CertificateCredential.""" + # Mock the Azure identity credential instance + mock_credential_instance = MagicMock() + mock_certificate_credential.return_value = mock_credential_instance + + # Mock the bearer token provider + mock_token_provider = MagicMock(return_value="mock-certificate-token") + mock_get_bearer_token_provider.return_value = mock_token_provider + + # Call the function + result = get_azure_ad_token_provider() + + # Assertions + assert callable(result) + mock_certificate_credential.assert_called_once_with( + client_id="test-client-id", + tenant_id="test-tenant-id", + certificate_path="/path/to/cert.pem", + ) + mock_get_bearer_token_provider.assert_called_once_with( + mock_credential_instance, "https://cognitiveservices.azure.com/.default" + ) + + # Test that the returned callable works + token = result() + assert token == "mock-certificate-token" From ae33113908e38b4709c9e0e9d6862e7fb92c217b Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 21:39:13 -0700 Subject: [PATCH 26/36] Update feature_request.yml --- .github/ISSUE_TEMPLATE/feature_request.yml | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 72943d0e6a2..2f37082d1c8 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -22,16 +22,10 @@ body: description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too. validations: required: true - - type: dropdown - id: ml-ops-team + - type: markdown attributes: - label: Are you a ML Ops Team? - description: This helps us prioritize your requests correctly - options: - - "No" - - "Yes" - validations: - required: true + value: | + * litellm is hiring a founding backend engineer, do you want to join us ? https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer - type: input id: contact attributes: From 329e69f610b17a685733187feab15dc64072b850 Mon Sep 17 00:00:00 2001 From: Ishaan Jaff Date: Thu, 22 May 2025 21:42:12 -0700 Subject: [PATCH 27/36] Update feature_request.yml (#11078) --- .github/ISSUE_TEMPLATE/feature_request.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 2f37082d1c8..13a2132ec95 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -22,10 +22,16 @@ body: description: Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., "I'm working on X and would like Y to be possible". If this is related to another GitHub issue, please link here too. validations: required: true - - type: markdown + - type: dropdown + id: hiring-interest attributes: - value: | - * litellm is hiring a founding backend engineer, do you want to join us ? https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer + label: LiteLLM is hiring a founding backend engineer, are you interested in joining us and shipping to all our users? + description: If yes, apply here - https://www.ycombinator.com/companies/litellm/jobs/6uvoBp3-founding-backend-engineer + options: + - "No" + - "Yes" + validations: + required: true - type: input id: contact attributes: From 64f325b92e0212c06c698e18586805a31a7caf15 Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 22 May 2025 22:36:19 -0700 Subject: [PATCH 28/36] adds tzdata (#10796) (#11052) With tzdata installed, the environment variable `TZ` will be respected by Python's datetime module. This means that users can specify the timezone they want LiteLLM to use. Co-authored-by: Simon Stone --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3a74c46e688..b972aab0961 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,7 @@ FROM $LITELLM_RUNTIME_IMAGE AS runtime USER root # Install runtime dependencies -RUN apk add --no-cache openssl +RUN apk add --no-cache openssl tzdata WORKDIR /app # Copy the current directory contents into the container at /app @@ -74,5 +74,5 @@ EXPOSE 4000/tcp ENTRYPOINT ["docker/prod_entrypoint.sh"] -# Append "--detailed_debug" to the end of CMD to view detailed debug logs +# Append "--detailed_debug" to the end of CMD to view detailed debug logs CMD ["--port", "4000"] From d4eec9558bc5b7d522dbe20f70d3ff8935bbb844 Mon Sep 17 00:00:00 2001 From: Martin Liu <1459760+martin-liu@users.noreply.github.com> Date: Thu, 22 May 2025 22:40:21 -0700 Subject: [PATCH 29/36] =?UTF-8?q?Fix=20proxy=5Fcli.py:=20avoid=20overridin?= =?UTF-8?q?g=20DATABASE=5FURL=20when=20it=E2=80=99s=20already=20provided.?= =?UTF-8?q?=20(#11076)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- litellm/proxy/proxy_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/proxy_cli.py b/litellm/proxy/proxy_cli.py index 4c022991f11..0a9ff2f064b 100644 --- a/litellm/proxy/proxy_cli.py +++ b/litellm/proxy/proxy_cli.py @@ -659,7 +659,7 @@ def run_server( # noqa: PLR0915 **key_management_settings ) database_url = general_settings.get("database_url", None) - if database_url is None: + if database_url is None and os.getenv("DATABASE_URL") is None: # Check if all required variables are provided database_host = os.getenv("DATABASE_HOST") database_username = os.getenv("DATABASE_USERNAME") From f1cc2d544eb0ac339765858d6a0cdb41651ea1ee Mon Sep 17 00:00:00 2001 From: Gunjan Solanki <44227165+gunjan-solanki@users.noreply.github.com> Date: Fri, 23 May 2025 11:15:14 +0530 Subject: [PATCH 30/36] feat(helm): Add loadBalancerClass support for LoadBalancer services (#11064) * feat(helm): Add loadBalancerClass support for LoadBalancer services Adds the ability to specify a loadBalancerClass when using LoadBalancer service type. This enables integration with custom load balancer implementations like Tailscale. * fixup! feat(helm): Add loadBalancerClass support for LoadBalancer services --- deploy/charts/litellm-helm/Chart.yaml | 2 +- deploy/charts/litellm-helm/README.md | 1 + .../litellm-helm/templates/service.yaml | 3 + .../litellm-helm/tests/service_tests.yaml | 116 ++++++++++++++++++ deploy/charts/litellm-helm/values.yaml | 3 + 5 files changed, 124 insertions(+), 1 deletion(-) create mode 100644 deploy/charts/litellm-helm/tests/service_tests.yaml diff --git a/deploy/charts/litellm-helm/Chart.yaml b/deploy/charts/litellm-helm/Chart.yaml index 5de591fd730..bd63ca6bfca 100644 --- a/deploy/charts/litellm-helm/Chart.yaml +++ b/deploy/charts/litellm-helm/Chart.yaml @@ -18,7 +18,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.4.3 +version: 0.4.4 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/deploy/charts/litellm-helm/README.md b/deploy/charts/litellm-helm/README.md index a0ba5781dfd..31bda3f7d79 100644 --- a/deploy/charts/litellm-helm/README.md +++ b/deploy/charts/litellm-helm/README.md @@ -34,6 +34,7 @@ If `db.useStackgresOperator` is used (not yet implemented): | `serviceAccount.create` | Whether or not to create a Kubernetes Service Account for this deployment. The default is `false` because LiteLLM has no need to access the Kubernetes API. | `false` | | `service.type` | Kubernetes Service type (e.g. `LoadBalancer`, `ClusterIP`, etc.) | `ClusterIP` | | `service.port` | TCP port that the Kubernetes Service will listen on. Also the TCP port within the Pod that the proxy will listen on. | `4000` | +| `service.loadBalancerClass` | Optional LoadBalancer implementation class (only used when `service.type` is `LoadBalancer`) | `""` | | `ingress.*` | See [values.yaml](./values.yaml) for example settings | N/A | | `proxy_config.*` | See [values.yaml](./values.yaml) for default settings. See [example_config_yaml](../../../litellm/proxy/example_config_yaml/) for configuration examples. | N/A | | `extraContainers[]` | An array of additional containers to be deployed as sidecars alongside the LiteLLM Proxy. | `[]` | diff --git a/deploy/charts/litellm-helm/templates/service.yaml b/deploy/charts/litellm-helm/templates/service.yaml index d8d81e78c89..11812208929 100644 --- a/deploy/charts/litellm-helm/templates/service.yaml +++ b/deploy/charts/litellm-helm/templates/service.yaml @@ -10,6 +10,9 @@ metadata: {{- include "litellm.labels" . | nindent 4 }} spec: type: {{ .Values.service.type }} + {{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerClass }} + loadBalancerClass: {{ .Values.service.loadBalancerClass }} + {{- end }} ports: - port: {{ .Values.service.port }} targetPort: http diff --git a/deploy/charts/litellm-helm/tests/service_tests.yaml b/deploy/charts/litellm-helm/tests/service_tests.yaml new file mode 100644 index 00000000000..43ed0180bc8 --- /dev/null +++ b/deploy/charts/litellm-helm/tests/service_tests.yaml @@ -0,0 +1,116 @@ +suite: Service Configuration Tests +templates: + - service.yaml +tests: + - it: should create a default ClusterIP service + template: service.yaml + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - equal: + path: spec.ports[0].port + value: 4000 + - equal: + path: spec.ports[0].targetPort + value: http + - equal: + path: spec.ports[0].protocol + value: TCP + - equal: + path: spec.ports[0].name + value: http + - isNull: + path: spec.loadBalancerClass + + - it: should create a NodePort service when specified + template: service.yaml + set: + service.type: NodePort + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: NodePort + - isNull: + path: spec.loadBalancerClass + + - it: should create a LoadBalancer service when specified + template: service.yaml + set: + service.type: LoadBalancer + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: LoadBalancer + - isNull: + path: spec.loadBalancerClass + + - it: should add loadBalancerClass when specified with LoadBalancer type + template: service.yaml + set: + service.type: LoadBalancer + service.loadBalancerClass: tailscale + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: LoadBalancer + - equal: + path: spec.loadBalancerClass + value: tailscale + + - it: should not add loadBalancerClass when specified with ClusterIP type + template: service.yaml + set: + service.type: ClusterIP + service.loadBalancerClass: tailscale + asserts: + - isKind: + of: Service + - equal: + path: spec.type + value: ClusterIP + - isNull: + path: spec.loadBalancerClass + + - it: should use custom port when specified + template: service.yaml + set: + service.port: 8080 + asserts: + - equal: + path: spec.ports[0].port + value: 8080 + + - it: should add service annotations when specified + template: service.yaml + set: + service.annotations: + cloud.google.com/load-balancer-type: "Internal" + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + asserts: + - isKind: + of: Service + - equal: + path: metadata.annotations + value: + cloud.google.com/load-balancer-type: "Internal" + service.beta.kubernetes.io/aws-load-balancer-internal: "true" + + - it: should use the correct selector labels + template: service.yaml + asserts: + - isNotNull: + path: spec.selector + - equal: + path: spec.selector + value: + app.kubernetes.io/name: litellm + app.kubernetes.io/instance: RELEASE-NAME diff --git a/deploy/charts/litellm-helm/values.yaml b/deploy/charts/litellm-helm/values.yaml index 0440e28eed0..213db35a20f 100644 --- a/deploy/charts/litellm-helm/values.yaml +++ b/deploy/charts/litellm-helm/values.yaml @@ -56,6 +56,9 @@ environmentConfigMaps: [] service: type: ClusterIP port: 4000 + # If service type is `LoadBalancer` you can + # optionally specify loadBalancerClass + # loadBalancerClass: tailscale ingress: enabled: false From b350cd306ade2fad8d51754f39706500b9e05dba Mon Sep 17 00:00:00 2001 From: Emerson Gomes Date: Fri, 23 May 2025 00:45:50 -0500 Subject: [PATCH 31/36] Add Azure Mistral Medium 25.05 (#11063) * Add Azure Mistral Medium 25.05 * fix provider --- model_prices_and_context_window.json | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index eca0bbcfc89..a37a2de9cf5 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -3117,6 +3117,18 @@ "supports_function_calling": true, "source": "https://azuremarketplace.microsoft.com/en/marketplace/apps/000-000.mistral-nemo-12b-2407?tab=PlansAndPrice" }, + "azure_ai/mistral-medium-2505": { + "max_tokens": 8191, + "max_input_tokens": 131072, + "max_output_tokens": 8191, + "input_cost_per_token": 0.0000004, + "output_cost_per_token": 0.000002, + "litellm_provider": "azure_ai", + "mode": "chat", + "supports_function_calling": true, + "supports_assistant_prefill": true, + "supports_tool_choice": true + }, "azure_ai/mistral-large": { "max_tokens": 8191, "max_input_tokens": 32000, From c1a4d3a7045c6684771178910f595067ec682ae2 Mon Sep 17 00:00:00 2001 From: bepotp Date: Fri, 23 May 2025 07:54:15 +0200 Subject: [PATCH 32/36] fix:Databricks Claude 3.7 Sonnet output token cost: $17.85/M instead of (#11007) $178.5/M Co-authored-by: Tommy PLANEL --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index eca0bbcfc89..ad53729a148 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -12557,7 +12557,7 @@ "max_output_tokens": 128000, "input_cost_per_token": 0.0000025, "input_dbu_cost_per_token": 0.00003571, - "output_cost_per_token": 0.00017857, + "output_cost_per_token": 0.000017857, "output_db_cost_per_token": 0.000214286, "litellm_provider": "databricks", "mode": "chat", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index a37a2de9cf5..0a37f9fb99e 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -12569,7 +12569,7 @@ "max_output_tokens": 128000, "input_cost_per_token": 0.0000025, "input_dbu_cost_per_token": 0.00003571, - "output_cost_per_token": 0.00017857, + "output_cost_per_token": 0.000017857, "output_db_cost_per_token": 0.000214286, "litellm_provider": "databricks", "mode": "chat", From e2d147102d62fe1824ac4b7140ad8a08bd151427 Mon Sep 17 00:00:00 2001 From: daarko10 <34120174+daarko10@users.noreply.github.com> Date: Fri, 23 May 2025 08:54:56 +0300 Subject: [PATCH 33/36] Fix/openrouter stream usage id 8913 (#11004) * Add handling and verification for 'usage' field in OpenRouter chat transformations and streaming responses. * Ensure consistent response ID by using valid ID from any chunk. * Remove redundant comments from OpenRouter chat transformation tests and logic. * Remove this from here as I'm opening a new pr * Reverting space * Remove redundant assertions from OpenRouter chat transformation test --- litellm/llms/openrouter/chat/transformation.py | 1 + .../chat/test_openrouter_chat_transformation.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/litellm/llms/openrouter/chat/transformation.py b/litellm/llms/openrouter/chat/transformation.py index 77f402a1317..e3f9d5c3dd0 100644 --- a/litellm/llms/openrouter/chat/transformation.py +++ b/litellm/llms/openrouter/chat/transformation.py @@ -120,6 +120,7 @@ class OpenRouterChatCompletionStreamingHandler(BaseModelResponseIterator): id=chunk["id"], object="chat.completion.chunk", created=chunk["created"], + usage=chunk.get("usage"), model=chunk["model"], choices=new_choices, ) diff --git a/tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py b/tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py index de0b284f0a3..ecdcadfb735 100644 --- a/tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py +++ b/tests/litellm/llms/openrouter/chat/test_openrouter_chat_transformation.py @@ -26,6 +26,11 @@ class TestOpenRouterChatCompletionStreamingHandler: "id": "test_id", "created": 1234567890, "model": "test_model", + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + "total_tokens": 30 + }, "choices": [ {"delta": {"content": "test content", "reasoning": "test reasoning"}} ], @@ -39,6 +44,9 @@ class TestOpenRouterChatCompletionStreamingHandler: assert result.object == "chat.completion.chunk" assert result.created == 1234567890 assert result.model == "test_model" + assert result.usage.prompt_tokens == chunk["usage"]["prompt_tokens"] + assert result.usage.completion_tokens == chunk["usage"]["completion_tokens"] + assert result.usage.total_tokens == chunk["usage"]["total_tokens"] assert len(result.choices) == 1 assert result.choices[0]["delta"]["reasoning_content"] == "test reasoning" From db4183715a27c66e20a45cd1e6028fff4d595ada Mon Sep 17 00:00:00 2001 From: Tornike Gurgenidze Date: Fri, 23 May 2025 09:55:46 +0400 Subject: [PATCH 34/36] feat: add embeddings to CustomLLM (#10980) * feat: add embeddings to CustomLLM * feat: add aembedding to custom llm --- litellm/llms/custom_llm.py | 26 +++++++- litellm/main.py | 24 ++++++++ tests/local_testing/test_custom_llm.py | 82 +++++++++++++++++++++++++- 3 files changed, 130 insertions(+), 2 deletions(-) diff --git a/litellm/llms/custom_llm.py b/litellm/llms/custom_llm.py index a2d04b1838d..390258e4e82 100644 --- a/litellm/llms/custom_llm.py +++ b/litellm/llms/custom_llm.py @@ -14,7 +14,7 @@ import httpx from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler from litellm.types.utils import GenericStreamingChunk -from litellm.utils import ImageResponse, ModelResponse +from litellm.utils import ImageResponse, ModelResponse, EmbeddingResponse from .base import BaseLLM @@ -152,6 +152,30 @@ class CustomLLM(BaseLLM): ) -> ImageResponse: raise CustomLLMError(status_code=500, message="Not implemented yet!") + def embedding( + self, + model: str, + input: list, + model_response: EmbeddingResponse, + print_verbose: Callable, + logging_obj: Any, + optional_params: dict, + litellm_params=None, + ) -> EmbeddingResponse: + raise CustomLLMError(status_code=500, message="Not implemented yet!") + + async def aembedding( + self, + model: str, + input: list, + model_response: EmbeddingResponse, + print_verbose: Callable, + logging_obj: Any, + optional_params: dict, + litellm_params=None, + ) -> EmbeddingResponse: + raise CustomLLMError(status_code=500, message="Not implemented yet!") + def custom_chat_llm_router( async_fn: bool, stream: Optional[bool], custom_llm: CustomLLM diff --git a/litellm/main.py b/litellm/main.py index 1c1f4879cc8..44611e203f0 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -4027,6 +4027,30 @@ def embedding( # noqa: PLR0915 client=client, aembedding=aembedding, ) + elif ( + custom_llm_provider in litellm._custom_providers + ): + custom_handler: Optional[CustomLLM] = None + for item in litellm.custom_provider_map: + if item["provider"] == custom_llm_provider: + custom_handler = item["custom_handler"] + + if custom_handler is None: + raise LiteLLMUnknownProvider( + model=model, custom_llm_provider=custom_llm_provider + ) + + handler_fn = custom_handler.embedding if not aembedding else custom_handler.aembedding + + response = handler_fn( + model=model, + input=input, + logging_obj=logging, + optional_params=optional_params, + model_response=EmbeddingResponse(), + print_verbose=print_verbose, + litellm_params=litellm_params + ) else: raise LiteLLMUnknownProvider( model=model, custom_llm_provider=custom_llm_provider diff --git a/tests/local_testing/test_custom_llm.py b/tests/local_testing/test_custom_llm.py index beb1e3332dd..77f4544afa1 100644 --- a/tests/local_testing/test_custom_llm.py +++ b/tests/local_testing/test_custom_llm.py @@ -44,7 +44,7 @@ from litellm import ( image_generation, ) from litellm.utils import ModelResponseIterator -from litellm.types.utils import ImageResponse, ImageObject +from litellm.types.utils import ImageResponse, ImageObject, EmbeddingResponse from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler @@ -257,6 +257,53 @@ class MyCustomLLM(CustomLLM): response_ms=1000, ) + def embedding( + self, + model: str, + input: list, + model_response: EmbeddingResponse, + print_verbose: Callable, + logging_obj: Any, + optional_params: dict, + litellm_params=None, + aembedding=None, + ) -> EmbeddingResponse: + model_response.model = model + + model_response.data = [ + { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": i, + } + for i, _ in enumerate(input) + ] + + return model_response + + async def aembedding( + self, + model: str, + input: list, + model_response: EmbeddingResponse, + print_verbose: Callable, + logging_obj: Any, + optional_params: dict, + litellm_params=None, + ) -> EmbeddingResponse: + model_response.model = model + + model_response.data = [ + { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": i, + } + for i, _ in enumerate(input) + ] + + return model_response + def test_get_llm_provider(): """""" @@ -452,3 +499,36 @@ def test_get_supported_openai_params(): response = get_supported_openai_params(model="my-custom-llm/my-fake-model") assert response is not None + +def test_simple_embedding(): + my_custom_llm = MyCustomLLM() + litellm.custom_provider_map = [ + {"provider": "custom_llm", "custom_handler": my_custom_llm} + ] + resp = litellm.embedding( + model="custom_llm/my-fake-model", + input=["good morning from litellm", "good night from litellm"] + ) + + assert resp.data[1] == { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": 1, + } + +@pytest.mark.asyncio +async def test_simple_aembedding(): + my_custom_llm = MyCustomLLM() + litellm.custom_provider_map = [ + {"provider": "custom_llm", "custom_handler": my_custom_llm} + ] + resp = await litellm.aembedding( + model="custom_llm/my-fake-model", + input=["good morning from litellm", "good night from litellm"] + ) + + assert resp.data[1] == { + "object": "embedding", + "embedding": [0.1, 0.2, 0.3], + "index": 1, + } \ No newline at end of file From 5f6928bd50c839d15fdac320b57683a58f38c1db Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 22 May 2025 23:04:37 -0700 Subject: [PATCH 35/36] Enable switching between custom auth and litellm api key auth + Fix `/customer/update` for max budgets (#11070) * feat(user_api_key_auth.py): (enterprise) allow user to enable custom auth + litellm api key auth makes it easy to migrate to proxy * fix(proxy/_types.py): allow setting 'spend' for new customer * fix(customer_endpoints.py): fix updating max budget on `/customer/update` Fixes https://github.com/BerriAI/litellm/issues/6920 * test(test_customer_endpoints.py): add unit tests for customer update endpoint * fix: fix linting error * fix(custom_auth_auto.py): fix ruff check * fix(customer_endpoints.py): fix documentation --- .../proxy/auth/user_api_key_auth.py | 30 +++++++ .../litellm_enterprise/proxy/proxy_server.py | 25 ++++++ .../types/proxy/proxy_server.py | 5 ++ litellm/proxy/_new_secret_config.yaml | 6 +- litellm/proxy/_types.py | 2 + litellm/proxy/auth/user_api_key_auth.py | 18 +++- litellm/proxy/custom_auth_auto.py | 18 ++++ .../customer_endpoints.py | 63 ++++++++++++-- litellm/proxy/proxy_server.py | 7 +- .../test_customer_endpoints.py | 87 +++++++++++++++++++ 10 files changed, 251 insertions(+), 10 deletions(-) create mode 100644 enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py create mode 100644 enterprise/litellm_enterprise/proxy/proxy_server.py create mode 100644 enterprise/litellm_enterprise/types/proxy/proxy_server.py create mode 100644 litellm/proxy/custom_auth_auto.py create mode 100644 tests/litellm/proxy/management_endpoints/test_customer_endpoints.py diff --git a/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py b/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py new file mode 100644 index 00000000000..37bab50971e --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/auth/user_api_key_auth.py @@ -0,0 +1,30 @@ +from typing import Any, Optional + +from fastapi import Request + +from litellm._logging import verbose_proxy_logger +from litellm.proxy._types import UserAPIKeyAuth + + +async def enterprise_custom_auth( + request: Request, api_key: str, user_custom_auth: Any +) -> Optional[UserAPIKeyAuth]: + from litellm_enterprise.proxy.proxy_server import custom_auth_settings + + if custom_auth_settings is None: + return None + + if custom_auth_settings["mode"] == "on": + return await user_custom_auth(request, api_key) + elif custom_auth_settings["mode"] == "off": + return None + elif custom_auth_settings["mode"] == "auto": + try: + return await user_custom_auth(request, api_key) + except Exception as e: + verbose_proxy_logger.debug( + f"Error in custom auth, checking litellm auth: {e}" + ) + return None + else: + raise ValueError(f"Invalid mode: {custom_auth_settings['mode']}") diff --git a/enterprise/litellm_enterprise/proxy/proxy_server.py b/enterprise/litellm_enterprise/proxy/proxy_server.py new file mode 100644 index 00000000000..481d65a9433 --- /dev/null +++ b/enterprise/litellm_enterprise/proxy/proxy_server.py @@ -0,0 +1,25 @@ +from typing import Optional + +from litellm_enterprise.types.proxy.proxy_server import CustomAuthSettings + +custom_auth_settings: Optional[CustomAuthSettings] = None + + +class EnterpriseProxyConfig: + async def load_custom_auth_settings( + self, general_settings: dict + ) -> CustomAuthSettings: + print(f"General settings: {general_settings}") + custom_auth_settings = general_settings.get("custom_auth_settings", None) + print(f"Custom auth settings: {custom_auth_settings}") + if custom_auth_settings is not None: + custom_auth_settings = CustomAuthSettings( + mode=custom_auth_settings.get("mode"), + ) + print(f"Custom auth settings: {custom_auth_settings}") + return custom_auth_settings + + async def load_enterprise_config(self, general_settings: dict) -> None: + global custom_auth_settings + custom_auth_settings = await self.load_custom_auth_settings(general_settings) + return None diff --git a/enterprise/litellm_enterprise/types/proxy/proxy_server.py b/enterprise/litellm_enterprise/types/proxy/proxy_server.py new file mode 100644 index 00000000000..497be59c4b9 --- /dev/null +++ b/enterprise/litellm_enterprise/types/proxy/proxy_server.py @@ -0,0 +1,5 @@ +from typing import Literal, TypedDict + + +class CustomAuthSettings(TypedDict): + mode: Literal["on", "off", "auto"] diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index c995567ed13..eac4a69e03c 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -78,9 +78,9 @@ model_list: prompt_label: "latest" api_key: os.environ/OPENAI_API_KEY -litellm_settings: - callbacks: ["langfuse"] - general_settings: store_model_in_db: true store_prompts_in_spend_logs: true + custom_auth: custom_auth_auto.user_api_key_auth + custom_auth_settings: + mode: "auto" \ No newline at end of file diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 1dcf417623a..1a5af39188b 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -940,6 +940,7 @@ class NewCustomerRequest(BudgetNewRequest): alias: Optional[str] = None # human-friendly alias blocked: bool = False # allow/disallow requests for this end-user budget_id: Optional[str] = None # give either a budget_id or max_budget + spend: Optional[float] = None allowed_model_region: Optional[ AllowedModelRegion ] = None # require all user requests to use models in this specific region @@ -1224,6 +1225,7 @@ class TeamRequest(LiteLLMPydanticObjectBase): class LiteLLM_BudgetTable(LiteLLMPydanticObjectBase): """Represents user-controllable params for a LiteLLM_BudgetTable record""" + budget_id: Optional[str] = None soft_budget: Optional[float] = None max_budget: Optional[float] = None max_parallel_requests: Optional[int] = None diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index 91536e08cd7..6007700ac2d 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -55,6 +55,16 @@ from litellm.proxy.utils import PrismaClient, ProxyLogging from litellm.secret_managers.main import get_secret_bool from litellm.types.services import ServiceTypes +try: + from litellm_enterprise.proxy.auth.user_api_key_auth import ( + enterprise_custom_auth as _enterprise_custom_auth, + ) + + enterprise_custom_auth: Optional[Callable] = _enterprise_custom_auth +except ImportError as e: + verbose_proxy_logger.debug(f"Error in enterprise custom auth: {e}") + enterprise_custom_auth = None + user_api_key_service_logger_obj = ServiceLogging() # used for tracking latency on OTEL custom_litellm_key_header = APIKeyHeader( @@ -346,7 +356,13 @@ async def _user_api_key_auth_builder( # noqa: PLR0915 ) ### USER-DEFINED AUTH FUNCTION ### - if user_custom_auth is not None: + if enterprise_custom_auth is not None: + response = await enterprise_custom_auth( + request=request, api_key=api_key, user_custom_auth=user_custom_auth + ) + if response is not None: + return UserAPIKeyAuth.model_validate(response) + elif user_custom_auth is not None: response = await user_custom_auth(request=request, api_key=api_key) # type: ignore return UserAPIKeyAuth.model_validate(response) diff --git a/litellm/proxy/custom_auth_auto.py b/litellm/proxy/custom_auth_auto.py new file mode 100644 index 00000000000..df6391021c1 --- /dev/null +++ b/litellm/proxy/custom_auth_auto.py @@ -0,0 +1,18 @@ +""" +Example custom auth function. + +This will allow all keys starting with "my-custom-key" to pass through. +""" +from fastapi import Request + +from litellm.proxy._types import UserAPIKeyAuth + + +async def user_api_key_auth(request: Request, api_key: str) -> UserAPIKeyAuth: + try: + if api_key.startswith("my-custom-key"): + return UserAPIKeyAuth(api_key=api_key) + else: + raise Exception("Invalid API key") + except Exception: + raise Exception("Invalid API key") diff --git a/litellm/proxy/management_endpoints/customer_endpoints.py b/litellm/proxy/management_endpoints/customer_endpoints.py index 1f6f846bc77..b75e3644e1c 100644 --- a/litellm/proxy/management_endpoints/customer_endpoints.py +++ b/litellm/proxy/management_endpoints/customer_endpoints.py @@ -185,6 +185,7 @@ async def new_end_user( - model_max_budget: Optional[dict] - [Not Implemented Yet] Specify max budget for a given model. Example: {"openai/gpt-4o-mini": {"max_budget": 100.0, "budget_duration": "1d"}} - max_parallel_requests: Optional[int] - [Not Implemented Yet] Specify max parallel requests for a given customer. - soft_budget: Optional[float] - [Not Implemented Yet] Get alerts when customer crosses given budget, doesn't block requests. + - spend: Optional[float] - Specify initial spend for a given customer. - Allow specifying allowed regions @@ -424,13 +425,65 @@ async def update_end_user( ): # models default to [], spend defaults to 0, we should not reset these values non_default_values[k] = v - ## ADD USER, IF NEW ## + ## Get end user table data ## + end_user_table_data = await prisma_client.db.litellm_endusertable.find_first( + where={"user_id": data.user_id}, include={"litellm_budget_table": True} + ) + + if end_user_table_data is None: + raise HTTPException( + status_code=400, + detail={ + "error": "End User Id={} does not exist in db".format(data.user_id) + }, + ) + + end_user_table_data_typed = LiteLLM_EndUserTable( + **end_user_table_data.model_dump() + ) + + ## Get budget table data ## + end_user_budget_table = end_user_table_data_typed.litellm_budget_table + + ## Get all params for budget table ## + budget_table_data = {} + update_end_user_table_data = {} + for k, v in non_default_values.items(): + if k in LiteLLM_BudgetTable.model_fields.keys(): + budget_table_data[k] = v + + if k in LiteLLM_EndUserTable.model_fields.keys(): + update_end_user_table_data[k] = v + + ## Check if budget id is set ## + if budget_table_data: + if end_user_budget_table is None: + ## Create new budget ## + budget_table_data_record = ( + await prisma_client.db.litellm_budgettable.create( + data=budget_table_data, include={"litellm_endusertable": True} + ) + ) + + update_end_user_table_data[ + "budget_id" + ] = budget_table_data_record.budget_id + else: + ## Update existing budget ## + budget_table_data_record = ( + await prisma_client.db.litellm_budgettable.update( + where={"budget_id": end_user_budget_table.budget_id}, + data=budget_table_data, + ) + ) + + ## Update user table, with update params + new budget id (if set) ## verbose_proxy_logger.debug("/customer/update: Received data = %s", data) if data.user_id is not None and len(data.user_id) > 0: - non_default_values["user_id"] = data.user_id # type: ignore + update_end_user_table_data["user_id"] = data.user_id # type: ignore verbose_proxy_logger.debug("In update customer, user_id condition block.") response = await prisma_client.db.litellm_endusertable.update( - where={"user_id": data.user_id}, data=non_default_values # type: ignore + where={"user_id": data.user_id}, data=update_end_user_table_data, include={"litellm_budget_table": True} # type: ignore ) if response is None: raise ValueError( @@ -444,13 +497,13 @@ async def update_end_user( raise ValueError(f"user_id is required, passed user_id = {data.user_id}") # update based on remaining passed in values + except Exception as e: - verbose_proxy_logger.error( + verbose_proxy_logger.exception( "litellm.proxy.proxy_server.update_end_user(): Exception occured - {}".format( str(e) ) ) - verbose_proxy_logger.debug(traceback.format_exc()) if isinstance(e, HTTPException): raise ProxyException( message=getattr(e, "detail", f"Internal Server Error({str(e)})"), diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 1c457baf2f9..c132b12ac6c 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -395,10 +395,12 @@ except Exception: # Import enterprise routes try: from litellm_enterprise.proxy.enterprise_routes import router as _enterprise_router + from litellm_enterprise.proxy.proxy_server import EnterpriseProxyConfig enterprise_router = _enterprise_router + enterprise_proxy_config: Optional[EnterpriseProxyConfig] = EnterpriseProxyConfig() except ImportError: - pass + enterprise_proxy_config = None ################### server_root_path = os.getenv("SERVER_ROOT_PATH", "") @@ -1863,6 +1865,9 @@ class ProxyConfig: value=custom_sso, config_file_path=config_file_path ) + if enterprise_proxy_config is not None: + await enterprise_proxy_config.load_enterprise_config(general_settings) + ## pass through endpoints if general_settings.get("pass_through_endpoints", None) is not None: await initialize_pass_through_endpoints( diff --git a/tests/litellm/proxy/management_endpoints/test_customer_endpoints.py b/tests/litellm/proxy/management_endpoints/test_customer_endpoints.py new file mode 100644 index 00000000000..6382976c361 --- /dev/null +++ b/tests/litellm/proxy/management_endpoints/test_customer_endpoints.py @@ -0,0 +1,87 @@ +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi import FastAPI, HTTPException +from fastapi.testclient import TestClient + +from litellm.proxy._types import ( + LiteLLM_BudgetTable, + LiteLLM_EndUserTable, + LitellmUserRoles, +) +from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.proxy.management_endpoints.customer_endpoints import router +from litellm.proxy.proxy_server import ProxyException + +app = FastAPI() +app.include_router(router) +client = TestClient(app) + + +@pytest.fixture +def mock_prisma_client(): + with patch("litellm.proxy.proxy_server.prisma_client") as mock: + yield mock + + +@pytest.fixture +def mock_user_api_key_auth(): + with patch("litellm.proxy.proxy_server.user_api_key_auth") as mock: + mock.return_value = UserAPIKeyAuth( + user_id="test-user", user_role=LitellmUserRoles.PROXY_ADMIN + ) + yield mock + + +def test_update_customer_success(mock_prisma_client, mock_user_api_key_auth): + # Mock the database responses + mock_end_user = LiteLLM_EndUserTable( + user_id="test-user-1", alias="Test User", blocked=False + ) + updated_mock_end_user = LiteLLM_EndUserTable( + user_id="test-user-1", alias="Updated Test User", blocked=False + ) + + # Mock the find_first response + mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock( + return_value=mock_end_user + ) + + # Mock the update response + mock_prisma_client.db.litellm_endusertable.update = AsyncMock( + return_value=updated_mock_end_user + ) + + # Test data + test_data = {"user_id": "test-user-1", "alias": "Updated Test User"} + + # Make the request + response = client.post( + "/customer/update", json=test_data, headers={"Authorization": "Bearer test-key"} + ) + + # Assert response + assert response.status_code == 200 + assert response.json()["user_id"] == "test-user-1" + assert response.json()["alias"] == "Updated Test User" + + +def test_update_customer_not_found(mock_prisma_client, mock_user_api_key_auth): + # Mock the database response to return None (user not found) + mock_prisma_client.db.litellm_endusertable.find_first = AsyncMock(return_value=None) + + # Test data + test_data = {"user_id": "non-existent-user", "alias": "Test User"} + + # Make the request + try: + response = client.post( + "/customer/update", + json=test_data, + headers={"Authorization": "Bearer test-key"}, + ) + except Exception as e: + print(e, type(e)) + assert isinstance(e, ProxyException) + assert int(e.code) == 400 + assert "End User Id=non-existent-user does not exist in db" in e.message From e9b7059af4d0aa0ad3da418628f34c1bd02251fa Mon Sep 17 00:00:00 2001 From: Krish Dholakia Date: Thu, 22 May 2025 23:05:45 -0700 Subject: [PATCH 36/36] Litellm add file validation (#11081) * fix: cleanup print statement * feat(managed_files.py): add auth check on managed files Implemented for file retrieve + delete calls * feat(files_endpoints.py): support returning files by model name enables managed file support * feat(managed_files/): filter list of files by the ones created by user prevents user from seeing another file * test: update test * fix(files_endpoints.py): list_files - always default to provider based routing * build: add new table to prisma schema --- enterprise/enterprise_hooks/managed_files.py | 105 +++++++++++++++++- .../migration.sql | 32 ++++++ .../litellm_proxy_extras/schema.prisma | 19 +++- litellm/llms/base_llm/files/transformation.py | 2 + litellm/proxy/_new_secret_config.yaml | 6 +- litellm/proxy/_types.py | 3 + .../openai_files_endpoints/files_endpoints.py | 60 ++++++++-- litellm/proxy/schema.prisma | 3 + litellm/router.py | 5 + schema.prisma | 19 +++- .../enterprise_hooks/test_managed_files.py | 31 +++++- .../llms/azure/test_azure_common_utils.py | 1 + 12 files changed, 265 insertions(+), 21 deletions(-) create mode 100644 litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql diff --git a/enterprise/enterprise_hooks/managed_files.py b/enterprise/enterprise_hooks/managed_files.py index 480ead78386..7410ad793b2 100644 --- a/enterprise/enterprise_hooks/managed_files.py +++ b/enterprise/enterprise_hooks/managed_files.py @@ -5,7 +5,7 @@ import asyncio import base64 import json import uuid -from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Union, cast +from typing import TYPE_CHECKING, Any, Dict, List, Literal, Optional, Tuple, Union, cast from fastapi import HTTPException @@ -26,8 +26,10 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( ) from litellm.types.llms.openai import ( AllMessageValues, + AsyncCursorPage, ChatCompletionFileObject, CreateFileRequest, + FileObject, OpenAIFileObject, OpenAIFilesPurpose, ) @@ -67,6 +69,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_object: OpenAIFileObject, litellm_parent_otel_span: Optional[Span], model_mappings: Dict[str, str], + user_api_key_dict: UserAPIKeyAuth, ) -> None: verbose_logger.info( f"Storing LiteLLM Managed File object with id={file_id} in cache" @@ -75,6 +78,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): unified_file_id=file_id, file_object=file_object, model_mappings=model_mappings, + flat_model_file_ids=list(model_mappings.values()), + created_by=user_api_key_dict.user_id, + updated_by=user_api_key_dict.user_id, ) await self.internal_usage_cache.async_set_cache( key=file_id, @@ -87,6 +93,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "unified_file_id": file_id, "file_object": file_object.model_dump_json(), "model_mappings": json.dumps(model_mappings), + "flat_model_file_ids": list(model_mappings.values()), + "created_by": user_api_key_dict.user_id, + "updated_by": user_api_key_dict.user_id, } ) @@ -169,6 +178,18 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): ) return initial_value.file_object + async def can_user_call_unified_file_id( + self, unified_file_id: str, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + ## check if the user has access to the unified file id + user_id = user_api_key_dict.user_id + managed_file = await self.prisma_client.db.litellm_managedfiletable.find_first( + where={"unified_file_id": unified_file_id} + ) + if managed_file: + return managed_file.created_by == user_id + return False + async def can_user_call_unified_object_id( self, unified_object_id: str, user_api_key_dict: UserAPIKeyAuth ) -> bool: @@ -184,6 +205,44 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): return managed_object.created_by == user_id return False + async def get_user_created_file_ids( + self, user_api_key_dict: UserAPIKeyAuth, model_object_ids: List[str] + ) -> List[OpenAIFileObject]: + """ + Get all file ids created by the user for a list of model object ids + + Returns: + - List of OpenAIFileObject's + """ + file_ids = await self.prisma_client.db.litellm_managedfiletable.find_many( + where={ + "created_by": user_api_key_dict.user_id, + "flat_model_file_ids": {"hasSome": model_object_ids}, + } + ) + return [OpenAIFileObject(**file_object.file_object) for file_object in file_ids] + + async def check_managed_file_id_access( + self, data: Dict, user_api_key_dict: UserAPIKeyAuth + ) -> bool: + retrieve_file_id = cast(Optional[str], data.get("file_id")) + potential_file_id = ( + _is_base64_encoded_unified_file_id(retrieve_file_id) + if retrieve_file_id + else False + ) + if potential_file_id and retrieve_file_id: + if await self.can_user_call_unified_file_id( + retrieve_file_id, user_api_key_dict + ): + return True + else: + raise HTTPException( + status_code=403, + detail=f"User {user_api_key_dict.user_id} does not have access to the file {retrieve_file_id}", + ) + return False + async def async_pre_call_hook( self, user_api_key_dict: UserAPIKeyAuth, @@ -200,6 +259,9 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): "rerank", "acreate_batch", "aretrieve_batch", + "acreate_file", + "afile_list", + "afile_delete", "afile_content", "acreate_fine_tuning_job", "aretrieve_fine_tuning_job", @@ -211,9 +273,14 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): - Detect litellm_proxy/ file_id - add dictionary of mappings of litellm_proxy/ file_id -> provider_file_id => {litellm_proxy/file_id: {"model_id": id, "file_id": provider_file_id}} """ - print( - "CALLS ASYNC PRE CALL HOOK - DATA={}, CALL_TYPE={}".format(data, call_type) - ) + ### HANDLE FILE ACCESS ### - ensure user has access to the file + if ( + call_type == CallTypes.afile_content.value + or call_type == CallTypes.afile_delete.value + ): + await self.check_managed_file_id_access(data, user_api_key_dict) + + ### HANDLE TRANSFORMATIONS ### if call_type == CallTypes.completion.value: messages = data.get("messages") if messages: @@ -298,7 +365,6 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): [input_file_id], user_api_key_dict.parent_otel_span ) - print("DATA={}".format(data)) return data async def async_pre_call_deployment_hook( @@ -416,6 +482,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): llm_router: Router, target_model_names_list: List[str], litellm_parent_otel_span: Span, + user_api_key_dict: UserAPIKeyAuth, ) -> OpenAIFileObject: responses = await self.create_file_for_each_model( llm_router=llm_router, @@ -448,6 +515,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): file_object=response, litellm_parent_otel_span=litellm_parent_otel_span, model_mappings=model_mappings, + user_api_key_dict=user_api_key_dict, ) return response @@ -560,6 +628,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): async def async_post_call_success_hook( self, data: Dict, user_api_key_dict: UserAPIKeyAuth, response: LLMResponseTypes ) -> Any: + print(f"response: {response}, type: {type(response)}") if isinstance(response, LiteLLMBatch): ## Check if unified_file_id is in the response unified_file_id = response._hidden_params.get( @@ -619,6 +688,31 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): user_api_key_dict=user_api_key_dict, ) ) + elif isinstance(response, AsyncCursorPage): + """ + For listing files, filter for the ones created by the user + """ + print("INSIDE ASYNC CURSOR PAGE BLOCK") + ## check if file object + if hasattr(response, "data") and isinstance(response.data, list): + if all( + isinstance(file_object, FileObject) for file_object in response.data + ): + ## Get all file id's + ## Check which file id's were created by the user + ## Filter the response to only include the files created by the user + ## Return the filtered response + file_ids = [ + file_object.id + for file_object in cast(List[FileObject], response.data) # type: ignore + ] + user_created_file_ids = await self.get_user_created_file_ids( + user_api_key_dict, file_ids + ) + ## Filter the response to only include the files created by the user + response.data = user_created_file_ids # type: ignore + return response + return response return response async def afile_retrieve( @@ -638,6 +732,7 @@ class _PROXY_LiteLLMManagedFiles(CustomLogger, BaseFileEndpoints): litellm_parent_otel_span: Optional[Span], **data: Dict, ) -> List[OpenAIFileObject]: + """Handled in files_endpoints.py""" return [] async def afile_delete( diff --git a/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql new file mode 100644 index 00000000000..95fb8372458 --- /dev/null +++ b/litellm-proxy-extras/litellm_proxy_extras/migrations/20250522223020_managed_object_table/migration.sql @@ -0,0 +1,32 @@ +-- AlterTable +ALTER TABLE "LiteLLM_ManagedFileTable" ADD COLUMN "created_by" TEXT, +ADD COLUMN "flat_model_file_ids" TEXT[] DEFAULT ARRAY[]::TEXT[], +ADD COLUMN "updated_by" TEXT; + +-- CreateTable +CREATE TABLE "LiteLLM_ManagedObjectTable" ( + "id" TEXT NOT NULL, + "unified_object_id" TEXT NOT NULL, + "model_object_id" TEXT NOT NULL, + "file_object" JSONB NOT NULL, + "file_purpose" TEXT NOT NULL, + "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP, + "created_by" TEXT, + "updated_at" TIMESTAMP(3) NOT NULL, + "updated_by" TEXT, + + CONSTRAINT "LiteLLM_ManagedObjectTable_pkey" PRIMARY KEY ("id") +); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_ManagedObjectTable_unified_object_id_key" ON "LiteLLM_ManagedObjectTable"("unified_object_id"); + +-- CreateIndex +CREATE UNIQUE INDEX "LiteLLM_ManagedObjectTable_model_object_id_key" ON "LiteLLM_ManagedObjectTable"("model_object_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_ManagedObjectTable_unified_object_id_idx" ON "LiteLLM_ManagedObjectTable"("unified_object_id"); + +-- CreateIndex +CREATE INDEX "LiteLLM_ManagedObjectTable_model_object_id_idx" ON "LiteLLM_ManagedObjectTable"("model_object_id"); + diff --git a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma index 1d6f3b52118..58064abd1dc 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/schema.prisma +++ b/litellm-proxy-extras/litellm_proxy_extras/schema.prisma @@ -453,13 +453,30 @@ model LiteLLM_ManagedFileTable { id String @id @default(uuid()) unified_file_id String @unique // The base64 encoded unified file ID file_object Json // Stores the OpenAIFileObject - model_mappings Json // Stores the mapping of model_id -> provider_file_id + model_mappings Json + flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id created_at DateTime @default(now()) + created_by String? updated_at DateTime @updatedAt + updated_by String? @@index([unified_file_id]) } +model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use the + id String @id @default(uuid()) + unified_object_id String @unique // The base64 encoded unified file ID + model_object_id String @unique // the id returned by the backend API provider + file_object Json // Stores the OpenAIFileObject + file_purpose String // either 'batch' or 'fine-tune' + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @updatedAt + updated_by String? + + @@index([unified_object_id]) + @@index([model_object_id]) +} model LiteLLM_ManagedVectorStoresTable { vector_store_id String @id diff --git a/litellm/llms/base_llm/files/transformation.py b/litellm/llms/base_llm/files/transformation.py index 4d749af21e1..38a6dc48092 100644 --- a/litellm/llms/base_llm/files/transformation.py +++ b/litellm/llms/base_llm/files/transformation.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union import httpx +from litellm.proxy._types import UserAPIKeyAuth from litellm.types.llms.openai import ( AllMessageValues, CreateFileRequest, @@ -115,6 +116,7 @@ class BaseFileEndpoints(ABC): llm_router: Router, target_model_names_list: List[str], litellm_parent_otel_span: Span, + user_api_key_dict: UserAPIKeyAuth, ) -> OpenAIFileObject: pass diff --git a/litellm/proxy/_new_secret_config.yaml b/litellm/proxy/_new_secret_config.yaml index eac4a69e03c..a255f32f5e3 100644 --- a/litellm/proxy/_new_secret_config.yaml +++ b/litellm/proxy/_new_secret_config.yaml @@ -2,10 +2,10 @@ model_list: - model_name: "gemini-2.0-flash-gemini" litellm_params: model: gemini/gemini-2.0-flash - - model_name: "gpt-4o-mini-openai" + - model_name: "gpt-4.1-openai" litellm_params: - model: gpt-4.1-mini-2025-04-14 - api_key: os.environ/OPENAI_API_KEY_2 + model: gpt-4.1 + api_key: os.environ/OPENAI_API_KEY model_info: access_groups: ["default-openai-models"] - model_name: "gpt-4o-realtime-preview" diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 1a5af39188b..c1feb3dc330 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2883,6 +2883,9 @@ class LiteLLM_ManagedFileTable(LiteLLMPydanticObjectBase): unified_file_id: str file_object: OpenAIFileObject model_mappings: Dict[str, str] + flat_model_file_ids: List[str] + created_by: Optional[str] + updated_by: Optional[str] class LiteLLM_ManagedObjectTable(LiteLLMPydanticObjectBase): diff --git a/litellm/proxy/openai_files_endpoints/files_endpoints.py b/litellm/proxy/openai_files_endpoints/files_endpoints.py index 3c2c3d80dcb..79345021490 100644 --- a/litellm/proxy/openai_files_endpoints/files_endpoints.py +++ b/litellm/proxy/openai_files_endpoints/files_endpoints.py @@ -179,6 +179,7 @@ async def route_create_file( create_file_request=_create_file_request, target_model_names_list=target_model_names_list, litellm_parent_otel_span=user_api_key_dict.parent_otel_span, + user_api_key_dict=user_api_key_dict, ) else: # get configs for custom_llm_provider @@ -869,6 +870,7 @@ async def list_files( fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), provider: Optional[str] = None, + target_model_names: Optional[str] = None, purpose: Optional[str] = None, ): """ @@ -885,8 +887,8 @@ async def list_files( ``` """ from litellm.proxy.proxy_server import ( - add_litellm_data_to_request, general_settings, + llm_router, proxy_config, proxy_logging_obj, version, @@ -894,24 +896,62 @@ async def list_files( data: Dict = {} try: - custom_llm_provider = ( - provider - or await get_custom_llm_provider_from_request_body(request=request) - or "openai" - ) # Include original request and headers in the data - data = await add_litellm_data_to_request( - data=data, + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + ( + data, + litellm_logging_obj, + ) = await base_llm_response_processor.common_processing_pre_call_logic( request=request, general_settings=general_settings, user_api_key_dict=user_api_key_dict, version=version, + proxy_logging_obj=proxy_logging_obj, proxy_config=proxy_config, + route_type=CallTypes.alist_fine_tuning_jobs.value, ) - response = await litellm.afile_list( - custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore + response: Optional[Any] = None + if target_model_names and isinstance(target_model_names, str): + target_model_names_list = target_model_names.split(",") + if len(target_model_names_list) != 1: + raise HTTPException( + status_code=400, + detail="target_model_names on list files must be a list of one model name. Example: ['gpt-4o']", + ) + ## Use router to list fine-tuning jobs for that model + if llm_router is None: + raise HTTPException( + status_code=500, + detail="LLM Router not initialized. Ensure models added to proxy.", + ) + data["model"] = target_model_names_list[0] + response = await llm_router.afile_list( + **data, + ) + else: + custom_llm_provider = ( + provider + or await get_custom_llm_provider_from_request_body(request=request) + or "openai" + ) + + response = await litellm.afile_list( + custom_llm_provider=custom_llm_provider, purpose=purpose, **data # type: ignore + ) + + if response is None: + raise HTTPException( + status_code=500, + detail="Either 'provider' or 'target_model_names' must be provided e.g. `?target_model_names=gpt-4o`", + ) + + ## POST CALL HOOKS ### + _response = await proxy_logging_obj.post_call_success_hook( + data=data, user_api_key_dict=user_api_key_dict, response=response ) + if _response is not None and isinstance(_response, OpenAIFileObject): + response = _response ### ALERTING ### asyncio.create_task( diff --git a/litellm/proxy/schema.prisma b/litellm/proxy/schema.prisma index e97dc7d2ae1..58064abd1dc 100644 --- a/litellm/proxy/schema.prisma +++ b/litellm/proxy/schema.prisma @@ -454,8 +454,11 @@ model LiteLLM_ManagedFileTable { unified_file_id String @unique // The base64 encoded unified file ID file_object Json // Stores the OpenAIFileObject model_mappings Json + flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id created_at DateTime @default(now()) + created_by String? updated_at DateTime @updatedAt + updated_by String? @@index([unified_file_id]) } diff --git a/litellm/router.py b/litellm/router.py index 6556791a84c..0dd52234000 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -764,6 +764,9 @@ class Router: self.aretrieve_fine_tuning_job = self.factory_function( litellm.aretrieve_fine_tuning_job, call_type="aretrieve_fine_tuning_job" ) + self.afile_list = self.factory_function( + litellm.afile_list, call_type="alist_files" + ) def validate_fallbacks(self, fallback_param: Optional[List]): """ @@ -3185,6 +3188,7 @@ class Router: "acancel_fine_tuning_job", "alist_fine_tuning_jobs", "aretrieve_fine_tuning_job", + "alist_files", ] = "assistants", ): """ @@ -3237,6 +3241,7 @@ class Router: "acancel_fine_tuning_job", "alist_fine_tuning_jobs", "aretrieve_fine_tuning_job", + "alist_files", ): return await self._ageneric_api_call_with_fallbacks( original_function=original_function, diff --git a/schema.prisma b/schema.prisma index 1d6f3b52118..b415a777359 100644 --- a/schema.prisma +++ b/schema.prisma @@ -453,13 +453,30 @@ model LiteLLM_ManagedFileTable { id String @id @default(uuid()) unified_file_id String @unique // The base64 encoded unified file ID file_object Json // Stores the OpenAIFileObject - model_mappings Json // Stores the mapping of model_id -> provider_file_id + model_mappings Json + flat_model_file_ids String[] @default([]) // Flat list of model file id's - for faster querying of model id -> unified file id created_at DateTime @default(now()) + created_by String? updated_at DateTime @updatedAt + updated_by String? @@index([unified_file_id]) } +model LiteLLM_ManagedObjectTable { // for batches or finetuning jobs which use managed files + id String @id @default(uuid()) + unified_object_id String @unique // The base64 encoded unified object ID + model_object_id String @unique // the id returned by the backend API provider + file_object Json // Stores the OpenAIFileObject + file_purpose String // either 'batch' or 'fine-tune' + created_at DateTime @default(now()) + created_by String? + updated_at DateTime @updatedAt + updated_by String? + + @@index([unified_object_id]) + @@index([model_object_id]) +} model LiteLLM_ManagedVectorStoresTable { vector_store_id String @id diff --git a/tests/enterprise/enterprise_hooks/test_managed_files.py b/tests/enterprise/enterprise_hooks/test_managed_files.py index 04a2717f788..81e27d941fa 100644 --- a/tests/enterprise/enterprise_hooks/test_managed_files.py +++ b/tests/enterprise/enterprise_hooks/test_managed_files.py @@ -3,13 +3,14 @@ import os import sys import pytest +from fastapi import HTTPException from fastapi.testclient import TestClient sys.path.insert( 0, os.path.abspath("../../..") ) # Adds the parent directory to the system path -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch from enterprise.enterprise_hooks.managed_files import _PROXY_LiteLLMManagedFiles from litellm.caching import DualCache @@ -240,3 +241,31 @@ async def test_async_pre_call_hook_for_unified_finetuning_job(): response = await proxy_managed_files.async_pre_call_hook(**data) assert response["fine_tuning_job_id"] == "ftjob-jTBys7bVsbyZDOwL9GlpYqXR" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("call_type", ["afile_content", "afile_delete"]) +async def test_can_user_call_unified_file_id(call_type): + """ + Test that on file retrieve, delete we check if the user has access to the file + """ + from litellm.proxy._types import UserAPIKeyAuth + + prisma_client = AsyncMock() + return_value = MagicMock() + return_value.created_by = "123" + prisma_client.db.litellm_managedfiletable.find_first.return_value = return_value + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + MagicMock(), prisma_client=prisma_client + ) + unified_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxmMTNlNDAzZS01YWM3LTRhZjktOGQzNS0wNDgwZDMxOTgyYTg7dGFyZ2V0X21vZGVsX25hbWVzLGdwdC00by1taW5pLW9wZW5haTtsbG1fb3V0cHV0X2ZpbGVfaWQsZmlsZS1Ib3UxZDFXc3c1SDNKcjFMYllpZDJiO2xsbV9vdXRwdXRfZmlsZV9tb2RlbF9pZCxmODBiNWU2NzQ1NzdkNjkyMjM4YmVhNTIxZDdiMGI5ZGYyY2FmMTEwMTU2YmU5YzBjM2NjMmNkNTBjOTM1ZDI0" + + with pytest.raises(HTTPException) as e: + await proxy_managed_files.async_pre_call_hook( + user_api_key_dict=UserAPIKeyAuth( + user_id="456", parent_otel_span=MagicMock() + ), + cache=MagicMock(), + data={"file_id": unified_file_id}, + call_type=call_type, + ) diff --git a/tests/litellm/llms/azure/test_azure_common_utils.py b/tests/litellm/llms/azure/test_azure_common_utils.py index 03d9d252198..34f0dc3a973 100644 --- a/tests/litellm/llms/azure/test_azure_common_utils.py +++ b/tests/litellm/llms/azure/test_azure_common_utils.py @@ -395,6 +395,7 @@ def test_select_azure_base_url_called(setup_mocks): "acancel_fine_tuning_job", "alist_fine_tuning_jobs", "aretrieve_fine_tuning_job", + "afile_list", ] ], )