From da72a812c9dd57a4acef5fbcbe378e4101bd843b Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 14:23:34 -0700 Subject: [PATCH 01/16] test_bedrock_passthrough_router --- ...odel_prices_and_context_window_backup.json | 8 +-- .../test_bedrock_completion.py | 54 +++++++++++++++++++ 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index caca89d23f8..03a3399c717 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -21896,8 +21896,8 @@ "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_batches": 3.75e-05, @@ -21913,8 +21913,8 @@ "input_cost_per_token_batches": 7.5e-06, "litellm_provider": "vertex_ai-anthropic_models", "max_input_tokens": 200000, - "max_output_tokens": 4096, - "max_tokens": 4096, + "max_output_tokens": 32000, + "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 7.5e-05, "output_cost_per_token_batches": 3.75e-05, diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index d41448727d5..81593fb3f41 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -3232,6 +3232,60 @@ async def test_bedrock_passthrough(sync_mode: bool): assert response.status_code == 200 +@pytest.mark.asyncio +async def test_bedrock_passthrough_router(): + """ + Test bedrock passthrough using litellm.Router with async mode. + Tests that the router: + 1. Resolves the router model name to the actual deployment + 2. Replaces the router model name in the endpoint with the actual deployment model + """ + import litellm + from litellm import Router + + litellm._turn_on_debug() + + router = Router( + model_list=[ + { + "model_name": "special-bedrock-model", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + }, + } + ] + ) + + data = { + "max_tokens": 512, + "messages": [{"role": "user", "content": "Hey"}], + "system": [ + { + "type": "text", + "text": "Analyze if this message indicates a new conversation topic. If it does, extract a 2-3 word title that captures the new topic. Format your response as a JSON object with two fields: 'isNewTopic' (boolean) and 'title' (string, or null if isNewTopic is false). Only include these fields, no other text.", + } + ], + "temperature": 0, + "metadata": { + "user_id": "5dd07c33da27e6d2968d94ea20bf47a7b090b6b158b82328d54da2909a108e84" + }, + "anthropic_version": "bedrock-2023-05-31", + "anthropic_beta": ["claude-code-20250219"], + } + + # Endpoint uses the router model name which should be replaced with actual deployment + response = await router.allm_passthrough_route( + model="special-bedrock-model", + method="POST", + endpoint="/model/special-bedrock-model/invoke", + data=data, + ) + + print(response.text) + + assert response.status_code == 200 + + @pytest.mark.asyncio async def test_bedrock_converse__streaming_passthrough(monkeypatch): import litellm From 74a35914c626b38869047efa640a6c273053eb2a Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 14:31:13 -0700 Subject: [PATCH 02/16] _add_deployment_model_to_endpoint_for_llm_passthrough_route --- litellm/router.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/litellm/router.py b/litellm/router.py index b136dcbae83..fbf8a1aa413 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2740,6 +2740,21 @@ class Router: ) ) raise e + + def _add_deployment_model_to_endpoint_for_llm_passthrough_route( + self, kwargs: Dict[str, Any], + model: str, + model_name: str + ) -> Dict[str, Any]: + """ + Add the deployment model to the endpoint for LLM passthrough route. + + e.g for bedrock invoke users can pass endpoint as /model/special-bedrock-model/invoke + it should be actually sent as /model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke + """ + if "endpoint" in kwargs and kwargs["endpoint"]: + kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) + return kwargs async def _ageneric_api_call_with_fallbacks_helper( self, model: str, original_generic_function: Callable, **kwargs @@ -2772,6 +2787,7 @@ class Router: model_name = data["model"] self.total_calls[model_name] += 1 + self._add_deployment_model_to_endpoint_for_llm_passthrough_route(kwargs=kwargs, model=model, model_name=model_name) ### get custom response = original_generic_function( **{ @@ -2850,6 +2866,12 @@ class Router: self.total_calls[model_name] += 1 + # For passthrough routes, use the actual model from deployment + # and swap model name in endpoint if present + if "endpoint" in kwargs and kwargs["endpoint"]: + kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) + kwargs["model"] = model_name + # Perform pre-call checks for routing strategy self.routing_strategy_pre_call_checks(deployment=deployment) From 902cf69ffd951152de45fddaf8f7f1e6c3a5aeb5 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 14:59:00 -0700 Subject: [PATCH 03/16] fixes for async pass throughs --- litellm/passthrough/main.py | 114 ++++++++++++++++++++++++++---------- 1 file changed, 82 insertions(+), 32 deletions(-) diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index b4a76822022..207dd5667d6 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -54,12 +54,7 @@ async def allm_passthrough_route( cookies: Optional[CookieTypes] = None, client: Optional[Union[HTTPHandler, AsyncHTTPHandler]] = None, **kwargs, -) -> Union[ - httpx.Response, - Coroutine[Any, Any, httpx.Response], - Generator[Any, Any, Any], - AsyncGenerator[Any, Any], -]: +) -> Union[httpx.Response, AsyncGenerator[Any, Any]]: """ Async: Reranks a list of documents based on their relevance to the query """ @@ -111,20 +106,23 @@ async def allm_passthrough_route( func_with_context = partial(ctx.run, func) init_response = await loop.run_in_executor(None, func_with_context) + # Since allm_passthrough_route=True, we always get a coroutine from _async_passthrough_request if asyncio.iscoroutine(init_response): response = await init_response - try: - response.raise_for_status() - except httpx.HTTPStatusError as e: - error_text = await e.response.aread() - error_text_str = error_text.decode("utf-8") - raise Exception(error_text_str) - + # Only call raise_for_status if it's a Response object (not a generator) + if isinstance(response, httpx.Response): + try: + response.raise_for_status() + except httpx.HTTPStatusError as e: + error_text = await e.response.aread() + error_text_str = error_text.decode("utf-8") + raise Exception(error_text_str) + + return response else: - response = init_response - - return response + # This shouldn't happen when allm_passthrough_route=True, but handle it for type safety + raise Exception("Expected coroutine from async passthrough route") except Exception as e: # For passthrough routes, we need to get the provider config to properly handle errors @@ -186,6 +184,7 @@ def llm_passthrough_route( ) -> Union[ httpx.Response, Coroutine[Any, Any, httpx.Response], + Coroutine[Any, Any, Union[httpx.Response, AsyncGenerator[Any, Any]]], Generator[Any, Any, Any], AsyncGenerator[Any, Any], ]: @@ -200,8 +199,10 @@ def llm_passthrough_route( from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager + _is_async = allm_passthrough_route + if client is None: - if allm_passthrough_route: + if _is_async: client = litellm.module_level_aclient else: client = litellm.module_level_client @@ -302,24 +303,40 @@ def llm_passthrough_route( # Update logging object with streaming status litellm_logging_obj.stream = is_streaming_request + ## LOGGING PRE-CALL + request_data = data if data else json + litellm_logging_obj.pre_call( + input=request_data, + api_key=provider_api_key, + additional_args={ + "complete_input_dict": request_data, + "api_base": str(updated_url), + "headers": headers, + }, + ) + try: - response = client.client.send(request=request, stream=is_streaming_request) - if asyncio.iscoroutine(response): - if is_streaming_request: - return _async_streaming(response, litellm_logging_obj, provider_config) - else: - return response - response.raise_for_status() - - if ( - hasattr(response, "iter_bytes") and is_streaming_request - ): # yield the chunk, so we can store it in the logging object - - return _sync_streaming(response, litellm_logging_obj, provider_config) + if _is_async: + # Return the coroutine to be awaited by the caller + return _async_passthrough_request( + client=client, + request=request, + is_streaming_request=is_streaming_request, + litellm_logging_obj=litellm_logging_obj, + provider_config=provider_config, + ) else: + # Sync path - client.client.send returns Response directly + response: httpx.Response = client.client.send(request=request, stream=is_streaming_request) # type: ignore + response.raise_for_status() - # For non-streaming responses, yield the entire response - return response + if ( + hasattr(response, "iter_bytes") and is_streaming_request + ): # yield the chunk, so we can store it in the logging object + return _sync_streaming(response, litellm_logging_obj, provider_config) + else: + # For non-streaming responses, yield the entire response + return response except Exception as e: if provider_config is None: raise e @@ -329,6 +346,39 @@ def llm_passthrough_route( ) +async def _async_passthrough_request( + client: Union[HTTPHandler, AsyncHTTPHandler], + request: httpx.Request, + is_streaming_request: bool, + litellm_logging_obj: "LiteLLMLoggingObj", + provider_config: "BasePassthroughConfig", +) -> Union[httpx.Response, AsyncGenerator[Any, Any]]: + """ + Handle async passthrough requests. + Uses async client to send request and properly handles streaming. + """ + # client.client.send returns a coroutine for async clients + response_result = client.client.send(request=request, stream=is_streaming_request) + + # Check if it's a coroutine and await it + if asyncio.iscoroutine(response_result): + if is_streaming_request: + # Pass the coroutine to _async_streaming which will await it + return _async_streaming( + response=response_result, + litellm_logging_obj=litellm_logging_obj, + provider_config=provider_config, + ) + else: + response = await response_result + await response.aread() + response.raise_for_status() + return response + else: + # Fallback for sync-like behavior (shouldn't happen in async path) + raise Exception("Expected coroutine from async client") + + def _sync_streaming( response: httpx.Response, litellm_logging_obj: "LiteLLMLoggingObj", From e5956ff0d48046973b74e51d923b8378c73b2ff8 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 15:08:06 -0700 Subject: [PATCH 04/16] handle_bedrock_passthrough_router_model --- .../llm_passthrough_endpoints.py | 133 +++++++++++++++++- 1 file changed, 128 insertions(+), 5 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index d07bfbb11ae..2361a3ae6e5 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -8,7 +8,7 @@ Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. import json import os -from typing import Optional, cast +from typing import Any, AsyncGenerator, Optional, Union, cast import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket @@ -482,6 +482,97 @@ async def anthropic_proxy_route( return received_value +async def handle_bedrock_passthrough_router_model( + model: str, + endpoint: str, + request: Request, + request_body: dict, + llm_router: litellm.Router, +) -> Union[Response, StreamingResponse]: + """ + Handle Bedrock passthrough for router models (models defined in config.yaml). + + This helper delegates to llm_router.allm_passthrough_route for proper credential + and configuration management from the router. + + Args: + model: The router model name (e.g., "aws/anthropic/bedrock-claude-3-5-sonnet-v1") + endpoint: The Bedrock endpoint path (e.g., "/model/{modelId}/invoke") + request: The FastAPI request object + request_body: The parsed request body + llm_router: The LiteLLM router instance + + Returns: + Response or StreamingResponse depending on endpoint type + """ + # Detect streaming based on endpoint + BEDROCK_STREAMING_ENDPOINTS = ["invoke-with-response-stream", "converse-stream"] + is_streaming = False + if any(route in endpoint for route in BEDROCK_STREAMING_ENDPOINTS): + is_streaming = True + + verbose_proxy_logger.debug( + f"Bedrock router passthrough: model='{model}', endpoint='{endpoint}', streaming={is_streaming}" + ) + + # Call router passthrough + result = await llm_router.allm_passthrough_route( + model=model, + method=request.method, + endpoint=endpoint, + request_query_params=request.query_params, + request_headers=dict(request.headers), + stream=is_streaming, + content=None, + data=None, + files=None, + json=( + request_body + if request.headers.get("content-type") == "application/json" + else None + ), + params=None, + headers=None, + cookies=None, + ) + + # Handle streaming response + if is_streaming: + import inspect + + if inspect.isasyncgen(result): + # AsyncGenerator case + return StreamingResponse( + content=result, + status_code=200, + headers={"content-type": "application/vnd.amazon.eventstream"}, + ) + else: + # httpx.Response case + result = cast(httpx.Response, result) + return StreamingResponse( + content=result.aiter_bytes(), + status_code=result.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=result.headers, + custom_headers=None, + ), + ) + + # Handle non-streaming response + result = cast(httpx.Response, result) + content = await result.aread() + + return Response( + content=content, + status_code=result.status_code, + headers=HttpPassThroughEndpointHelpers.get_response_headers( + headers=result.headers, + custom_headers=None, + ), + ) + + async def handle_bedrock_count_tokens( endpoint: str, request: Request, @@ -560,6 +651,15 @@ async def bedrock_llm_proxy_route( ): """ Handles Bedrock LLM API calls. + + Supports both direct Bedrock models and router models from config.yaml. + + Endpoints: + - /model/{modelId}/invoke + - /model/{modelId}/invoke-with-response-stream + - /model/{modelId}/converse + - /model/{modelId}/converse-stream + - /model/application-inference-profile/{profileId}/{action} """ from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing from litellm.proxy.proxy_server import ( @@ -588,24 +688,47 @@ async def bedrock_llm_proxy_route( request_body=request_body, ) - data: Dict[str, Any] = {} - base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + # Extract model from endpoint path try: endpoint_parts = endpoint.split("/") if "application-inference-profile" in endpoint: - # For application-inference-profile, include the profile ID part as well + # Format: model/application-inference-profile/{profile-id}/{action} model = "/".join(endpoint_parts[1:3]) else: + # Format: model/{modelId}/{action} model = endpoint_parts[1] except Exception: raise HTTPException( status_code=400, detail={ - "error": "Model missing from endpoint. Expected format: /model//. Got: " + "error": "Model missing from endpoint. Expected format: /model/{modelId}/{action}. Got: " + endpoint, }, ) + # Check if this is a router model (from config.yaml) + is_router_model = is_passthrough_request_using_router_model( + request_body={"model": model}, llm_router=llm_router + ) + + # If router model, use dedicated router passthrough handler + if is_router_model and llm_router: + return await handle_bedrock_passthrough_router_model( + model=model, + endpoint=endpoint, + request=request, + request_body=request_body, + llm_router=llm_router, + ) + + # Fall back to existing implementation for direct Bedrock models + verbose_proxy_logger.debug( + f"Bedrock passthrough: Using direct Bedrock model '{model}' for endpoint '{endpoint}'" + ) + + data: Dict[str, Any] = {} + base_llm_response_processor = ProxyBaseLLMRequestProcessing(data=data) + data["method"] = request.method data["endpoint"] = endpoint data["data"] = request_body From cc5eac496541f64ba1efde6ef6a529a8357c2020 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 15:10:54 -0700 Subject: [PATCH 05/16] fix _add_deployment_model_to_endpoint_for_llm_passthrough_route --- litellm/proxy/proxy_config.yaml | 7 ++- litellm/router.py | 18 +++++++- tests/test_litellm/test_router.py | 71 +++++++++++++++++++++++++++++++ 3 files changed, 93 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 0cf9556d909..14befaf2ccf 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -2,5 +2,8 @@ model_list: - model_name: mistral/* litellm_params: model: mistral/* - - + - model_name: special-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock \ No newline at end of file diff --git a/litellm/router.py b/litellm/router.py index fbf8a1aa413..1e7acd01db0 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -2753,7 +2753,23 @@ class Router: it should be actually sent as /model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke """ if "endpoint" in kwargs and kwargs["endpoint"]: - kwargs["endpoint"] = kwargs["endpoint"].replace(model, model_name) + # For provider-specific endpoints, strip the provider prefix from model_name + # e.g., "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0" -> "us.anthropic.claude-3-5-sonnet-20240620-v1:0" + from litellm import get_llm_provider + + try: + # get_llm_provider returns (model_without_prefix, provider, api_key, api_base) + stripped_model_name, _, _, _ = get_llm_provider( + model=model_name, + custom_llm_provider=kwargs.get("custom_llm_provider"), + api_base=kwargs.get("api_base"), + ) + replacement_model_name = stripped_model_name + except Exception: + # If get_llm_provider fails, fall back to using model_name as-is + replacement_model_name = model_name + + kwargs["endpoint"] = kwargs["endpoint"].replace(model, replacement_model_name) return kwargs async def _ageneric_api_call_with_fallbacks_helper( diff --git a/tests/test_litellm/test_router.py b/tests/test_litellm/test_router.py index 877a9d069e7..3e825e935d1 100644 --- a/tests/test_litellm/test_router.py +++ b/tests/test_litellm/test_router.py @@ -1548,3 +1548,74 @@ def test_get_deployment_model_info_base_model_merge_priority(): assert result["key"] == "gpt-4" print("✓ Base model merge priority test passed!") + + +def test_add_deployment_model_to_endpoint_for_llm_passthrough_route(): + """ + Test that _add_deployment_model_to_endpoint_for_llm_passthrough_route correctly strips bedrock provider prefix + """ + router = litellm.Router( + model_list=[ + { + "model_name": "special-bedrock-model", + "litellm_params": { + "model": "bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + }, + } + ], + ) + + # Test Case 1: Bedrock model with provider prefix - should strip "bedrock/" prefix + kwargs = { + "endpoint": "/model/special-bedrock-model/invoke", + "custom_llm_provider": "bedrock", + } + result = router._add_deployment_model_to_endpoint_for_llm_passthrough_route( + kwargs=kwargs, + model="special-bedrock-model", + model_name="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + ) + assert ( + result["endpoint"] == "/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke" + ), f"Expected '/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke', got '{result['endpoint']}'" + + # Test Case 2: Bedrock invoke-with-response-stream endpoint + kwargs = { + "endpoint": "/model/special-bedrock-model/invoke-with-response-stream", + "custom_llm_provider": "bedrock", + } + result = router._add_deployment_model_to_endpoint_for_llm_passthrough_route( + kwargs=kwargs, + model="special-bedrock-model", + model_name="bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0", + ) + assert ( + result["endpoint"] == "/model/us.anthropic.claude-3-5-sonnet-20240620-v1:0/invoke-with-response-stream" + ), f"Expected streaming endpoint with stripped prefix, got '{result['endpoint']}'" + + # Test Case 3: Bedrock converse endpoint + kwargs = { + "endpoint": "/model/bedrock-model/converse", + "custom_llm_provider": "bedrock", + } + result = router._add_deployment_model_to_endpoint_for_llm_passthrough_route( + kwargs=kwargs, + model="bedrock-model", + model_name="bedrock/us.meta.llama3-8b-instruct-v1:0", + ) + assert ( + result["endpoint"] == "/model/us.meta.llama3-8b-instruct-v1:0/converse" + ), f"Expected '/model/us.meta.llama3-8b-instruct-v1:0/converse', got '{result['endpoint']}'" + + # Test Case 4: Bedrock provider prefix auto-detected from model_name + kwargs = { + "endpoint": "/model/router-model/invoke", + } + result = router._add_deployment_model_to_endpoint_for_llm_passthrough_route( + kwargs=kwargs, + model="router-model", + model_name="bedrock/us.meta.llama3-8b-instruct-v1:0", + ) + assert ( + result["endpoint"] == "/model/us.meta.llama3-8b-instruct-v1:0/invoke" + ), f"Expected '/model/us.meta.llama3-8b-instruct-v1:0/invoke', got '{result['endpoint']}'" From e17636d4a6fad44cd3faf1f2fb866df51dfb92a6 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 15:26:32 -0700 Subject: [PATCH 06/16] add _extract_model_from_bedrock_endpoint --- .../llm_passthrough_endpoints.py | 79 +++++++++++++++---- litellm/proxy/proxy_config.yaml | 6 +- 2 files changed, 65 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 2361a3ae6e5..c6cd29ffdeb 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -482,6 +482,63 @@ async def anthropic_proxy_route( return received_value +# Bedrock endpoint actions - consolidated list used for model extraction and streaming detection +BEDROCK_ENDPOINT_ACTIONS = { + "invoke", + "invoke-with-response-stream", + "converse", + "converse-stream", + "count_tokens", + "count-tokens", +} + +BEDROCK_STREAMING_ACTIONS = {"invoke-with-response-stream", "converse-stream"} + + +def _extract_model_from_bedrock_endpoint(endpoint: str) -> str: + """ + Extract model name from Bedrock endpoint path. + + Handles model names with slashes (e.g., aws/anthropic/bedrock-claude-3-5-sonnet-v1) + by finding the action in the endpoint and extracting everything between "model" and the action. + + Args: + endpoint: The endpoint path (e.g., "/model/aws/anthropic/model-name/invoke") + + Returns: + The extracted model name (e.g., "aws/anthropic/model-name") + + Raises: + ValueError: If model cannot be extracted from endpoint + """ + try: + endpoint_parts = endpoint.split("/") + + if "application-inference-profile" in endpoint: + # Format: model/application-inference-profile/{profile-id}/{action} + return "/".join(endpoint_parts[1:3]) + + # Format: model/{modelId}/{action} + # Find the index of the action in the endpoint parts + action_index = None + for idx, part in enumerate(endpoint_parts): + if part in BEDROCK_ENDPOINT_ACTIONS: + action_index = idx + break + + if action_index is not None and action_index > 1: + # Join all parts between "model" and the action + return "/".join(endpoint_parts[1:action_index]) + + # Fallback to taking everything after "model" if no action found + return "/".join(endpoint_parts[1:]) + + except Exception as e: + raise ValueError( + f"Model missing from endpoint. Expected format: /model/{{modelId}}/{{action}}. Got: {endpoint}" + ) from e + + async def handle_bedrock_passthrough_router_model( model: str, endpoint: str, @@ -506,10 +563,7 @@ async def handle_bedrock_passthrough_router_model( Response or StreamingResponse depending on endpoint type """ # Detect streaming based on endpoint - BEDROCK_STREAMING_ENDPOINTS = ["invoke-with-response-stream", "converse-stream"] - is_streaming = False - if any(route in endpoint for route in BEDROCK_STREAMING_ENDPOINTS): - is_streaming = True + is_streaming = any(action in endpoint for action in BEDROCK_STREAMING_ACTIONS) verbose_proxy_logger.debug( f"Bedrock router passthrough: model='{model}', endpoint='{endpoint}', streaming={is_streaming}" @@ -688,22 +742,13 @@ async def bedrock_llm_proxy_route( request_body=request_body, ) - # Extract model from endpoint path + # Extract model from endpoint path using helper try: - endpoint_parts = endpoint.split("/") - if "application-inference-profile" in endpoint: - # Format: model/application-inference-profile/{profile-id}/{action} - model = "/".join(endpoint_parts[1:3]) - else: - # Format: model/{modelId}/{action} - model = endpoint_parts[1] - except Exception: + model = _extract_model_from_bedrock_endpoint(endpoint=endpoint) + except ValueError as e: raise HTTPException( status_code=400, - detail={ - "error": "Model missing from endpoint. Expected format: /model/{modelId}/{action}. Got: " - + endpoint, - }, + detail={"error": str(e)}, ) # Check if this is a router model (from config.yaml) diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 14befaf2ccf..1d4fc0937ec 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -4,6 +4,6 @@ model_list: model: mistral/* - model_name: special-bedrock-model litellm_params: - model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 - aws_region_name: us-west-2 - custom_llm_provider: bedrock \ No newline at end of file + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock \ No newline at end of file From 51f1907e982dc79b2ad8f9475761cd65d8788577 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 15:36:51 -0700 Subject: [PATCH 07/16] test_bedrock_error_handling_returns_actual_error --- .../test_llm_pass_through_endpoints.py | 56 ++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 239f83b21ad..38d5c5cdb6d 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -18,12 +18,12 @@ import litellm from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( BaseOpenAIPassThroughHandler, RouteChecks, + bedrock_llm_proxy_route, create_pass_through_route, llm_passthrough_factory_proxy_route, - vllm_proxy_route, vertex_discovery_proxy_route, vertex_proxy_route, - bedrock_llm_proxy_route, + vllm_proxy_route, ) from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials @@ -996,6 +996,58 @@ class TestBedrockLLMProxyRoute: assert call_kwargs["model"] == "anthropic.claude-3-sonnet-20240229-v1:0" assert result == "success" + @pytest.mark.asyncio + async def test_bedrock_error_handling_returns_actual_error(self): + """ + Test that when Bedrock API returns an error, it is properly propagated to the user + instead of being returned as a generic "Internal Server Error". + """ + from fastapi import HTTPException + + from litellm.llms.base_llm.chat.transformation import BaseLLMException + from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( + handle_bedrock_passthrough_router_model, + ) + + mock_request = Mock() + mock_request.method = "POST" + mock_request.headers = {"content-type": "application/json"} + mock_request.query_params = {} + + mock_request_body = { + "messages": [ + { + "role": "user", + "content": [{"textaaa": "Hello"}] + } + ] + } + + bedrock_error_message = '{"message":"ContentBlock object at messages.0.content.0 must set one of the following keys: text, image, toolUse, toolResult, document, video."}' + + mock_llm_router = Mock() + mock_llm_router.allm_passthrough_route = AsyncMock( + side_effect=BaseLLMException( + status_code=400, + message=bedrock_error_message + ) + ) + + endpoint = "model/test-model/converse" + model = "test-model" + + with pytest.raises(HTTPException) as exc_info: + await handle_bedrock_passthrough_router_model( + model=model, + endpoint=endpoint, + request=mock_request, + request_body=mock_request_body, + llm_router=mock_llm_router, + ) + + assert exc_info.value.status_code == 400 + assert "ContentBlock object at messages.0.content.0 must set one of the following keys" in str(exc_info.value.detail) + class TestLLMPassthroughFactoryProxyRoute: @pytest.mark.asyncio From bcf53c6cebd7c3d512a9644ae96a26361ca20f9f Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 15:40:42 -0700 Subject: [PATCH 08/16] working - errors from bedrock through pass throughs --- litellm/passthrough/main.py | 13 ++-- .../llm_passthrough_endpoints.py | 59 +++++++++++++------ litellm/proxy/proxy_config.yaml | 5 ++ .../test_llm_pass_through_endpoints.py | 20 ++++--- 4 files changed, 64 insertions(+), 33 deletions(-) diff --git a/litellm/passthrough/main.py b/litellm/passthrough/main.py index 207dd5667d6..cc57ceac50e 100644 --- a/litellm/passthrough/main.py +++ b/litellm/passthrough/main.py @@ -112,20 +112,19 @@ async def allm_passthrough_route( # Only call raise_for_status if it's a Response object (not a generator) if isinstance(response, httpx.Response): - try: - response.raise_for_status() - except httpx.HTTPStatusError as e: - error_text = await e.response.aread() - error_text_str = error_text.decode("utf-8") - raise Exception(error_text_str) + response.raise_for_status() return response else: # This shouldn't happen when allm_passthrough_route=True, but handle it for type safety raise Exception("Expected coroutine from async passthrough route") + except httpx.HTTPStatusError as e: + # For HTTP errors, re-raise as-is to preserve the original error details + # The caller (e.g., proxy layer) can handle conversion to appropriate response format + raise e except Exception as e: - # For passthrough routes, we need to get the provider config to properly handle errors + # For other exceptions, use provider-specific error handling from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index c6cd29ffdeb..11fc3babd66 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -570,25 +570,46 @@ async def handle_bedrock_passthrough_router_model( ) # Call router passthrough - result = await llm_router.allm_passthrough_route( - model=model, - method=request.method, - endpoint=endpoint, - request_query_params=request.query_params, - request_headers=dict(request.headers), - stream=is_streaming, - content=None, - data=None, - files=None, - json=( - request_body - if request.headers.get("content-type") == "application/json" - else None - ), - params=None, - headers=None, - cookies=None, - ) + try: + result = await llm_router.allm_passthrough_route( + model=model, + method=request.method, + endpoint=endpoint, + request_query_params=request.query_params, + request_headers=dict(request.headers), + stream=is_streaming, + content=None, + data=None, + files=None, + json=( + request_body + if request.headers.get("content-type") == "application/json" + else None + ), + params=None, + headers=None, + cookies=None, + ) + except httpx.HTTPStatusError as e: + # Handle HTTP errors from the provider by converting to HTTPException + error_body = await e.response.aread() + error_text = error_body.decode("utf-8") + + raise HTTPException( + status_code=e.response.status_code, + detail={"error": error_text}, + ) + except Exception as e: + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + # If it's a BaseLLMException (from non-HTTP errors), convert to HTTPException + if isinstance(e, BaseLLMException): + raise HTTPException( + status_code=e.status_code, + detail={"error": e.message}, + ) + # Re-raise any other exceptions + raise e # Handle streaming response if is_streaming: diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 1d4fc0937ec..13532fbe7fe 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -3,6 +3,11 @@ model_list: litellm_params: model: mistral/* - model_name: special-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock + - model_name: aws/anthropic/bedrock-claude-3-5-sonnet-v1 litellm_params: model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 aws_region_name: us-west-2 diff --git a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 38d5c5cdb6d..6eeca946190 100644 --- a/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -1004,7 +1004,6 @@ class TestBedrockLLMProxyRoute: """ from fastapi import HTTPException - from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( handle_bedrock_passthrough_router_model, ) @@ -1025,14 +1024,21 @@ class TestBedrockLLMProxyRoute: bedrock_error_message = '{"message":"ContentBlock object at messages.0.content.0 must set one of the following keys: text, image, toolUse, toolResult, document, video."}' - mock_llm_router = Mock() - mock_llm_router.allm_passthrough_route = AsyncMock( - side_effect=BaseLLMException( - status_code=400, - message=bedrock_error_message - ) + # Create a mock httpx.Response for the error + mock_error_response = Mock(spec=httpx.Response) + mock_error_response.status_code = 400 + mock_error_response.aread = AsyncMock(return_value=bedrock_error_message.encode('utf-8')) + + # Create the HTTPStatusError + mock_http_error = httpx.HTTPStatusError( + message="Bad Request", + request=Mock(spec=httpx.Request), + response=mock_error_response, ) + mock_llm_router = Mock() + mock_llm_router.allm_passthrough_route = AsyncMock(side_effect=mock_http_error) + endpoint = "model/test-model/converse" model = "test-model" From f87c86806c64e31ae5b6750fe4525a75098a5105 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 16:13:01 -0700 Subject: [PATCH 09/16] docs invoke --- docs/my-website/docs/pass_through/bedrock.md | 203 +++++++++++++++++-- 1 file changed, 184 insertions(+), 19 deletions(-) diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md index 48502864d78..c709013f4ab 100644 --- a/docs/my-website/docs/pass_through/bedrock.md +++ b/docs/my-website/docs/pass_through/bedrock.md @@ -11,18 +11,49 @@ Pass-through endpoints for Bedrock - call provider-specific endpoint, in native Just replace `https://bedrock-runtime.{aws_region_name}.amazonaws.com` with `LITELLM_PROXY_BASE_URL/bedrock` 🚀 -#### **Example Usage** +## Overview + +LiteLLM supports two ways to call Bedrock endpoints: + +### 1. **Using config.yaml** (Recommended for model endpoints) + +Define your Bedrock models in `config.yaml` and reference them by name. The proxy handles authentication and routing. + +**Use for**: `/converse`, `/converse-stream`, `/invoke`, `/invoke-with-response-stream` + +```yaml +model_list: + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock +``` + ```bash -curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \ --H 'Authorization: Bearer anything' \ +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \ +-H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ --d '{ - "messages": [ - {"role": "user", - "content": [{"text": "Hello"}] - } - ] -}' +-d '{"messages": [{"role": "user", "content": [{"text": "Hello"}]}]}' +``` + +### 2. **Direct passthrough** (For non-model endpoints) + +Set AWS credentials via environment variables and call Bedrock endpoints directly. + +**Use for**: Guardrails, Knowledge Bases, Agents, and other non-model endpoints + +```bash +export AWS_ACCESS_KEY_ID="" +export AWS_SECRET_ACCESS_KEY="" +export AWS_REGION_NAME="us-west-2" +``` + +```bash +curl "http://0.0.0.0:4000/bedrock/guardrail/my-guardrail-id/version/1/apply" \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{"contents": [{"text": {"text": "Hello"}}], "source": "INPUT"}' ``` Supports **ALL** Bedrock Endpoints (including streaming). @@ -33,39 +64,139 @@ Supports **ALL** Bedrock Endpoints (including streaming). Let's call the Bedrock [`/converse` endpoint](https://docs.aws.amazon.com/bedrock/latest/APIReference/API_runtime_Converse.html) -1. Add AWS Keys to your environment +1. Create a `config.yaml` file with your Bedrock model + +```yaml +model_list: + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock +``` + +Set your AWS credentials: ```bash export AWS_ACCESS_KEY_ID="" # Access key export AWS_SECRET_ACCESS_KEY="" # Secret access key -export AWS_REGION_NAME="" # us-east-1, us-east-2, us-west-1, us-west-2 ``` 2. Start LiteLLM Proxy ```bash -litellm +litellm --config config.yaml # RUNNING on http://0.0.0.0:4000 ``` 3. Test it! -Let's call the Bedrock converse endpoint +Let's call the Bedrock converse endpoint using the model name from config: ```bash -curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \ --H 'Authorization: Bearer anything' \ +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \ +-H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ -d '{ "messages": [ - {"role": "user", - "content": [{"text": "Hello"}] + { + "role": "user", + "content": [{"text": "Hello, how are you?"}] + } + ], + "inferenceConfig": { + "maxTokens": 100 } - ] }' ``` +## Setup with config.yaml + +Use config.yaml to define Bedrock models and use them via passthrough endpoints. + +### 1. Define models in config.yaml + +```yaml +model_list: + - model_name: my-claude-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock + + - model_name: my-cohere-model + litellm_params: + model: bedrock/cohere.command-r-v1:0 + aws_region_name: us-east-1 + custom_llm_provider: bedrock +``` + +### 2. Start proxy with config + +```bash +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### 3. Call Bedrock Converse endpoint + +Use the `model_name` from config in the URL path: + +```bash +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/converse' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "messages": [ + { + "role": "user", + "content": [{"text": "Hello, how are you?"}] + } + ], + "inferenceConfig": { + "temperature": 0.5, + "maxTokens": 100 + } +}' +``` + +### 4. Call Bedrock Converse Stream endpoint + +For streaming responses, use the `/converse-stream` endpoint: + +```bash +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/converse-stream' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "messages": [ + { + "role": "user", + "content": [{"text": "Tell me a short story"}] + } + ], + "inferenceConfig": { + "temperature": 0.7, + "maxTokens": 200 + } +}' +``` + +### Supported Bedrock Endpoints with config.yaml + +When using models from config.yaml, you can call any Bedrock endpoint: + +| Endpoint | Description | Example | +|----------|-------------|---------| +| `/model/{model_name}/converse` | Converse API | `http://0.0.0.0:4000/bedrock/model/my-claude-model/converse` | +| `/model/{model_name}/converse-stream` | Streaming Converse | `http://0.0.0.0:4000/bedrock/model/my-claude-model/converse-stream` | +| `/model/{model_name}/invoke` | Legacy Invoke API | `http://0.0.0.0:4000/bedrock/model/my-claude-model/invoke` | +| `/model/{model_name}/invoke-with-response-stream` | Legacy Streaming | `http://0.0.0.0:4000/bedrock/model/my-claude-model/invoke-with-response-stream` | + +The proxy automatically resolves the `model_name` to the actual Bedrock model ID and region configured in your `config.yaml`. + ## Examples @@ -114,6 +245,22 @@ curl -X POST 'https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.comma ### **Example 2: Apply Guardrail** +**Setup**: Set AWS credentials for direct passthrough + +```bash +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +export AWS_REGION_NAME="us-west-2" +``` + +Start proxy: + +```bash +litellm + +# RUNNING on http://0.0.0.0:4000 +``` + #### LiteLLM Proxy Call ```bash @@ -142,6 +289,24 @@ curl "https://bedrock-runtime.us-west-2.amazonaws.com/guardrail/guardrailIdentif ### **Example 3: Query Knowledge Base** +**Setup**: Set AWS credentials for direct passthrough + +```bash +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +export AWS_REGION_NAME="us-west-2" +``` + +Start proxy: + +```bash +litellm + +# RUNNING on http://0.0.0.0:4000 +``` + +#### LiteLLM Proxy Call + ```bash curl -X POST "http://0.0.0.0:4000/bedrock/knowledgebases/{knowledgeBaseId}/retrieve" \ -H 'Authorization: Bearer sk-anything' \ From c2b69e371c64363d4e71424f4b5cf8440415ce20 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 16:13:08 -0700 Subject: [PATCH 10/16] docs pass through --- docs/my-website/docs/pass_through/bedrock.md | 24 ++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md index c709013f4ab..70b0ec16d5e 100644 --- a/docs/my-website/docs/pass_through/bedrock.md +++ b/docs/my-website/docs/pass_through/bedrock.md @@ -411,25 +411,41 @@ curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' Call Bedrock Agents via LiteLLM proxy +**Setup**: Set AWS credentials on your LiteLLM proxy server + +```bash +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +export AWS_REGION_NAME="us-west-2" +``` + +Start proxy: + +```bash +litellm + +# RUNNING on http://0.0.0.0:4000 +``` + +**Usage from Python**: + ```python import os import boto3 from botocore.config import Config -# # Define your proxy endpoint +# Define your proxy endpoint proxy_endpoint = "http://0.0.0.0:4000/bedrock" # 👈 your proxy base url -# # Create a Config object with the proxy # Custom headers custom_headers = { 'litellm_user_api_key': 'Bearer sk-1234', # 👈 your proxy api key } - +# Use fake credentials in client (proxy handles real auth) os.environ["AWS_ACCESS_KEY_ID"] = "my-fake-key-id" os.environ["AWS_SECRET_ACCESS_KEY"] = "my-fake-access-key" - # Create the client runtime_client = boto3.client( service_name="bedrock-agent-runtime", From fe5dd23a847157ff6558ab3986d7aa98e11ee0ff Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 16:17:02 -0700 Subject: [PATCH 11/16] docs boto3 pass thoughs --- docs/my-website/docs/pass_through/bedrock.md | 107 ++++++++++++++++++- litellm/proxy/proxy_config.yaml | 11 ++ 2 files changed, 113 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md index 70b0ec16d5e..412e523f4f0 100644 --- a/docs/my-website/docs/pass_through/bedrock.md +++ b/docs/my-website/docs/pass_through/bedrock.md @@ -5,7 +5,7 @@ Pass-through endpoints for Bedrock - call provider-specific endpoint, in native | Feature | Supported | Notes | |-------|-------|-------| | Cost Tracking | ✅ | For `/invoke` and `/converse` endpoints | -| Logging | ✅ | works across all integrations | +| Load Balancing | ✅ | You can load balance `/invoke`, `/converse` routes across multiple deployments| Logging | ✅ | works across all integrations | | End-user Tracking | ❌ | [Tell us if you need this](https://github.com/BerriAI/litellm/issues/new) | | Streaming | ✅ | | @@ -21,7 +21,7 @@ Define your Bedrock models in `config.yaml` and reference them by name. The prox **Use for**: `/converse`, `/converse-stream`, `/invoke`, `/invoke-with-response-stream` -```yaml +```yaml showLineNumbers model_list: - model_name: my-bedrock-model litellm_params: @@ -30,7 +30,7 @@ model_list: custom_llm_provider: bedrock ``` -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ @@ -43,13 +43,13 @@ Set AWS credentials via environment variables and call Bedrock endpoints directl **Use for**: Guardrails, Knowledge Bases, Agents, and other non-model endpoints -```bash +```bash showLineNumbers export AWS_ACCESS_KEY_ID="" export AWS_SECRET_ACCESS_KEY="" export AWS_REGION_NAME="us-west-2" ``` -```bash +```bash showLineNumbers curl "http://0.0.0.0:4000/bedrock/guardrail/my-guardrail-id/version/1/apply" \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ @@ -197,6 +197,103 @@ When using models from config.yaml, you can call any Bedrock endpoint: The proxy automatically resolves the `model_name` to the actual Bedrock model ID and region configured in your `config.yaml`. +### Load Balancing Across Multiple Deployments + +Define multiple Bedrock deployments with the same `model_name` to enable automatic load balancing. + +#### 1. Define multiple deployments in config.yaml + +```yaml showLineNumbers +model_list: + # First deployment - us-west-2 + - model_name: my-claude-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock + + # Second deployment - us-east-1 (load balanced) + - model_name: my-claude-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-east-1 + custom_llm_provider: bedrock +``` + +#### 2. Start proxy with config + +```bash showLineNumbers +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +#### 3. Call the endpoint - requests are automatically load balanced + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/invoke' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "anthropic_version": "bedrock-2023-05-31" +}' +``` + +The proxy will automatically distribute requests across both `us-west-2` and `us-east-1` deployments. This works for all Bedrock endpoints: `/invoke`, `/invoke-with-response-stream`, `/converse`, and `/converse-stream`. + +#### Using boto3 SDK with load balancing + +You can also call the load-balanced endpoint using the boto3 SDK: + +```python showLineNumbers +import boto3 +import json + +# Point boto3 to the LiteLLM proxy +bedrock_runtime = boto3.client( + service_name='bedrock-runtime', + region_name='us-west-2', # Can be any region + endpoint_url='http://0.0.0.0:4000/bedrock' +) + +# Custom header for authentication +def add_custom_headers(request, **kwargs): + request.headers.update({'litellm_user_api_key': 'Bearer sk-1234'}) + +# Register the event to inject headers before sending request +bedrock_runtime.meta.events.register('before-send.*.*', add_custom_headers) + +# Call the load-balanced model +response = bedrock_runtime.invoke_model( + modelId='my-claude-model', # Your model_name from config.yaml + contentType='application/json', + accept='application/json', + body=json.dumps({ + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "anthropic_version": "bedrock-2023-05-31" + }) +) + +# Parse response +response_body = json.loads(response['body'].read()) +print(response_body['content'][0]['text']) +``` + +The proxy will automatically load balance your boto3 requests across all configured deployments. + ## Examples diff --git a/litellm/proxy/proxy_config.yaml b/litellm/proxy/proxy_config.yaml index 13532fbe7fe..cca136811f7 100644 --- a/litellm/proxy/proxy_config.yaml +++ b/litellm/proxy/proxy_config.yaml @@ -11,4 +11,15 @@ model_list: litellm_params: model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 aws_region_name: us-west-2 + custom_llm_provider: bedrock + # Load balancing test - multiple deployments with same model_name + - model_name: load-balanced-claude + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + custom_llm_provider: bedrock + - model_name: load-balanced-claude + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-east-1 custom_llm_provider: bedrock \ No newline at end of file From 8c5d6891d4be75891d376b7a6b307dcd3443f8ce Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 16:18:13 -0700 Subject: [PATCH 12/16] docs pt --- docs/my-website/docs/pass_through/bedrock.md | 52 ++++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md index 412e523f4f0..8edb42f672a 100644 --- a/docs/my-website/docs/pass_through/bedrock.md +++ b/docs/my-website/docs/pass_through/bedrock.md @@ -66,7 +66,7 @@ Let's call the Bedrock [`/converse` endpoint](https://docs.aws.amazon.com/bedroc 1. Create a `config.yaml` file with your Bedrock model -```yaml +```yaml showLineNumbers model_list: - model_name: my-bedrock-model litellm_params: @@ -77,14 +77,14 @@ model_list: Set your AWS credentials: -```bash +```bash showLineNumbers export AWS_ACCESS_KEY_ID="" # Access key export AWS_SECRET_ACCESS_KEY="" # Secret access key ``` 2. Start LiteLLM Proxy -```bash +```bash showLineNumbers litellm --config config.yaml # RUNNING on http://0.0.0.0:4000 @@ -94,7 +94,7 @@ litellm --config config.yaml Let's call the Bedrock converse endpoint using the model name from config: -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ @@ -117,7 +117,7 @@ Use config.yaml to define Bedrock models and use them via passthrough endpoints. ### 1. Define models in config.yaml -```yaml +```yaml showLineNumbers model_list: - model_name: my-claude-model litellm_params: @@ -134,7 +134,7 @@ model_list: ### 2. Start proxy with config -```bash +```bash showLineNumbers litellm --config config.yaml # RUNNING on http://0.0.0.0:4000 @@ -144,7 +144,7 @@ litellm --config config.yaml Use the `model_name` from config in the URL path: -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/converse' \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ @@ -166,7 +166,7 @@ curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/converse' \ For streaming responses, use the `/converse-stream` endpoint: -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-claude-model/converse-stream' \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ @@ -312,7 +312,7 @@ Key Changes: #### LiteLLM Proxy Call -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \ -H 'Authorization: Bearer sk-anything' \ -H 'Content-Type: application/json' \ @@ -327,7 +327,7 @@ curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' #### Direct Bedrock API Call -```bash +```bash showLineNumbers curl -X POST 'https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.command-r-v1:0/converse' \ -H 'Authorization: AWS4-HMAC-SHA256..' \ -H 'Content-Type: application/json' \ @@ -344,7 +344,7 @@ curl -X POST 'https://bedrock-runtime.us-west-2.amazonaws.com/model/cohere.comma **Setup**: Set AWS credentials for direct passthrough -```bash +```bash showLineNumbers export AWS_ACCESS_KEY_ID="your-access-key" export AWS_SECRET_ACCESS_KEY="your-secret-key" export AWS_REGION_NAME="us-west-2" @@ -352,7 +352,7 @@ export AWS_REGION_NAME="us-west-2" Start proxy: -```bash +```bash showLineNumbers litellm # RUNNING on http://0.0.0.0:4000 @@ -360,7 +360,7 @@ litellm #### LiteLLM Proxy Call -```bash +```bash showLineNumbers curl "http://0.0.0.0:4000/bedrock/guardrail/guardrailIdentifier/version/guardrailVersion/apply" \ -H 'Authorization: Bearer sk-anything' \ -H 'Content-Type: application/json' \ @@ -373,7 +373,7 @@ curl "http://0.0.0.0:4000/bedrock/guardrail/guardrailIdentifier/version/guardrai #### Direct Bedrock API Call -```bash +```bash showLineNumbers curl "https://bedrock-runtime.us-west-2.amazonaws.com/guardrail/guardrailIdentifier/version/guardrailVersion/apply" \ -H 'Authorization: AWS4-HMAC-SHA256..' \ -H 'Content-Type: application/json' \ @@ -388,7 +388,7 @@ curl "https://bedrock-runtime.us-west-2.amazonaws.com/guardrail/guardrailIdentif **Setup**: Set AWS credentials for direct passthrough -```bash +```bash showLineNumbers export AWS_ACCESS_KEY_ID="your-access-key" export AWS_SECRET_ACCESS_KEY="your-secret-key" export AWS_REGION_NAME="us-west-2" @@ -396,7 +396,7 @@ export AWS_REGION_NAME="us-west-2" Start proxy: -```bash +```bash showLineNumbers litellm # RUNNING on http://0.0.0.0:4000 @@ -404,7 +404,7 @@ litellm #### LiteLLM Proxy Call -```bash +```bash showLineNumbers curl -X POST "http://0.0.0.0:4000/bedrock/knowledgebases/{knowledgeBaseId}/retrieve" \ -H 'Authorization: Bearer sk-anything' \ -H 'Content-Type: application/json' \ @@ -425,7 +425,7 @@ curl -X POST "http://0.0.0.0:4000/bedrock/knowledgebases/{knowledgeBaseId}/retri #### Direct Bedrock API Call -```bash +```bash showLineNumbers curl -X POST "https://bedrock-agent-runtime.us-west-2.amazonaws.com/knowledgebases/{knowledgeBaseId}/retrieve" \ -H 'Authorization: AWS4-HMAC-SHA256..' \ -H 'Content-Type: application/json' \ @@ -456,7 +456,7 @@ Use this, to avoid giving developers the raw AWS Keys, but still letting them us 1. Setup environment -```bash +```bash showLineNumbers export DATABASE_URL="" export LITELLM_MASTER_KEY="" export AWS_ACCESS_KEY_ID="" # Access key @@ -464,7 +464,7 @@ export AWS_SECRET_ACCESS_KEY="" # Secret access key export AWS_REGION_NAME="" # us-east-1, us-east-2, us-west-1, us-west-2 ``` -```bash +```bash showLineNumbers litellm # RUNNING on http://0.0.0.0:4000 @@ -472,7 +472,7 @@ litellm 2. Generate virtual key -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/key/generate' \ -H 'Authorization: Bearer sk-1234' \ -H 'Content-Type: application/json' \ @@ -481,7 +481,7 @@ curl -X POST 'http://0.0.0.0:4000/key/generate' \ Expected Response -```bash +```bash showLineNumbers { ... "key": "sk-1234ewknldferwedojwojw" @@ -491,7 +491,7 @@ Expected Response 3. Test it! -```bash +```bash showLineNumbers curl -X POST 'http://0.0.0.0:4000/bedrock/model/cohere.command-r-v1:0/converse' \ -H 'Authorization: Bearer sk-1234ewknldferwedojwojw' \ -H 'Content-Type: application/json' \ @@ -510,7 +510,7 @@ Call Bedrock Agents via LiteLLM proxy **Setup**: Set AWS credentials on your LiteLLM proxy server -```bash +```bash showLineNumbers export AWS_ACCESS_KEY_ID="your-access-key" export AWS_SECRET_ACCESS_KEY="your-secret-key" export AWS_REGION_NAME="us-west-2" @@ -518,7 +518,7 @@ export AWS_REGION_NAME="us-west-2" Start proxy: -```bash +```bash showLineNumbers litellm # RUNNING on http://0.0.0.0:4000 @@ -526,7 +526,7 @@ litellm **Usage from Python**: -```python +```python showLineNumbers import os import boto3 from botocore.config import Config From 79516563a9d3a2a5b81d636eaadf45c4f0a04e80 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 16:25:27 -0700 Subject: [PATCH 13/16] docs add /invoke and /converse routes --- docs/my-website/docs/bedrock_converse.md | 149 +++++++++++++++++++++++ docs/my-website/docs/bedrock_invoke.md | 143 ++++++++++++++++++++++ docs/my-website/sidebars.js | 2 + 3 files changed, 294 insertions(+) create mode 100644 docs/my-website/docs/bedrock_converse.md create mode 100644 docs/my-website/docs/bedrock_invoke.md diff --git a/docs/my-website/docs/bedrock_converse.md b/docs/my-website/docs/bedrock_converse.md new file mode 100644 index 00000000000..4458b57a51d --- /dev/null +++ b/docs/my-website/docs/bedrock_converse.md @@ -0,0 +1,149 @@ +# /converse + +Call Bedrock's `/converse` endpoint through LiteLLM Proxy. + +| Feature | Supported | +|---------|-----------| +| Cost Tracking | ✅ | +| Logging | ✅ | +| Streaming | ✅ via `/converse-stream` | +| Load Balancing | ✅ | + +## Quick Start + +### 1. Setup config.yaml + +```yaml showLineNumbers +model_list: + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # reads from environment + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock +``` + +Set AWS credentials in your environment: + +```bash showLineNumbers +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +``` + +### 2. Start Proxy + +```bash showLineNumbers +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### 3. Call /converse endpoint + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "messages": [ + { + "role": "user", + "content": [{"text": "Hello, how are you?"}] + } + ], + "inferenceConfig": { + "temperature": 0.5, + "maxTokens": 100 + } +}' +``` + +## Streaming + +For streaming responses, use `/converse-stream`: + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/converse-stream' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "messages": [ + { + "role": "user", + "content": [{"text": "Tell me a short story"}] + } + ], + "inferenceConfig": { + "temperature": 0.7, + "maxTokens": 200 + } +}' +``` + +## Load Balancing + +Define multiple deployments with the same `model_name` for automatic load balancing: + +```yaml showLineNumbers +model_list: + # Deployment 1 - us-west-2 + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock + + # Deployment 2 - us-east-1 + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-east-1 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock +``` + +The proxy automatically distributes requests across both regions. + +## Using boto3 SDK + +```python showLineNumbers +import boto3 +import json + +bedrock_runtime = boto3.client( + service_name='bedrock-runtime', + region_name='us-west-2', + endpoint_url='http://0.0.0.0:4000/bedrock' +) + +def add_custom_headers(request, **kwargs): + request.headers.update({'litellm_user_api_key': 'Bearer sk-1234'}) + +bedrock_runtime.meta.events.register('before-send.*.*', add_custom_headers) + +response = bedrock_runtime.converse( + modelId='my-bedrock-model', + messages=[ + { + "role": "user", + "content": [{"text": "Hello, how are you?"}] + } + ], + inferenceConfig={ + "temperature": 0.5, + "maxTokens": 100 + } +) + +print(response['output']['message']['content'][0]['text']) +``` + +## More Info + +For complete documentation including Guardrails, Knowledge Bases, and Agents, see: +- [Full Bedrock Passthrough Docs](./pass_through/bedrock) + diff --git a/docs/my-website/docs/bedrock_invoke.md b/docs/my-website/docs/bedrock_invoke.md new file mode 100644 index 00000000000..6d86778bc47 --- /dev/null +++ b/docs/my-website/docs/bedrock_invoke.md @@ -0,0 +1,143 @@ +# /invoke + +Call Bedrock's `/invoke` endpoint through LiteLLM Proxy. + +| Feature | Supported | +|---------|-----------| +| Cost Tracking | ✅ | +| Logging | ✅ | +| Streaming | ✅ via `/invoke-with-response-stream` | +| Load Balancing | ✅ | + +## Quick Start + +### 1. Setup config.yaml + +```yaml showLineNumbers +model_list: + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID # reads from environment + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock +``` + +Set AWS credentials in your environment: + +```bash showLineNumbers +export AWS_ACCESS_KEY_ID="your-access-key" +export AWS_SECRET_ACCESS_KEY="your-secret-key" +``` + +### 2. Start Proxy + +```bash showLineNumbers +litellm --config config.yaml + +# RUNNING on http://0.0.0.0:4000 +``` + +### 3. Call /invoke endpoint + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/invoke' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Hello, how are you?" + } + ], + "anthropic_version": "bedrock-2023-05-31" +}' +``` + +## Streaming + +For streaming responses, use `/invoke-with-response-stream`: + +```bash showLineNumbers +curl -X POST 'http://0.0.0.0:4000/bedrock/model/my-bedrock-model/invoke-with-response-stream' \ +-H 'Authorization: Bearer sk-1234' \ +-H 'Content-Type: application/json' \ +-d '{ + "max_tokens": 100, + "messages": [ + { + "role": "user", + "content": "Tell me a short story" + } + ], + "anthropic_version": "bedrock-2023-05-31" +}' +``` + +## Load Balancing + +Define multiple deployments with the same `model_name` for automatic load balancing: + +```yaml showLineNumbers +model_list: + # Deployment 1 - us-west-2 + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-west-2 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock + + # Deployment 2 - us-east-1 + - model_name: my-bedrock-model + litellm_params: + model: bedrock/us.anthropic.claude-3-5-sonnet-20240620-v1:0 + aws_region_name: us-east-1 + aws_access_key_id: os.environ/AWS_ACCESS_KEY_ID + aws_secret_access_key: os.environ/AWS_SECRET_ACCESS_KEY + custom_llm_provider: bedrock +``` + +The proxy automatically distributes requests across both regions. + +## Using boto3 SDK + +```python showLineNumbers +import boto3 +import json + +bedrock_runtime = boto3.client( + service_name='bedrock-runtime', + region_name='us-west-2', + endpoint_url='http://0.0.0.0:4000/bedrock' +) + +def add_custom_headers(request, **kwargs): + request.headers.update({'litellm_user_api_key': 'Bearer sk-1234'}) + +bedrock_runtime.meta.events.register('before-send.*.*', add_custom_headers) + +response = bedrock_runtime.invoke_model( + modelId='my-bedrock-model', + contentType='application/json', + accept='application/json', + body=json.dumps({ + "max_tokens": 100, + "messages": [{"role": "user", "content": "Hello"}], + "anthropic_version": "bedrock-2023-05-31" + }) +) + +response_body = json.loads(response['body'].read()) +print(response_body['content'][0]['text']) +``` + +## More Info + +For complete documentation including Guardrails, Knowledge Bases, and Agents, see: +- [Full Bedrock Passthrough Docs](./pass_through/bedrock) + diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js index 71f730c4542..e5d1ab81ac4 100644 --- a/docs/my-website/sidebars.js +++ b/docs/my-website/sidebars.js @@ -347,6 +347,8 @@ const sidebars = { ] }, "moderation", + "bedrock_invoke", + "bedrock_converse", "ocr", { type: "category", From bbf9d6a57b9cb9483ec555268abd61f3f4127764 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 16:34:56 -0700 Subject: [PATCH 14/16] docs boto3 instructions --- docs/my-website/docs/bedrock_converse.md | 14 ++++++++------ docs/my-website/docs/bedrock_invoke.md | 14 ++++++++------ docs/my-website/docs/pass_through/bedrock.md | 15 +++++++-------- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/docs/my-website/docs/bedrock_converse.md b/docs/my-website/docs/bedrock_converse.md index 4458b57a51d..cf66b1a50a6 100644 --- a/docs/my-website/docs/bedrock_converse.md +++ b/docs/my-website/docs/bedrock_converse.md @@ -113,20 +113,22 @@ The proxy automatically distributes requests across both regions. ```python showLineNumbers import boto3 import json +import os +# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy) +os.environ['AWS_ACCESS_KEY_ID'] = 'dummy' +os.environ['AWS_SECRET_ACCESS_KEY'] = 'dummy' +os.environ['AWS_BEARER_TOKEN_BEDROCK'] = "sk-1234" # your litellm proxy api key + +# Point boto3 to the LiteLLM proxy bedrock_runtime = boto3.client( service_name='bedrock-runtime', region_name='us-west-2', endpoint_url='http://0.0.0.0:4000/bedrock' ) -def add_custom_headers(request, **kwargs): - request.headers.update({'litellm_user_api_key': 'Bearer sk-1234'}) - -bedrock_runtime.meta.events.register('before-send.*.*', add_custom_headers) - response = bedrock_runtime.converse( - modelId='my-bedrock-model', + modelId='my-bedrock-model', # Your model_name from config.yaml messages=[ { "role": "user", diff --git a/docs/my-website/docs/bedrock_invoke.md b/docs/my-website/docs/bedrock_invoke.md index 6d86778bc47..6f29f1d51c3 100644 --- a/docs/my-website/docs/bedrock_invoke.md +++ b/docs/my-website/docs/bedrock_invoke.md @@ -109,20 +109,22 @@ The proxy automatically distributes requests across both regions. ```python showLineNumbers import boto3 import json +import os +# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy) +os.environ['AWS_ACCESS_KEY_ID'] = 'dummy' +os.environ['AWS_SECRET_ACCESS_KEY'] = 'dummy' +os.environ['AWS_BEARER_TOKEN_BEDROCK'] = "sk-1234" # your litellm proxy api key + +# Point boto3 to the LiteLLM proxy bedrock_runtime = boto3.client( service_name='bedrock-runtime', region_name='us-west-2', endpoint_url='http://0.0.0.0:4000/bedrock' ) -def add_custom_headers(request, **kwargs): - request.headers.update({'litellm_user_api_key': 'Bearer sk-1234'}) - -bedrock_runtime.meta.events.register('before-send.*.*', add_custom_headers) - response = bedrock_runtime.invoke_model( - modelId='my-bedrock-model', + modelId='my-bedrock-model', # Your model_name from config.yaml contentType='application/json', accept='application/json', body=json.dumps({ diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md index 8edb42f672a..631c9a1d4e9 100644 --- a/docs/my-website/docs/pass_through/bedrock.md +++ b/docs/my-website/docs/pass_through/bedrock.md @@ -255,21 +255,20 @@ You can also call the load-balanced endpoint using the boto3 SDK: ```python showLineNumbers import boto3 import json +import os + +# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy) +os.environ['AWS_ACCESS_KEY_ID'] = 'dummy' +os.environ['AWS_SECRET_ACCESS_KEY'] = 'dummy' +os.environ['AWS_BEARER_TOKEN_BEDROCK'] = "sk-1234" # your litellm proxy api key # Point boto3 to the LiteLLM proxy bedrock_runtime = boto3.client( service_name='bedrock-runtime', - region_name='us-west-2', # Can be any region + region_name='us-west-2', endpoint_url='http://0.0.0.0:4000/bedrock' ) -# Custom header for authentication -def add_custom_headers(request, **kwargs): - request.headers.update({'litellm_user_api_key': 'Bearer sk-1234'}) - -# Register the event to inject headers before sending request -bedrock_runtime.meta.events.register('before-send.*.*', add_custom_headers) - # Call the load-balanced model response = bedrock_runtime.invoke_model( modelId='my-claude-model', # Your model_name from config.yaml From 0bef295b15d1513463975fa57ac2d13152d33713 Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 16:36:24 -0700 Subject: [PATCH 15/16] doc fix --- docs/my-website/docs/pass_through/bedrock.md | 39 ++++++-------------- 1 file changed, 11 insertions(+), 28 deletions(-) diff --git a/docs/my-website/docs/pass_through/bedrock.md b/docs/my-website/docs/pass_through/bedrock.md index 631c9a1d4e9..b8d20d77da0 100644 --- a/docs/my-website/docs/pass_through/bedrock.md +++ b/docs/my-website/docs/pass_through/bedrock.md @@ -527,42 +527,26 @@ litellm ```python showLineNumbers import os -import boto3 -from botocore.config import Config +import boto3 -# Define your proxy endpoint -proxy_endpoint = "http://0.0.0.0:4000/bedrock" # 👈 your proxy base url - -# Custom headers -custom_headers = { - 'litellm_user_api_key': 'Bearer sk-1234', # 👈 your proxy api key -} - -# Use fake credentials in client (proxy handles real auth) -os.environ["AWS_ACCESS_KEY_ID"] = "my-fake-key-id" -os.environ["AWS_SECRET_ACCESS_KEY"] = "my-fake-access-key" +# Set dummy AWS credentials (required by boto3, but not used by LiteLLM proxy) +os.environ["AWS_ACCESS_KEY_ID"] = "dummy" +os.environ["AWS_SECRET_ACCESS_KEY"] = "dummy" +os.environ["AWS_BEARER_TOKEN_BEDROCK"] = "sk-1234" # your litellm proxy api key # Create the client runtime_client = boto3.client( service_name="bedrock-agent-runtime", region_name="us-west-2", - endpoint_url=proxy_endpoint + endpoint_url="http://0.0.0.0:4000/bedrock" ) -# Custom header injection -def inject_custom_headers(request, **kwargs): - request.headers.update(custom_headers) - -# Attach the event to inject custom headers before the request is sent -runtime_client.meta.events.register('before-send.*.*', inject_custom_headers) - - response = runtime_client.invoke_agent( - agentId="L1RT58GYRW", - agentAliasId="MFPSBCXYTW", - sessionId="12345", - inputText="Who do you know?" - ) + agentId="L1RT58GYRW", + agentAliasId="MFPSBCXYTW", + sessionId="12345", + inputText="Who do you know?" +) completion = "" @@ -571,5 +555,4 @@ for event in response.get("completion"): completion += chunk["bytes"].decode() print(completion) - ``` From a31a8108b536c8dc3265da1473852c858282ca6c Mon Sep 17 00:00:00 2001 From: Ishaan Jaffer Date: Thu, 16 Oct 2025 16:45:14 -0700 Subject: [PATCH 16/16] fix pass thu ruff check --- .../proxy/pass_through_endpoints/llm_passthrough_endpoints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index 11fc3babd66..6f9f04e5cc2 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -8,7 +8,7 @@ Use litellm with Anthropic SDK, Vertex AI SDK, Cohere SDK, etc. import json import os -from typing import Any, AsyncGenerator, Optional, Union, cast +from typing import Any, Optional, Union, cast import httpx from fastapi import APIRouter, Depends, HTTPException, Request, Response, WebSocket