diff --git a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py index 2545fe0ed7d..7ab363886b0 100644 --- a/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py +++ b/litellm/llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py @@ -29,14 +29,14 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert """ if "Authorization" not in headers: vertex_ai_project = ( - optional_params.pop("vertex_project", None) - or optional_params.pop("vertex_ai_project", None) + litellm_params.pop("vertex_project", None) + or litellm_params.pop("vertex_ai_project", None) or litellm.vertex_project or get_secret_str("VERTEXAI_PROJECT") ) vertex_credentials = ( - optional_params.pop("vertex_credentials", None) - or optional_params.pop("vertex_ai_credentials", None) + litellm_params.pop("vertex_credentials", None) + or litellm_params.pop("vertex_ai_credentials", None) or get_secret_str("VERTEXAI_CREDENTIALS") ) @@ -50,7 +50,7 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert api_base = self.get_complete_vertex_url( custom_api_base=api_base, - vertex_location=optional_params.pop("vertex_location", None), + vertex_location=litellm_params.pop("vertex_location", None), vertex_project=vertex_ai_project, project_id=project_id, partner=VertexPartnerProvider.claude, @@ -93,4 +93,8 @@ class VertexAIPartnerModelsAnthropicMessagesConfig(AnthropicMessagesConfig, Vert ) anthropic_messages_request["anthropic_version"] = "vertex-2023-10-16" + + anthropic_messages_request.pop( + "model", None + ) # do not pass model in request body to vertex ai return anthropic_messages_request diff --git a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py index afac84326fe..131a0031c9c 100644 --- a/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py +++ b/litellm/proxy/pass_through_endpoints/llm_provider_handlers/vertex_passthrough_logging_handler.py @@ -86,6 +86,7 @@ class VertexPassthroughLoggingHandler: vertex_image_generation_class = VertexImageGeneration() model = VertexPassthroughLoggingHandler.extract_model_from_url(url_route) + _json_response = httpx_response.json() litellm_prediction_response: Union[ @@ -121,6 +122,20 @@ class VertexPassthroughLoggingHandler: "result": litellm_prediction_response, "kwargs": kwargs, } + elif "rawPredict" in url_route or "streamRawPredict" in url_route: + model = VertexPassthroughLoggingHandler.extract_model_from_url(url_route) + + _json_response = httpx_response.json() + + litellm_prediction_response = ModelResponse() + + logging_obj.model = model + logging_obj.model_call_details["model"] = logging_obj.model + + return { + "result": None, + "kwargs": kwargs, + } else: return { "result": None, diff --git a/litellm/proxy/pass_through_endpoints/success_handler.py b/litellm/proxy/pass_through_endpoints/success_handler.py index 09f0e3847a2..4b098a818d7 100644 --- a/litellm/proxy/pass_through_endpoints/success_handler.py +++ b/litellm/proxy/pass_through_endpoints/success_handler.py @@ -35,6 +35,8 @@ class PassThroughEndpointLogging: "generateContent", "streamGenerateContent", "predict", + "rawPredict", + "streamRawPredict", ] # Anthropic @@ -101,9 +103,9 @@ class PassThroughEndpointLogging: standard_logging_response_object: Optional[ PassThroughEndpointLoggingResultValues ] = None - logging_obj.model_call_details["passthrough_logging_payload"] = ( - passthrough_logging_payload - ) + logging_obj.model_call_details[ + "passthrough_logging_payload" + ] = passthrough_logging_payload if self.is_vertex_route(url_route): vertex_passthrough_logging_handler_result = ( VertexPassthroughLoggingHandler.vertex_passthrough_handler(