fix(vertex_ai): replace custom model names with actual Vertex AI model names in passthrough URLs (#19948)

When the passthrough URL already contains project and location, the code was skipping the deployment lookup and forwarding the URL as-is to Vertex AI. For custom model names like gcp/google/gemini-2.5-flash, Vertex AI returned 404 because it only knows the actual model name (gemini-2.5-flash). The fix makes the deployment lookup always run, so the custom model name gets replaced with the actual Vertex AI model name before forwarding.
2026-09-15 23:31:29 +00:00 · 2026-01-29 01:22:16 +01:00 · 2026-01-29 01:22:16 +01:00 · df5d5664e2
commit df5d5664e2
parent 3816570313
2 changed files with 97 additions and 15 deletions
--- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
+++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py
@ -1610,24 +1610,34 @@ async def _base_vertex_proxy_route(
        vertex_location=vertex_location,
    )

-    if vertex_project is None or vertex_location is None:
-        # Check if model is in router config
-        model_id = get_vertex_model_id_from_url(endpoint)
-        if model_id:
-            from litellm.proxy.proxy_server import llm_router
+    # Check if model is in router config - always do this to resolve custom model names
+    model_id = get_vertex_model_id_from_url(endpoint)
+    if model_id:
+        from litellm.proxy.proxy_server import llm_router

-            if llm_router:
-                try:
-                    # Use the dedicated pass-through deployment selection method to automatically filter use_in_pass_through=True
-                    deployment = llm_router.get_available_deployment_for_pass_through(model=model_id)
-                    if deployment:
-                        litellm_params = deployment.get("litellm_params", {})
+        if llm_router:
+            try:
+                # Use the dedicated pass-through deployment selection method to automatically filter use_in_pass_through=True
+                deployment = llm_router.get_available_deployment_for_pass_through(model=model_id)
+                if deployment:
+                    litellm_params = deployment.get("litellm_params", {})
+                    if vertex_project is None:
                        vertex_project = litellm_params.get("vertex_project")
+                    if vertex_location is None:
                        vertex_location = litellm_params.get("vertex_location")
-                except Exception as e:
-                    verbose_proxy_logger.debug(
-                        f"Error getting available deployment for model {model_id}: {e}"
-                    )
+
+                    # Replace custom model name with actual Vertex AI model name in the endpoint
+                    # e.g., "gcp/google/gemini-3-pro" -> "gemini-3-pro"
+                    actual_model = litellm_params.get("model", "")
+                    if "/" in actual_model:
+                        actual_model = actual_model.split("/", 1)[1]
+                    if actual_model and model_id != actual_model:
+                        encoded_endpoint = encoded_endpoint.replace(model_id, actual_model)
+                        endpoint = endpoint.replace(model_id, actual_model)
+            except Exception as e:
+                verbose_proxy_logger.debug(
+                    f"Error getting available deployment for model {model_id}: {e}"
+                )

    vertex_credentials = passthrough_endpoint_router.get_vertex_credentials(
        project_id=vertex_project,
--- a/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
+++ b/tests/test_litellm/proxy/pass_through_endpoints/test_vertex_passthrough_load_balancing.py
@ -447,3 +447,75 @@ def test_forward_headers_from_request_x_pass_prefix():
    assert "x-pass-anthropic-beta" not in result
    assert "x-pass-custom-header" not in result

+
+@pytest.mark.asyncio
+async def test_vertex_passthrough_custom_model_name_replaced_in_url():
+    """
+    Test that when a passthrough URL contains a custom model_name (e.g., gcp/google/gemini-3-pro),
+    the URL is rewritten to use the actual Vertex AI model name (e.g., gemini-3-pro)
+    before being forwarded to Vertex AI.
+
+    This prevents 404 errors from Vertex AI when custom model names are used in the config.
+
+    Config example:
+        model_name: gcp/google/gemini-3-pro
+        litellm_params:
+          model: vertex_ai/gemini-3-pro
+          vertex_project: "my-project"
+          vertex_location: "global"
+          use_in_pass_through: true
+    """
+    mock_request = MagicMock()
+    mock_response = MagicMock()
+    mock_handler = MagicMock()
+
+    # Deployment with custom model_name but real vertex model
+    mock_deployment = {
+        "litellm_params": {
+            "model": "vertex_ai/gemini-3-pro",
+            "vertex_project": "nv-gcpllmgwit-20250411173346",
+            "vertex_location": "global",
+            "use_in_pass_through": True,
+        }
+    }
+    mock_router = MagicMock()
+    mock_router.get_available_deployment_for_pass_through.return_value = mock_deployment
+
+    # The URL contains project/location AND a custom model name with slashes
+    test_endpoint = "v1/projects/nv-gcpllmgwit-20250411173346/locations/global/publishers/google/models/gcp/google/gemini-3-pro:generateContent"
+
+    with patch("litellm.proxy.proxy_server.llm_router", mock_router), \
+         patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.passthrough_endpoint_router") as mock_pt_router, \
+         patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints._prepare_vertex_auth_headers", new_callable=AsyncMock) as mock_prep_headers, \
+         patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route") as mock_create_route, \
+         patch("litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.user_api_key_auth", new_callable=AsyncMock) as mock_auth:
+
+        mock_pt_router.get_vertex_credentials.return_value = MagicMock()
+        mock_prep_headers.return_value = ({}, "https://global-aiplatform.googleapis.com", False, "nv-gcpllmgwit-20250411173346", "global")
+        mock_endpoint_func = AsyncMock()
+        mock_create_route.return_value = mock_endpoint_func
+        mock_auth.return_value = {}
+
+        mock_handler.get_default_base_target_url.return_value = "https://global-aiplatform.googleapis.com"
+
+        await _base_vertex_proxy_route(
+            endpoint=test_endpoint,
+            request=mock_request,
+            fastapi_response=mock_response,
+            get_vertex_pass_through_handler=mock_handler,
+        )
+
+        # Verify the router was called with the custom model name (extracted from URL)
+        mock_router.get_available_deployment_for_pass_through.assert_called_once_with(
+            model="gcp/google/gemini-3-pro"
+        )
+
+        # Verify the target URL passed to create_pass_through_route contains
+        # the REAL Vertex AI model name, not the custom one
+        create_route_call = mock_create_route.call_args
+        target_url = create_route_call.kwargs.get("target", "")
+        assert "gcp/google/gemini-3-pro" not in target_url, \
+            f"Custom model name should have been replaced in target URL. Got: {target_url}"
+        assert "gemini-3-pro" in target_url, \
+            f"Actual Vertex AI model name should be in target URL. Got: {target_url}"
+