feat(typesafe): add TypeSafe Jev passthrough with logging and cost tracking

Backport of #41607 to stable/1.101.x.
Cherry-picked from deb9d8aedd (main).

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>

gateway/routes/allowlist.py

@@ -95,6 +95,7 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = (
     "/langfuse/",
     "/vllm/",
     "/mistral/",
+    "/typesafe/",
     "/groq/",
     "/voyage/",
     "/cursor/",

litellm/model_prices_and_context_window_backup.json

@@ -63110,5 +63110,26 @@
         "supports_tool_choice": false,
         "supports_response_schema": true,
         "supports_vision": false
+    },
+    "typesafe/jev-1.13.0": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
+    "typesafe/jev-latest": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
+    "typesafe/jev-preview": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
     }
 }

litellm/proxy/_types.py

@@ -473,6 +473,7 @@ class LiteLLMRoutes(enum.Enum):
         "/eu.assemblyai",
         "/vllm",
         "/mistral",
+        "/typesafe",
         "/milvus",
         "/gigachat",
         "/watsonx",

litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py

@@ -501,6 +501,42 @@ async def mistral_proxy_route(
     return received_value
 
 
+@router.api_route(
+    "/typesafe/{endpoint:path}",
+    methods=["GET", "POST"],  # mutable-ok: FastAPI route metadata requires a list
+    tags=["TypeSafe AI Pass-through", "pass-through"],  # mutable-ok: FastAPI route metadata requires a list
+)
+async def typesafe_proxy_route(
+    endpoint: str,
+    request: Request,
+    fastapi_response: Response,
+    user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)],
+):
+    """[Docs](https://docs.litellm.ai/docs/pass_through/typesafe)"""
+    base_target_url: Final = get_secret_str("TYPESAFE_API_BASE") or "https://api.typesafe.ai"
+    encoded_endpoint: Final = httpx.URL(endpoint).path
+    normalized_endpoint: Final = encoded_endpoint if encoded_endpoint.startswith("/") else f"/{encoded_endpoint}"
+    base_url: Final = httpx.URL(base_target_url)
+    updated_url: Final = base_url.copy_with(
+        path=HttpPassThroughEndpointHelpers.join_base_and_endpoint_path(base_url, normalized_endpoint),
+    )
+    typesafe_api_key: Final = passthrough_endpoint_router.get_credentials(
+        custom_llm_provider="typesafe",
+        region_name=None,
+    )
+    endpoint_func: Final = create_pass_through_route(
+        endpoint=endpoint,
+        target=str(updated_url),
+        custom_headers={  # mutable-ok: pass-through request headers require a mutable mapping
+            "Authorization": f"Bearer {typesafe_api_key}",
+            "Content-Type": "application/json",
+        },
+        custom_llm_provider="typesafe",
+        is_streaming_request=False,
+    )
+    return await endpoint_func(request, fastapi_response, user_api_key_dict)
+
+
 @router.api_route(
     "/milvus/{endpoint:path}",
     methods=["GET", "POST", "PUT", "DELETE", "PATCH"],

litellm/proxy/pass_through_endpoints/llm_provider_handlers/typesafe_passthrough_logging_handler.py

@@ -0,0 +1,117 @@
+from collections.abc import Mapping
+from datetime import datetime
+from typing import Final
+
+import httpx
+from pydantic import BaseModel, TypeAdapter, ValidationError
+
+import litellm
+from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
+from litellm.litellm_core_utils.litellm_logging import (
+    get_standard_logging_object_payload,  # pyright: ignore[reportUnknownVariableType]  # legacy helper has an untyped signature
+)
+from litellm.proxy._types import PassThroughEndpointLoggingTypedDict
+from litellm.types.utils import ModelResponse, StandardPassThroughResponseObject, Usage
+
+
+class _TypeSafeUsage(BaseModel):
+    input_tokens: int = 0
+    output_tokens: int = 0
+
+
+class _TypeSafeResponse(BaseModel):
+    model: str | None = None
+    usage: _TypeSafeUsage | None = None
+
+
+class _RegistryPricing(BaseModel):
+    input_cost_per_token: float = 0.0
+    output_cost_per_token: float = 0.0
+
+
+_TYPESAFE_RESPONSE_ADAPTER: Final = TypeAdapter(_TypeSafeResponse)
+_REGISTRY_PRICING_ADAPTER: Final = TypeAdapter(_RegistryPricing)
+
+
+def _parse_typesafe_response(response_body: Mapping[str, object]) -> _TypeSafeResponse:
+    try:
+        return _TYPESAFE_RESPONSE_ADAPTER.validate_python(response_body)
+    except ValidationError:
+        return _TypeSafeResponse()
+
+
+def _pricing_for(model_keys: tuple[str, ...]) -> _RegistryPricing:
+    for model_key in model_keys:
+        if model_key not in litellm.model_cost:  # pyright: ignore[reportUnknownMemberType]  # registry is dynamically typed
+            continue
+        try:
+            return _REGISTRY_PRICING_ADAPTER.validate_python(
+                litellm.model_cost[model_key]  # pyright: ignore[reportUnknownMemberType]  # registry is dynamically typed
+            )
+        except ValidationError:
+            continue
+    return _RegistryPricing()
+
+
+class TypeSafePassthroughLoggingHandler:
+    @staticmethod
+    def typesafe_passthrough_handler(
+        httpx_response: httpx.Response,
+        response_body: Mapping[str, object],
+        logging_obj: LiteLLMLoggingObj,
+        url_route: str,
+        result: str,
+        start_time: datetime,
+        end_time: datetime,
+        cache_hit: bool,
+        request_body: Mapping[str, object],
+        **kwargs: object,
+    ) -> PassThroughEndpointLoggingTypedDict:
+        response: Final = _parse_typesafe_response(response_body)
+        response_model: Final = response.model
+        request_model_value: Final = request_body.get("model")
+        request_model: Final = request_model_value if isinstance(request_model_value, str) else None
+        logged_model: Final = response_model or request_model or "unknown"
+        model_name: Final = f"typesafe/{logged_model}"
+        usage: Final = response.usage or _TypeSafeUsage()
+        input_tokens: Final = usage.input_tokens
+        output_tokens: Final = usage.output_tokens
+        candidate_model_keys: Final = tuple(
+            f"typesafe/{model}" for model in (response_model, request_model) if model is not None
+        )
+        pricing: Final = _pricing_for(candidate_model_keys)
+        response_cost: Final = (
+            input_tokens * pricing.input_cost_per_token + output_tokens * pricing.output_cost_per_token
+        )
+        usage_object: Final = Usage(
+            prompt_tokens=input_tokens,
+            completion_tokens=output_tokens,
+            total_tokens=input_tokens + output_tokens,
+        )
+        updated_kwargs: Final = {  # mutable-ok: pass-through logging contract requires mutable kwargs
+            **kwargs,
+            "model": model_name,
+            "custom_llm_provider": "typesafe",
+            "response_cost": response_cost,
+            "combined_usage_object": usage_object,
+        }
+        logging_obj.model_call_details.update(
+            model=model_name,
+            custom_llm_provider="typesafe",
+            response_cost=response_cost,
+        )
+        standard_logging_object: Final = get_standard_logging_object_payload(
+            kwargs=updated_kwargs,
+            init_response_obj=ModelResponse(model=model_name, usage=usage_object),
+            start_time=start_time,
+            end_time=end_time,
+            logging_obj=logging_obj,
+            status="success",
+        )
+        return {  # mutable-ok: pass-through logging contract requires mutable result
+            "result": StandardPassThroughResponseObject(response=result),
+            "kwargs": {  # mutable-ok: pass-through logging contract requires mutable kwargs
+                **updated_kwargs,
+                "standard_logging_object": standard_logging_object,
+            },
+        }

litellm/proxy/pass_through_endpoints/success_handler.py

@@ -1,5 +1,6 @@
 import json
 from datetime import datetime
+from types import MappingProxyType
 from typing import Any, Final
 from urllib.parse import urlparse
 
@@ -256,6 +257,25 @@ class PassThroughEndpointLogging:
             )
             standard_logging_response_object = comprehend_medical_handler_result["result"]  # rebind-ok: elif-chain
             kwargs = comprehend_medical_handler_result["kwargs"]  # rebind-ok: elif-chain contract
+        elif self.is_typesafe_route(custom_llm_provider):
+            from .llm_provider_handlers.typesafe_passthrough_logging_handler import (
+                TypeSafePassthroughLoggingHandler,
+            )
+
+            typesafe_handler_result: Final = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
+                httpx_response=httpx_response,
+                response_body=response_body if isinstance(response_body, dict) else MappingProxyType({}),
+                logging_obj=logging_obj,
+                url_route=url_route,
+                result=result,
+                start_time=start_time,
+                end_time=end_time,
+                cache_hit=cache_hit,
+                request_body=request_body,
+                **kwargs,
+            )
+            standard_logging_response_object = typesafe_handler_result["result"]
+            kwargs = typesafe_handler_result["kwargs"]
         elif self.is_vertex_ai_live_route(url_route):
             from .llm_provider_handlers.vertex_ai_live_passthrough_logging_handler import (
                 VertexAILivePassthroughLoggingHandler,
@@ -387,6 +407,9 @@ class PassThroughEndpointLogging:
     def is_comprehend_medical_route(self, custom_llm_provider: str | None) -> bool:
         return custom_llm_provider == "comprehendmedical"
 
+    def is_typesafe_route(self, custom_llm_provider: str | None) -> bool:
+        return custom_llm_provider == "typesafe"
+
     def is_langfuse_route(self, url_route: str):
         parsed_url: Final = urlparse(url_route)
         for route in self.TRACKED_LANGFUSE_ROUTES:

litellm/types/utils.py

@@ -336,6 +336,7 @@ class ModelInfoBase(ProviderSpecificModelInfo, total=False):
             "chat",
             "audio_transcription",
             "responses",
+            "evaluation",
             "ocr",
             "realtime",
         ]

model_prices_and_context_window.json

@@ -63110,5 +63110,26 @@
         "supports_tool_choice": false,
         "supports_response_schema": true,
         "supports_vision": false
+    },
+    "typesafe/jev-1.13.0": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
+    "typesafe/jev-latest": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
+    },
+    "typesafe/jev-preview": {
+        "input_cost_per_token": 4.2e-08,
+        "litellm_provider": "typesafe",
+        "mode": "evaluation",
+        "output_cost_per_token": 0.0,
+        "source": "https://docs.typesafe.ai/models"
     }
 }

model_prices_and_context_window.schema.json

@@ -411,6 +411,7 @@
             "chat",
             "completion",
             "embedding",
+            "evaluation",
             "guardrail",
             "image_edit",
             "image_generation",

tests/test_litellm/proxy/pass_through_endpoints/llm_provider_handlers/test_typesafe_passthrough_logging_handler.py

@@ -0,0 +1,134 @@
+from datetime import datetime
+from unittest.mock import MagicMock
+
+import httpx
+import pytest
+
+import litellm
+from litellm.proxy.pass_through_endpoints.llm_provider_handlers.typesafe_passthrough_logging_handler import (
+    TypeSafePassthroughLoggingHandler,
+)
+from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging
+
+
+@pytest.fixture(autouse=True)
+def local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
+
+
+def _response() -> httpx.Response:
+    return httpx.Response(
+        200,
+        request=httpx.Request("POST", "https://api.typesafe.ai/v1/systemone"),
+        json={"model": "jev-1.13.0"},
+    )
+
+
+def _logging_obj() -> MagicMock:
+    logging_obj = MagicMock()
+    logging_obj.model_call_details = {}
+    return logging_obj
+
+
+def _handler_result(response_body: dict, request_body: dict) -> dict:
+    return TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
+        httpx_response=_response(),
+        response_body=response_body,
+        logging_obj=_logging_obj(),
+        url_route="https://api.typesafe.ai/v1/systemone",
+        result='{"answers": {}}',
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body=request_body,
+    )
+
+
+def test_uses_registry_pricing_and_standard_usage():
+    logging_obj = _logging_obj()
+    model_key = "typesafe/jev-1.13.0"
+    model_cost = litellm.model_cost[model_key]
+    response = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
+        httpx_response=_response(),
+        response_body={"model": "jev-1.13.0", "usage": {"input_tokens": 312, "output_tokens": 48}},
+        logging_obj=logging_obj,
+        url_route="https://api.typesafe.ai/v1/systemone",
+        result='{"answers": {}}',
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body={"model": "jev-latest"},
+    )
+
+    expected_cost = 312 * model_cost["input_cost_per_token"] + 48 * model_cost["output_cost_per_token"]
+    assert response["kwargs"]["response_cost"] == pytest.approx(expected_cost)
+    assert response["kwargs"]["combined_usage_object"].prompt_tokens == 312
+    assert response["kwargs"]["combined_usage_object"].completion_tokens == 48
+    assert response["kwargs"]["combined_usage_object"].total_tokens == 360
+
+
+def test_falls_back_to_request_model_when_response_model_is_missing():
+    result = _handler_result(
+        {"usage": {"input_tokens": 10, "output_tokens": 2}},
+        {"model": "jev-latest"},
+    )
+
+    model_cost = litellm.model_cost["typesafe/jev-latest"]
+    expected_cost = 10 * model_cost["input_cost_per_token"] + 2 * model_cost["output_cost_per_token"]
+    assert result["kwargs"]["model"] == "typesafe/jev-latest"
+    assert result["kwargs"]["response_cost"] == pytest.approx(expected_cost)
+
+
+def test_call_naming_no_model_is_logged_as_unknown_and_never_priced_as_a_registry_model():
+    result = _handler_result({"usage": {"input_tokens": 10, "output_tokens": 2}}, {})
+
+    assert result["kwargs"]["model"] == "typesafe/unknown"
+    assert result["kwargs"]["response_cost"] == 0.0
+
+
+def test_missing_usage_is_zero_cost():
+    result = _handler_result({"model": "jev-1.13.0"}, {"model": "jev-latest"})
+
+    assert result["kwargs"]["response_cost"] == 0.0
+
+
+def test_records_model_provider_and_cost_on_logging_details():
+    logging_obj = _logging_obj()
+    result = TypeSafePassthroughLoggingHandler.typesafe_passthrough_handler(
+        httpx_response=_response(),
+        response_body={"model": "jev-1.13.0", "usage": {"input_tokens": 1, "output_tokens": 0}},
+        logging_obj=logging_obj,
+        url_route="https://api.typesafe.ai/v1/systemone",
+        result="{}",
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        request_body={"model": "jev-latest"},
+    )
+
+    assert result["kwargs"]["model"] == "typesafe/jev-1.13.0"
+    assert result["kwargs"]["custom_llm_provider"] == "typesafe"
+    assert result["kwargs"]["response_cost"] > 0
+    assert logging_obj.model_call_details["model"] == "typesafe/jev-1.13.0"
+    assert logging_obj.model_call_details["custom_llm_provider"] == "typesafe"
+    assert logging_obj.model_call_details["response_cost"] == result["kwargs"]["response_cost"]
+
+
+def test_success_handler_dispatches_to_typesafe_handler():
+    logging_obj = _logging_obj()
+    normalized = PassThroughEndpointLogging().normalize_llm_passthrough_logging_payload(
+        httpx_response=_response(),
+        response_body={"model": "jev-1.13.0", "usage": {"input_tokens": 1, "output_tokens": 0}},
+        request_body={"model": "jev-latest"},
+        logging_obj=logging_obj,
+        url_route="https://api.typesafe.ai/v1/systemone",
+        result="{}",
+        start_time=datetime.now(),
+        end_time=datetime.now(),
+        cache_hit=False,
+        custom_llm_provider="typesafe",
+    )
+
+    assert normalized["kwargs"]["custom_llm_provider"] == "typesafe"
+    assert normalized["kwargs"]["model"] == "typesafe/jev-1.13.0"

tests/test_litellm/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py

@@ -8,6 +8,7 @@ from types import MappingProxyType, SimpleNamespace
 from typing import Final
 from unittest import mock
 from unittest.mock import AsyncMock, MagicMock, Mock, patch
+from urllib.parse import parse_qs
 
 import httpx
 import pytest
@@ -37,6 +38,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import (
     milvus_proxy_route,
     mistral_proxy_route,
     openai_proxy_route,
+    typesafe_proxy_route,
     vertex_discovery_proxy_route,
     vertex_proxy_route,
     vllm_proxy_route,
@@ -5206,3 +5208,51 @@ class TestAzureRouterModelStreamingKeepalive:
 
         assert result.headers["x-upstream"] == "kept"
         assert chunks == [b"data: hello\n\n"]
+
+
+class TestTypeSafePassthroughRoute:
+    @staticmethod
+    def _request(body: object, query_params: Mapping[str, str] | None = None) -> MagicMock:
+        request = MagicMock(spec=Request)
+        request.method = "POST"
+        request.query_params = query_params or {}
+        request.json = AsyncMock(return_value=body)
+        return request
+
+    @pytest.mark.asyncio
+    async def test_forwards_target_auth_headers_provider_and_query(self, monkeypatch):
+        monkeypatch.setenv("TYPESAFE_API_KEY", "typesafe-test-key")
+        monkeypatch.setenv("TYPESAFE_API_BASE", "https://typesafe.example/base")
+
+        async def fake_upstream(request, *_args):
+            target: Final = create_route.call_args.kwargs["target"]
+            upstream_url: Final = httpx.URL(target).copy_merge_params(request.query_params)
+            return {"upstream_query": parse_qs(upstream_url.query.decode())}
+
+        endpoint_func = AsyncMock(side_effect=fake_upstream)
+        create_route = Mock(return_value=endpoint_func)
+        monkeypatch.setattr(
+            "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.create_pass_through_route",
+            create_route,
+        )
+
+        request = self._request({"state": "x"}, {"trace": "yes"})
+        result = await typesafe_proxy_route(
+            endpoint="v1/systemone",
+            request=request,
+            fastapi_response=MagicMock(spec=Response),
+            user_api_key_dict=UserAPIKeyAuth(api_key="virtual-key"),
+        )
+
+        assert result == {"upstream_query": {"trace": ["yes"]}}
+        endpoint_func.assert_awaited_once()
+        create_route.assert_called_once_with(
+            endpoint="v1/systemone",
+            target="https://typesafe.example/base/v1/systemone",
+            custom_headers={
+                "Authorization": "Bearer typesafe-test-key",
+                "Content-Type": "application/json",
+            },
+            custom_llm_provider="typesafe",
+            is_streaming_request=False,
+        )

tests/test_litellm/test_typesafe_model_metadata.py

@@ -0,0 +1,17 @@
+import pytest
+
+import litellm
+
+
+@pytest.fixture(autouse=True)
+def local_model_cost_map(monkeypatch: pytest.MonkeyPatch):
+    monkeypatch.setenv("LITELLM_LOCAL_MODEL_COST_MAP", "True")
+    monkeypatch.setattr(litellm, "model_cost", litellm.get_model_cost_map(url=""))
+
+
+def test_typesafe_models_share_pricing_and_provider_metadata():
+    entries = [litellm.model_cost[f"typesafe/{model}"] for model in ("jev-1.13.0", "jev-latest", "jev-preview")]
+
+    assert {entry["input_cost_per_token"] for entry in entries} == {entries[0]["input_cost_per_token"]}
+    assert {entry["output_cost_per_token"] for entry in entries} == {entries[0]["output_cost_per_token"]}
+    assert {entry["litellm_provider"] for entry in entries} == {"typesafe"}

tests/test_litellm/test_utils.py

@@ -1005,6 +1005,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid():
                         "container",
                         "image_edit",
                         "embedding",
+                        "evaluation",
                         "guardrail",
                         "image_generation",
                         "video_generation",

ui/litellm-dashboard/src/lib/http/schema.d.ts

@@ -16168,6 +16168,30 @@ export interface paths {
         patch: operations["toolset_mcp_route_toolset__toolset_name__mcp_patch"];
         trace?: never;
     };
+    "/typesafe/{endpoint}": {
+        parameters: {
+            query?: never;
+            header?: never;
+            path?: never;
+            cookie?: never;
+        };
+        /**
+         * Typesafe Proxy Route
+         * @description [Docs](https://docs.litellm.ai/docs/pass_through/typesafe)
+         */
+        get: operations["typesafe_proxy_route_typesafe__endpoint__get"];
+        put?: never;
+        /**
+         * Typesafe Proxy Route
+         * @description [Docs](https://docs.litellm.ai/docs/pass_through/typesafe)
+         */
+        post: operations["typesafe_proxy_route_typesafe__endpoint__post"];
+        delete?: never;
+        options?: never;
+        head?: never;
+        patch?: never;
+        trace?: never;
+    };
     "/update/default_team_settings": {
         parameters: {
             query?: never;
@@ -59601,6 +59625,68 @@ export interface operations {
             };
         };
     };
+    typesafe_proxy_route_typesafe__endpoint__get: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                endpoint: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
+    typesafe_proxy_route_typesafe__endpoint__post: {
+        parameters: {
+            query?: never;
+            header?: never;
+            path: {
+                endpoint: string;
+            };
+            cookie?: never;
+        };
+        requestBody?: never;
+        responses: {
+            /** @description Successful Response */
+            200: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": unknown;
+                };
+            };
+            /** @description Validation Error */
+            422: {
+                headers: {
+                    [name: string]: unknown;
+                };
+                content: {
+                    "application/json": components["schemas"]["HTTPValidationError"];
+                };
+            };
+        };
+    };
     update_default_team_settings_update_default_team_settings_patch: {
         parameters: {
             query?: never;