diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 85557090ae9..ca8e2af18e6 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -76,6 +76,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "itpm", "otpm", "use_xai_oauth", + "chatgpt_auth_file", "github_copilot_auth_type", "github_copilot_host", "github_copilot_user_session", diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py index 04dd0eca967..19c1d5c414c 100644 --- a/litellm/litellm_core_utils/get_llm_provider_logic.py +++ b/litellm/litellm_core_utils/get_llm_provider_logic.py @@ -850,7 +850,9 @@ def _get_openai_compatible_provider_info( api_base, dynamic_api_key, custom_llm_provider, - ) = litellm.ChatGPTConfig().get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider) + ) = litellm.ChatGPTConfig().get_openai_compatible_provider_info( + model, api_base, api_key, custom_llm_provider, litellm_params + ) elif custom_llm_provider == "novita": api_base = api_base or get_secret("NOVITA_API_BASE") or "https://api.novita.ai/v3/openai" dynamic_api_key = api_key or get_secret_str("NOVITA_API_KEY") diff --git a/litellm/litellm_core_utils/get_provider_specific_headers.py b/litellm/litellm_core_utils/get_provider_specific_headers.py index bc325c3c702..62dfcb073e5 100644 --- a/litellm/litellm_core_utils/get_provider_specific_headers.py +++ b/litellm/litellm_core_utils/get_provider_specific_headers.py @@ -1,10 +1,23 @@ from collections.abc import Mapping, Sequence from typing import Final +from pydantic import TypeAdapter + from litellm.types.utils import ProviderSpecificHeader +_PROVIDER_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter( + ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None +) + class ProviderSpecificHeaderUtils: + @staticmethod + def forwarded_authorization(provider_specific_header: object, custom_llm_provider: str | None) -> str | None: + headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + _PROVIDER_HEADERS.validate_python(provider_specific_header), custom_llm_provider + ) + return next((value for key, value in headers.items() if key.lower() == "authorization"), None) + @staticmethod def get_provider_specific_headers( provider_specific_header: ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None, diff --git a/litellm/llms/chatgpt/authenticator.py b/litellm/llms/chatgpt/authenticator.py index 4723e4c0200..692f6820ff4 100644 --- a/litellm/llms/chatgpt/authenticator.py +++ b/litellm/llms/chatgpt/authenticator.py @@ -1,7 +1,9 @@ import json import os import time -from typing import Final +from collections.abc import Mapping +from functools import lru_cache +from typing import TYPE_CHECKING, Final import httpx from pydantic import JsonValue, TypeAdapter, ValidationError @@ -28,6 +30,9 @@ from .common_utils import ( extract_chatgpt_account_id, ) +if TYPE_CHECKING: + from litellm.types.router import GenericLiteLLMParams + TOKEN_EXPIRY_SKEW_SECONDS: Final = 60 TOKEN_REFRESH_TIMEOUT_SECONDS: Final = 30 DEVICE_CODE_TIMEOUT_SECONDS: Final = 15 * 60 @@ -41,6 +46,26 @@ def _optional_str(value: JsonValue | None) -> str | None: return value if isinstance(value, str) else None +def get_chatgpt_auth_file( + litellm_params: "Mapping[str, object] | GenericLiteLLMParams | None", +) -> str | None: + if litellm_params is None: + return None + value: Final = ( + litellm_params.get("chatgpt_auth_file") + if isinstance(litellm_params, Mapping) + else litellm_params.chatgpt_auth_file + ) + if not isinstance(value, str) or not value: + return None + return os.path.abspath(os.path.expanduser(value)) + + +@lru_cache(maxsize=128) +def get_cached_authenticator(auth_file: str) -> "Authenticator": + return Authenticator(auth_file=auth_file) + + def _token_refresh_timeout() -> httpx.Timeout: configured: Final = get_configured_request_timeout() seconds: Final = ( @@ -50,13 +75,13 @@ def _token_refresh_timeout() -> httpx.Timeout: class Authenticator: - def __init__(self) -> None: - self.token_dir = os.getenv( - "CHATGPT_TOKEN_DIR", - os.path.expanduser("~/.config/litellm/chatgpt"), + def __init__(self, auth_file: str | None = None) -> None: + default_auth_file: Final = os.path.join( + os.getenv("CHATGPT_TOKEN_DIR", os.path.expanduser("~/.config/litellm/chatgpt")), + os.getenv("CHATGPT_AUTH_FILE", "auth.json"), ) - self.auth_file = os.path.join(self.token_dir, os.getenv("CHATGPT_AUTH_FILE", "auth.json")) - self._ensure_token_dir() + self.auth_file = os.path.abspath(os.path.expanduser(auth_file or default_auth_file)) + self.token_dir = os.path.dirname(self.auth_file) def get_api_base(self) -> str: return os.getenv("CHATGPT_API_BASE") or os.getenv("OPENAI_CHATGPT_API_BASE") or CHATGPT_API_BASE @@ -126,6 +151,7 @@ class Authenticator: def _write_auth_file(self, data: JsonObject) -> None: try: + self._ensure_token_dir() with open(self.auth_file, "w") as f: json.dump(data, f) except OSError as exc: diff --git a/litellm/llms/chatgpt/chat/transformation.py b/litellm/llms/chatgpt/chat/transformation.py index 43ba7094b48..5f6e88160fc 100644 --- a/litellm/llms/chatgpt/chat/transformation.py +++ b/litellm/llms/chatgpt/chat/transformation.py @@ -1,14 +1,20 @@ +from collections.abc import Mapping from typing import Any, Final from litellm.exceptions import AuthenticationError from litellm.llms.openai.openai import OpenAIConfig from litellm.types.llms.openai import AllMessageValues +from litellm.types.router import GenericLiteLLMParams -from ..authenticator import Authenticator +from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file from ..common_utils import ( GetAccessTokenError, + chatgpt_client_api_base, ensure_chatgpt_session_id, + get_chatgpt_client_credential, get_chatgpt_default_headers, + get_chatgpt_request_credential, + strip_chatgpt_client_credential_headers, ) from .streaming_utils import ChatGPTToolCallNormalizer @@ -24,6 +30,23 @@ class ChatGPTConfig(OpenAIConfig): super().__init__() self.authenticator = authenticator if authenticator is not None else Authenticator() + def _resolve_authenticator( + self, litellm_params: Mapping[str, object] | GenericLiteLLMParams | None + ) -> Authenticator: + auth_file: Final = get_chatgpt_auth_file(litellm_params) + return get_cached_authenticator(auth_file) if auth_file else self.authenticator + + @staticmethod + def _get_access_token_or_raise(authenticator: Authenticator, model: str, llm_provider: str) -> str: + try: + return authenticator.get_access_token() + except GetAccessTokenError as e: + raise AuthenticationError( + model=model, + llm_provider=llm_provider, + message=str(e), + ) + def api_base_without_login(self) -> str: return self.authenticator.get_api_base() @@ -33,18 +56,16 @@ class ChatGPTConfig(OpenAIConfig): api_base: str | None, api_key: str | None, custom_llm_provider: str, + litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None, ) -> tuple[str | None, str | None, str]: + client_credential: Final = get_chatgpt_request_credential(litellm_params) + if client_credential is not None: + return chatgpt_client_api_base(api_base), client_credential.access_token, custom_llm_provider if api_key: return api_base or self.api_base_without_login(), api_key, custom_llm_provider - dynamic_api_base: Final = self.api_base_without_login() - try: - dynamic_api_key: Final = self.authenticator.get_access_token() - except GetAccessTokenError as e: - raise AuthenticationError( - model=model, - llm_provider=custom_llm_provider, - message=str(e), - ) + authenticator: Final = self._resolve_authenticator(litellm_params) + dynamic_api_base: Final = authenticator.get_api_base() + dynamic_api_key: Final = self._get_access_token_or_raise(authenticator, model, custom_llm_provider) return dynamic_api_base, dynamic_api_key, custom_llm_provider def get_openai_compatible_provider_info( @@ -53,26 +74,43 @@ class ChatGPTConfig(OpenAIConfig): api_base: str | None, api_key: str | None, custom_llm_provider: str, + litellm_params: Mapping[str, object] | GenericLiteLLMParams | None = None, ) -> tuple[str | None, str | None, str]: - return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider) + return self._get_openai_compatible_provider_info(model, api_base, api_key, custom_llm_provider, litellm_params) def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, messages: list[AllMessageValues], optional_params: dict, - litellm_params: dict, + litellm_params: dict[str, object], api_key: str | None = None, api_base: str | None = None, - ) -> dict: + ) -> dict[str, str]: + client_credential: Final = get_chatgpt_client_credential(headers) + if client_credential is not None: + return dict( + get_chatgpt_default_headers( + client_credential.access_token, + client_credential.account_id, + ensure_chatgpt_session_id(litellm_params), + ), + **strip_chatgpt_client_credential_headers(headers), + ) + authenticator: Final = self._resolve_authenticator(litellm_params) + resolved_api_key: Final = ( + self._get_access_token_or_raise(authenticator, model, "chatgpt") + if get_chatgpt_auth_file(litellm_params) + else api_key + ) validated_headers: Final = super().validate_environment( - headers, model, messages, optional_params, litellm_params, api_key, api_base + headers, model, messages, optional_params, litellm_params, resolved_api_key, api_base ) - account_id: Final = self.authenticator.get_account_id() + account_id: Final = authenticator.get_account_id() session_id: Final = ensure_chatgpt_session_id(litellm_params) - default_headers: Final = get_chatgpt_default_headers(api_key or "", account_id, session_id) + default_headers: Final = get_chatgpt_default_headers(resolved_api_key or "", account_id, session_id) return {**default_headers, **validated_headers} def post_stream_processing(self, stream: Any) -> ChatGPTToolCallNormalizer: diff --git a/litellm/llms/chatgpt/common_utils.py b/litellm/llms/chatgpt/common_utils.py index 6c20bbbc0d5..ec7fbfa92da 100644 --- a/litellm/llms/chatgpt/common_utils.py +++ b/litellm/llms/chatgpt/common_utils.py @@ -6,16 +6,28 @@ import base64 import json import os import platform -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass from types import MappingProxyType -from typing import Any, Final, TypeAlias +from typing import TYPE_CHECKING, Final, TypeAlias +from urllib.parse import urlsplit from uuid import uuid4 import httpx from pydantic import JsonValue, TypeAdapter +from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.types.utils import ProviderSpecificHeader + +if TYPE_CHECKING: + from litellm.types.router import GenericLiteLLMParams + +_SCOPED_HEADERS: Final[TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None]] = TypeAdapter( + ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None +) +_STRING_HEADERS: Final = TypeAdapter(Mapping[str, str]) +_PARAMS: Final = TypeAdapter(Mapping[str, object]) # OAuth + API constants (derived from openai/codex) CHATGPT_AUTH_BASE: Final = "https://auth.openai.com" @@ -241,7 +253,7 @@ def get_chatgpt_default_headers( access_token: str, account_id: str | None, session_id: str | None = None, -) -> dict: +) -> dict[str, str]: originator: Final = get_chatgpt_originator() user_agent: Final = get_chatgpt_user_agent(originator) headers: Final = { @@ -280,10 +292,17 @@ def extract_chatgpt_account_id(token: str | None) -> str | None: return account_id if isinstance(account_id, str) and account_id else None +def _bearer_token(value: str) -> str: + scheme, separator, token = value.strip().partition(" ") + if not separator: + return scheme + return token.strip() if scheme.lower() == "bearer" else "" + + def is_chatgpt_oauth_key(value: str | None) -> bool: if value is None: return False - return OPENAI_AUTH_CLAIM_KEY in decode_jwt_claims(value.removeprefix("Bearer ")) + return OPENAI_AUTH_CLAIM_KEY in decode_jwt_claims(_bearer_token(value)) @dataclass(frozen=True, slots=True) @@ -300,7 +319,7 @@ def get_chatgpt_client_credential(headers: Mapping[str, str]) -> ChatGPTClientCr auth_header: Final = _header_value(headers, "authorization") if auth_header is None or not is_chatgpt_oauth_key(auth_header): return None - access_token: Final = auth_header.removeprefix("Bearer ") + access_token: Final = _bearer_token(auth_header) account_id: Final = _header_value(headers, CHATGPT_ACCOUNT_ID_HEADER.lower()) or extract_chatgpt_account_id( access_token ) @@ -330,26 +349,50 @@ def strip_chatgpt_client_credential_headers(headers: Mapping[str, str]) -> Mappi ) +def get_chatgpt_request_credential( + params: "Mapping[str, object] | GenericLiteLLMParams | None", +) -> ChatGPTClientCredential | None: + normalized: Final = _normalize_litellm_params(params) + scoped: Final = normalized.get("provider_specific_header") + extra: Final = normalized.get("extra_headers") + headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( + provider_specific_header=_SCOPED_HEADERS.validate_python(scoped), custom_llm_provider="chatgpt" + ) + return get_chatgpt_client_credential(headers) or get_chatgpt_client_credential( + _STRING_HEADERS.validate_python(extra) if extra is not None else MappingProxyType({}) + ) + + +def chatgpt_client_api_base(api_base: str | None) -> str: + base: Final = api_base or CHATGPT_API_BASE + parsed: Final = urlsplit(base) + if ( + parsed.scheme != "https" + or parsed.hostname != "chatgpt.com" + or parsed.port not in (None, 443) + or parsed.username + or parsed.password + ): + raise ChatGPTAuthError( + status_code=400, message="Client ChatGPT credentials require the official HTTPS ChatGPT endpoint" + ) + return base + + def get_chatgpt_default_instructions() -> str: return os.getenv("CHATGPT_DEFAULT_INSTRUCTIONS") or CHATGPT_DEFAULT_INSTRUCTIONS -def _normalize_litellm_params(litellm_params: Any | None) -> dict: - if litellm_params is None: - return {} - if isinstance(litellm_params, dict): - return litellm_params - if hasattr(litellm_params, "model_dump"): - try: - return litellm_params.model_dump() - except Exception: - return {} - if hasattr(litellm_params, "dict"): - try: - return litellm_params.dict() - except Exception: - return {} - return {} +def _normalize_litellm_params(litellm_params: object) -> Mapping[str, object]: + if isinstance(litellm_params, Mapping): + return _PARAMS.validate_python(litellm_params) + serializer: Final = getattr(litellm_params, "model_dump", None) or getattr(litellm_params, "dict", None) + if not callable(serializer): + return MappingProxyType({}) + try: + return _PARAMS.validate_python(serializer()) + except Exception: + return MappingProxyType({}) def get_chatgpt_session_id(litellm_params: object) -> str | None: @@ -358,8 +401,8 @@ def get_chatgpt_session_id(litellm_params: object) -> str | None: value = params.get(key) if value: return str(value) - metadata: Final = params.get("metadata") - if isinstance(metadata, dict): + metadata: Final = _normalize_litellm_params(params.get("metadata")) + if metadata: value = metadata.get("session_id") if value: return str(value) diff --git a/litellm/llms/chatgpt/responses/transformation.py b/litellm/llms/chatgpt/responses/transformation.py index ef33fb3304f..5909492c215 100644 --- a/litellm/llms/chatgpt/responses/transformation.py +++ b/litellm/llms/chatgpt/responses/transformation.py @@ -25,11 +25,12 @@ from litellm.types.llms.openai import ( from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import LlmProviders -from ..authenticator import Authenticator +from ..authenticator import Authenticator, get_cached_authenticator, get_chatgpt_auth_file from ..common_utils import ( CHATGPT_API_BASE, GetAccessTokenError, ensure_chatgpt_session_id, + extract_chatgpt_account_id, get_chatgpt_client_credential, get_chatgpt_default_headers, get_chatgpt_default_instructions, @@ -51,27 +52,34 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): def custom_llm_provider(self) -> LlmProviders: return LlmProviders.CHATGPT + def _resolve_authenticator(self, litellm_params: GenericLiteLLMParams | None) -> Authenticator: + auth_file: Final = get_chatgpt_auth_file(litellm_params) + return get_cached_authenticator(auth_file) if auth_file else self.authenticator + def validate_environment( self, - headers: dict, + headers: dict[str, str], model: str, litellm_params: GenericLiteLLMParams | None, - ) -> dict: + ) -> dict[str, str]: session_id: Final = ensure_chatgpt_session_id(litellm_params) client_credential: Final = get_chatgpt_client_credential(headers) if client_credential is not None: - return { # mutable-ok: validate_environment returns the request header dict + return { **get_chatgpt_default_headers(client_credential.access_token, client_credential.account_id, session_id), **strip_chatgpt_client_credential_headers(headers), } configured_api_key: Final = litellm_params.api_key if litellm_params is not None else None - if configured_api_key: - return { # mutable-ok: validate_environment returns the request header dict - **get_chatgpt_default_headers(configured_api_key, None, session_id), + if configured_api_key and not get_chatgpt_auth_file(litellm_params): + return { + **get_chatgpt_default_headers( + configured_api_key, extract_chatgpt_account_id(configured_api_key), session_id + ), **headers, } + authenticator: Final = self._resolve_authenticator(litellm_params) try: - access_token: Final = self.authenticator.get_access_token() + access_token: Final = authenticator.get_access_token() except GetAccessTokenError as e: raise AuthenticationError( model=model, @@ -79,7 +87,7 @@ class ChatGPTResponsesAPIConfig(OpenAIResponsesAPIConfig): message=str(e), ) - account_id: Final = self.authenticator.get_account_id() + account_id: Final = authenticator.get_account_id() default_headers: Final = get_chatgpt_default_headers(access_token, account_id, session_id) return {**default_headers, **headers} diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 42db3c2f377..905f0f22e43 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -199,25 +199,23 @@ from litellm.utils import ( def _merge_headers_case_insensitive( - headers: dict[str, str], extra_headers: Mapping[str, object] | None -) -> dict[str, str]: - if not extra_headers: - return headers - overridden: Final = frozenset(name.lower() for name in extra_headers) - return { # mutable-ok: httpx request headers are a plain dict - **{name: value for name, value in headers.items() if name.lower() not in overridden}, - **{name: str(value) for name, value in extra_headers.items()}, + headers: Mapping[str, object], extra_headers: Mapping[str, object] | None +) -> dict[str, str]: # mutable-ok: legacy HTTP handler methods require a dict of headers + overridden: Final = frozenset(name.lower() for name in extra_headers or {}) + return { + **{name: str(value) for name, value in headers.items() if name.lower() not in overridden}, + **{name: str(value) for name, value in (extra_headers or {}).items()}, } def _client_request_headers( - response_api_optional_request_params: ResponsesAPIOptionalRequestParams, + response_api_optional_request_params: Mapping[str, object], extra_headers: Mapping[str, object] | None, -) -> dict: - return { # mutable-ok: validate_environment takes the request header dict - **(response_api_optional_request_params.get("extra_headers", {}) or {}), - **(extra_headers or {}), - } +) -> dict[str, str]: # mutable-ok: provider validate_environment methods require a dict + from litellm.litellm_core_utils.core_helpers import as_str_mapping + + optional_headers: Final = as_str_mapping(response_api_optional_request_params.get("extra_headers")) + return _merge_headers_case_insensitive(optional_headers or {}, extra_headers) def _rust_responses_websocket_enabled( @@ -2928,7 +2926,7 @@ class BaseLLMHTTPHandler: else: sync_httpx_client = client - headers = _merge_headers_case_insensitive( + headers = _merge_headers_case_insensitive( # rebind-ok: validation replaces the method's incoming headers responses_api_provider_config.validate_environment( headers=_client_request_headers(response_api_optional_request_params, extra_headers), model=model, diff --git a/litellm/main.py b/litellm/main.py index f30d93361e5..011617670b4 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -677,7 +677,8 @@ async def acompletion( } if custom_llm_provider is None: _supplemental: Final = cast( # cast-ok: kwargs values are request-scoped objects - "dict[str, object]", {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} + "dict[str, object]", + {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs}, ) _, custom_llm_provider, _, _ = get_llm_provider( model=model, @@ -5678,7 +5679,9 @@ def completion( if deployment_id is not None: # azure llms model = deployment_id custom_llm_provider = "azure" - _supplemental_provider_params: Final = {k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS if k in kwargs} + _supplemental_provider_params: Final = { + k: kwargs[k] for k in OPTIONAL_KWARGS_KEYS | {"provider_specific_header"} if k in kwargs + } model, custom_llm_provider, dynamic_api_key, api_base = get_llm_provider( model=model, custom_llm_provider=custom_llm_provider, @@ -5998,6 +6001,7 @@ def completion( tpm=kwargs.get("tpm"), rpm=kwargs.get("rpm"), use_xai_oauth=kwargs.get("use_xai_oauth", False), + chatgpt_auth_file=routing_kwargs.get("chatgpt_auth_file"), gigachat_scope=kwargs.get("gigachat_scope"), gigachat_auth_url=kwargs.get("gigachat_auth_url"), gigachat_access_token=kwargs.get("gigachat_access_token"), diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 25f900a5f9f..30ede935cfa 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -736,7 +736,9 @@ def is_request_body_safe( def _reject_request_credential_params(litellm_params: Mapping[str, object], *, manages_deployments: bool) -> None: if manages_deployments: - reject_server_owned_wif_fields(litellm_params) + reject_server_owned_wif_fields( + MappingProxyType({key: value for key, value in litellm_params.items() if key != "chatgpt_auth_file"}) + ) else: reject_server_owned_wif_params(litellm_params) reject_federated_credential_reference(litellm_params) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 458636bd6e0..00af4daf684 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -4299,9 +4299,7 @@ def add_provider_specific_headers_to_request( for header, value in headers.items() if header.lower() == "authorization" and is_anthropic_oauth_key(value) } - chatgpt_oauth_credential_headers: Final = dict( # mutable-ok: ProviderSpecificHeader.extra_headers is a dict field - select_chatgpt_client_credential_headers(headers) - ) + chatgpt_oauth_credential_headers: Final = dict(select_chatgpt_client_credential_headers(headers)) typesafe_danger_zone_headers: Final = { header: value @@ -4337,7 +4335,7 @@ def add_provider_specific_headers_to_request( if scoped_headers: data["provider_specific_header"] = scoped_headers[0] if len(scoped_headers) == 1 else scoped_headers - return bool(anthropic_oauth_credential_headers) + return bool(anthropic_oauth_credential_headers or chatgpt_oauth_credential_headers) def _add_otel_traceparent_to_data(data: dict, request: Request): diff --git a/litellm/responses/main.py b/litellm/responses/main.py index 19bf8d77d48..269a10426ab 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -705,10 +705,12 @@ def _api_base_kwarg(kwargs: Mapping[str, object]) -> str | None: return api_base if isinstance(api_base, str) else None -def _dispatched_model_name(model: str, custom_llm_provider: str, api_base: str | None) -> str: - provider_model, _, _, _ = litellm.get_llm_provider( - model=model, custom_llm_provider=custom_llm_provider, api_base=api_base - ) +def _dispatched_model_name( + model: str, custom_llm_provider: str, api_base: str | None, litellm_params: GenericLiteLLMParams | None = None +) -> str: + from litellm.litellm_core_utils.get_llm_provider_logic import resolve_model_and_provider_for_metadata + + provider_model, _ = resolve_model_and_provider_for_metadata(model=model, custom_llm_provider=custom_llm_provider) return _strip_responses_routing_prefix(provider_model) @@ -718,6 +720,7 @@ def _will_bridge_to_chat_completions( use_chat_completions_api: bool, model_info: object, api_base: str | None, + litellm_params: GenericLiteLLMParams | None = None, ) -> bool: """``_bridges_to_chat_completions`` for callers running before the provider config is resolved. @@ -734,7 +737,7 @@ def _will_bridge_to_chat_completions( return True return _bridges_to_chat_completions( _resolve_responses_api_provider_config( - _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base), + _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base, litellm_params), custom_llm_provider, model_info, api_base, @@ -750,12 +753,15 @@ def provider_stores_the_conversation(model: str, custom_llm_provider: str | None first MCP round runs as an internal call that writes no spend log at all. """ api_base: Final = _api_base_kwarg(kwargs) + litellm_params: Final = GenericLiteLLMParams(**kwargs) return not _will_bridge_to_chat_completions( model, - custom_llm_provider or litellm.get_llm_provider(model=model, api_base=api_base)[1], + custom_llm_provider + or litellm.get_llm_provider(model=model, api_base=api_base, litellm_params=litellm_params)[1], bool(kwargs.get("use_chat_completions_api")), kwargs.get("model_info"), api_base, + litellm_params, ) @@ -936,6 +942,7 @@ async def aresponses( bool(kwargs.get("use_chat_completions_api")), kwargs.get("model_info"), _api_base_kwarg(kwargs), + GenericLiteLLMParams(**kwargs), ), ): _stamp_injection_points_for_the_hook(kwargs, model, custom_llm_provider, input, tools) @@ -1051,11 +1058,14 @@ def _resolve_prompt_swapped_provider( kwargs: Mapping[str, object], prompt_id: str | None, ) -> str: - swapped_provider: Final = litellm.get_llm_provider(model=swapped_model)[1] + litellm_params: Final = GenericLiteLLMParams(**kwargs) + swapped_provider: Final = litellm.get_llm_provider(model=swapped_model, litellm_params=litellm_params)[1] if kwargs.get("api_key") is None and kwargs.get("api_base") is None: return swapped_provider try: - original_provider: Final = custom_llm_provider or litellm.get_llm_provider(model=original_model)[1] + original_provider: Final = ( + custom_llm_provider or litellm.get_llm_provider(model=original_model, litellm_params=litellm_params)[1] + ) except litellm.BadRequestError: return swapped_provider if swapped_provider == original_provider: @@ -1108,6 +1118,7 @@ def _apply_prompt_management_to_responses_call( use_chat_completions_api, kwargs.get("model_info"), _api_base_kwarg(kwargs), + GenericLiteLLMParams(**kwargs), ), ): _stamp_injection_points_for_the_hook(kwargs, model, custom_llm_provider, input, tools) @@ -1514,8 +1525,16 @@ def responses( use_chat_completions_api=use_chat_completions_api, ) - # get llm provider logic + from litellm.litellm_core_utils.core_helpers import attribute_or_none + litellm_params: Final = GenericLiteLLMParams(**kwargs) + provider_headers: Final = ResponsesAPIRequestUtils.merge_provider_specific_headers( + extra_headers=extra_headers, + provider_specific_header=attribute_or_none(litellm_params, "provider_specific_header"), + custom_llm_provider=custom_llm_provider, + ) + local_vars["extra_headers"] = provider_headers + affinity_api_base: Final = ( get_openai_credentials(api_base=litellm_params.api_base).api_base if custom_llm_provider == "openai" @@ -1525,7 +1544,7 @@ def responses( try: effective_extra_headers: Final = ( add_provider_affinity_header( - headers=extra_headers or MappingProxyType({}), + headers=provider_headers or MappingProxyType({}), litellm_params=MappingProxyType( { "provider_affinity_header": litellm_params.provider_affinity_header, @@ -1538,7 +1557,7 @@ def responses( default_header=default_affinity_header, ) if litellm_params.provider_affinity_header is not None or default_affinity_header is not None - else extra_headers + else provider_headers ) except ValueError as affinity_error: raise litellm.BadRequestError( @@ -1558,13 +1577,6 @@ def responses( local_vars=local_vars, ) - extra_headers = ResponsesAPIRequestUtils.merge_provider_specific_headers( - extra_headers=extra_headers, - provider_specific_header=kwargs.get("provider_specific_header"), - custom_llm_provider=custom_llm_provider, - ) - local_vars["extra_headers"] = extra_headers - ######################################################### # Update input and tools with provider-specific file IDs if managed files are used ######################################################### diff --git a/litellm/responses/utils.py b/litellm/responses/utils.py index 6f60ed2f0f4..2d0873a218a 100644 --- a/litellm/responses/utils.py +++ b/litellm/responses/utils.py @@ -10,9 +10,9 @@ from typing_extensions import TypeIs # noqa: TID251 # narrows untyped wire pay import litellm from litellm._logging import verbose_logger -from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.litellm_core_utils.core_helpers import as_str_mapping, attribute_or_none, iter_items from litellm.litellm_core_utils.dot_notation_indexing import delete_nested_value, is_nested_path +from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig, ReasoningSummaryLevel from litellm.types.llms.openai import ( AllMessageValues, @@ -36,7 +36,9 @@ from litellm.types.utils import ( text_tokens_without_nested_reasoning, ) -_PROVIDER_SPECIFIC_HEADER_ADAPTER: Final = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None) +_PROVIDER_SPECIFIC_HEADER_ADAPTER: Final[ + TypeAdapter[ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None] +] = TypeAdapter(ProviderSpecificHeader | Sequence[ProviderSpecificHeader] | None) class _UsageExtraFields(ExtraItemsTypedDict, total=False, extra_items=object): @@ -239,19 +241,17 @@ class ResponsesAPIRequestUtils: @staticmethod def merge_provider_specific_headers( - extra_headers: dict[str, object] | None, + extra_headers: Mapping[str, object] | None, provider_specific_header: object, custom_llm_provider: str | None, - ) -> dict[str, object] | None: + ) -> dict[str, object] | None: # mutable-ok: legacy Responses request fields require a dict scoped_headers: Final = ProviderSpecificHeaderUtils.get_provider_specific_headers( provider_specific_header=_PROVIDER_SPECIFIC_HEADER_ADAPTER.validate_python(provider_specific_header), custom_llm_provider=custom_llm_provider, ) - if not scoped_headers: - return extra_headers - if not extra_headers: - return scoped_headers - return {**extra_headers, **scoped_headers} # mutable-ok: extra_headers is the request's mutable header dict + if extra_headers is None and not scoped_headers: + return None + return {**(extra_headers or {}), **scoped_headers} @staticmethod def _check_valid_arg( diff --git a/litellm/router.py b/litellm/router.py index 7b8c258b5b9..5f432a25131 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -98,6 +98,7 @@ from litellm.litellm_core_utils.credential_accessor import CredentialAccessor, r from litellm.litellm_core_utils.dd_tracing import tracer from litellm.litellm_core_utils.get_llm_provider_logic import ( declared_authenticating_provider, + inferred_provider, is_registered_custom_provider, resolve_model_and_provider_for_metadata, ) @@ -4799,9 +4800,28 @@ class Router: deployment_litellm_model_name = deployment["litellm_params"]["model"] deployment_api_base = deployment["litellm_params"].get("api_base") deployment_model_name: Final = deployment["model_name"] - if is_clientside_credential(request_kwargs=kwargs): + from litellm.litellm_core_utils.get_provider_specific_headers import ProviderSpecificHeaderUtils + + request_context: Final = cast( # cast-ok: the router's legacy request dict has string keys + Mapping[str, object], kwargs + ) + deployment_context: Final = _LITELLM_PARAMS_ADAPTER.validate_python(deployment["litellm_params"]) + declared_provider: Final = deployment_context.get("custom_llm_provider") + declared_model: Final = deployment_context.get("model") + forwarded_authorization: Final = ProviderSpecificHeaderUtils.forwarded_authorization( + request_context.get("provider_specific_header"), + declared_provider + if isinstance(declared_provider, str) + else inferred_provider(declared_model if isinstance(declared_model, str) else None), + ) + if is_clientside_credential(request_kwargs=kwargs) or forwarded_authorization is not None: deployment_pydantic_obj: Final = self._handle_clientside_credential( - deployment=deployment, kwargs=kwargs, function_name=function_name + deployment=deployment, + kwargs={ + **kwargs, + **({"api_key": forwarded_authorization} if forwarded_authorization is not None else {}), + }, + function_name=function_name, ) model_info = deployment_pydantic_obj.model_info.model_dump() deployment_litellm_model_name = deployment_pydantic_obj.litellm_params.model @@ -11048,6 +11068,10 @@ class Router: custom_llm_provider = None dynamic_api_key = None api_base = None + elif declared_authenticating_provider(litellm_model, deployment.litellm_params.custom_llm_provider) is not None: + _model, custom_llm_provider = ( # rebind-ok: each branch resolves the deployment provider + self._model_group_llm_provider(deployment.litellm_params) + ) else: # check if model provider in supported providers ( diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 089f21a33c1..717b74e5741 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -91,6 +91,7 @@ class ProviderConnection: litellm_credential_name: str | None = None configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None use_xai_oauth: bool | None = None + chatgpt_auth_file: str | None = None github_copilot_auth_type: str | None = None github_copilot_host: str | None = None github_copilot_user_session: object | None = None diff --git a/litellm/types/router.py b/litellm/types/router.py index cd1389aea9d..ba9b82e2a57 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -572,6 +572,10 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): default=False, description="Use stored xAI OAuth credentials when no xAI API key is configured.", ) + chatgpt_auth_file: str | None = Field( + default=None, + description="Path to a ChatGPT OAuth auth.json for this deployment.", + ) fireworks_forward_user_id: bool | None = Field( default=None, description="Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests.", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index cc7ef851111..4f39741a079 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4360,6 +4360,7 @@ server_owned_wif_litellm_params: Final = ( + openai_wif_litellm_params + oauth_token_exchange_litellm_params + github_copilot_oauth_litellm_params + + ("chatgpt_auth_file",) ) secret_bearing_wif_litellm_params: Final = tuple(sorted(WIF_SECRET_BEARING_KEYS)) diff --git a/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py b/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py index 40ffc38c2b2..6d1097a90fb 100644 --- a/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py +++ b/tests/unit/llms/chatgpt/chat/test_chatgpt_chat_transformation.py @@ -57,3 +57,444 @@ class TestChatGPTChatProviderInfo: with pytest.raises(AuthenticationError, match="not logged in"): _provider_info(config, None, None) + + +import asyncio +import json +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import httpx +import respx + +import litellm +from litellm.integrations.dotprompt.dotprompt_manager import DotpromptManager +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.chatgpt.chat.transformation import ChatGPTConfig +from litellm.types.router import GenericLiteLLMParams + + +def _write_auth_record(path: Path, token: str, account_id: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "access_token": token, + "account_id": account_id, + "expires_at": 4102444800, + } + ) + ) + + +@pytest.fixture +def auth_files(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> tuple[Path, Path]: + monkeypatch.setattr("time.time", lambda: 1700000000) + default_dir: Final = tmp_path / "default" + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(default_dir)) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + _write_auth_record(default_dir / "auth.json", "token-default", "acct-default") + file_a: Final = tmp_path / "account-a" / "auth.json" + file_b: Final = tmp_path / "account-b" / "auth.json" + _write_auth_record(file_a, "token-a", "acct-a") + _write_auth_record(file_b, "token-b", "acct-b") + return file_a, file_b + + +class TestChatGPTMultiAccountChat: + def test_validate_environment_uses_per_model_auth_file(self, auth_files: tuple[Path, Path]) -> None: + file_a, file_b = auth_files + config: Final = ChatGPTConfig() + + headers_a: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + messages=[], + optional_params={}, + litellm_params={"chatgpt_auth_file": str(file_a)}, + api_key="token-default", + ) + headers_b: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + messages=[], + optional_params={}, + litellm_params={"chatgpt_auth_file": str(file_b)}, + api_key="token-default", + ) + + assert headers_a["Authorization"] == "Bearer token-a" + assert headers_a["ChatGPT-Account-Id"] == "acct-a" + assert headers_b["Authorization"] == "Bearer token-b" + assert headers_b["ChatGPT-Account-Id"] == "acct-b" + + def test_validate_environment_defaults_to_global_auth_file(self, auth_files: tuple[Path, Path]) -> None: + config: Final = ChatGPTConfig() + + headers: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + messages=[], + optional_params={}, + litellm_params={}, + api_key="token-default", + ) + + assert headers["Authorization"] == "Bearer token-default" + assert headers["ChatGPT-Account-Id"] == "acct-default" + + def test_provider_info_uses_per_model_auth_file(self, auth_files: tuple[Path, Path]) -> None: + file_a, _ = auth_files + config: Final = ChatGPTConfig() + + _, per_model_key, _ = config._get_openai_compatible_provider_info( + "gpt-6.1-sol", None, None, "chatgpt", {"chatgpt_auth_file": str(file_a)} + ) + _, default_key, _ = config._get_openai_compatible_provider_info("gpt-6.1-sol", None, None, "chatgpt") + + assert per_model_key == "token-a" + assert default_key == "token-default" + + def test_get_llm_provider_threads_auth_file(self, auth_files: tuple[Path, Path]) -> None: + file_a, _ = auth_files + + model, provider, dynamic_api_key, _ = get_llm_provider( + model="chatgpt/gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(chatgpt_auth_file=str(file_a)), + ) + + assert model == "gpt-6.1-sol" + assert provider == "chatgpt" + assert dynamic_api_key == "token-a" + + +def _subscription_response(request: httpx.Request) -> httpx.Response: + account: Final = request.headers["ChatGPT-Account-Id"] + assert request.headers["Authorization"] == f"Bearer token-{account.removeprefix('acct-')}" + assert "chatgpt_auth_file" not in request.content.decode() + message: Final = { + "id": "msg_test", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": account, "annotations": []}], + } + response: Final = { + "id": "resp_test", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6.1-sol", + "output": [message], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + } + events: Final = ( + {"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}, + { + "type": "response.output_text.delta", + "item_id": "msg_test", + "output_index": 0, + "content_index": 0, + "delta": account, + }, + {"type": "response.output_item.done", "output_index": 0, "item": message}, + {"type": "response.completed", "response": response}, + ) + return httpx.Response( + 200, + text="".join(f"data: {json.dumps(event)}\n\n" for event in events), + headers={"content-type": "text/event-stream"}, + ) + + +@pytest.mark.parametrize("blocked_default", [False, True]) +@pytest.mark.parametrize("with_prompt", [False, True]) +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("through_router", [False, True]) +@pytest.mark.asyncio +async def test_requests_use_selected_subscription( + auth_files: tuple[Path, Path], + endpoint: str, + through_router: bool, + with_prompt: bool, + blocked_default: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + default_dir: Final = auth_files[0].parent / "missing-default" + if blocked_default: + default_dir.write_text("ordinary file") + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(default_dir / "unused")) + if with_prompt: + monkeypatch.setattr( + litellm, + "callbacks", + [ + DotpromptManager( + prompt_id="account-check", + prompt_data={"content": "Saved account prompt", "metadata": {"model": "chatgpt/gpt-6.1-sol"}}, + ) + ], + ) + router: Final = ( + litellm.Router( + model_list=[ + { + "model_name": f"account-{index}", + "litellm_params": {"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(path)}, + } + for index, path in enumerate(auth_files) + ] + ) + if through_router + else None + ) + + async def request(index: int, path: Path) -> str: + params: Final = MappingProxyType( + { + "model": f"account-{index}" if router else "chatgpt/gpt-6.1-sol", + **({"chatgpt_auth_file": str(path)} if router is None else {}), + **({"prompt_id": "account-check"} if with_prompt else {}), + } + ) + if endpoint == "chat": + result: Final = await (router.acompletion if router else litellm.acompletion)( + **params, messages=[{"role": "user", "content": "Hello"}] + ) + return result.choices[0].message.content + response: Final = await (router.aresponses if router else litellm.aresponses)( + **params, input=[{"role": "user", "content": "Hello"}] + ) + return response.output[0].content[0].text + + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=_subscription_response) + results: Final = await asyncio.gather(*(request(index, path) for index, path in enumerate(auth_files))) + assert results == ["acct-a", "acct-b"] + if with_prompt: + assert all("Saved account prompt" in call.request.content.decode() for call in transport.calls) + + +def _client_token(account: str) -> str: + import base64 + + payload: Final = ( + base64.urlsafe_b64encode(json.dumps({"https://api.openai.com/auth": {"chatgpt_account_id": account}}).encode()) + .rstrip(b"=") + .decode() + ) + return f"header.{payload}.signature" + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("configured_login", [False, True]) +@pytest.mark.parametrize("with_prompt", [False, True]) +@pytest.mark.parametrize("stream", [False, True]) +@pytest.mark.asyncio +async def test_client_login_keeps_its_account_across_gateway_deployments( + auth_files: tuple[Path, Path], + endpoint: str, + configured_login: bool, + with_prompt: bool, + stream: bool, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(auth_files[0].parent / "no-gateway-login")) + if with_prompt: + monkeypatch.setattr( + litellm, + "callbacks", + [ + DotpromptManager( + prompt_id="client-account-check", + prompt_data={"content": "Saved client prompt", "metadata": {"model": "chatgpt/gpt-6.1-sol"}}, + ) + ], + ) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "subscriptions", + "litellm_params": { + "model": "chatgpt/gpt-6.1-sol", + **({"chatgpt_auth_file": str(path)} if configured_login else {}), + }, + } + for path in auth_files + ], + num_retries=0, + ) + + def reply(request: httpx.Request) -> httpx.Response: + account: Final = request.headers["ChatGPT-Account-Id"] + assert request.headers["Authorization"] == f"Bearer {_client_token(account)}" + return _subscription_response( + httpx.Request( + "POST", + request.url, + content=request.content, + headers={ + **{key: value for key, value in request.headers.items() if key.lower() != "authorization"}, + "Authorization": f"Bearer token-{account.removeprefix('acct-')}", + }, + ) + ) + + async def call(account: str) -> str: + params: Final = MappingProxyType( + { + "model": "subscriptions", + "stream": stream, + **({"prompt_id": "client-account-check"} if with_prompt else {}), + "provider_specific_header": { + "custom_llm_provider": "chatgpt", + "extra_headers": {"Authorization": f"Bearer {_client_token(account)}"}, + }, + } + ) + response: Final = await ( + router.acompletion(**params, messages=[{"role": "user", "content": "Hello"}]) + if endpoint == "chat" + else router.aresponses(**params, input=[{"role": "user", "content": "Hello"}]) + ) + if stream: + chunks: Final = [part async for part in response] + return ( + "".join(part.choices[0].delta.content or "" for part in chunks) + if endpoint == "chat" + else "".join( + getattr(part, "delta", "") + for part in chunks + if getattr(part, "type", "") == "response.output_text.delta" + ) + ) + return response.choices[0].message.content if endpoint == "chat" else response.output[0].content[0].text + + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").mock(side_effect=reply) + results: Final = await asyncio.gather(call("acct-client-a"), call("acct-client-b")) + assert results == ["acct-client-a", "acct-client-b"] + assert len(transport.calls) == 2 + + +@pytest.mark.asyncio +async def test_one_client_limit_does_not_cool_down_other_subscriptions( + auth_files: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + {"model_name": "pool", "litellm_params": {"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(path)}} + for path in auth_files + ], + num_retries=0, + allowed_fails=0, + cooldown_time=60, + ) + headers: Final = { + "custom_llm_provider": "chatgpt", + "extra_headers": {"Authorization": f"Bearer {_client_token('acct-limited')}"}, + } + with respx.mock as transport: + route: Final = transport.post("https://chatgpt.com/backend-api/codex/responses") + route.respond( + 429, + headers={"retry-after": "60"}, + json={"error": {"message": "Usage limit reached", "type": "rate_limit_error"}}, + ) + for _ in range(2): + with pytest.raises(litellm.RateLimitError): + await router.aresponses(model="pool", input="Hello", provider_specific_header=headers) + await asyncio.sleep(0) + route.mock(side_effect=_subscription_response) + response: Final = await router.aresponses(model="pool", input="Hello") + assert response.output[0].content[0].text in ("acct-a", "acct-b") + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize( + "target", ["https://example.com/v1", "http://chatgpt.com/backend-api/codex", "https://chatgpt.com.example.com/v1"] +) +@pytest.mark.asyncio +async def test_client_login_cannot_be_redirected_to_another_host(endpoint: str, target: str) -> None: + params: Final = MappingProxyType( + { + "model": "chatgpt/gpt-6.1-sol", + "api_base": target, + "provider_specific_header": { + "custom_llm_provider": "chatgpt", + "extra_headers": {"Authorization": f"Bearer {_client_token('acct-client')}"}, + }, + } + ) + with respx.mock as transport: + request: Final = ( + litellm.acompletion(**params, messages=[{"role": "user", "content": "Hello"}]) + if endpoint == "chat" + else litellm.aresponses(**params, input="Hello") + ) + with pytest.raises(litellm.BadRequestError, match="official HTTPS ChatGPT endpoint"): + await request + assert len(transport.calls) == 0 + + +@pytest.mark.asyncio +async def test_cross_provider_fallback_does_not_receive_the_chatgpt_login( + auth_files: tuple[Path, Path], monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "subscription", + "litellm_params": {"model": "chatgpt/gpt-6.1-sol", "chatgpt_auth_file": str(auth_files[0])}, + }, + { + "model_name": "api-fallback", + "litellm_params": {"model": "openai/gpt-6.1-sol", "api_key": "sk-example-fallback"}, + }, + ], + fallbacks=[{"subscription": ["api-fallback"]}], + num_retries=0, + ) + with respx.mock as transport: + transport.post("https://chatgpt.com/backend-api/codex/responses").respond( + 429, json={"error": {"message": "Usage limit reached", "type": "rate_limit_error"}} + ) + fallback: Final = transport.post("https://api.openai.com/v1/responses").respond( + 200, + json={ + "id": "resp_fallback", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": "gpt-6.1-sol", + "output": [ + { + "id": "msg_fallback", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "fallback", "annotations": []}], + } + ], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + response: Final = await router.aresponses( + model="subscription", + input="Hello", + provider_specific_header={ + "custom_llm_provider": "chatgpt", + "extra_headers": { + "Authorization": f"Bearer {_client_token('acct-client')}", + "ChatGPT-Account-Id": "acct-client", + }, + }, + ) + assert response.output[0].content[0].text == "fallback" + assert fallback.calls.last.request.headers["Authorization"] == "Bearer sk-example-fallback" + assert "ChatGPT-Account-Id" not in fallback.calls.last.request.headers diff --git a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py index bd60372de3a..84ef5045c60 100644 --- a/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py +++ b/tests/unit/llms/chatgpt/responses/test_chatgpt_responses_transformation.py @@ -7,6 +7,7 @@ Source: litellm/llms/chatgpt/responses/transformation.py import base64 import json from collections.abc import Generator +from pathlib import Path from typing import Final from unittest.mock import MagicMock, patch @@ -400,9 +401,13 @@ class TestChatGPTResponsesAPITransformation: def _fake_chatgpt_oauth_token(account_id: str) -> str: header = base64.urlsafe_b64encode(b'{"alg":"RS256"}').rstrip(b"=").decode() - payload = base64.urlsafe_b64encode( - json.dumps({"https://api.openai.com/auth": {"chatgpt_account_id": account_id}}).encode() - ).rstrip(b"=").decode() + payload = ( + base64.urlsafe_b64encode( + json.dumps({"https://api.openai.com/auth": {"chatgpt_account_id": account_id}}).encode() + ) + .rstrip(b"=") + .decode() + ) return f"{header}.{payload}.fake-signature" @@ -447,7 +452,9 @@ class TestChatGPTClientCredentialForwarding: assert headers["ChatGPT-Account-Id"] == "acct-from-header" assert [name for name in headers if name.lower() == "chatgpt-account-id"] == ["ChatGPT-Account-Id"] - @pytest.mark.parametrize("bearer", ["Bearer sk-proj-fake-platform-key", "Bearer not-a-jwt", "Bearer a.bm90LWpzb24.c"]) + @pytest.mark.parametrize( + "bearer", ["Bearer sk-proj-fake-platform-key", "Bearer not-a-jwt", "Bearer a.bm90LWpzb24.c"] + ) def test_non_chatgpt_bearer_falls_back_to_configured_key(self, bearer): authenticator = _gateway_authenticator() @@ -496,3 +503,59 @@ class TestChatGPTClientCredentialForwarding: assert [h["Authorization"] for h in forwarded] == [f"Bearer {alice}", f"Bearer {bob}", f"Bearer {alice}"] assert [h["ChatGPT-Account-Id"] for h in forwarded] == ["acct-alice", "acct-bob", "acct-alice"] + + +class TestChatGPTResponsesMultiAccount: + @pytest.fixture(autouse=True) + def fixed_clock(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + + @staticmethod + def _write_auth_record(path: Path, token: str, account_id: str) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text( + json.dumps( + { + "access_token": token, + "account_id": account_id, + "expires_at": 4102444800, + } + ) + ) + + def test_validate_environment_uses_per_model_auth_file( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + default_dir: Final = tmp_path / "default" + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(default_dir)) + monkeypatch.delenv("CHATGPT_AUTH_FILE", raising=False) + self._write_auth_record(default_dir / "auth.json", "token-default", "acct-default") + file_a: Final = tmp_path / "account-a" / "auth.json" + file_b: Final = tmp_path / "account-b" / "auth.json" + self._write_auth_record(file_a, "token-a", "acct-a") + self._write_auth_record(file_b, "token-b", "acct-b") + + config: Final = ChatGPTResponsesAPIConfig() + + headers_a: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(chatgpt_auth_file=str(file_a)), + ) + headers_b: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(chatgpt_auth_file=str(file_b)), + ) + headers_default: Final = config.validate_environment( + headers={}, + model="gpt-6.1-sol", + litellm_params=GenericLiteLLMParams(), + ) + + assert headers_a["Authorization"] == "Bearer token-a" + assert headers_a["ChatGPT-Account-Id"] == "acct-a" + assert headers_b["Authorization"] == "Bearer token-b" + assert headers_b["ChatGPT-Account-Id"] == "acct-b" + assert headers_default["Authorization"] == "Bearer token-default" + assert headers_default["ChatGPT-Account-Id"] == "acct-default" diff --git a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py index 471a95f87ae..48e55e166e8 100644 --- a/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py +++ b/tests/unit/llms/chatgpt/test_chatgpt_authenticator.py @@ -2,6 +2,8 @@ import base64 import json import time from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Final from unittest.mock import MagicMock, mock_open, patch import pytest @@ -11,8 +13,11 @@ from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS, HTTP_HANDLER_CONN from litellm.llms.chatgpt.authenticator import ( TOKEN_REFRESH_TIMEOUT_SECONDS, Authenticator, + get_cached_authenticator, + get_chatgpt_auth_file, ) from litellm.llms.chatgpt.common_utils import GetAccessTokenError +from litellm.types.router import GenericLiteLLMParams def _make_jwt(payload: dict) -> str: @@ -180,3 +185,98 @@ class TestChatGPTAuthenticator: ): assert authenticator.get_account_id() is None mock_write.assert_not_called() + + +def _write_auth_record(path: Path, token: str, account_id: str) -> None: + path.write_text( + json.dumps( + { + "access_token": token, + "account_id": account_id, + "expires_at": 4102444800, + } + ) + ) + + +class TestChatGPTMultiAccountAuthenticator: + @pytest.fixture(autouse=True) + def fixed_clock(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("time.time", lambda: 1700000000) + + def test_explicit_auth_file_overrides_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "env-dir")) + monkeypatch.setenv("CHATGPT_AUTH_FILE", "env.json") + custom_file: Final = tmp_path / "account-a" / "auth.json" + + authenticator: Final = Authenticator(auth_file=str(custom_file)) + + assert authenticator.auth_file == str(custom_file) + assert authenticator.token_dir == str(custom_file.parent) + assert not custom_file.parent.exists() + authenticator._write_auth_file({"access_token": "token-a"}) + assert json.loads(custom_file.read_text())["access_token"] == "token-a" + + def test_default_auth_file_from_env(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("CHATGPT_TOKEN_DIR", str(tmp_path / "env-dir")) + monkeypatch.setenv("CHATGPT_AUTH_FILE", "env.json") + + authenticator: Final = Authenticator() + + assert authenticator.auth_file == str(tmp_path / "env-dir" / "env.json") + + def test_bare_filename_auth_file_resolves_against_cwd( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + + authenticator: Final = Authenticator(auth_file="auth.json") + + assert authenticator.auth_file == str(tmp_path / "auth.json") + assert authenticator.token_dir == str(tmp_path) + + def test_two_auth_files_are_isolated(self, tmp_path: Path) -> None: + file_a: Final = tmp_path / "account-a.json" + file_b: Final = tmp_path / "account-b.json" + _write_auth_record(file_a, "token-a", "acct-a") + _write_auth_record(file_b, "token-b", "acct-b") + + authenticator_a: Final = Authenticator(auth_file=str(file_a)) + authenticator_b: Final = Authenticator(auth_file=str(file_b)) + + assert authenticator_a.get_access_token() == "token-a" + assert authenticator_a.get_account_id() == "acct-a" + assert authenticator_b.get_access_token() == "token-b" + assert authenticator_b.get_account_id() == "acct-b" + + def test_get_cached_authenticator_reuses_instance_per_path(self, tmp_path: Path) -> None: + file_a: Final = tmp_path / "account-a.json" + file_b: Final = tmp_path / "account-b.json" + + authenticator_a: Final = get_cached_authenticator(str(file_a)) + + assert get_cached_authenticator(str(file_a)) is authenticator_a + assert get_cached_authenticator(str(file_b)) is not authenticator_a + + def test_get_chatgpt_auth_file(self) -> None: + assert get_chatgpt_auth_file(None) is None + assert get_chatgpt_auth_file({}) is None + assert get_chatgpt_auth_file({"chatgpt_auth_file": ""}) is None + assert get_chatgpt_auth_file({"chatgpt_auth_file": "/a/auth.json"}) == "/a/auth.json" + assert get_chatgpt_auth_file(GenericLiteLLMParams()) is None + assert get_chatgpt_auth_file(GenericLiteLLMParams(chatgpt_auth_file="/b/auth.json")) == "/b/auth.json" + + def test_relative_and_absolute_auth_file_share_cached_authenticator( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.chdir(tmp_path) + get_cached_authenticator.cache_clear() + auth_file: Final = tmp_path / "x" / "auth.json" + auth_file.parent.mkdir(parents=True) + auth_file.write_text("{}") + + relative_key: Final = get_chatgpt_auth_file({"chatgpt_auth_file": "x/auth.json"}) + absolute_key: Final = get_chatgpt_auth_file({"chatgpt_auth_file": str(auth_file)}) + assert relative_key is not None and absolute_key is not None + + assert get_cached_authenticator(relative_key) is get_cached_authenticator(absolute_key) diff --git a/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py b/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py index 9fdb1428877..6b280d97089 100644 --- a/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py +++ b/tests/unit/llms/chatgpt/test_chatgpt_common_utils.py @@ -76,3 +76,15 @@ class TestSelectChatGPTClientCredentialHeaders: credential = get_chatgpt_client_credential({"authorization": f"Bearer {CHATGPT_JWT}"}) assert credential is not None assert credential.account_id == "acct-alice" + + +@pytest.mark.parametrize("scheme", ["Bearer", "bearer", "BEARER"]) +def test_bearer_scheme_is_case_insensitive(scheme): + credential = get_chatgpt_client_credential({"authorization": f"{scheme} {CHATGPT_JWT}"}) + assert credential is not None + assert credential.access_token == CHATGPT_JWT + assert credential.account_id == "acct-alice" + + +def test_other_authentication_schemes_are_not_chatgpt_logins(): + assert not is_chatgpt_oauth_key(f"Basic {CHATGPT_JWT}") diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index ee1ae43fd3a..10a0f974a42 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -3626,6 +3626,18 @@ class TestIsRequestBodySafeBlocksEndpointTargetingFields: ) +@pytest.mark.parametrize("allow_client_side_credentials", [False, True]) +@pytest.mark.parametrize("nested", [False, True]) +def test_request_cannot_select_chatgpt_auth_file(allow_client_side_credentials: bool, nested: bool) -> None: + selector: Final = {"chatgpt_auth_file": "/tokens/account-b/auth.json"} + with pytest.raises(ValueError, match="chatgpt_auth_file"): + is_request_body_safe( + request_body={"model": "chatgpt-account-a", **({"extra_body": selector} if nested else selector)}, + general_settings={"allow_client_side_credentials": allow_client_side_credentials}, + llm_router=None, + model="chatgpt-account-a", + ) + class TestIsRequestBodySafeBlocksBedrockProjectOverride: """``aws_bedrock_project_id`` pins a deployment to a Bedrock project so that project's data-retention policy applies to its requests. A @@ -5304,3 +5316,27 @@ async def test_a_non_numeric_max_request_size_mb_is_ignored_without_premium(): patch("litellm.proxy.proxy_server.premium_user", False), ): assert await check_if_request_size_is_safe(request=_request_with_content_length(1024)) is True + + +@pytest.mark.parametrize("allow_client_side_credentials", [False, True]) +@pytest.mark.parametrize("route", ["/model/new", "/model/update", "/model/account-a/update", "/health/test_connection"]) +def test_chatgpt_model_configuration_reaches_role_authorization( + allow_client_side_credentials: bool, route: str +) -> None: + body: Final = {"litellm_params": {"model": "chatgpt/gpt-6-luna", "chatgpt_auth_file": "/tokens/account-a/auth.json"}} + assert is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": allow_client_side_credentials}, + llm_router=None, + model="", + route=route, + ) + for inference_route in ("/v1/chat/completions", "/v1/responses"): + with pytest.raises(ValueError, match="chatgpt_auth_file"): + is_request_body_safe( + request_body=body, + general_settings={"allow_client_side_credentials": allow_client_side_credentials}, + llm_router=None, + model="", + route=inference_route, + ) diff --git a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py index 303b8e060ed..a718426eea6 100644 --- a/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_model_management_endpoints.py @@ -8933,7 +8933,14 @@ class TestNonAdminCannotPersistWifFieldsOnModel: mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() @pytest.mark.asyncio - async def test_patch_model_non_admin_cannot_set_wif_field(self): + @pytest.mark.parametrize( + "field,value", + [ + ("anthropic_keycloak_token_url", "https://identity.example.com/token"), + ("chatgpt_auth_file", "/tokens/account-b/auth.json"), + ], + ) + async def test_patch_model_non_admin_cannot_set_wif_field(self, field: str, value: str) -> None: from litellm.proxy.management_endpoints.model_management_endpoints import ( patch_model, ) @@ -8979,13 +8986,13 @@ class TestNonAdminCannotPersistWifFieldsOnModel: model_id="m1", patch_data=updateDeployment( litellm_params=updateLiteLLMParams( - anthropic_keycloak_token_url="https://attacker.example/token", + **{field: value}, ) ), user_api_key_dict=non_admin, ) err = exc_info.value - assert getattr(err, "param", "") == "anthropic_keycloak_token_url" + assert getattr(err, "param", "") == field mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() @pytest.mark.asyncio @@ -9044,7 +9051,14 @@ class TestNonAdminCannotPersistWifFieldsOnModel: mock_prisma.db.litellm_proxymodeltable.update.assert_not_called() @pytest.mark.asyncio - async def test_patch_model_admin_can_set_wif_field(self): + @pytest.mark.parametrize( + "field,value", + [ + ("anthropic_keycloak_token_url", "https://identity.example.com/token"), + ("chatgpt_auth_file", "/tokens/account-b/auth.json"), + ], + ) + async def test_patch_model_admin_can_set_wif_field(self, field: str, value: str) -> None: from litellm.proxy.management_endpoints.model_management_endpoints import ( patch_model, ) @@ -9095,7 +9109,7 @@ class TestNonAdminCannotPersistWifFieldsOnModel: model_id="m1", patch_data=updateDeployment( litellm_params=updateLiteLLMParams( - anthropic_keycloak_token_url="https://keycloak.internal/token", + **{field: value}, ) ), user_api_key_dict=admin, diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index b0086662f1c..740342f54d9 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -94,6 +94,7 @@ CONNECTION_NAMES: Final = ( "github_copilot_host", "configurable_clientside_auth_params", "use_xai_oauth", + "chatgpt_auth_file", "fireworks_forward_user_id", "aws_batch_role_arn", "aws_bedrock_control_plane_endpoint", diff --git a/tests/unit/types/test_router.py b/tests/unit/types/test_router.py index 2892a829eab..7855eba0042 100644 --- a/tests/unit/types/test_router.py +++ b/tests/unit/types/test_router.py @@ -16,8 +16,8 @@ from litellm.types.router import ( holds_secret_pointer, reject_server_owned_wif_params, server_owned_wif_fields_named, - server_owned_wif_fields_written, server_owned_wif_fields_present, + server_owned_wif_fields_written, ) from litellm.types.utils import ( CustomPricingLiteLLMParams, @@ -367,6 +367,7 @@ def test_server_owned_registry_includes_anthropic_openai_and_oauth_token_exchang + openai_wif_litellm_params + oauth_token_exchange_litellm_params + github_copilot_oauth_litellm_params + + ("chatgpt_auth_file",) ) assert set(openai_wif_litellm_params) == { "openai_identity_provider_id", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 262e8b6deae..011e131c858 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35309,6 +35309,11 @@ export interface components { cache_read_input_token_cost_priority?: number | null; /** Cache Read Input Token Cost Ultrafast */ cache_read_input_token_cost_ultrafast?: number | null; + /** + * Chatgpt Auth File + * @description Path to a ChatGPT OAuth auth.json for this deployment. + */ + chatgpt_auth_file?: string | null; /** Citation Cost Per Token */ citation_cost_per_token?: number | null; /** Client Id */ @@ -50660,6 +50665,11 @@ export interface components { cache_read_input_token_cost_priority?: number | null; /** Cache Read Input Token Cost Ultrafast */ cache_read_input_token_cost_ultrafast?: number | null; + /** + * Chatgpt Auth File + * @description Path to a ChatGPT OAuth auth.json for this deployment. + */ + chatgpt_auth_file?: string | null; /** Citation Cost Per Token */ citation_cost_per_token?: number | null; /** Client Id */