mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-13 23:11:40 +00:00
chore(typing): clear basedpyright Any errors in proxy, streaming, and logging modules
Replace Any-typed seams with concrete types across key management, team endpoints, proxy server/utils, streaming handler, llm http handler, and responses/mcp modules. Lowers ruff-strict, type-discipline, and basedpyright-code budget ceilings to match.
This commit is contained in:
mateo-berri
parent
23de7a15d9
commit
5ff4f70a39
14 changed files with 2953 additions and 2897 deletions
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"reportAny": {
|
||||
"limit": 31256
|
||||
"limit": 29310
|
||||
},
|
||||
"reportArgumentType": {
|
||||
"limit": 2645
|
||||
"limit": 2633
|
||||
},
|
||||
"reportAssignmentType": {
|
||||
"limit": 329
|
||||
|
|
@ -18,13 +18,13 @@
|
|||
"limit": 59
|
||||
},
|
||||
"reportDeprecated": {
|
||||
"limit": 325
|
||||
"limit": 322
|
||||
},
|
||||
"reportDuplicateImport": {
|
||||
"limit": 42
|
||||
"limit": 39
|
||||
},
|
||||
"reportExplicitAny": {
|
||||
"limit": 10208
|
||||
"limit": 8957
|
||||
},
|
||||
"reportFunctionMemberAccess": {
|
||||
"limit": 11
|
||||
|
|
@ -54,10 +54,10 @@
|
|||
"limit": 0
|
||||
},
|
||||
"reportMissingParameterType": {
|
||||
"limit": 5869
|
||||
"limit": 5851
|
||||
},
|
||||
"reportMissingTypeArgument": {
|
||||
"limit": 15861
|
||||
"limit": 15840
|
||||
},
|
||||
"reportMissingTypeStubs": {
|
||||
"limit": 41
|
||||
|
|
@ -99,19 +99,19 @@
|
|||
"limit": 0
|
||||
},
|
||||
"reportUnknownArgumentType": {
|
||||
"limit": 45357
|
||||
"limit": 44980
|
||||
},
|
||||
"reportUnknownLambdaType": {
|
||||
"limit": 113
|
||||
},
|
||||
"reportUnknownMemberType": {
|
||||
"limit": 40477
|
||||
"limit": 40408
|
||||
},
|
||||
"reportUnknownParameterType": {
|
||||
"limit": 20338
|
||||
"limit": 20308
|
||||
},
|
||||
"reportUnknownVariableType": {
|
||||
"limit": 32047
|
||||
"limit": 31987
|
||||
},
|
||||
"reportUnnecessaryCast": {
|
||||
"limit": 177
|
||||
|
|
@ -138,7 +138,7 @@
|
|||
"limit": 204
|
||||
},
|
||||
"reportUnusedImport": {
|
||||
"limit": 1003
|
||||
"limit": 1000
|
||||
},
|
||||
"reportUnusedVariable": {
|
||||
"limit": 1297
|
||||
|
|
|
|||
|
|
@ -17,39 +17,43 @@ until they're actually needed.
|
|||
|
||||
import importlib
|
||||
import sys
|
||||
from typing import Any, Optional, cast, Callable
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any, Optional, cast
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from tiktoken import Encoding
|
||||
|
||||
# Import all the data structures that define what can be lazy-loaded
|
||||
# These are just lists of names and maps of where to find them
|
||||
from ._lazy_imports_registry import (
|
||||
# Name tuples
|
||||
COST_CALCULATOR_NAMES,
|
||||
LITELLM_LOGGING_NAMES,
|
||||
UTILS_NAMES,
|
||||
TOKEN_COUNTER_NAMES,
|
||||
LLM_CLIENT_CACHE_NAMES,
|
||||
BEDROCK_TYPES_NAMES,
|
||||
TYPES_UTILS_NAMES,
|
||||
CACHING_NAMES,
|
||||
HTTP_HANDLER_NAMES,
|
||||
DOTPROMPT_NAMES,
|
||||
LLM_CONFIG_NAMES,
|
||||
TYPES_NAMES,
|
||||
LLM_PROVIDER_LOGIC_NAMES,
|
||||
UTILS_MODULE_NAMES,
|
||||
# Import maps
|
||||
_UTILS_IMPORT_MAP,
|
||||
_COST_CALCULATOR_IMPORT_MAP,
|
||||
_TYPES_UTILS_IMPORT_MAP,
|
||||
_TOKEN_COUNTER_IMPORT_MAP,
|
||||
_BEDROCK_TYPES_IMPORT_MAP,
|
||||
_CACHING_IMPORT_MAP,
|
||||
_LITELLM_LOGGING_IMPORT_MAP,
|
||||
_COST_CALCULATOR_IMPORT_MAP,
|
||||
_DOTPROMPT_IMPORT_MAP,
|
||||
_TYPES_IMPORT_MAP,
|
||||
_LITELLM_LOGGING_IMPORT_MAP,
|
||||
_LLM_CONFIGS_IMPORT_MAP,
|
||||
_LLM_PROVIDER_LOGIC_IMPORT_MAP,
|
||||
_TOKEN_COUNTER_IMPORT_MAP,
|
||||
_TYPES_IMPORT_MAP,
|
||||
_TYPES_UTILS_IMPORT_MAP,
|
||||
# Import maps
|
||||
_UTILS_IMPORT_MAP,
|
||||
_UTILS_MODULE_IMPORT_MAP,
|
||||
BEDROCK_TYPES_NAMES,
|
||||
CACHING_NAMES,
|
||||
# Name tuples
|
||||
COST_CALCULATOR_NAMES,
|
||||
DOTPROMPT_NAMES,
|
||||
HTTP_HANDLER_NAMES,
|
||||
LITELLM_LOGGING_NAMES,
|
||||
LLM_CLIENT_CACHE_NAMES,
|
||||
LLM_CONFIG_NAMES,
|
||||
LLM_PROVIDER_LOGIC_NAMES,
|
||||
TOKEN_COUNTER_NAMES,
|
||||
TYPES_NAMES,
|
||||
TYPES_UTILS_NAMES,
|
||||
UTILS_MODULE_NAMES,
|
||||
UTILS_NAMES,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -77,10 +81,10 @@ def _get_utils_globals() -> dict:
|
|||
# They're separate from the main lazy import system because they have specific use cases
|
||||
|
||||
# Lazy loader for default encoding - avoids importing heavy tiktoken library at startup
|
||||
_default_encoding: Optional[Any] = None
|
||||
_default_encoding: Optional["Encoding"] = None
|
||||
|
||||
|
||||
def _get_default_encoding() -> Any:
|
||||
def _get_default_encoding() -> "Encoding":
|
||||
"""
|
||||
Lazily load and cache the default OpenAI encoding.
|
||||
|
||||
|
|
@ -99,7 +103,7 @@ def _get_default_encoding() -> Any:
|
|||
|
||||
|
||||
# Lazy loader for get_modified_max_tokens to avoid importing token_counter at module import time
|
||||
_get_modified_max_tokens_func: Optional[Any] = None
|
||||
_get_modified_max_tokens_func: Any | None = None
|
||||
|
||||
|
||||
def _get_modified_max_tokens() -> Any:
|
||||
|
|
@ -123,7 +127,7 @@ def _get_modified_max_tokens() -> Any:
|
|||
|
||||
|
||||
# Lazy loader for token_counter to avoid importing token_counter module at module import time
|
||||
_token_counter_new_func: Optional[Any] = None
|
||||
_token_counter_new_func: Any | None = None
|
||||
|
||||
|
||||
def _get_token_counter_new() -> Any:
|
||||
|
|
@ -153,7 +157,7 @@ def _get_token_counter_new() -> Any:
|
|||
# This registry maps attribute names (like "ModelResponse") to handler functions
|
||||
# It's built once the first time someone accesses a lazy-loaded attribute
|
||||
# Example: {"ModelResponse": _lazy_import_utils, "Cache": _lazy_import_caching, ...}
|
||||
_LAZY_IMPORT_REGISTRY: Optional[dict[str, Callable[[str], Any]]] = None
|
||||
_LAZY_IMPORT_REGISTRY: dict[str, Callable[[str], Any]] | None = None
|
||||
|
||||
|
||||
def _get_lazy_import_registry() -> dict[str, Callable[[str], Any]]:
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -6,16 +6,11 @@ import logging
|
|||
import threading
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import AsyncIterator, Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from typing import (
|
||||
Any,
|
||||
AsyncIterator,
|
||||
Callable,
|
||||
Dict,
|
||||
Iterator,
|
||||
List,
|
||||
NoReturn,
|
||||
Optional,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
|
@ -36,15 +31,13 @@ from litellm.types.llms.openai import OpenAIChatCompletionChunk
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import (
|
||||
Delta,
|
||||
)
|
||||
from litellm.types.utils import GenericStreamingChunk as GChunk
|
||||
from litellm.types.utils import (
|
||||
LlmProviders,
|
||||
ModelResponse,
|
||||
ModelResponseStream,
|
||||
StreamingChoices,
|
||||
Usage,
|
||||
)
|
||||
from litellm.types.utils import GenericStreamingChunk as GChunk
|
||||
|
||||
from ..exceptions import OpenAIError
|
||||
from .core_helpers import map_finish_reason, process_response_headers
|
||||
|
|
@ -105,7 +98,7 @@ class _ProviderChunkParsed:
|
|||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _ProviderChunkEarlyReturn:
|
||||
value: Any
|
||||
value: ModelResponseStream | None
|
||||
|
||||
|
||||
_ProviderChunkResult = Union[_ProviderChunkParsed, _ProviderChunkEarlyReturn]
|
||||
|
|
@ -116,11 +109,11 @@ class CustomStreamWrapper:
|
|||
self,
|
||||
completion_stream,
|
||||
model,
|
||||
logging_obj: Any,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
logging_obj: LiteLLMLoggingObject,
|
||||
custom_llm_provider: str | None = None,
|
||||
stream_options=None,
|
||||
make_call: Optional[Callable] = None,
|
||||
_response_headers: Optional[dict] = None,
|
||||
make_call: Callable | None = None,
|
||||
_response_headers: dict | None = None,
|
||||
):
|
||||
self.model = model
|
||||
self.make_call = make_call
|
||||
|
|
@ -139,9 +132,9 @@ class CustomStreamWrapper:
|
|||
self.sent_last_thinking_block = False
|
||||
self.thinking_content = ""
|
||||
|
||||
self.system_fingerprint: Optional[str] = None
|
||||
self.received_finish_reason: Optional[str] = None
|
||||
self.intermittent_finish_reason: Optional[str] = None # finish reasons that show up mid-stream
|
||||
self.system_fingerprint: str | None = None
|
||||
self.received_finish_reason: str | None = None
|
||||
self.intermittent_finish_reason: str | None = None # finish reasons that show up mid-stream
|
||||
self.special_tokens = [
|
||||
"<|assistant|>",
|
||||
"<|system|>",
|
||||
|
|
@ -154,7 +147,7 @@ class CustomStreamWrapper:
|
|||
self.holding_chunk = ""
|
||||
self.complete_response = ""
|
||||
self.response_uptil_now = ""
|
||||
_model_info: Dict = litellm_params.model_info or {}
|
||||
_model_info: dict = litellm_params.model_info or {}
|
||||
|
||||
_api_base = get_api_base(
|
||||
model=model or "",
|
||||
|
|
@ -171,7 +164,7 @@ class CustomStreamWrapper:
|
|||
) # GUARANTEE OPENAI HEADERS IN RESPONSE
|
||||
|
||||
self._response_headers = _response_headers
|
||||
self.response_id: Optional[str] = None
|
||||
self.response_id: str | None = None
|
||||
self.logging_loop = None
|
||||
self.rules = Rules()
|
||||
self.stream_options = stream_options or getattr(logging_obj, "stream_options", None)
|
||||
|
|
@ -179,14 +172,14 @@ class CustomStreamWrapper:
|
|||
self.sent_stream_usage = False
|
||||
self.send_stream_usage = True if self.check_send_stream_usage(self.stream_options) else False
|
||||
self.tool_call = False
|
||||
self.chunks: List = [] # keep track of the returned chunks - used for calculating the input/output tokens for stream options
|
||||
self.chunks: list = [] # keep track of the returned chunks - used for calculating the input/output tokens for stream options
|
||||
self._repeated_messages_count = 1
|
||||
self.is_function_call = self.check_is_function_call(logging_obj=logging_obj)
|
||||
self.created: Optional[int] = None
|
||||
self._last_returned_hidden_params: Optional[dict] = None
|
||||
self.created: int | None = None
|
||||
self._last_returned_hidden_params: dict | None = None
|
||||
|
||||
_cached_logging_provider = self.logging_obj.model_call_details.get("custom_llm_provider", None)
|
||||
self._cached_logging_llm_provider: Optional[str] = _cached_logging_provider
|
||||
self._cached_logging_llm_provider: str | None = _cached_logging_provider
|
||||
_effective_model = model or ""
|
||||
if custom_llm_provider == "openai" and custom_llm_provider != _cached_logging_provider:
|
||||
_effective_model = "{}/{}".format(_cached_logging_provider, _effective_model)
|
||||
|
|
@ -195,12 +188,12 @@ class CustomStreamWrapper:
|
|||
# Snapshot assumes self._hidden_params is populated from litellm_params
|
||||
# at init and never mutated during the stream. If that ever changes,
|
||||
# this cache must be removed.
|
||||
self._base_hidden_params: Dict[str, Any] = {
|
||||
self._base_hidden_params: dict[str, Any] = {
|
||||
**self._hidden_params,
|
||||
"response_cost": None,
|
||||
}
|
||||
|
||||
self._post_streaming_hooks: Optional[List] = None
|
||||
self._post_streaming_hooks: list | None = None
|
||||
|
||||
def _check_max_streaming_duration(self) -> None:
|
||||
"""Raise litellm.Timeout if the stream has exceeded LITELLM_MAX_STREAMING_DURATION_SECONDS."""
|
||||
|
|
@ -243,7 +236,7 @@ class CustomStreamWrapper:
|
|||
e,
|
||||
)
|
||||
|
||||
def check_send_stream_usage(self, stream_options: Optional[dict]):
|
||||
def check_send_stream_usage(self, stream_options: dict | None):
|
||||
return stream_options is not None and stream_options.get("include_usage", False) is True
|
||||
|
||||
def check_is_function_call(self, logging_obj) -> bool:
|
||||
|
|
@ -314,7 +307,7 @@ class CustomStreamWrapper:
|
|||
llm_provider="",
|
||||
)
|
||||
|
||||
def check_special_tokens(self, chunk: str, finish_reason: Optional[str]):
|
||||
def check_special_tokens(self, chunk: str, finish_reason: str | None):
|
||||
"""
|
||||
Output parse <s> / </s> special tokens for sagemaker + hf streaming.
|
||||
"""
|
||||
|
|
@ -596,7 +589,7 @@ class CustomStreamWrapper:
|
|||
except Exception as e:
|
||||
raise e
|
||||
|
||||
def handle_baseten_chunk(self, chunk):
|
||||
def handle_baseten_chunk(self, chunk) -> str:
|
||||
try:
|
||||
chunk = chunk.decode("utf-8")
|
||||
if len(chunk) > 0:
|
||||
|
|
@ -665,12 +658,12 @@ class CustomStreamWrapper:
|
|||
except Exception as e:
|
||||
raise e
|
||||
|
||||
def model_response_creator(self, chunk: Optional[dict] = None, hidden_params: Optional[dict] = None):
|
||||
def model_response_creator(self, chunk: dict | None = None, hidden_params: dict | None = None):
|
||||
_model = self._cached_model_name
|
||||
_logging_obj_llm_provider = self._cached_logging_llm_provider
|
||||
|
||||
if chunk is None:
|
||||
args: Dict[str, Any] = {"model": _model}
|
||||
args: dict[str, Any] = {"model": _model}
|
||||
else:
|
||||
chunk.pop("model", None)
|
||||
args = {"model": _model}
|
||||
|
|
@ -744,7 +737,7 @@ class CustomStreamWrapper:
|
|||
|
||||
def copy_model_response_level_provider_specific_fields(
|
||||
self,
|
||||
original_chunk: Union[ModelResponseStream, OpenAIChatCompletionChunk],
|
||||
original_chunk: ModelResponseStream | OpenAIChatCompletionChunk,
|
||||
model_response: ModelResponseStream,
|
||||
) -> ModelResponseStream:
|
||||
"""
|
||||
|
|
@ -759,9 +752,9 @@ class CustomStreamWrapper:
|
|||
|
||||
def is_chunk_non_empty(
|
||||
self,
|
||||
completion_obj: Dict[str, Any],
|
||||
completion_obj: dict[str, Any],
|
||||
model_response: ModelResponseStream,
|
||||
response_obj: Dict[str, Any],
|
||||
response_obj: dict[str, Any],
|
||||
) -> bool:
|
||||
if (
|
||||
"content" in completion_obj
|
||||
|
|
@ -885,9 +878,9 @@ class CustomStreamWrapper:
|
|||
|
||||
def return_processed_chunk_logic( # noqa: C901
|
||||
self,
|
||||
completion_obj: Dict[str, Any],
|
||||
completion_obj: dict[str, Any],
|
||||
model_response: ModelResponseStream,
|
||||
response_obj: Dict[str, Any],
|
||||
response_obj: dict[str, Any],
|
||||
):
|
||||
from litellm.litellm_core_utils.core_helpers import (
|
||||
preserve_upstream_non_openai_attributes,
|
||||
|
|
@ -947,7 +940,7 @@ class CustomStreamWrapper:
|
|||
if response_obj.get("provider_specific_fields") is not None:
|
||||
completion_obj["provider_specific_fields"] = response_obj["provider_specific_fields"]
|
||||
model_response.choices[0].delta = Delta(**completion_obj)
|
||||
_index: Optional[int] = completion_obj.get("index")
|
||||
_index: int | None = completion_obj.get("index")
|
||||
if _index is not None:
|
||||
model_response.choices[0].index = _index
|
||||
|
||||
|
|
@ -1728,7 +1721,7 @@ class CustomStreamWrapper:
|
|||
print_verbose(
|
||||
f"PROCESSED CHUNK PRE CHUNK CREATOR: {chunk.decode('utf-8', errors='replace') if isinstance(chunk, bytes) else chunk}; custom_llm_provider: {self.custom_llm_provider}"
|
||||
)
|
||||
response: Optional[ModelResponseStream] = self.chunk_creator(chunk=chunk)
|
||||
response: ModelResponseStream | None = self.chunk_creator(chunk=chunk)
|
||||
print_verbose(f"PROCESSED CHUNK POST CHUNK CREATOR: {response}")
|
||||
|
||||
if response is None:
|
||||
|
|
@ -1916,7 +1909,7 @@ class CustomStreamWrapper:
|
|||
|
||||
elif self.custom_llm_provider == "gemini" and hasattr(chunk, "parts") and len(chunk.parts) == 0:
|
||||
continue
|
||||
processed_chunk: Optional[ModelResponseStream] = self.chunk_creator(chunk=chunk)
|
||||
processed_chunk: ModelResponseStream | None = self.chunk_creator(chunk=chunk)
|
||||
if processed_chunk is None:
|
||||
continue
|
||||
|
||||
|
|
@ -2137,7 +2130,7 @@ class CustomStreamWrapper:
|
|||
return
|
||||
try:
|
||||
partial_response = litellm.stream_chunk_builder(chunks=self.chunks)
|
||||
usage = cast(Optional[Usage], getattr(partial_response, "usage", None))
|
||||
usage = cast(Usage | None, getattr(partial_response, "usage", None))
|
||||
if usage is None:
|
||||
return
|
||||
self.logging_obj.model_call_details["combined_usage_object"] = usage
|
||||
|
|
@ -2178,7 +2171,7 @@ class CustomStreamWrapper:
|
|||
except Exception as mapping_error:
|
||||
mapped_exception = mapping_error
|
||||
|
||||
def _normalize_status_code(exc: Exception) -> Optional[int]:
|
||||
def _normalize_status_code(exc: Exception) -> int | None:
|
||||
"""Best-effort status_code extraction."""
|
||||
try:
|
||||
code = getattr(exc, "status_code", None)
|
||||
|
|
@ -2218,7 +2211,7 @@ class CustomStreamWrapper:
|
|||
)
|
||||
|
||||
@staticmethod
|
||||
def _strip_sse_data_from_chunk(chunk: Optional[str]) -> Optional[str]:
|
||||
def _strip_sse_data_from_chunk(chunk: str | None) -> str | None:
|
||||
"""
|
||||
Strips the 'data: ' prefix from Server-Sent Events (SSE) chunks.
|
||||
|
||||
|
|
@ -2254,7 +2247,7 @@ class CustomStreamWrapper:
|
|||
return chunk
|
||||
|
||||
|
||||
def calculate_total_usage(chunks: List[ModelResponse]) -> Usage:
|
||||
def calculate_total_usage(chunks: list[ModelResponse]) -> Usage:
|
||||
"""Assume most recent usage chunk has total usage uptil then."""
|
||||
prompt_tokens: int = 0
|
||||
completion_tokens: int = 0
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
547
litellm/main.py
547
litellm/main.py
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -3,9 +3,6 @@
|
|||
import logging
|
||||
from typing import (
|
||||
Any,
|
||||
List,
|
||||
Optional,
|
||||
Union,
|
||||
cast,
|
||||
)
|
||||
|
||||
|
|
@ -18,10 +15,10 @@ from litellm.utils import CustomStreamWrapper
|
|||
|
||||
|
||||
def _add_mcp_metadata_to_response(
|
||||
response: Union[ModelResponse, CustomStreamWrapper],
|
||||
openai_tools: Optional[List],
|
||||
tool_calls: Optional[List] = None,
|
||||
tool_results: Optional[List] = None,
|
||||
response: ModelResponse | CustomStreamWrapper,
|
||||
openai_tools: list | None,
|
||||
tool_calls: list | None = None,
|
||||
tool_results: list | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Add MCP metadata to response's provider_specific_fields.
|
||||
|
|
@ -80,10 +77,10 @@ def _add_mcp_metadata_to_response(
|
|||
|
||||
async def acompletion_with_mcp(
|
||||
model: str,
|
||||
messages: List,
|
||||
tools: Optional[List] = None,
|
||||
messages: list,
|
||||
tools: list | None = None,
|
||||
**kwargs: Any,
|
||||
) -> Union[ModelResponse, CustomStreamWrapper]:
|
||||
) -> ModelResponse | CustomStreamWrapper:
|
||||
"""
|
||||
Async completion with MCP integration.
|
||||
|
||||
|
|
@ -229,10 +226,10 @@ async def acompletion_with_mcp(
|
|||
self.openai_tools = openai_tools
|
||||
self.base_call_args = base_call_args
|
||||
self.request_tags = request_tags
|
||||
self.collected_chunks: List[ModelResponseStream] = []
|
||||
self.tool_calls: Optional[List] = None
|
||||
self.tool_results: Optional[List] = None
|
||||
self.complete_response: Optional[ModelResponse] = None
|
||||
self.collected_chunks: list[ModelResponseStream] = []
|
||||
self.tool_calls: list | None = None
|
||||
self.tool_results: list | None = None
|
||||
self.complete_response: ModelResponse | None = None
|
||||
self.stream_exhausted = False
|
||||
self.tool_execution_done = False
|
||||
self.follow_up_stream = None
|
||||
|
|
@ -503,12 +500,12 @@ async def acompletion_with_mcp(
|
|||
# Create a wrapper class that delegates to our custom iterator
|
||||
# We'll use a simple approach: just replace the __aiter__ method
|
||||
class MCPStreamWrapper(CustomStreamWrapper):
|
||||
def __init__(self, original_wrapper, custom_iterator):
|
||||
def __init__(self, original_wrapper: CustomStreamWrapper, custom_iterator):
|
||||
# Initialize with the same parameters as original wrapper
|
||||
super().__init__(
|
||||
completion_stream=None,
|
||||
model=getattr(original_wrapper, "model", "unknown"),
|
||||
logging_obj=getattr(original_wrapper, "logging_obj", None),
|
||||
logging_obj=original_wrapper.logging_obj,
|
||||
custom_llm_provider=getattr(original_wrapper, "custom_llm_provider", None),
|
||||
stream_options=getattr(original_wrapper, "stream_options", None),
|
||||
make_call=getattr(original_wrapper, "make_call", None),
|
||||
|
|
|
|||
|
|
@ -5,10 +5,11 @@ import json
|
|||
import time
|
||||
import traceback
|
||||
import uuid
|
||||
from collections.abc import Mapping
|
||||
from datetime import datetime
|
||||
from functools import lru_cache
|
||||
from types import MappingProxyType
|
||||
from typing import Any, Dict, List, Literal, Mapping, Optional
|
||||
from typing import Any, Literal
|
||||
|
||||
import httpx
|
||||
from openai._streaming import SSEDecoder
|
||||
|
|
@ -29,7 +30,11 @@ from litellm.litellm_core_utils.llm_response_utils.response_metadata import (
|
|||
from litellm.litellm_core_utils.thread_pool_executor import executor
|
||||
from litellm.llms.base_llm.responses.transformation import BaseResponsesAPIConfig
|
||||
from litellm.responses.utils import ResponseAPILoggingUtils, ResponsesAPIRequestUtils
|
||||
from litellm.types.llms.openai import ResponsesAPIStreamEvents
|
||||
from litellm.types.llms.openai import (
|
||||
PART_UNION_TYPES,
|
||||
ResponsesAPIResponse,
|
||||
ResponsesAPIStreamEvents,
|
||||
)
|
||||
from litellm.types.utils import CallTypes
|
||||
from litellm.utils import async_post_call_success_deployment_hook
|
||||
|
||||
|
|
@ -41,7 +46,7 @@ def _get_openai_response_types():
|
|||
return openai_types
|
||||
|
||||
|
||||
def _log_background_task_failure(task: "asyncio.Task[Any]", *, task_name: str) -> None:
|
||||
def _log_background_task_failure(task: asyncio.Task[None], *, task_name: str) -> None:
|
||||
if task.cancelled():
|
||||
return
|
||||
exception = task.exception()
|
||||
|
|
@ -78,7 +83,7 @@ _ERROR_CODE_HTTP_STATUS: Mapping[str, int] = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
def _error_event_fields(error_obj: object) -> tuple[str, Optional[str], Optional[str]]:
|
||||
def _error_event_fields(error_obj: object) -> tuple[str, str | None, str | None]:
|
||||
if isinstance(error_obj, dict):
|
||||
raw_message = error_obj.get("message")
|
||||
raw_type = error_obj.get("type")
|
||||
|
|
@ -97,7 +102,7 @@ def _error_event_fields(error_obj: object) -> tuple[str, Optional[str], Optional
|
|||
return message, error_type, code
|
||||
|
||||
|
||||
def _status_code_for_error_fields(error_type: Optional[str], error_code: Optional[str]) -> int:
|
||||
def _status_code_for_error_fields(error_type: str | None, error_code: str | None) -> int:
|
||||
fields = tuple(field for field in (error_code, error_type) if field is not None)
|
||||
if any(field.startswith("rate_limit") or field == "insufficient_quota" for field in fields):
|
||||
return 429
|
||||
|
|
@ -118,34 +123,34 @@ class BaseResponsesAPIStreamingIterator:
|
|||
self,
|
||||
response: httpx.Response,
|
||||
model: str,
|
||||
responses_api_provider_config: Optional[BaseResponsesAPIConfig],
|
||||
responses_api_provider_config: BaseResponsesAPIConfig | None,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
request_data: Optional[Dict[str, Any]] = None,
|
||||
call_type: Optional[str] = None,
|
||||
litellm_metadata: dict[str, Any] | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
request_data: dict[str, Any] | None = None,
|
||||
call_type: str | None = None,
|
||||
):
|
||||
self.response = response
|
||||
self.model = model
|
||||
self.logging_obj = logging_obj
|
||||
self.finished = False
|
||||
self.responses_api_provider_config = responses_api_provider_config
|
||||
self.completed_response: Optional[Any] = None
|
||||
self.completed_response: Any | None = None
|
||||
self.start_time = getattr(logging_obj, "start_time", datetime.now())
|
||||
self._failure_handled = False # Track if failure handler has been called
|
||||
self._yielded_first_chunk = False
|
||||
self._generated_content = ""
|
||||
self._completed_response_cached = False
|
||||
self._completed_response_logged = False
|
||||
self._completed_response_cache_hit: Optional[bool] = None
|
||||
self._completed_response_cache_hit: bool | None = None
|
||||
self._persist_completed_response_before_logging = True
|
||||
self._stream_created_time: float = time.time()
|
||||
|
||||
# track request context for hooks
|
||||
self.litellm_metadata = litellm_metadata
|
||||
self.custom_llm_provider = custom_llm_provider
|
||||
self.request_data: Dict[str, Any] = request_data or {}
|
||||
self.call_type: Optional[str] = call_type
|
||||
self.request_data: dict[str, Any] = request_data or {}
|
||||
self.call_type: str | None = call_type
|
||||
|
||||
# set hidden params for response headers (e.g., x-litellm-model-id)
|
||||
# This matches the stream wrapper in litellm/litellm_core_utils/streaming_handler.py
|
||||
|
|
@ -153,7 +158,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
model=model or "",
|
||||
optional_params=self.logging_obj.model_call_details.get("litellm_params", {}),
|
||||
)
|
||||
_model_info: Dict = litellm_metadata.get("model_info", {}) if litellm_metadata else {}
|
||||
_model_info: dict = litellm_metadata.get("model_info", {}) if litellm_metadata else {}
|
||||
self._hidden_params = {
|
||||
"model_id": _model_info.get("id", None),
|
||||
"api_base": _api_base,
|
||||
|
|
@ -175,7 +180,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
llm_provider=self.custom_llm_provider or "",
|
||||
)
|
||||
|
||||
def _process_chunk(self, chunk) -> Optional[Any]:
|
||||
def _process_chunk(self, chunk: str) -> Any | None:
|
||||
"""Process a single chunk of data from the stream"""
|
||||
if not chunk:
|
||||
return None
|
||||
|
|
@ -298,14 +303,12 @@ class BaseResponsesAPIStreamingIterator:
|
|||
self.completed_response = openai_responses_api_chunk
|
||||
# Add cost to usage object if include_cost_in_streaming_usage is True
|
||||
if litellm.include_cost_in_streaming_usage and self.logging_obj is not None:
|
||||
response_obj: Optional[Any] = getattr(openai_responses_api_chunk, "response", None)
|
||||
response_obj: Any | None = getattr(openai_responses_api_chunk, "response", None)
|
||||
if response_obj:
|
||||
usage_obj: Optional[Any] = getattr(response_obj, "usage", None)
|
||||
usage_obj: Any | None = getattr(response_obj, "usage", None)
|
||||
if usage_obj is not None:
|
||||
try:
|
||||
cost: Optional[float] = self.logging_obj._response_cost_calculator(
|
||||
result=response_obj
|
||||
)
|
||||
cost: float | None = self.logging_obj._response_cost_calculator(result=response_obj)
|
||||
if cost is not None:
|
||||
setattr(usage_obj, "cost", cost)
|
||||
except Exception:
|
||||
|
|
@ -403,10 +406,10 @@ class BaseResponsesAPIStreamingIterator:
|
|||
)
|
||||
self._handle_failure(exception)
|
||||
|
||||
def _record_failed_response_usage(self, response_obj: Optional[Any]) -> None:
|
||||
def _record_failed_response_usage(self, response_obj: ResponsesAPIResponse | None) -> None:
|
||||
if response_obj is None or self.logging_obj is None:
|
||||
return
|
||||
usage_obj = getattr(response_obj, "usage", None)
|
||||
usage_obj = response_obj.usage
|
||||
if usage_obj is None:
|
||||
return
|
||||
try:
|
||||
|
|
@ -453,7 +456,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
is_pre_first_chunk=not self._yielded_first_chunk,
|
||||
)
|
||||
|
||||
def _get_completed_response_object(self) -> Optional[Any]:
|
||||
def _get_completed_response_object(self) -> ResponsesAPIResponse | None:
|
||||
openai_types = _get_openai_response_types()
|
||||
completed_response = self.completed_response
|
||||
if isinstance(completed_response, openai_types.ResponsesAPIResponse):
|
||||
|
|
@ -535,7 +538,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
"""
|
||||
try:
|
||||
# Align with chat pipeline: use logging_obj model_call_details + call_type
|
||||
typed_call_type: Optional[CallTypes] = None
|
||||
typed_call_type: CallTypes | None = None
|
||||
if self.call_type is not None:
|
||||
try:
|
||||
typed_call_type = CallTypes(self.call_type)
|
||||
|
|
@ -579,7 +582,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
if self.completed_response is None:
|
||||
return
|
||||
|
||||
request_payload: Dict[str, Any] = {}
|
||||
request_payload: dict[str, Any] = {}
|
||||
if isinstance(self.request_data, dict):
|
||||
request_payload.update(self.request_data)
|
||||
try:
|
||||
|
|
@ -609,7 +612,7 @@ class BaseResponsesAPIStreamingIterator:
|
|||
pass
|
||||
|
||||
try:
|
||||
typed_call_type: Optional[CallTypes] = None
|
||||
typed_call_type: CallTypes | None = None
|
||||
if self.call_type is not None:
|
||||
try:
|
||||
typed_call_type = CallTypes(self.call_type)
|
||||
|
|
@ -689,10 +692,10 @@ class ResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
model: str,
|
||||
responses_api_provider_config: BaseResponsesAPIConfig,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
request_data: Optional[Dict[str, Any]] = None,
|
||||
call_type: Optional[str] = None,
|
||||
litellm_metadata: dict[str, Any] | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
request_data: dict[str, Any] | None = None,
|
||||
call_type: str | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
response,
|
||||
|
|
@ -771,10 +774,10 @@ class SyncResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
model: str,
|
||||
responses_api_provider_config: BaseResponsesAPIConfig,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
request_data: Optional[Dict[str, Any]] = None,
|
||||
call_type: Optional[str] = None,
|
||||
litellm_metadata: dict[str, Any] | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
request_data: dict[str, Any] | None = None,
|
||||
call_type: str | None = None,
|
||||
):
|
||||
super().__init__(
|
||||
response,
|
||||
|
|
@ -858,10 +861,10 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
model: str,
|
||||
responses_api_provider_config: BaseResponsesAPIConfig,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
request_data: Optional[Dict[str, Any]] = None,
|
||||
call_type: Optional[str] = None,
|
||||
litellm_metadata: dict[str, Any] | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
request_data: dict[str, Any] | None = None,
|
||||
call_type: str | None = None,
|
||||
):
|
||||
transformed = responses_api_provider_config.transform_response_api_response(
|
||||
model=model,
|
||||
|
|
@ -882,7 +885,7 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
|
||||
def _set_events_from_response(
|
||||
self,
|
||||
transformed: Any,
|
||||
transformed: ResponsesAPIResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> None:
|
||||
self._events = _build_synthetic_response_events(
|
||||
|
|
@ -925,10 +928,10 @@ class MockResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
||||
def __init__(
|
||||
self,
|
||||
response: Any,
|
||||
response: ResponsesAPIResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
request_data: Optional[Dict[str, Any]] = None,
|
||||
call_type: Optional[str] = None,
|
||||
request_data: dict[str, Any] | None = None,
|
||||
call_type: str | None = None,
|
||||
):
|
||||
BaseResponsesAPIStreamingIterator.__init__(
|
||||
self,
|
||||
|
|
@ -943,13 +946,13 @@ class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
)
|
||||
self._completed_response_cache_hit = True
|
||||
self._persist_completed_response_before_logging = False
|
||||
self._events: List[Any] = []
|
||||
self._events: list[Any] = []
|
||||
self._idx = 0
|
||||
self._set_events_from_response(transformed=response, logging_obj=logging_obj)
|
||||
|
||||
def _set_events_from_response(
|
||||
self,
|
||||
transformed: Any,
|
||||
transformed: ResponsesAPIResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
) -> None:
|
||||
self._events = _build_synthetic_response_events(
|
||||
|
|
@ -989,7 +992,7 @@ class CachedResponsesAPIStreamingIterator(BaseResponsesAPIStreamingIterator):
|
|||
return evt
|
||||
|
||||
|
||||
def _dump_response_object(obj: Any) -> Dict[str, Any]:
|
||||
def _dump_response_object(obj: Any) -> dict[str, Any]:
|
||||
if hasattr(obj, "model_dump"):
|
||||
return obj.model_dump()
|
||||
if isinstance(obj, dict):
|
||||
|
|
@ -1002,7 +1005,7 @@ def _build_response_status_event(
|
|||
"response.created",
|
||||
"response.in_progress",
|
||||
],
|
||||
transformed: Any,
|
||||
transformed: ResponsesAPIResponse,
|
||||
) -> Any:
|
||||
openai_types = _get_openai_response_types()
|
||||
in_progress_response = transformed.model_copy(
|
||||
|
|
@ -1019,11 +1022,11 @@ def _build_content_part_done_event(
|
|||
item_id: str,
|
||||
output_index: int,
|
||||
content_index: int,
|
||||
part_payload: Dict[str, Any],
|
||||
) -> Optional[Any]:
|
||||
part_payload: dict[str, Any],
|
||||
) -> Any | None:
|
||||
openai_types = _get_openai_response_types()
|
||||
part_type = part_payload.get("type")
|
||||
part: Any
|
||||
part: PART_UNION_TYPES
|
||||
if part_type == "output_text":
|
||||
annotations = [
|
||||
openai_types.BaseLiteLLMOpenAIResponseObject(**annotation)
|
||||
|
|
@ -1059,11 +1062,11 @@ def _build_content_part_done_event(
|
|||
|
||||
def _add_text_like_part_events(
|
||||
*,
|
||||
events: List[Any],
|
||||
events: list[Any],
|
||||
item_id: str,
|
||||
output_index: int,
|
||||
content_index: int,
|
||||
part_payload: Dict[str, Any],
|
||||
part_payload: dict[str, Any],
|
||||
chunk_size: int,
|
||||
) -> None:
|
||||
openai_types = _get_openai_response_types()
|
||||
|
|
@ -1125,22 +1128,22 @@ def _add_text_like_part_events(
|
|||
|
||||
def _build_synthetic_response_events(
|
||||
*,
|
||||
transformed: Any,
|
||||
transformed: ResponsesAPIResponse,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
chunk_size: int,
|
||||
) -> List[Any]:
|
||||
) -> list[Any]:
|
||||
openai_types = _get_openai_response_types()
|
||||
if litellm.include_cost_in_streaming_usage and logging_obj is not None:
|
||||
usage_obj: Optional[Any] = getattr(transformed, "usage", None)
|
||||
usage_obj = transformed.usage
|
||||
if usage_obj is not None:
|
||||
try:
|
||||
cost: Optional[float] = logging_obj._response_cost_calculator(result=transformed)
|
||||
cost: float | None = logging_obj._response_cost_calculator(result=transformed)
|
||||
if cost is not None:
|
||||
setattr(usage_obj, "cost", cost)
|
||||
usage_obj.cost = cost
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
events: List[Any] = [
|
||||
events: list[Any] = [
|
||||
_build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_CREATED, transformed),
|
||||
_build_response_status_event(openai_types.ResponsesAPIStreamEvents.RESPONSE_IN_PROGRESS, transformed),
|
||||
]
|
||||
|
|
@ -1297,26 +1300,26 @@ class ResponsesWebSocketStreaming:
|
|||
websocket: Any,
|
||||
backend_ws: Any,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
request_data: Optional[Dict] = None,
|
||||
first_message: Optional[str] = None,
|
||||
guardrail_callbacks: Optional[List[Any]] = None,
|
||||
output_guardrail_callbacks: Optional[List[Any]] = None,
|
||||
authorized_model: Optional[str] = None,
|
||||
user_api_key_dict: Any | None = None,
|
||||
request_data: dict | None = None,
|
||||
first_message: str | None = None,
|
||||
guardrail_callbacks: list[Any] | None = None,
|
||||
output_guardrail_callbacks: list[Any] | None = None,
|
||||
authorized_model: str | None = None,
|
||||
):
|
||||
self.websocket = websocket
|
||||
self.backend_ws = backend_ws
|
||||
self.logging_obj = logging_obj
|
||||
self.user_api_key_dict = user_api_key_dict
|
||||
self.request_data: Dict = request_data or {}
|
||||
self.messages: list[Dict] = []
|
||||
self.input_messages: list[Dict[str, str]] = []
|
||||
self.request_data: dict = request_data or {}
|
||||
self.messages: list[dict] = []
|
||||
self.input_messages: list[dict[str, str]] = []
|
||||
self.first_message = first_message
|
||||
self.guardrail_callbacks: List[Any] = guardrail_callbacks or []
|
||||
self.output_guardrail_callbacks: List[Any] = output_guardrail_callbacks or []
|
||||
self.guardrail_callbacks: list[Any] = guardrail_callbacks or []
|
||||
self.output_guardrail_callbacks: list[Any] = output_guardrail_callbacks or []
|
||||
# Model name authorized at connection time; enforced on every
|
||||
# response.create frame to prevent deployment-substitution attacks.
|
||||
self.authorized_model: Optional[str] = authorized_model
|
||||
self.authorized_model: str | None = authorized_model
|
||||
|
||||
def _should_store_event(self, event_obj: dict) -> bool:
|
||||
return event_obj.get("type") in RESPONSES_WS_LOGGED_EVENT_TYPES
|
||||
|
|
@ -1592,7 +1595,7 @@ class ResponsesWebSocketStreaming:
|
|||
if not self.guardrail_callbacks:
|
||||
return response_str
|
||||
|
||||
pii_tokens: Dict[str, str] = (self.request_data.get("metadata") or {}).get("pii_tokens", {})
|
||||
pii_tokens: dict[str, str] = (self.request_data.get("metadata") or {}).get("pii_tokens", {})
|
||||
if not pii_tokens:
|
||||
return response_str
|
||||
|
||||
|
|
@ -1797,22 +1800,22 @@ class ManagedResponsesWebSocketHandler:
|
|||
self,
|
||||
websocket: Any,
|
||||
model: str,
|
||||
logging_obj: "LiteLLMLoggingObj",
|
||||
user_api_key_dict: Optional[Any] = None,
|
||||
litellm_metadata: Optional[Dict[str, Any]] = None,
|
||||
api_key: Optional[str] = None,
|
||||
api_base: Optional[str] = None,
|
||||
timeout: Optional[float] = None,
|
||||
custom_llm_provider: Optional[str] = None,
|
||||
first_message: Optional[str] = None,
|
||||
logging_obj: LiteLLMLoggingObj,
|
||||
user_api_key_dict: Any | None = None,
|
||||
litellm_metadata: dict[str, Any] | None = None,
|
||||
api_key: str | None = None,
|
||||
api_base: str | None = None,
|
||||
timeout: float | None = None,
|
||||
custom_llm_provider: str | None = None,
|
||||
first_message: str | None = None,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
self.websocket = websocket
|
||||
self.model = model
|
||||
self.logging_obj = logging_obj
|
||||
self.user_api_key_dict = user_api_key_dict
|
||||
self.litellm_metadata: Dict[str, Any] = litellm_metadata or {}
|
||||
self.model_group: Optional[str] = self.litellm_metadata.get("model_group") or self.litellm_metadata.get(
|
||||
self.litellm_metadata: dict[str, Any] = litellm_metadata or {}
|
||||
self.model_group: str | None = self.litellm_metadata.get("model_group") or self.litellm_metadata.get(
|
||||
"deployment_model_name"
|
||||
)
|
||||
self.api_key = api_key
|
||||
|
|
@ -1822,19 +1825,19 @@ class ManagedResponsesWebSocketHandler:
|
|||
self._connection_provider = self._resolve_provider(model) or custom_llm_provider
|
||||
self.first_message = first_message
|
||||
# Carry through safe pass-through kwargs (e.g. extra_headers)
|
||||
self.extra_kwargs: Dict[str, Any] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS}
|
||||
self.extra_kwargs: dict[str, Any] = {k: v for k, v in kwargs.items() if k not in _MANAGED_WS_SKIP_KWARGS}
|
||||
# In-memory session history: response_id → full accumulated message list.
|
||||
# Keyed by the DECODED (pre-encoding) response ID from response.completed.
|
||||
# This avoids the async DB-write race condition where spend logs haven't
|
||||
# been committed yet when the next response.create arrives.
|
||||
self._session_history: Dict[str, List[Dict[str, Any]]] = {}
|
||||
self._session_history: dict[str, list[dict[str, Any]]] = {}
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Internal helpers
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _serialize_chunk(chunk: Any) -> Optional[str]:
|
||||
def _serialize_chunk(chunk: Any) -> str | None:
|
||||
"""Serialize a streaming chunk to a JSON string for WebSocket transmission."""
|
||||
try:
|
||||
if hasattr(chunk, "model_dump_json"):
|
||||
|
|
@ -1856,7 +1859,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
except Exception:
|
||||
pass
|
||||
|
||||
def _get_history_messages(self, previous_response_id: str) -> List[Dict[str, Any]]:
|
||||
def _get_history_messages(self, previous_response_id: str) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Return accumulated message history for *previous_response_id*.
|
||||
|
||||
|
|
@ -1867,7 +1870,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
raw_id = decoded.get("response_id", previous_response_id)
|
||||
return list(self._session_history.get(raw_id, []))
|
||||
|
||||
def _store_history(self, response_id: str, messages: List[Dict[str, Any]]) -> None:
|
||||
def _store_history(self, response_id: str, messages: list[dict[str, Any]]) -> None:
|
||||
"""
|
||||
Store the complete accumulated message history for *response_id*.
|
||||
|
||||
|
|
@ -1877,13 +1880,13 @@ class ManagedResponsesWebSocketHandler:
|
|||
self._session_history[response_id] = messages
|
||||
|
||||
@staticmethod
|
||||
def _extract_response_id(completed_event: Dict[str, Any]) -> Optional[str]:
|
||||
def _extract_response_id(completed_event: dict[str, Any]) -> str | None:
|
||||
"""
|
||||
Pull the raw (decoded) response ID out of a ``response.completed`` event.
|
||||
Returns *None* if the event doesn't contain a usable ID.
|
||||
"""
|
||||
resp_obj = completed_event.get("response", {})
|
||||
encoded_id: Optional[str] = resp_obj.get("id") if isinstance(resp_obj, dict) else None
|
||||
encoded_id: str | None = resp_obj.get("id") if isinstance(resp_obj, dict) else None
|
||||
if not encoded_id:
|
||||
return None
|
||||
decoded = ResponsesAPIRequestUtils._decode_responses_api_response_id(encoded_id)
|
||||
|
|
@ -1891,8 +1894,8 @@ class ManagedResponsesWebSocketHandler:
|
|||
|
||||
@staticmethod
|
||||
def _extract_output_messages(
|
||||
completed_event: Dict[str, Any],
|
||||
) -> List[Dict[str, Any]]:
|
||||
completed_event: dict[str, Any],
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Convert the output items in a ``response.completed`` event into
|
||||
Responses API message dicts suitable for the next turn's ``input``.
|
||||
|
|
@ -1900,7 +1903,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
resp_obj = completed_event.get("response", {})
|
||||
if not isinstance(resp_obj, dict):
|
||||
return []
|
||||
messages: List[Dict[str, Any]] = []
|
||||
messages: list[dict[str, Any]] = []
|
||||
for item in resp_obj.get("output", []) or []:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
|
|
@ -1927,7 +1930,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
return messages
|
||||
|
||||
@staticmethod
|
||||
def _input_to_messages(input_val: Any) -> List[Dict[str, Any]]:
|
||||
def _input_to_messages(input_val: Any) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Normalise the ``input`` field of a ``response.create`` event to a list
|
||||
of Responses API message dicts.
|
||||
|
|
@ -1948,7 +1951,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
# _process_response_create sub-methods
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
async def _parse_message(self, raw_message: str) -> Optional[Dict[str, Any]]:
|
||||
async def _parse_message(self, raw_message: str) -> dict[str, Any] | None:
|
||||
"""Parse raw WS text; return the message dict or None (JSON error / ignored type)."""
|
||||
try:
|
||||
msg_obj = json.loads(raw_message)
|
||||
|
|
@ -1961,14 +1964,14 @@ class ManagedResponsesWebSocketHandler:
|
|||
return msg_obj
|
||||
|
||||
@staticmethod
|
||||
def _is_warmup_frame(msg_obj: Dict[str, Any]) -> bool:
|
||||
def _is_warmup_frame(msg_obj: dict[str, Any]) -> bool:
|
||||
"""Return True for a response.create whose generate flag is false."""
|
||||
nested = msg_obj.get("response")
|
||||
source = nested if isinstance(nested, dict) and nested else msg_obj
|
||||
return source.get("generate") is False
|
||||
|
||||
@staticmethod
|
||||
def _is_warmup_response_id(response_id: Optional[str]) -> bool:
|
||||
def _is_warmup_response_id(response_id: str | None) -> bool:
|
||||
"""Return True for synthetic warmup IDs that only exist on this connection."""
|
||||
if not response_id:
|
||||
return False
|
||||
|
|
@ -1977,13 +1980,13 @@ class ManagedResponsesWebSocketHandler:
|
|||
return str(raw_id).startswith(_WARMUP_RESPONSE_ID_PREFIX)
|
||||
|
||||
@staticmethod
|
||||
def _warmup_source_params(msg_obj: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def _warmup_source_params(msg_obj: dict[str, Any]) -> dict[str, Any]:
|
||||
nested = msg_obj.get("response")
|
||||
if isinstance(nested, dict) and nested:
|
||||
return nested
|
||||
return {k: v for k, v in msg_obj.items() if k != "type"}
|
||||
|
||||
def _build_warmup_response(self, msg_obj: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def _build_warmup_response(self, msg_obj: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Build a minimal completed Responses API object for a warmup ack."""
|
||||
source = self._warmup_source_params(msg_obj)
|
||||
wire_model = source.get("model") or self.model_group or self.model
|
||||
|
|
@ -2001,7 +2004,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
},
|
||||
}
|
||||
|
||||
async def _send_warmup_ack(self, msg_obj: Dict[str, Any]) -> None:
|
||||
async def _send_warmup_ack(self, msg_obj: dict[str, Any]) -> None:
|
||||
"""
|
||||
Acknowledge a generate=false prewarm without calling the provider.
|
||||
|
||||
|
|
@ -2024,14 +2027,14 @@ class ManagedResponsesWebSocketHandler:
|
|||
await self.websocket.send_text(serialized)
|
||||
|
||||
@staticmethod
|
||||
def _build_base_call_kwargs(msg_obj: Dict[str, Any]) -> Dict[str, Any]:
|
||||
def _build_base_call_kwargs(msg_obj: dict[str, Any]) -> dict[str, Any]:
|
||||
"""
|
||||
Extract Responses API params from the event, handling both wire formats:
|
||||
Nested: {"type": "response.create", "response": {"input": [...], ...}}
|
||||
Flat: {"type": "response.create", "input": [...], "model": "...", ...}
|
||||
"""
|
||||
nested = msg_obj.get("response")
|
||||
response_params: Dict[str, Any] = (
|
||||
response_params: dict[str, Any] = (
|
||||
nested if isinstance(nested, dict) and nested else {k: v for k, v in msg_obj.items() if k != "type"}
|
||||
)
|
||||
return {
|
||||
|
|
@ -2042,10 +2045,10 @@ class ManagedResponsesWebSocketHandler:
|
|||
|
||||
def _apply_history(
|
||||
self,
|
||||
call_kwargs: Dict[str, Any],
|
||||
previous_response_id: Optional[str],
|
||||
current_messages: List[Dict[str, Any]],
|
||||
prior_history: List[Dict[str, Any]],
|
||||
call_kwargs: dict[str, Any],
|
||||
previous_response_id: str | None,
|
||||
current_messages: list[dict[str, Any]],
|
||||
prior_history: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Prepend in-memory turn history, or fall back to DB-based reconstruction."""
|
||||
if not previous_response_id:
|
||||
|
|
@ -2074,7 +2077,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
call_kwargs["previous_response_id"] = previous_response_id
|
||||
|
||||
@staticmethod
|
||||
def _resolve_provider(model: Optional[str]) -> Optional[str]:
|
||||
def _resolve_provider(model: str | None) -> str | None:
|
||||
"""Resolve the LLM provider for a model string, or None if unresolvable."""
|
||||
if not model:
|
||||
return None
|
||||
|
|
@ -2086,7 +2089,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
except Exception:
|
||||
return None
|
||||
|
||||
def _same_provider(self, model: Optional[str]) -> bool:
|
||||
def _same_provider(self, model: str | None) -> bool:
|
||||
"""Return True if model uses the same LLM provider as the connection model."""
|
||||
if model is None or model == self.model:
|
||||
return True
|
||||
|
|
@ -2095,7 +2098,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
return False
|
||||
return event_provider == self._connection_provider
|
||||
|
||||
def _inject_credentials(self, call_kwargs: Dict[str, Any], model: Optional[str] = None) -> None:
|
||||
def _inject_credentials(self, call_kwargs: dict[str, Any], model: str | None = None) -> None:
|
||||
"""Inject connection-level credentials and metadata into call_kwargs."""
|
||||
if self.api_key is not None:
|
||||
call_kwargs["api_key"] = self.api_key
|
||||
|
|
@ -2114,7 +2117,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
call_kwargs["litellm_metadata"] = dict(self.litellm_metadata)
|
||||
|
||||
@staticmethod
|
||||
def _update_proxy_request(call_kwargs: Dict[str, Any], model: str) -> None:
|
||||
def _update_proxy_request(call_kwargs: dict[str, Any], model: str) -> None:
|
||||
"""Update proxy_server_request body so spend logs record the full request."""
|
||||
proxy_server_request = (call_kwargs.get("litellm_metadata") or {}).get("proxy_server_request") or {}
|
||||
if not isinstance(proxy_server_request, dict):
|
||||
|
|
@ -2133,7 +2136,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
call_kwargs.setdefault("litellm_params", {})
|
||||
call_kwargs["litellm_params"]["proxy_server_request"] = proxy_server_request
|
||||
|
||||
async def _stream_and_forward(self, model: str, call_kwargs: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
async def _stream_and_forward(self, model: str, call_kwargs: dict[str, Any]) -> dict[str, Any] | None:
|
||||
"""
|
||||
Stream ``litellm.aresponses`` and forward every chunk over the WebSocket.
|
||||
|
||||
|
|
@ -2141,7 +2144,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
directly (before serialization) to avoid a redundant JSON round-trip on
|
||||
every chunk. Returns the completed event dict, or ``None``.
|
||||
"""
|
||||
completed_event: Optional[Dict[str, Any]] = None
|
||||
completed_event: dict[str, Any] | None = None
|
||||
stream_response = await litellm.aresponses(model=model, **call_kwargs)
|
||||
async for chunk in stream_response: # type: ignore[union-attr]
|
||||
if chunk is None:
|
||||
|
|
@ -2165,9 +2168,9 @@ class ManagedResponsesWebSocketHandler:
|
|||
|
||||
def _save_turn_history(
|
||||
self,
|
||||
completed_event: Optional[Dict[str, Any]],
|
||||
prior_history: List[Dict[str, Any]],
|
||||
current_messages: List[Dict[str, Any]],
|
||||
completed_event: dict[str, Any] | None,
|
||||
prior_history: list[dict[str, Any]],
|
||||
current_messages: list[dict[str, Any]],
|
||||
) -> None:
|
||||
"""Store this turn in in-memory history for future previous_response_id lookups."""
|
||||
if completed_event is None:
|
||||
|
|
@ -2236,7 +2239,7 @@ class ManagedResponsesWebSocketHandler:
|
|||
else:
|
||||
model = requested_model
|
||||
|
||||
previous_response_id: Optional[str] = call_kwargs.pop("previous_response_id", None)
|
||||
previous_response_id: str | None = call_kwargs.pop("previous_response_id", None)
|
||||
current_messages = self._input_to_messages(call_kwargs.get("input"))
|
||||
|
||||
# Fetch history once; reused in both _apply_history and _save_turn_history
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
{
|
||||
"ANN001": {
|
||||
"limit": 3118
|
||||
"limit": 3100
|
||||
},
|
||||
"ANN002": {
|
||||
"limit": 69
|
||||
|
|
@ -9,10 +9,10 @@
|
|||
"limit": 831
|
||||
},
|
||||
"ANN201": {
|
||||
"limit": 2138
|
||||
"limit": 2135
|
||||
},
|
||||
"ANN202": {
|
||||
"limit": 944
|
||||
"limit": 941
|
||||
},
|
||||
"ANN204": {
|
||||
"limit": 724
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
"limit": 130
|
||||
},
|
||||
"ANN401": {
|
||||
"limit": 2009
|
||||
"limit": 1808
|
||||
},
|
||||
"ASYNC230": {
|
||||
"limit": 14
|
||||
|
|
@ -42,7 +42,7 @@
|
|||
"limit": 84
|
||||
},
|
||||
"B010": {
|
||||
"limit": 194
|
||||
"limit": 191
|
||||
},
|
||||
"B018": {
|
||||
"limit": 5
|
||||
|
|
@ -60,7 +60,7 @@
|
|||
"limit": 4
|
||||
},
|
||||
"BLE001": {
|
||||
"limit": 2899
|
||||
"limit": 2893
|
||||
},
|
||||
"C401": {
|
||||
"limit": 11
|
||||
|
|
@ -123,7 +123,7 @@
|
|||
"limit": 52
|
||||
},
|
||||
"I001": {
|
||||
"limit": 270
|
||||
"limit": 265
|
||||
},
|
||||
"LOG015": {
|
||||
"limit": 8
|
||||
|
|
@ -135,7 +135,7 @@
|
|||
"limit": 30
|
||||
},
|
||||
"PERF401": {
|
||||
"limit": 142
|
||||
"limit": 139
|
||||
},
|
||||
"PERF402": {
|
||||
"limit": 9
|
||||
|
|
@ -222,7 +222,7 @@
|
|||
"limit": 38
|
||||
},
|
||||
"RET504": {
|
||||
"limit": 716
|
||||
"limit": 698
|
||||
},
|
||||
"RUF010": {
|
||||
"limit": 874
|
||||
|
|
@ -306,7 +306,7 @@
|
|||
"limit": 9
|
||||
},
|
||||
"TID251": {
|
||||
"limit": 2652
|
||||
"limit": 2627
|
||||
},
|
||||
"TRY002": {
|
||||
"limit": 547
|
||||
|
|
@ -324,10 +324,10 @@
|
|||
"limit": 879
|
||||
},
|
||||
"UP006": {
|
||||
"limit": 12135
|
||||
"limit": 11225
|
||||
},
|
||||
"UP007": {
|
||||
"limit": 2526
|
||||
"limit": 2129
|
||||
},
|
||||
"UP008": {
|
||||
"limit": 5
|
||||
|
|
@ -354,15 +354,15 @@
|
|||
"limit": 4
|
||||
},
|
||||
"UP035": {
|
||||
"limit": 2232
|
||||
"limit": 2191
|
||||
},
|
||||
"UP036": {
|
||||
"limit": 4
|
||||
},
|
||||
"UP037": {
|
||||
"limit": 105
|
||||
"limit": 102
|
||||
},
|
||||
"UP045": {
|
||||
"limit": 17805
|
||||
"limit": 16130
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,9 +1,9 @@
|
|||
{
|
||||
"LIT001": {
|
||||
"limit": 23250
|
||||
"limit": 23244
|
||||
},
|
||||
"LIT002": {
|
||||
"limit": 27277
|
||||
"limit": 27176
|
||||
},
|
||||
"LIT003": {
|
||||
"limit": 292
|
||||
|
|
@ -24,6 +24,6 @@
|
|||
"limit": 1004
|
||||
},
|
||||
"LIT009": {
|
||||
"limit": 2473
|
||||
"limit": 2431
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue