From 60a325e4038367fa6330a603f7715df454c5d581 Mon Sep 17 00:00:00 2001 From: Alexsander Hamir Date: Mon, 8 Dec 2025 05:38:21 -0800 Subject: [PATCH] Document missing environment variables and fix incorrect types (#17649) * fix: correct type annotations for anthropic streaming handlers - Fix return type of _handle_accumulated_json_chunk from Optional[GenericStreamingChunk] to Optional[ModelResponseStream] - Fix return type of _parse_sse_data from Optional[GenericStreamingChunk] to Optional[ModelResponseStream] - Add type annotation for output_items in background_streaming.py These changes align type annotations with actual return values from chunk_parser() which returns ModelResponseStream. * docs: add missing ONYX_API_KEY and ONYX_API_BASE to environment variables reference - Add ONYX_API_BASE documentation entry - Add ONYX_API_KEY documentation entry - Fixes test_env_keys.py test failure --- docs/my-website/docs/proxy/config_settings.md | 2 ++ litellm/llms/anthropic/chat/handler.py | 8 ++++---- litellm/proxy/response_polling/background_streaming.py | 3 ++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/my-website/docs/proxy/config_settings.md b/docs/my-website/docs/proxy/config_settings.md index 65b1c4afdbc..c52b5d571b6 100644 --- a/docs/my-website/docs/proxy/config_settings.md +++ b/docs/my-website/docs/proxy/config_settings.md @@ -739,6 +739,8 @@ router_settings: | OPENMETER_API_ENDPOINT | API endpoint for OpenMeter integration | OPENMETER_API_KEY | API key for OpenMeter services | OPENMETER_EVENT_TYPE | Type of events sent to OpenMeter +| ONYX_API_BASE | Base URL for Onyx Security AI Guard service (defaults to https://ai-guard.onyx.security) +| ONYX_API_KEY | API key for Onyx Security AI Guard service | OTEL_ENDPOINT | OpenTelemetry endpoint for traces | OTEL_EXPORTER_OTLP_ENDPOINT | OpenTelemetry endpoint for traces | OTEL_ENVIRONMENT_NAME | Environment name for OpenTelemetry diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 5c084e0f70f..2dfee889fa4 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -874,7 +874,7 @@ class ModelResponseIterator: def _handle_accumulated_json_chunk( self, data_str: str - ) -> Optional[GenericStreamingChunk]: + ) -> Optional[ModelResponseStream]: """ Handle partial JSON chunks by accumulating them until valid JSON is received. @@ -885,7 +885,7 @@ class ModelResponseIterator: data_str: The JSON string to parse (without "data:" prefix) Returns: - GenericStreamingChunk if JSON is complete, None if still accumulating + ModelResponseStream if JSON is complete, None if still accumulating """ # Accumulate JSON data self.accumulated_json += data_str @@ -899,7 +899,7 @@ class ModelResponseIterator: # If it's not valid JSON yet, continue to the next chunk return None - def _parse_sse_data(self, str_line: str) -> Optional[GenericStreamingChunk]: + def _parse_sse_data(self, str_line: str) -> Optional[ModelResponseStream]: """ Parse SSE data line, handling both complete and partial JSON chunks. @@ -907,7 +907,7 @@ class ModelResponseIterator: str_line: The SSE line starting with "data:" Returns: - GenericStreamingChunk if parsing succeeded, None if accumulating partial JSON + ModelResponseStream if parsing succeeded, None if accumulating partial JSON """ data_str = str_line[5:] # Remove "data:" prefix diff --git a/litellm/proxy/response_polling/background_streaming.py b/litellm/proxy/response_polling/background_streaming.py index b0dcb69a82e..aa14a737ac1 100644 --- a/litellm/proxy/response_polling/background_streaming.py +++ b/litellm/proxy/response_polling/background_streaming.py @@ -9,6 +9,7 @@ https://platform.openai.com/docs/api-reference/responses-streaming """ import asyncio import json +from typing import Any, Dict from fastapi import Request, Response @@ -85,7 +86,7 @@ async def background_streaming_task( # noqa: PLR0915 # Process streaming response following OpenAI events format # https://platform.openai.com/docs/api-reference/responses-streaming - output_items = {} # Track output items by ID + output_items: Dict[str, Dict[str, Any]] = {} # Track output items by ID accumulated_text = {} # Track accumulated text deltas by (item_id, content_index) # ResponsesAPIResponse fields to extract from response.completed