From ed9c02d2d9606d3d96f548ed11713f1abaf7bf63 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:08:26 -0700 Subject: [PATCH 01/65] fix(ollama): one stream id, 400 on non-text tool content, separated tool results, no empty message item (#45177) * fix(ollama): one stream id, 400 on non-text tool content, separated tool results, no empty message item The ollama/ text stream minted a fresh chatcmpl- id per chunk; every chunk now carries the one id of its response. A tool message with int content raised a TypeError inside ollama_pt and answered 500; it now answers 400 naming the message index and the type it carried. Merged user, tool, and function turns and the text parts of one message were joined with no separator; they are joined with a newline. A streamed tool call with no text rebuilds to content null instead of an empty string, and the Responses bridge no longer opens a message item on a leading empty delta, so a /v1/responses stream on a tool call completes with the function_call alone, as the non-stream call does * fix(ollama): defer the BadRequestError annotation so litellm imports on Python 3.10 to 3.13 * fix(ollama): read the user run lazily, type the prompt helpers, and 400 on a non-string text part * fix(ollama): 400 on a text or image_url part that carries no value * fix(responses): stream bridged output items at contiguous indexes and reject non-object ollama content parts The chat-completions bridge reserved output_index 0 for the message item, so a tool-only stream announced its function_call at index 1 with no item at 0 and the OpenAI SDK's responses.stream() accumulator raised IndexError. Items now take the next index in emission order, reasoning included, and response.completed lists the output in that streamed order. ollama_pt answers 400 for a content part that is not an object and for an image_url object without a url string, instead of a 500 or a silent drop. * fix(responses): list only announced items in the bridged completed snapshot The Responses bridge drops a message item from response.completed when the stream never announced one and its text is empty, so a tool-only stream and a reasoning-only stream end with the items the client saw. The chunk builder's rebuilt content goes back to what main does; aligning it with the non-stream shape is its own change for every provider's consumers * style(responses): sort the itertools import with the other stdlib imports * test(integration): add ollama prompt-tools and responses bridge audit cells --- .../prompt_templates/factory.py | 112 +++- .../llms/ollama/completion/transformation.py | 9 +- .../streaming_iterator.py | 92 ++- .../test_ollama_prompt_tools_wire.py | 618 +++++++++++++++++- ...sponses_bridge_stream_output_items_wire.py | 248 +++++++ ...llm_core_utils_prompt_templates_factory.py | 78 +++ .../test_ollama_completion_transformation.py | 22 + .../test_litellm_completion_responses.py | 6 +- .../test_streaming_iterator_transformation.py | 169 ++++- 9 files changed, 1293 insertions(+), 61 deletions(-) create mode 100644 tests/integration/providers/test_responses_bridge_stream_output_items_wire.py diff --git a/litellm/litellm_core_utils/prompt_templates/factory.py b/litellm/litellm_core_utils/prompt_templates/factory.py index 4580f9bd01b..3b0cf0ce238 100644 --- a/litellm/litellm_core_utils/prompt_templates/factory.py +++ b/litellm/litellm_core_utils/prompt_templates/factory.py @@ -206,36 +206,108 @@ def _handle_ollama_system_message(messages: list, prompt: str, msg_i: int) -> tu return system_content_str, msg_i +_OLLAMA_USER_ROLES: Final = frozenset({"user", "tool", "function"}) + + +def _ollama_bad_message(model: str, message: AllMessageValues, msg_i: int, detail: str) -> "litellm.BadRequestError": + return litellm.BadRequestError( + message=BAD_MESSAGE_ERROR_STR + f"the {message['role']} message at index {msg_i} {detail}", + model=model, + llm_provider="ollama", + ) + + +def _ollama_content_part(model: str, message: AllMessageValues, part: object, msg_i: int) -> tuple[str, str]: + match part: + case {"type": "text", "text": str() as text}: + return text, "" + case {"type": "text", "text": bad_text}: + raise _ollama_bad_message( + model, message, msg_i, f"has a {type(bad_text).__name__} text part; text must be a string" + ) + case {"type": "text"}: + raise _ollama_bad_message(model, message, msg_i, "has a text part with no text; text must be a string") + case {"type": "image_url", "image_url": str() as image_url}: + return "", image_url + case {"type": "image_url", "image_url": {"url": str() as image_url}}: + return "", image_url + case {"type": "image_url", "image_url": dict()}: + raise _ollama_bad_message( + model, + message, + msg_i, + "has an image_url object without a url string; image_url must be a URL string or an object with a url", + ) + case {"type": "image_url", "image_url": bad_image_url}: + raise _ollama_bad_message( + model, + message, + msg_i, + f"has a {type(bad_image_url).__name__} image_url; image_url must be a URL string or an object with a url", + ) + case {"type": "image_url"}: + raise _ollama_bad_message( + model, + message, + msg_i, + "has an image_url part with no image_url; image_url must be a URL string or an object with a url", + ) + case Mapping(): + return "", "" + case _: + raise _ollama_bad_message( + model, message, msg_i, f"has a {type(part).__name__} content part; content parts must be objects" + ) + + +def _ollama_user_message_parts( + model: str, message: AllMessageValues, msg_i: int +) -> tuple[tuple[str, ...], tuple[str, ...]]: + msg_content: Final = message.get("content") + if msg_content is None: + return (), () + if isinstance(msg_content, str): + return ((msg_content,) if msg_content else ()), () + if not isinstance(msg_content, list): + raise _ollama_bad_message( + model, + message, + msg_i, + f"has {type(msg_content).__name__} content; content must be a string or a list of content parts", + ) + parts: Final = tuple(_ollama_content_part(model, message, part, msg_i) for part in msg_content) + texts: Final = tuple(text for text, _ in parts if text) + image_urls: Final = tuple(image_url for _, image_url in parts if image_url) + return texts, image_urls + + +def _ollama_user_turn(model: str, messages: Sequence[AllMessageValues], msg_i: int) -> tuple[str, tuple[str, ...], int]: + user_run: Final = tuple( + itertools.takewhile( + lambda message: message["role"] in _OLLAMA_USER_ROLES, (messages[i] for i in range(msg_i, len(messages))) + ) + ) + user_parts: Final = tuple( + _ollama_user_message_parts(model, message, msg_i + offset) for offset, message in enumerate(user_run) + ) + user_content_str: Final = "\n".join("\n".join(texts) for texts, _ in user_parts if texts) + image_urls: Final = tuple(itertools.chain.from_iterable(urls for _, urls in user_parts)) + return user_content_str, image_urls, len(user_run) + + def ollama_pt( model: str, messages: list ) -> ( str | OllamaVisionModelObject ): # https://github.com/ollama/ollama/blob/af4cf55884ac54b9e637cd71dadfe9b7a5685877/docs/modelfile.md#template - user_message_types: Final = {"user", "tool", "function"} msg_i = 0 images: Final = [] prompt = "" while msg_i < len(messages): init_msg_i = msg_i - user_content_str = "" - ## MERGE CONSECUTIVE USER CONTENT ## - while msg_i < len(messages) and messages[msg_i]["role"] in user_message_types: - msg_content = messages[msg_i].get("content") - if msg_content: - if isinstance(msg_content, list): - for m in msg_content: - if m.get("type", "") == "image_url": - if isinstance(m["image_url"], str): - images.append(m["image_url"]) - elif isinstance(m["image_url"], dict): - images.append(m["image_url"]["url"]) - elif m.get("type", "") == "text": - user_content_str += m["text"] - else: - # Tool message content will always be a string - user_content_str += msg_content - - msg_i += 1 + user_content_str, image_urls, user_run_len = _ollama_user_turn(model, messages, msg_i) + images.extend(image_urls) + msg_i += user_run_len if user_content_str: prompt += f"### User:\n{user_content_str}\n\n" diff --git a/litellm/llms/ollama/completion/transformation.py b/litellm/llms/ollama/completion/transformation.py index c508c27c11a..dcae8524c4f 100644 --- a/litellm/llms/ollama/completion/transformation.py +++ b/litellm/llms/ollama/completion/transformation.py @@ -39,6 +39,7 @@ from litellm.types.utils import ( ModelResponseStream, ProviderField, StreamingChoices, + generate_id, ) from ..common_utils import OllamaError, OllamaModelInfo, convert_image @@ -524,6 +525,7 @@ class OllamaConfig(BaseConfig): class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): def __init__(self, streaming_response, sync_stream: bool, json_mode: bool | None = False): super().__init__(streaming_response, sync_stream, json_mode) + self.response_id: Final[str] = generate_id() self.started_reasoning_content: bool = False self.finished_reasoning_content: bool = False self.streamed_content: bool = False @@ -622,6 +624,7 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): content = self._hold_json_object_start(text) return ModelResponseStream( + id=self.response_id, choices=[ StreamingChoices( index=0, @@ -641,24 +644,26 @@ class OllamaTextCompletionResponseIterator(BaseModelResponseIterator): # Return reasoning content as ModelResponseStream so UIs can render it thinking_content: Final = chunk.get("thinking") or "" return ModelResponseStream( + id=self.response_id, choices=[ StreamingChoices( index=0, delta=Delta(reasoning_content=thinking_content), ) - ] + ], ) else: # In this case, 'thinking' is not present in the chunk, chunk["done"] is false, # and chunk["response"] is falsy (None or empty string), # but Ollama is just starting to stream, so it should be processed as a normal dict return ModelResponseStream( + id=self.response_id, choices=[ StreamingChoices( index=0, delta=Delta(reasoning_content=""), ) - ] + ], ) # raise Exception(f"Unable to parse ollama chunk - {chunk}") except Exception as e: diff --git a/litellm/responses/litellm_completion_transformation/streaming_iterator.py b/litellm/responses/litellm_completion_transformation/streaming_iterator.py index eccca70fde1..f06f8ff4951 100644 --- a/litellm/responses/litellm_completion_transformation/streaming_iterator.py +++ b/litellm/responses/litellm_completion_transformation/streaming_iterator.py @@ -1,6 +1,7 @@ import time import uuid from collections.abc import Sequence +from itertools import filterfalse from typing import Any, Final, cast import litellm @@ -81,6 +82,19 @@ def _delta_has_signed_thinking_block(delta: object) -> bool: return any(isinstance(b, dict) and (b.get("signature") or b.get("data")) for b in blocks) +def _delta_carries_output(delta: ChatCompletionDelta) -> bool: + fields: Final = ( + delta.content, + getattr(delta, "reasoning_content", None), + delta.tool_calls, + delta.function_call, + getattr(delta, "annotations", None), + getattr(delta, "images", None), + getattr(delta, "audio", None), + ) + return any(fields) or _delta_has_signed_thinking_block(delta) + + class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): """ Async iterator for processing streaming responses from the Responses API. @@ -119,7 +133,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self.completed_response = None self.final_text: str = "" self._cached_item_id: str | None = None - self._message_output_index: int = 0 + self._message_output_index: int | None = None + self._reasoning_output_index: int | None = None self._cached_response_id: str | None = None self._buffered_chunk: ModelResponseStream | None = None self._upstream_exhausted: bool = False @@ -130,7 +145,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._tool_item_id_by_call_id: dict[str, str] = {} # mutable-ok: filled per call id as tool call events stream self._tool_call_id_by_index: dict[int, str] = {} self._ambiguous_tool_call_indexes: set[int] = set() - self._next_tool_output_index: int = 1 # output_index=0 reserved for the message item + self._next_output_index: int = 0 self._final_tool_events_queued: bool = False self._sequence_number: int = 0 self._cached_reasoning_item_id: str | None = None @@ -155,11 +170,39 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): existing: Final = self._tool_output_index_by_call_id.get(call_id) if existing is not None: return existing - idx: Final = self._next_tool_output_index - self._next_tool_output_index += 1 + idx: Final = self._allocate_output_index() self._tool_output_index_by_call_id[call_id] = idx return idx + def _allocate_output_index(self) -> int: + idx: Final = self._next_output_index + self._next_output_index += 1 + return idx + + def _message_index(self) -> int: + if self._message_output_index is None: + self._message_output_index = self._allocate_output_index() + return self._message_output_index + + def _reasoning_index(self) -> int: + if self._reasoning_output_index is None: + self._reasoning_output_index = self._allocate_output_index() + return self._reasoning_output_index + + def _streamed_output_index(self, item: object) -> int | None: + match getattr(item, "type", None): + case "message": + return self._message_output_index + case "reasoning": + return self._reasoning_output_index + case _: + call_id: Final = getattr(item, "call_id", None) or str(getattr(item, "id", "")).removeprefix("ws_") + return self._tool_output_index_by_call_id.get(str(call_id)) + + def _streamed_output_position(self, item: object) -> tuple[bool, int]: + index: Final = self._streamed_output_index(item) + return (index is None, index or 0) + def _normalize_tool_call_index(self, tool_call: object) -> int | None: idx_raw: Final = tool_call.get("index") if isinstance(tool_call, dict) else getattr(tool_call, "index", None) if idx_raw is None: @@ -572,7 +615,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._sequence_number += 1 event: Final = OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, - output_index=self._message_output_index, + output_index=self._message_index(), item=BaseLiteLLMOpenAIResponseObject( **{ "id": self._cached_item_id, @@ -594,7 +637,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): event: Final = ContentPartAddedEvent( type=ResponsesAPIStreamEvents.CONTENT_PART_ADDED, item_id=self._cached_item_id, - output_index=self._message_output_index, + output_index=self._message_index(), content_index=0, part=BaseLiteLLMOpenAIResponseObject(**{"type": "output_text", "text": "", "annotations": []}), ) @@ -606,15 +649,10 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): self._cached_item_id = f"msg_{uuid.uuid4()}" self.sent_message_item_added_event = True self.sent_content_part_added_event = True - if self._cached_reasoning_item_id is not None: - self._message_output_index = self._next_tool_output_index - self._next_tool_output_index += 1 - else: - self._message_output_index = 0 self._sequence_number += 1 event: Final = OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, - output_index=self._message_output_index, + output_index=self._message_index(), item=BaseLiteLLMOpenAIResponseObject( **{ "id": self._cached_item_id, @@ -699,7 +737,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return ReasoningSummaryTextDoneEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DONE, item_id=reasoning_item_id, - output_index=0, + output_index=self._reasoning_index(), sequence_number=sequence_number, summary_index=0, text=reasoning_content, @@ -730,7 +768,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return ReasoningSummaryPartDoneEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_PART_DONE, item_id=reasoning_item_id, - output_index=0, + output_index=self._reasoning_index(), sequence_number=sequence_number, summary_index=0, part=BaseLiteLLMOpenAIResponseObject( @@ -748,7 +786,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return OutputTextDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DONE, item_id=self._cached_item_id, - output_index=self._message_output_index, + output_index=self._message_index(), content_index=0, text=getattr(litellm_complete_object.choices[0].message, "content", "") or "", ) @@ -775,7 +813,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return ContentPartDoneEvent( type=ResponsesAPIStreamEvents.CONTENT_PART_DONE, item_id=self._cached_item_id, - output_index=self._message_output_index, + output_index=self._message_index(), content_index=0, part=part, ) @@ -794,7 +832,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): ) return OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, - output_index=self._message_output_index, + output_index=self._message_index(), sequence_number=1, item=BaseLiteLLMOpenAIResponseObject( **{ @@ -841,7 +879,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): """ return OutputItemDoneEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, - output_index=0, + output_index=self._reasoning_index(), sequence_number=sequence_number, item=BaseLiteLLMOpenAIResponseObject( **{ @@ -939,6 +977,8 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): if not chunk.choices: return delta: Final = chunk.choices[0].delta + if chunk.choices[0].finish_reason is None and not _delta_carries_output(delta): + return self._sequence_number += 1 self.sent_output_item_added_event = True @@ -952,7 +992,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): event = OutputItemAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, - output_index=0, + output_index=self._reasoning_index(), item=BaseLiteLLMOpenAIResponseObject( **{ "id": self._cached_reasoning_item_id, @@ -1176,7 +1216,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): event = OutputTextAnnotationAddedEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED, item_id=item_id, - output_index=self._message_output_index, + output_index=self._message_index(), content_index=0, annotation_index=idx, annotation=annotation_dict, @@ -1196,7 +1236,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): return ReasoningSummaryTextDeltaEvent( type=ResponsesAPIStreamEvents.REASONING_SUMMARY_TEXT_DELTA, item_id=self._cached_reasoning_item_id, - output_index=0, + output_index=self._reasoning_index(), delta=reasoning_content, ) @@ -1209,7 +1249,7 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): text_delta_event: Final = OutputTextDeltaEvent( type=ResponsesAPIStreamEvents.OUTPUT_TEXT_DELTA, item_id=item_id, - output_index=self._message_output_index, + output_index=self._message_index(), content_index=0, delta=delta_content, ) @@ -1266,7 +1306,13 @@ class LiteLLMCompletionStreamingIterator(ResponsesAPIStreamingIterator): "reasoning", self._cached_reasoning_item_id, ) - return reasoning_aligned + streamed_items: Final = filterfalse(self._is_unstreamed_empty_message, reasoning_aligned) + return tuple(sorted(streamed_items, key=self._streamed_output_position)) + + def _is_unstreamed_empty_message(self, item: object) -> bool: + if getattr(item, "type", None) != "message" or self._message_output_index is not None: + return False + return not any(getattr(part, "text", None) for part in getattr(item, "content", None) or ()) def _emit_terminal_response_event( self, litellm_model_response: ModelResponse diff --git a/tests/integration/providers/test_ollama_prompt_tools_wire.py b/tests/integration/providers/test_ollama_prompt_tools_wire.py index 115fdc65e33..f3aa18cb366 100644 --- a/tests/integration/providers/test_ollama_prompt_tools_wire.py +++ b/tests/integration/providers/test_ollama_prompt_tools_wire.py @@ -1,3 +1,4 @@ +import asyncio import itertools import json import uuid @@ -8,10 +9,11 @@ from typing import Final import anthropic import openai import pytest -from openai.types.chat import ChatCompletionChunk +from openai.types import Completion +from openai.types.chat import ChatCompletion, ChatCompletionChunk from openai.types.chat.chat_completion_chunk import Choice as ChunkChoice from openai.types.chat.chat_completion_chunk import ChoiceDeltaToolCall -from integration._support.client import Gateway, eventually +from integration._support.client import Gateway, eventually, object_value from integration._support.database import read_rows from integration._support.wire import Reply, Request, Wire, wire_server from pydantic import JsonValue, TypeAdapter @@ -343,6 +345,7 @@ async def test_async_openai_sdk_stream_answers_the_tool_result_in_plain_text(gat extra_body={"cache": _NO_CACHE}, ) chunks: Final = [chunk async for chunk in stream] + assert {chunk.id for chunk in chunks} == {chunks[0].id} choices: Final = tuple(_stream_choices(chunks)) assert "".join(choice.delta.content or "" for choice in choices) == _ANSWER assert tuple(_delta_tool_calls(choices)) == () @@ -497,6 +500,7 @@ async def test_async_openai_sdk_responses_stream_emits_the_function_call_item(ga assert json.loads(item.arguments) == _ARGUMENTS completed: Final = [event for event in events if event.type == "response.completed"] assert len(completed) == 1 + assert [item.type for item in completed[0].response.output] == ["function_call"], completed[0].response.output final_calls: Final = [item for item in completed[0].response.output if item.type == "function_call"] assert [(item.name, json.loads(item.arguments)) for item in final_calls] == [("get_weather", _ARGUMENTS)] assert completed[0].response.output_text == "" @@ -640,8 +644,8 @@ def test_ollama_server_error_on_the_tool_result_turn_does_not_take_the_deploymen pytest.param("r" * 5120, "r" * 5120, id="5kb-string-forwarded-intact"), pytest.param( [{"type": "text", "text": "Paris: 22 degrees"}, {"type": "text", "text": "clear skies"}], - "Paris: 22 degreesclear skies", - id="text-parts-joined", + "Paris: 22 degrees\nclear skies", + id="text-parts-joined-by-newline", ), ], ) @@ -668,6 +672,7 @@ def test_the_same_tool_result_twice_is_forwarded_twice_under_one_instruction(gat prompt: Final = _prompt_of(_only_generate(wire)) _assert_instructed_once(prompt, "get_weather") assert prompt.count(_RESULT) == 2, prompt + assert f"### User:\n{_RESULT}\n{_RESULT}\n\n" in prompt, prompt assert prompt.count("### User:") == 2 and prompt.count("### Assistant:") == 1, prompt @@ -707,15 +712,16 @@ def test_a_non_function_json_answer_is_returned_as_text(gateway: Gateway) -> Non assert _spend_row(completion.id) == _billed(model) -def test_int_tool_result_content_fails_in_the_response_body_and_leaves_the_deployment_serving(gateway: Gateway) -> None: +def test_int_tool_result_content_is_a_400_naming_the_field_and_leaves_the_deployment_serving(gateway: Gateway) -> None: with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) code, text = _post( gateway, "/v1/chat/completions", {"model": model, "messages": _second_turn(22), "tools": [_WEATHER_TOOL]} ) - assert code >= 400, text + assert code == 400, text error: Final = _JSON_OBJECT.validate_json(text)["error"] - assert isinstance(error, dict) and isinstance(error["message"], str) and error["message"], text + assert isinstance(error, dict) and isinstance(error["message"], str), text + assert "content" in error["message"] and "tool message" in error["message"], text assert _generate_calls(wire) == () payload: Final = _post_chat(gateway, model, _second_turn(), tools=[_WEATHER_TOOL]) assert json.dumps(payload["choices"]).count(_ANSWER) == 1, payload @@ -794,3 +800,601 @@ def _deployments(gateway: Gateway) -> list[JsonValue]: data: Final = gateway.get("/model/info")["data"] assert isinstance(data, list) return data + + +_FOLLOW_UP: Final = "Is it windy there too?" +_TIME_CALL_ID: Final = "call_prompt_tools_2" +_TIME_RESULT: Final = "Paris: 14:05 local time" +_THINKING: Final = "The tool result already answers the question, so reply in plain text." +_ANTHROPIC_TIME_TOOL: Final[dict[str, JsonValue]] = { + "name": "get_time", + "description": "Local time for a city", + "input_schema": _PARAMETERS, +} +_RESPONSES_TIME_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "name": "get_time", + "description": "Local time for a city", + "parameters": _PARAMETERS, +} +_AUDIO_PART: Final[dict[str, JsonValue]] = {"type": "input_audio", "input_audio": {"data": "UklGRg==", "format": "wav"}} + + +def _ndjson_line(**fields: JsonValue) -> bytes: + return json.dumps({"model": _BACKEND, "created_at": "2026-10-07T00:00:00Z", **fields}).encode() + b"\n" + + +def _ndjson_reply(lines: Sequence[bytes]) -> Reply: + final: Final = _ndjson_line( + response="", done=True, done_reason="stop", prompt_eval_count=30, eval_count=12 + ) + return Reply(content_type="application/x-ndjson", chunks=(*lines, final)) + + +def _thinking_reply(thinking: str) -> Reply: + pieces: Final = tuple(thinking[index : index + 7] for index in range(0, len(thinking), 7)) + return _ndjson_reply( + ( + _ndjson_line(response="", done=False), + *(_ndjson_line(response="", thinking=piece, done=False) for piece in pieces), + ) + ) + + +def _completion_texts(chunks: Sequence[Completion]) -> Iterator[str]: + for chunk in chunks: + yield from (choice.text or "" for choice in chunk.choices) + + +def _raw_stream_frames(gateway: Gateway, path: str, body: dict[str, JsonValue]) -> tuple[str, ...]: + with gateway.client.stream( + "POST", path, json={**body, "cache": _NO_CACHE}, headers={"Authorization": f"Bearer {gateway.key}"} + ) as response: + assert response.status_code == 200, response.read() + return tuple(line.removeprefix("data: ") for line in response.iter_lines() if line.startswith("data: ")) + + +def _raw_choice_contents(choices: JsonValue) -> Iterator[str]: + assert isinstance(choices, list) + for choice in choices: + content: Final = object_value(object_value(choice)["delta"]).get("content") + if isinstance(content, str): + yield content + + +def _raw_delta_contents(payloads: Sequence[dict[str, JsonValue]]) -> Iterator[str]: + for payload in payloads: + yield from _raw_choice_contents(payload["choices"]) + + +def _cache_rows(model: str) -> list[dict[str, JsonValue]]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT request_id, cache_hit, status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,) + ), + lambda found: len(found) >= 2, + seconds=70, + ) + assert len(rows) == 2, rows + return rows + + +def _two_call_turn() -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": _QUESTION}, + { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": _CALL_ID, "name": "get_weather", "input": _ARGUMENTS}, + {"type": "tool_use", "id": _TIME_CALL_ID, "name": "get_time", "input": _ARGUMENTS}, + ], + }, + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": _CALL_ID, "content": _RESULT}, + {"type": "tool_result", "tool_use_id": _TIME_CALL_ID, "content": _TIME_RESULT}, + ], + }, + ] + + +def _anthropic_result_with_text() -> list[dict[str, JsonValue]]: + return [ + *_anthropic_second_turn()[:2], + { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": _CALL_ID, "content": _RESULT}, + {"type": "text", "text": _FOLLOW_UP}, + ], + }, + ] + + +def _anthropic_int_result() -> list[dict[str, JsonValue]]: + return [ + *_anthropic_second_turn()[:2], + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": _CALL_ID, "content": 22}]}, + ] + + +def _responses_two_outputs() -> list[dict[str, JsonValue]]: + return [ + {"role": "user", "content": _QUESTION}, + {"type": "function_call", "call_id": _CALL_ID, "name": "get_weather", "arguments": json.dumps(_ARGUMENTS)}, + {"type": "function_call", "call_id": _TIME_CALL_ID, "name": "get_time", "arguments": json.dumps(_ARGUMENTS)}, + {"type": "function_call_output", "call_id": _CALL_ID, "output": _RESULT}, + {"type": "function_call_output", "call_id": _TIME_CALL_ID, "output": _TIME_RESULT}, + ] + + +def _responses_int_output() -> list[dict[str, JsonValue]]: + return [*_responses_second_turn()[:2], {"type": "function_call_output", "call_id": _CALL_ID, "output": 22}] + + +def _error_message(text: str) -> str: + error: Final = _JSON_OBJECT.validate_json(text)["error"] + assert isinstance(error, dict) and isinstance(error["message"], str), text + return error["message"] + + +async def _chat_attempt(client: openai.AsyncOpenAI, model: str, messages: list[dict[str, JsonValue]]) -> ChatCompletion: + return await client.chat.completions.create( + model=model, + messages=messages, # pyright: ignore[reportArgumentType] # plain JSON messages + tools=[_WEATHER_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tool + extra_body={"cache": _NO_CACHE}, + ) + + +def test_openai_sdk_completions_stream_keeps_one_id(gateway: Gateway) -> None: + with _ollama_server(lambda _: _streamed_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + stream: Final = _openai_client(gateway).completions.create( + model=model, prompt=_QUESTION, stream=True, extra_body={"cache": _NO_CACHE} + ) + chunks: Final = list(stream) + assert {chunk.id for chunk in chunks} == {chunks[0].id}, [chunk.id for chunk in chunks] + assert "".join(_completion_texts(chunks)) == _ANSWER + body: Final = _only_generate(wire) + assert body["stream"] is True + assert body["model"] == _BACKEND + assert body["prompt"] == _QUESTION, body + assert _spend_row(chunks[0].id) == _billed(model) + + +def test_raw_chat_stream_frames_share_one_id_and_end_with_done(gateway: Gateway) -> None: + with _ollama_server(lambda _: _streamed_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + frames: Final = _raw_stream_frames( + gateway, + "/v1/chat/completions", + {"model": model, "messages": _second_turn(), "tools": [_WEATHER_TOOL], "stream": True}, + ) + assert frames[-1] == "[DONE]", frames + payloads: Final = [_JSON_OBJECT.validate_json(frame) for frame in frames[:-1]] + assert {payload["id"] for payload in payloads} == {payloads[0]["id"]}, [payload["id"] for payload in payloads] + assert "".join(_raw_delta_contents(payloads)) == _ANSWER + body: Final = _only_generate(wire) + assert body["stream"] is True + _assert_tool_turn(_prompt_of(body)) + identity: Final = payloads[0]["id"] + assert isinstance(identity, str) + assert _spend_row(identity) == _billed(model) + + +async def test_cached_stream_replay_keeps_one_id_and_reaches_ollama_once(gateway: Gateway) -> None: + messages: Final[list[dict[str, JsonValue]]] = [{"role": "user", "content": f"{_QUESTION} ({uuid.uuid4().hex})"}] + with _ollama_server(lambda _: _streamed_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + client: Final = _async_openai_client(gateway) + first: Final = [ + chunk + async for chunk in await client.chat.completions.create( + model=model, + messages=messages, # pyright: ignore[reportArgumentType] # plain JSON messages + stream=True, + ) + ] + assert {chunk.id for chunk in first} == {first[0].id}, [chunk.id for chunk in first] + assert "".join(choice.delta.content or "" for choice in _stream_choices(first)) == _ANSWER + assert _spend_row(first[0].id) == _billed(model) + second: Final = [ + chunk + async for chunk in await client.chat.completions.create( + model=model, + messages=messages, # pyright: ignore[reportArgumentType] # plain JSON messages + stream=True, + ) + ] + assert {chunk.id for chunk in second} == {second[0].id}, [chunk.id for chunk in second] + assert "".join(choice.delta.content or "" for choice in _stream_choices(second)) == _ANSWER + assert len(_generate_calls(wire)) == 1 + rows: Final = _cache_rows(model) + hits: Final = [row for row in rows if row["cache_hit"] == "True"] + assert len(hits) == 1, rows + hit_id: Final = hits[0]["request_id"] + assert isinstance(hit_id, str) and hit_id.startswith(second[0].id), rows + assert [row["request_id"] for row in rows if row["cache_hit"] != "True"] == [first[0].id], rows + assert {row["status"] for row in rows} == {"success"}, rows + + +def test_consecutive_user_messages_are_joined_by_a_newline(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + messages: Final[list[dict[str, JsonValue]]] = [*_first_turn(), {"role": "user", "content": _FOLLOW_UP}] + _post_chat(gateway, model, messages, tools=[_WEATHER_TOOL]) + prompt: Final = _prompt_of(_only_generate(wire)) + _assert_instructed_once(prompt, "get_weather") + assert f"### User:\n{_QUESTION}\n{_FOLLOW_UP}\n\n" in prompt, prompt + assert prompt.count("### User:") == 1, prompt + + +def test_a_tool_result_followed_by_user_text_is_joined_by_a_newline(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + messages: Final[list[dict[str, JsonValue]]] = [*_second_turn(), {"role": "user", "content": _FOLLOW_UP}] + payload: Final = _post_chat(gateway, model, messages, tools=[_WEATHER_TOOL]) + assert json.dumps(payload["choices"]).count(_ANSWER) == 1, payload + prompt: Final = _prompt_of(_only_generate(wire)) + _assert_instructed_once(prompt, "get_weather") + assert ( + f"### User:\n{_QUESTION}\n\n### Assistant:\n{_CALL_JSON}\n\n### User:\n{_RESULT}\n{_FOLLOW_UP}\n\n" in prompt + ), prompt + assert prompt.count("### User:") == 2, prompt + + +def test_anthropic_sdk_two_tool_results_in_one_turn_are_separated(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + message: Final = _anthropic_client(gateway).messages.create( + model=model, + max_tokens=64, + messages=_two_call_turn(), # pyright: ignore[reportArgumentType] # plain JSON messages + tools=[_ANTHROPIC_TOOL, _ANTHROPIC_TIME_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tools + extra_body={"cache": _NO_CACHE}, + ) + assert message.stop_reason == "end_turn" + assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", _ANSWER)] + prompt: Final = _prompt_of(_only_generate(wire)) + _assert_instructed_once(prompt, "get_weather", "get_time") + assert f"### User:\n{_RESULT}\n{_TIME_RESULT}\n\n" in prompt, prompt + assert prompt.count("### User:") == 2 and prompt.count("### Assistant:") == 1, prompt + assistant: Final = prompt.split("### Assistant:\n", 1)[1].split("### User:", 1)[0] + assert "get_weather" in assistant and "get_time" in assistant, assistant + assert _spend_row(message.id) == _billed(model) + + +async def test_async_anthropic_sdk_stream_separates_a_tool_result_from_user_text_in_one_turn( + gateway: Gateway, +) -> None: + with _ollama_server(lambda _: _streamed_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + async with _async_anthropic_client(gateway).messages.stream( + model=model, + max_tokens=64, + messages=_anthropic_result_with_text(), # pyright: ignore[reportArgumentType] # plain JSON messages + tools=[_ANTHROPIC_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tool + extra_body={"cache": _NO_CACHE}, + ) as stream: + texts: Final = [event.text async for event in stream if event.type == "text"] + final: Final = await stream.get_final_message() + assert "".join(texts) == _ANSWER + assert final.stop_reason == "end_turn" + body: Final = _only_generate(wire) + assert body["stream"] is True + prompt: Final = _prompt_of(body) + _assert_instructed_once(prompt, "get_weather") + assert ( + f"### User:\n{_QUESTION}\n\n### Assistant:\n{_CALL_JSON}\n\n### User:\n{_RESULT}\n{_FOLLOW_UP}\n\n" in prompt + ), prompt + assert _spend_row(final.id) == _billed(model) + + +def test_anthropic_sdk_int_tool_result_content_is_dropped_before_the_prompt(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + message: Final = _anthropic_client(gateway).messages.create( + model=model, + max_tokens=64, + messages=_anthropic_int_result(), # pyright: ignore[reportArgumentType] # plain JSON messages + tools=[_ANTHROPIC_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tool + extra_body={"cache": _NO_CACHE}, + ) + assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", _ANSWER)] + prompt: Final = _prompt_of(_only_generate(wire)) + _assert_instructed_once(prompt, "get_weather") + assert f"### User:\n{_QUESTION}\n\n### Assistant:\n{_CALL_JSON}\n\n### System:\n" in prompt, prompt + assert "22" not in prompt.split("### Assistant:\n", 1)[1], prompt + assert _spend_row(message.id) == _billed(model) + + +def test_openai_sdk_responses_two_function_outputs_are_separated(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = _openai_client(gateway).responses.create( + model=model, + input=_responses_two_outputs(), # pyright: ignore[reportArgumentType] # plain JSON input items + tools=[_RESPONSES_TOOL, _RESPONSES_TIME_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tools + store=False, + extra_body={"cache": _NO_CACHE}, + ) + assert [item.type for item in response.output] == ["message"] + assert response.output_text == _ANSWER + prompt: Final = _prompt_of(_only_generate(wire)) + _assert_instructed_once(prompt, "get_weather", "get_time") + assert f"### User:\n{_RESULT}\n{_TIME_RESULT}\n\n" in prompt, prompt + assert prompt.count("### User:") == 2 and prompt.count("### Assistant:") == 1, prompt + assert _model_spend_rows(model, 1)[0]["status"] == "success" + + +def test_openai_sdk_responses_int_function_output_reaches_the_prompt_as_text(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + response: Final = _openai_client(gateway).responses.create( + model=model, + input=_responses_int_output(), # pyright: ignore[reportArgumentType] # plain JSON input items + tools=[_RESPONSES_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tool + store=False, + extra_body={"cache": _NO_CACHE}, + ) + assert response.output_text == _ANSWER + _assert_tool_turn(_prompt_of(_only_generate(wire)), "22") + assert _model_spend_rows(model, 1)[0]["status"] == "success" + + +@pytest.mark.parametrize( + ("messages", "message_ref", "detail"), + [ + pytest.param(_second_turn(22), "tool message at index 2", "has int content", id="tool-int-content"), + pytest.param( + _second_turn({"text": _RESULT}), "tool message at index 2", "has dict content", id="tool-dict-content" + ), + pytest.param( + _second_turn([_RESULT]), "tool message at index 2", "has a str content part", id="tool-string-part" + ), + pytest.param( + _second_turn([{"type": "text", "text": 22}]), + "tool message at index 2", + "has a int text part", + id="tool-int-text-part", + ), + pytest.param( + _second_turn([{"type": "text"}]), + "tool message at index 2", + "has a text part with no text", + id="tool-text-part-without-text", + ), + pytest.param( + [{"role": "user", "content": [{"type": "text", "text": _QUESTION}, {"type": "image_url", "image_url": 22}]}], + "user message at index 0", + "has a int image_url", + id="user-int-image-url", + ), + pytest.param( + [{"role": "user", "content": [{"type": "image_url", "image_url": {"detail": "high"}}]}], + "user message at index 0", + "has an image_url object without a url string", + id="user-image-url-object-without-url", + ), + pytest.param( + [{"role": "user", "content": [{"type": "image_url"}]}], + "user message at index 0", + "has an image_url part with no image_url", + id="user-image-url-part-without-image-url", + ), + ], +) +def test_malformed_content_is_a_400_naming_the_message_and_field( + gateway: Gateway, messages: list[dict[str, JsonValue]], message_ref: str, detail: str +) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + code, text = _post(gateway, "/v1/chat/completions", {"model": model, "messages": messages, "tools": [_WEATHER_TOOL]}) + assert code == 400, text + assert f"the {message_ref} {detail}" in _error_message(text), text + assert _generate_calls(wire) == () + + +@pytest.mark.parametrize( + "content", + [ + pytest.param(22, id="user-int-content"), + pytest.param({"text": _QUESTION}, id="user-dict-content"), + pytest.param(["just a string"], id="user-string-part"), + ], +) +def test_non_list_user_content_is_rejected_before_it_reaches_ollama(gateway: Gateway, content: JsonValue) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + code, text = _post( + gateway, + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": content}], "tools": [_WEATHER_TOOL]}, + ) + assert code >= 400, text + assert _error_message(text), text + assert _generate_calls(wire) == () + payload: Final = _post_chat(gateway, model, _second_turn(), tools=[_WEATHER_TOOL]) + assert json.dumps(payload["choices"]).count(_ANSWER) == 1, payload + _assert_tool_turn(_prompt_of(_only_generate(wire))) + + +def test_malformed_content_400_writes_a_failure_spend_row_and_leaves_the_deployment_serving(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + code, text = _post( + gateway, "/v1/chat/completions", {"model": model, "messages": _second_turn(22), "tools": [_WEATHER_TOOL]} + ) + assert code == 400, text + assert "the tool message at index 2 has int content" in _error_message(text), text + assert [row["status"] for row in _model_spend_rows(model, 1)] == ["failure"] + assert _generate_calls(wire) == () + payload: Final = _post_chat(gateway, model, _second_turn(), tools=[_WEATHER_TOOL]) + assert json.dumps(payload["choices"]).count(_ANSWER) == 1, payload + _assert_tool_turn(_prompt_of(_only_generate(wire))) + assert sorted(str(row["status"]) for row in _model_spend_rows(model, 2)) == ["failure", "success"] + + +def test_malformed_content_on_an_unauthenticated_request_is_a_401_before_translation(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + code, text = _post( + gateway, + "/v1/chat/completions", + {"model": model, "messages": _second_turn(22), "tools": [_WEATHER_TOOL]}, + key=f"sk-not-a-key-{uuid.uuid4().hex}", + ) + assert code == 401, text + assert _generate_calls(wire) == () + + +def test_a_content_part_of_an_unknown_type_is_dropped_from_the_prompt(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + messages: Final[list[dict[str, JsonValue]]] = [ + {"role": "user", "content": [{"type": "text", "text": _QUESTION}, _AUDIO_PART]} + ] + _post_chat(gateway, model, messages, tools=[_WEATHER_TOOL]) + prompt: Final = _prompt_of(_only_generate(wire)) + _assert_instructed_once(prompt, "get_weather") + assert f"### User:\n{_QUESTION}\n\n" in prompt, prompt + assert "UklGRg==" not in prompt and "input_audio" not in prompt, prompt + + +@pytest.mark.parametrize( + "messages", + [ + pytest.param( + [ + { + "role": "user", + "content": [ + {"type": "text", "text": ""}, + {"type": "text", "text": _QUESTION}, + {"type": "text", "text": ""}, + ], + } + ], + id="empty-text-parts-dropped", + ), + pytest.param([{"role": "user", "content": None}, *_first_turn()], id="null-content-skipped"), + pytest.param([{"role": "user", "content": ""}, *_first_turn()], id="empty-string-skipped"), + ], +) +def test_empty_and_null_user_content_are_dropped_from_the_user_section( + gateway: Gateway, messages: list[dict[str, JsonValue]] +) -> None: + with _ollama_server(lambda _: _generate_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + _post_chat(gateway, model, messages, tools=[_WEATHER_TOOL]) + prompt: Final = _prompt_of(_only_generate(wire)) + _assert_instructed_once(prompt, "get_weather") + assert f"### User:\n{_QUESTION}\n\n" in prompt, prompt + assert prompt.count("### User:") == 1, prompt + + +async def test_async_openai_sdk_responses_stream_of_thinking_only_emits_one_reasoning_item(gateway: Gateway) -> None: + with _ollama_server(lambda _: _thinking_reply(_THINKING)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + stream: Final = await _async_openai_client(gateway).responses.create( + model=model, + input=_responses_second_turn(), # pyright: ignore[reportArgumentType] # plain JSON input items + tools=[_RESPONSES_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tool + store=False, + stream=True, + extra_body={"cache": _NO_CACHE}, + ) + events: Final = [event async for event in stream] + added: Final = [ + (event.item.type, event.output_index) for event in events if event.type == "response.output_item.added" + ] + assert added == [("reasoning", 0)], [event.type for event in events] + completed: Final = [event for event in events if event.type == "response.completed"] + assert len(completed) == 1, [event.type for event in events] + assert [item.type for item in completed[0].response.output] == ["reasoning"], completed[0].response.output + assert completed[0].response.output_text == "" + body: Final = _only_generate(wire) + assert body["stream"] is True + _assert_tool_turn(_prompt_of(body)) + assert _model_spend_rows(model, 1)[0]["status"] == "success" + + +async def test_async_openai_sdk_responses_stream_with_an_empty_answer_keeps_one_message_item(gateway: Gateway) -> None: + with _ollama_server(lambda _: _ndjson_reply(())) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + stream: Final = await _async_openai_client(gateway).responses.create( + model=model, + input=_responses_second_turn(), # pyright: ignore[reportArgumentType] # plain JSON input items + tools=[_RESPONSES_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tool + store=False, + stream=True, + extra_body={"cache": _NO_CACHE}, + ) + events: Final = [event async for event in stream] + added: Final = [ + (event.item.type, event.output_index) for event in events if event.type == "response.output_item.added" + ] + assert added == [("message", 0)], [event.type for event in events] + completed: Final = [event for event in events if event.type == "response.completed"] + assert len(completed) == 1, [event.type for event in events] + assert [item.type for item in completed[0].response.output] == ["message"], completed[0].response.output + assert completed[0].response.output_text == "" + body: Final = _only_generate(wire) + assert body["stream"] is True + _assert_tool_turn(_prompt_of(body)) + assert _model_spend_rows(model, 1)[0]["status"] == "success" + + +def test_openai_sdk_responses_stream_context_manager_gets_the_function_call_without_an_empty_message( + gateway: Gateway, +) -> None: + with _ollama_server(lambda _: _streamed_reply(_CALL_JSON)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + with _openai_client(gateway).responses.stream( + model=model, + input=_QUESTION, + tools=[_RESPONSES_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tool + store=False, + extra_body={"cache": _NO_CACHE}, + ) as stream: + events: Final = list(stream) + final: Final = stream.get_final_response() + added: Final = [ + (event.item.type, event.output_index) for event in events if event.type == "response.output_item.added" + ] + assert added == [("function_call", 0)], [event.type for event in events] + assert [item.type for item in final.output] == ["function_call"], final.output + call: Final = final.output[0] + assert call.type == "function_call" and call.name == "get_weather" + assert json.loads(call.arguments) == _ARGUMENTS + assert final.output_text == "" + body: Final = _only_generate(wire) + assert body["stream"] is True + _assert_instructed_once(_prompt_of(body), "get_weather") + assert _model_spend_rows(model, 1)[0]["status"] == "success" + + +async def test_concurrent_valid_and_malformed_turns_are_each_answered_in_their_own_shape(gateway: Gateway) -> None: + with _ollama_server(lambda _: _generate_reply(_ANSWER)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"ollama/{_BACKEND}", api_base=wire.url, api_key=_API_KEY) + client: Final = _async_openai_client(gateway) + results: Final = await asyncio.gather( + *(_chat_attempt(client, model, _second_turn(22 if index % 3 == 0 else _RESULT)) for index in range(12)), + return_exceptions=True, + ) + answers: Final = [result for result in results if isinstance(result, ChatCompletion)] + rejections: Final = [result for result in results if isinstance(result, openai.BadRequestError)] + assert (len(answers), len(rejections)) == (8, 4), results + assert {answer.choices[0].message.content for answer in answers} == {_ANSWER} + for rejection in rejections: + assert "the tool message at index 2 has int content" in str(rejection), rejection + prompts: Final = [_prompt_of(_JSON_OBJECT.validate_json(request.body)) for request in _generate_calls(wire)] + assert len(prompts) == 8, prompts + for prompt in prompts: + _assert_tool_turn(prompt) + statuses: Final = sorted(str(row["status"]) for row in _model_spend_rows(model, 12)) + assert statuses == ["failure"] * 4 + ["success"] * 8, statuses + for answer in answers: + assert _spend_row(answer.id) == _billed(model) diff --git a/tests/integration/providers/test_responses_bridge_stream_output_items_wire.py b/tests/integration/providers/test_responses_bridge_stream_output_items_wire.py new file mode 100644 index 00000000000..a325686e05a --- /dev/null +++ b/tests/integration/providers/test_responses_bridge_stream_output_items_wire.py @@ -0,0 +1,248 @@ +import json +import uuid +from collections.abc import Callable, Iterator, Sequence +from contextlib import contextmanager +from typing import Final + +import openai +from integration._support.client import Gateway, Scenario, eventually, object_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_BACKEND: Final = "qwen3-bridge-items" +_API_KEY: Final = "synthetic-hosted-vllm-key" +_QUESTION: Final = "What is the weather in Paris?" +_ANSWER: Final = "Paris is 22 degrees Celsius with clear skies." +_PREFACE: Final = "Checking the weather." +_REASONING: Final = "The user asks for the weather, so the weather tool applies." +_CALL_ID: Final = "call_bridge_items_1" +_ARGUMENTS: Final[dict[str, JsonValue]] = {"city": "Paris"} +_TOOL: Final[dict[str, JsonValue]] = { + "type": "function", + "name": "get_weather", + "description": "Weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}, +} +_NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True} +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_DISCOVERY_PROBE: Final = ("GET", "/v1/models") + + +def _frame(identity: str, delta: dict[str, JsonValue], finish_reason: str | None = None) -> bytes: + usage: Final = {"usage": {"prompt_tokens": 30, "completion_tokens": 12, "total_tokens": 42}} if finish_reason else {} + body: Final = { + "id": identity, + "object": "chat.completion.chunk", + "created": 1, + "model": _BACKEND, + "choices": [{"index": 0, "delta": delta, "finish_reason": finish_reason}], + **usage, + } + return b"data: " + json.dumps(body).encode() + b"\n\n" + + +def _sse_reply(identity: str, deltas: Sequence[dict[str, JsonValue]], finish_reason: str) -> Reply: + return Reply( + content_type="text/event-stream", + chunks=( + _frame(identity, {"role": "assistant", "content": ""}), + *(_frame(identity, delta) for delta in deltas), + _frame(identity, {}, finish_reason), + b"data: [DONE]\n\n", + ), + ) + + +def _tool_call_delta() -> dict[str, JsonValue]: + return { + "tool_calls": [ + { + "index": 0, + "id": _CALL_ID, + "type": "function", + "function": {"name": "get_weather", "arguments": json.dumps(_ARGUMENTS)}, + } + ] + } + + +def _is_discovery_probe(request: Request) -> bool: + return (request.method, request.target) == _DISCOVERY_PROBE + + +@contextmanager +def _vllm_server(respond: Callable[[Request], Reply]) -> Iterator[Wire]: + with wire_server( + lambda request: Reply(body=b'{"object":"list","data":[]}') if _is_discovery_probe(request) else respond(request) + ) as wire: + yield wire + + +def _bridged_vllm_model(scenario: Scenario, wire: Wire) -> str: + return scenario.model( + model=f"hosted_vllm/{_BACKEND}", api_base=wire.url + "/v1", api_key=_API_KEY, use_chat_completions_api=True + ) + + +def _only_streamed_chat(wire: Wire, *tool_names: str) -> dict[str, JsonValue]: + received: Final = tuple(request for request in wire.drain() if not _is_discovery_probe(request)) + assert [(request.method, request.target) for request in received] == [("POST", "/v1/chat/completions")] + assert received[0].headers["authorization"] == f"Bearer {_API_KEY}" + body: Final = _JSON_OBJECT.validate_json(received[0].body) + assert body["stream"] is True + tools: Final = body.get("tools", []) + assert isinstance(tools, list) + assert [object_value(object_value(tool)["function"])["name"] for tool in tools] == list(tool_names), tools + messages: Final = body["messages"] + assert isinstance(messages, list) and object_value(messages[-1])["content"] == _QUESTION, messages + return body + + +def _spend_statuses(model: str) -> list[JsonValue]: + rows: Final = eventually( + lambda: read_rows('SELECT status FROM "LiteLLM_SpendLogs" WHERE model_group=%s', (model,)), + lambda found: len(found) >= 1, + seconds=70, + ) + return [row["status"] for row in rows] + + +def _openai_client(gateway: Gateway) -> openai.OpenAI: + return openai.OpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) + + +def _async_openai_client(gateway: Gateway) -> openai.AsyncOpenAI: + return openai.AsyncOpenAI(base_url=str(gateway.client.base_url) + "/v1", api_key=gateway.key, max_retries=0) + + +def _raw_events(gateway: Gateway, body: dict[str, JsonValue]) -> list[dict[str, JsonValue]]: + with gateway.client.stream( + "POST", + "/v1/responses", + json={**body, "cache": _NO_CACHE}, + headers={"Authorization": f"Bearer {gateway.key}"}, + ) as response: + assert response.status_code == 200, response.read() + return [ + _JSON_OBJECT.validate_json(line.removeprefix("data: ")) + for line in response.iter_lines() + if line.startswith("data: {") + ] + + +def _raw_added_items(events: Sequence[dict[str, JsonValue]]) -> Iterator[tuple[JsonValue, JsonValue]]: + for event in events: + if event["type"] == "response.output_item.added": + yield object_value(event["item"])["type"], event["output_index"] + + +def _raw_completed_output_types(events: Sequence[dict[str, JsonValue]]) -> list[JsonValue]: + completed: Final = [event for event in events if event["type"] == "response.completed"] + assert len(completed) == 1, [event["type"] for event in events] + output: Final = object_value(completed[0]["response"])["output"] + assert isinstance(output, list) + return [object_value(item)["type"] for item in output] + + +def test_openai_sdk_responses_stream_with_a_tool_call_only_reply_announces_no_message_item(gateway: Gateway) -> None: + identity: Final = f"chatcmpl-bridge-{uuid.uuid4().hex}" + reply: Final = _sse_reply(identity, (_tool_call_delta(),), "tool_calls") + with _vllm_server(lambda _: reply) as wire, gateway.scenario() as scenario: + model: Final = _bridged_vllm_model(scenario, wire) + with _openai_client(gateway).responses.stream( + model=model, + input=_QUESTION, + tools=[_TOOL], # pyright: ignore[reportArgumentType] # plain JSON tool + store=False, + extra_body={"cache": _NO_CACHE}, + ) as stream: + events: Final = list(stream) + final: Final = stream.get_final_response() + added: Final = [ + (event.item.type, event.output_index) for event in events if event.type == "response.output_item.added" + ] + assert added == [("function_call", 0)], [event.type for event in events] + assert [item.type for item in final.output] == ["function_call"], final.output + call: Final = final.output[0] + assert call.type == "function_call" and call.name == "get_weather" and call.call_id == _CALL_ID + assert json.loads(call.arguments) == _ARGUMENTS + assert final.output_text == "" + created: Final = [event for event in events if event.type == "response.created"] + assert len(created) == 1 and created[0].response.id == final.id, [event.type for event in events] + _only_streamed_chat(wire, "get_weather") + assert _spend_statuses(model) == ["success"] + + +async def test_async_openai_sdk_responses_stream_text_reply_has_one_message_item_at_index_zero( + gateway: Gateway, +) -> None: + identity: Final = f"chatcmpl-bridge-{uuid.uuid4().hex}" + reply: Final = _sse_reply(identity, ({"content": _ANSWER[:17]}, {"content": _ANSWER[17:]}), "stop") + with _vllm_server(lambda _: reply) as wire, gateway.scenario() as scenario: + model: Final = _bridged_vllm_model(scenario, wire) + stream: Final = await _async_openai_client(gateway).responses.create( + model=model, input=_QUESTION, store=False, stream=True, extra_body={"cache": _NO_CACHE} + ) + events: Final = [event async for event in stream] + added: Final = [ + (event.item.type, event.output_index) for event in events if event.type == "response.output_item.added" + ] + assert added == [("message", 0)], [event.type for event in events] + done: Final = [(event.item.type, event.output_index) for event in events if event.type == "response.output_item.done"] + assert done == [("message", 0)], [event.type for event in events] + assert "".join(event.delta for event in events if event.type == "response.output_text.delta") == _ANSWER + completed: Final = [event for event in events if event.type == "response.completed"] + assert len(completed) == 1 and completed[0].response.output_text == _ANSWER + assert [item.type for item in completed[0].response.output] == ["message"], completed[0].response.output + created: Final = [event for event in events if event.type == "response.created"] + assert len(created) == 1 and created[0].response.id == completed[0].response.id + _only_streamed_chat(wire) + assert _spend_statuses(model) == ["success"] + + +async def test_async_openai_sdk_responses_stream_reasoning_then_text_gets_contiguous_output_indexes( + gateway: Gateway, +) -> None: + identity: Final = f"chatcmpl-bridge-{uuid.uuid4().hex}" + reply: Final = _sse_reply(identity, ({"reasoning_content": _REASONING}, {"content": _ANSWER}), "stop") + with _vllm_server(lambda _: reply) as wire, gateway.scenario() as scenario: + model: Final = _bridged_vllm_model(scenario, wire) + stream: Final = await _async_openai_client(gateway).responses.create( + model=model, input=_QUESTION, store=False, stream=True, extra_body={"cache": _NO_CACHE} + ) + events: Final = [event async for event in stream] + added: Final = [ + (event.item.type, event.output_index) for event in events if event.type == "response.output_item.added" + ] + assert added == [("reasoning", 0), ("message", 1)], [event.type for event in events] + assert "".join(event.delta for event in events if event.type == "response.output_text.delta") == _ANSWER + completed: Final = [event for event in events if event.type == "response.completed"] + assert len(completed) == 1 and completed[0].response.output_text == _ANSWER + assert [item.type for item in completed[0].response.output] == ["reasoning", "message"], ( + completed[0].response.output + ) + _only_streamed_chat(wire) + assert _spend_statuses(model) == ["success"] + + +def test_raw_responses_stream_text_then_tool_call_keeps_the_message_first(gateway: Gateway) -> None: + identity: Final = f"chatcmpl-bridge-{uuid.uuid4().hex}" + reply: Final = _sse_reply(identity, ({"content": _PREFACE}, _tool_call_delta()), "tool_calls") + with _vllm_server(lambda _: reply) as wire, gateway.scenario() as scenario: + model: Final = _bridged_vllm_model(scenario, wire) + events: Final = _raw_events( + gateway, {"model": model, "input": _QUESTION, "tools": [_TOOL], "store": False, "stream": True} + ) + assert list(_raw_added_items(events)) == [("message", 0), ("function_call", 1)], [e["type"] for e in events] + assert _raw_completed_output_types(events) == ["message", "function_call"] + deltas: Final = [event["delta"] for event in events if event["type"] == "response.output_text.delta"] + assert "".join(str(delta) for delta in deltas) == _PREFACE, deltas + done_calls: Final = [ + object_value(event["item"]) + for event in events + if event["type"] == "response.output_item.done" and object_value(event["item"])["type"] == "function_call" + ] + assert [(item["name"], item["call_id"]) for item in done_calls] == [("get_weather", _CALL_ID)], done_calls + _only_streamed_chat(wire, "get_weather") + assert _spend_statuses(model) == ["success"] diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py index 7962d5f6d73..56e97173eb3 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_factory.py @@ -207,6 +207,84 @@ def test_ollama_pt_consecutive_user_messages(): assert result["prompt"] == expected_prompt +def _ollama_tool_turn(*results: object) -> list[dict]: + call: Final = {"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": '{"city": "Paris"}'}} + return [ + {"role": "user", "content": "Weather in Paris?"}, + {"role": "assistant", "content": None, "tool_calls": [call]}, + *({"role": "tool", "tool_call_id": "call_1", "content": result} for result in results), + ] + + +@pytest.mark.parametrize( + ("results", "forwarded"), + [ + pytest.param(("Paris: 22 degrees", "Sky: clear"), "Paris: 22 degrees\nSky: clear", id="two-tool-messages"), + pytest.param( + ([{"type": "text", "text": "Paris: 22 degrees"}, {"type": "text", "text": "clear skies"}],), + "Paris: 22 degrees\nclear skies", + id="text-parts-of-one-tool-message", + ), + pytest.param( + ([{"type": "text", "text": "Paris: 22 degrees"}], "Sky: clear"), + "Paris: 22 degrees\nSky: clear", + id="text-part-then-string", + ), + pytest.param( + ([{"type": "text", "text": ""}, {"type": "text", "text": "clear skies"}],), + "clear skies", + id="empty-text-part-adds-no-blank-line", + ), + ], +) +def test_ollama_pt_separates_merged_tool_results_with_a_newline(results: tuple[object, ...], forwarded: str): + result: Final = ollama_pt(model="llama2", messages=_ollama_tool_turn(*results)) + + assert isinstance(result, dict) + assert result["prompt"].endswith(f"### User:\n{forwarded}\n\n"), result["prompt"] + + +@pytest.mark.parametrize("content", [22, 22.5, True, {"temperature": 22}], ids=type) +def test_ollama_pt_rejects_non_text_tool_content_as_a_bad_request(content: object): + with pytest.raises(litellm.BadRequestError) as excinfo: + ollama_pt(model="llama2", messages=_ollama_tool_turn(content)) + + assert excinfo.value.status_code == 400 + assert "content" in excinfo.value.message + assert "tool message at index 2" in excinfo.value.message + assert type(content).__name__ in excinfo.value.message + + +@pytest.mark.parametrize( + ("part", "expected_detail"), + ( + ({"type": "image_url", "image_url": None}, "NoneType image_url"), + ({"type": "text", "text": 22}, "int text part"), + ({"type": "text"}, "text part with no text"), + ({"type": "image_url"}, "image_url part with no image_url"), + ({"type": "image_url", "image_url": {"detail": "high"}}, "image_url object without a url string"), + ("hello", "str content part"), + ), + ids=( + "none-image-url", + "int-text", + "text-without-text", + "image-url-without-image-url", + "image-url-object-without-url", + "str-part", + ), +) +def test_ollama_pt_rejects_a_malformed_content_part_as_a_bad_request(part: object, expected_detail: str): + messages: Final = [{"role": "user", "content": [part]}] + + with pytest.raises(litellm.BadRequestError) as excinfo: + ollama_pt(model="llava", messages=messages) + + assert excinfo.value.status_code == 400 + assert "user message at index 0" in excinfo.value.message + assert expected_detail in excinfo.value.message + + @pytest.mark.asyncio async def test_anthropic_bedrock_thinking_blocks_with_none_content(): """ diff --git a/tests/unit/llms/ollama/test_ollama_completion_transformation.py b/tests/unit/llms/ollama/test_ollama_completion_transformation.py index 8558bf50bb9..2979cc7d572 100644 --- a/tests/unit/llms/ollama/test_ollama_completion_transformation.py +++ b/tests/unit/llms/ollama/test_ollama_completion_transformation.py @@ -596,6 +596,28 @@ class TestOllamaConfig: class TestOllamaTextCompletionResponseIterator: + def test_every_chunk_of_one_stream_carries_the_same_response_id(self): + iterator: Final = OllamaTextCompletionResponseIterator( + streaming_response=iter([]), sync_stream=True, json_mode=False + ) + ollama_chunks: Final = ( + {"model": "qwen3:0.6b", "created_at": "2026-10-07T00:00:00Z", "response": "", "done": False}, + {"model": "qwen3:0.6b", "created_at": "2026-10-07T00:00:00Z", "response": "", "thinking": "Hm", "done": False}, + {"model": "qwen3:0.6b", "created_at": "2026-10-07T00:00:00Z", "response": "Hel", "done": False}, + {"model": "qwen3:0.6b", "created_at": "2026-10-07T00:00:00Z", "response": "lo", "done": False}, + ) + + results: Final = tuple(iterator.chunk_parser(chunk) for chunk in ollama_chunks) + + ids: Final = {result.id for result in results if isinstance(result, ModelResponseStream)} + assert len(results) == len(ollama_chunks) and len(ids) == 1, ids + assert next(iter(ids)).startswith("chatcmpl-") + other: Final = OllamaTextCompletionResponseIterator( + streaming_response=iter([]), sync_stream=True, json_mode=False + ) + other_result: Final = other.chunk_parser(ollama_chunks[2]) + assert isinstance(other_result, ModelResponseStream) and other_result.id not in ids + def test_chunk_parser_with_thinking_field(self): """Test that chunks with 'thinking' field and empty 'response' are handled correctly.""" iterator = OllamaTextCompletionResponseIterator( diff --git a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py index 2090ff3c9fa..ac02354e4d9 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py +++ b/tests/unit/responses/litellm_completion_transformation/test_litellm_completion_responses.py @@ -3954,7 +3954,9 @@ class TestEnsureOutputItemContentPartAdded: iterator._tool_item_id_by_call_id = {} iterator._tool_call_id_by_index = {} iterator._ambiguous_tool_call_indexes = set() - iterator._next_tool_output_index = 1 + iterator._next_output_index = 0 + iterator._message_output_index = None + iterator._reasoning_output_index = None iterator._final_tool_events_queued = False iterator._custom_tool_names = set() iterator.responses_api_request = {} @@ -4456,7 +4458,7 @@ class TestEnsureOutputItemContentPartAdded: Choices( finish_reason=finish_reason, index=0, - message=Message(content="", role="assistant"), + message=Message(content="Partial answer", role="assistant"), ) ], usage=Usage(prompt_tokens=10, completion_tokens=1, total_tokens=11), diff --git a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py index a42eb74b0fa..cb764f0c17b 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py +++ b/tests/unit/responses/litellm_completion_transformation/test_streaming_iterator_transformation.py @@ -132,14 +132,14 @@ def test_tool_call_delta_is_emitted_as_responses_events(): evt1 = iterator._transform_chat_completion_chunk_to_response_api_chunk(chunk) assert evt1 is not None assert evt1.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED - assert evt1.output_index == 1 + assert evt1.output_index == 0 # The arguments are now chunked, so we get the first delta chunk evt2 = iterator._transform_chat_completion_chunk_to_response_api_chunk(chunk) assert evt2 is not None assert evt2.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DELTA assert evt2.item_id == "fc_call_1" - assert evt2.output_index == 1 + assert evt2.output_index == 0 # The delta will be a chunk of the arguments, not the full arguments assert len(evt2.delta) <= 10 # Chunks are max 10 characters @@ -391,7 +391,7 @@ def test_tool_calls_present_only_in_final_response_are_emitted_before_completed( # First common_done_event_logic call should yield tool events, not response.completed. evt1 = iterator.common_done_event_logic(sync_mode=True) assert evt1.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED - assert evt1.output_index == 1 + assert evt1.output_index == 0 # Now delta events are emitted (arguments split into chunks) # Collect all delta events @@ -412,12 +412,12 @@ def test_tool_calls_present_only_in_final_response_are_emitted_before_completed( # The last event should be FUNCTION_CALL_ARGUMENTS_DONE assert evt.type == ResponsesAPIStreamEvents.FUNCTION_CALL_ARGUMENTS_DONE assert evt.item_id == "fc_call_2" - assert evt.output_index == 1 + assert evt.output_index == 0 assert evt.arguments == '{"y":2}' evt_final = iterator.common_done_event_logic(sync_mode=True) assert evt_final.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE - assert evt_final.output_index == 1 + assert evt_final.output_index == 0 def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): @@ -472,7 +472,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): # First event should be OUTPUT_ITEM_ADDED assert evt is not None assert evt.type == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED - assert evt.output_index == 1 + assert evt.output_index == 0 assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__ # Collect all remaining delta events from the pending queue by creating empty chunks @@ -506,7 +506,7 @@ def test_tool_call_arguments_are_chunked_to_match_openai_behavior(): for evt in delta_events: assert len(evt.delta) <= 10 assert evt.item_id == "fc_call_test" - assert evt.output_index == 1 + assert evt.output_index == 0 assert hasattr(evt, "__dict__") and "sequence_number" in evt.__dict__ # Verify all deltas concatenated equal the original arguments @@ -978,6 +978,23 @@ def _reasoning_chunk(reasoning: str, finish_reason: str | None = None) -> ModelR ) +def _annotation_only_chunk() -> ModelResponseStream: + citation: Final = {"start_index": 0, "end_index": 2, "url": "https://example.com", "title": "Example"} + return ModelResponseStream( + id=CHAT_COMPLETION_ID, + created=1748575031, + model="claude-haiku-4-5", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + index=0, + delta=Delta(role="assistant", annotations=[{"type": "url_citation", "url_citation": citation}]), + finish_reason=None, + ) + ], + ) + + def _signature_only_thinking_chunk(signature: str) -> ModelResponseStream: return ModelResponseStream( id=CHAT_COMPLETION_ID, @@ -1034,6 +1051,68 @@ async def test_tool_only_stream_emits_no_message_item_events(sync_mode: bool): assert any(getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED for event in events) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.parametrize( + "leading_chunk", + [pytest.param(_chunk(""), id="empty-text-delta"), pytest.param(_reasoning_chunk(""), id="empty-reasoning-delta")], +) +@pytest.mark.asyncio +async def test_empty_leading_delta_does_not_open_a_message_item_ahead_of_a_tool_call( + sync_mode: bool, leading_chunk: ModelResponseStream +): + iterator: Final = _build_iterator([leading_chunk, _tool_call_chunk(), _chunk("", finish_reason="tool_calls")]) + + events: Final = await _collect_events(iterator, sync_mode) + + item_events: Final = [ + (event.type, event.item.type) + for event in events + if getattr(event, "type", None) + in (ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE) + ] + assert item_events == [ + (ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED, "function_call"), + (ResponsesAPIStreamEvents.OUTPUT_ITEM_DONE, "function_call"), + ] + completed: Final = [ + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ] + assert [item.type for item in completed[0].response.output] == ["function_call"] + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_empty_leading_delta_still_opens_the_message_item_for_the_first_text_delta(sync_mode: bool): + iterator: Final = _build_iterator([_chunk(""), _chunk("Hi"), _chunk("", finish_reason="stop")]) + + events: Final = await _collect_events(iterator, sync_mode) + + added_types: Final = [ + event.item.type + for event in events + if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + ] + assert added_types == ["message"] + completed: Final = [ + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ] + assert completed[0].response.output_text == "Hi" + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_annotation_only_leading_delta_opens_the_message_item_before_its_annotation(sync_mode: bool): + iterator: Final = _build_iterator([_annotation_only_chunk(), _chunk("Hi"), _chunk("", finish_reason="stop")]) + + events: Final = await _collect_events(iterator, sync_mode) + + event_types: Final = [getattr(event, "type", None) for event in events] + assert ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED in event_types + assert event_types.index(ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED) < event_types.index( + ResponsesAPIStreamEvents.OUTPUT_TEXT_ANNOTATION_ADDED + ) + + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_signature_only_thinking_streams_a_replayable_reasoning_item(sync_mode: bool): @@ -1152,6 +1231,82 @@ async def test_tool_then_reasoning_then_text_gives_message_its_own_output_index( assert len(output_indexes_by_item_id) == len(set(output_indexes_by_item_id.values())) +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_tool_only_stream_puts_the_function_call_at_output_index_zero(sync_mode: bool): + iterator: Final = _build_iterator([_tool_call_chunk(), _chunk("", finish_reason="tool_calls")]) + + events: Final = await _collect_events(iterator, sync_mode) + + indexed_events: Final = [event for event in events if hasattr(event, "output_index")] + assert indexed_events + assert {event.output_index for event in indexed_events} == {0} + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [item.type for item in completed.response.output] == ["function_call"] + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_completed_output_follows_the_streamed_output_indexes(sync_mode: bool): + iterator: Final = _build_iterator( + [ + _reasoning_chunk("thinking"), + _tool_call_chunk(), + _chunk("Hello"), + _chunk("!", finish_reason="stop"), + ] + ) + + events: Final = await _collect_events(iterator, sync_mode) + + added: Final = [ + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + ] + assert [event.output_index for event in added] == list(range(len(added))) + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [item.type for item in completed.response.output] == [event.item.type for event in added] + assert [item.id for item in completed.response.output] == [event.item.id for event in added] + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_reasoning_only_stream_lists_no_message_item_it_never_announced(sync_mode: bool): + iterator: Final = _build_iterator([_reasoning_chunk("thinking"), _chunk("", finish_reason="stop")]) + + events: Final = await _collect_events(iterator, sync_mode) + + added: Final = [ + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + ] + assert [event.item.type for event in added] == ["reasoning"] + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [item.type for item in completed.response.output] == ["reasoning"] + + +@pytest.mark.parametrize("sync_mode", [True, False]) +@pytest.mark.asyncio +async def test_empty_answer_stream_keeps_the_message_item_it_announced(sync_mode: bool): + iterator: Final = _build_iterator([_chunk("", finish_reason="stop")]) + + events: Final = await _collect_events(iterator, sync_mode) + + added: Final = [ + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.OUTPUT_ITEM_ADDED + ] + assert [event.item.type for event in added] == ["message"] + completed: Final = next( + event for event in events if getattr(event, "type", None) == ResponsesAPIStreamEvents.RESPONSE_COMPLETED + ) + assert [item.type for item in completed.response.output] == ["message"] + assert [item.id for item in completed.response.output] == [added[0].item.id] + + @pytest.mark.parametrize("sync_mode", [True, False]) @pytest.mark.asyncio async def test_plain_text_stream_announces_exactly_one_message_item(sync_mode: bool): From dc2a14a6564c79ef6a5ca337434bda996dd3ecd2 Mon Sep 17 00:00:00 2001 From: moe-berri Date: Wed, 7 Oct 2026 19:22:56 -0700 Subject: [PATCH 02/65] feat(lens): simplify deployment and first trace setup (#45230) * feat(lens): simplify deployment and first trace setup * test(lens): keep setup fixtures within lint budgets * fix(lens): preserve setup state and harden bundled storage startup * fix(lens): handle setup recovery and Helm endpoint boundaries --- .github/workflows/helm_unit_test.yml | 6 + deploy/lens/README.md | 59 +++--- deploy/lens/config.yaml | 2 + deploy/lens/configure.py | 73 +++++++ deploy/lens/test_configure.py | 52 +++++ helm/litellm-helm/templates/_helpers.tpl | 27 +++ helm/litellm-helm/templates/deployment.yaml | 4 +- .../templates/lens/clickhouse.yaml | 98 +++++++++ .../templates/lens/deployment.yaml | 12 +- helm/litellm-helm/templates/lens/secrets.yaml | 27 +++ .../tests/lens_endpoint_defaults_tests.yaml | 84 ++++++++ .../tests/lens_saved_secrets_tests.yaml | 45 ++++ .../tests/lens_service_tests.yaml | 2 +- helm/litellm-helm/tests/lens_setup_tests.yaml | 160 +++++++++++++++ helm/litellm-helm/values.yaml | 11 + helm/litellm/templates/_helpers.tpl | 30 ++- helm/litellm/templates/lens/clickhouse.yaml | 98 +++++++++ helm/litellm/templates/lens/deployment.yaml | 12 +- helm/litellm/templates/lens/secrets.yaml | 27 +++ .../tests/lens_endpoint_defaults_tests.yaml | 86 ++++++++ .../tests/lens_saved_secrets_tests.yaml | 47 +++++ helm/litellm/tests/lens_service_tests.yaml | 2 +- helm/litellm/tests/lens_setup_tests.yaml | 162 +++++++++++++++ helm/litellm/tests/lens_worker_tests.yaml | 13 +- helm/litellm/values.yaml | 11 + litellm/proxy/lens/endpoints.py | 25 ++- litellm/proxy/lens/ingestion.py | 2 + tests/unit/proxy/lens/test_endpoints.py | 18 ++ ui/litellm-dashboard/eslint-suppressions.json | 13 +- .../lens/LensSetup.integration.test.tsx | 67 +++++- .../InvestigationsView.integration.test.tsx | 6 +- .../lens/onboarding/LensGettingStarted.tsx | 6 +- .../lens/onboarding/OnboardingSteps.tsx | 35 ++-- .../TracingSetupCard.integration.test.tsx | 103 ++++++++-- .../onboarding/tracing/TracingSetupCard.tsx | 192 ++++++++++++------ .../AgentTracesSection.integration.test.tsx | 59 ++++-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 + 37 files changed, 1502 insertions(+), 184 deletions(-) create mode 100644 deploy/lens/configure.py create mode 100644 deploy/lens/test_configure.py create mode 100644 helm/litellm-helm/templates/lens/clickhouse.yaml create mode 100644 helm/litellm-helm/templates/lens/secrets.yaml create mode 100644 helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml create mode 100644 helm/litellm-helm/tests/lens_saved_secrets_tests.yaml create mode 100644 helm/litellm-helm/tests/lens_setup_tests.yaml create mode 100644 helm/litellm/templates/lens/clickhouse.yaml create mode 100644 helm/litellm/templates/lens/secrets.yaml create mode 100644 helm/litellm/tests/lens_endpoint_defaults_tests.yaml create mode 100644 helm/litellm/tests/lens_saved_secrets_tests.yaml create mode 100644 helm/litellm/tests/lens_setup_tests.yaml diff --git a/.github/workflows/helm_unit_test.yml b/.github/workflows/helm_unit_test.yml index f95848945a0..f3d7bdf36cc 100644 --- a/.github/workflows/helm_unit_test.yml +++ b/.github/workflows/helm_unit_test.yml @@ -22,6 +22,12 @@ jobs: with: persist-credentials: false + - name: Check Lens Compose configuration + run: | + python3 -m unittest discover -s deploy/lens -p 'test_*.py' + python3 deploy/lens/configure.py --version 1.2.3 --env-file "$RUNNER_TEMP/lens.env" + docker compose --env-file "$RUNNER_TEMP/lens.env" -f deploy/lens/stack.yaml config --quiet + - name: Set up Helm 3.11.1 uses: azure/setup-helm@1a275c3b69536ee54be43f2070a358922e12c8d4 # v4.3.1 with: diff --git a/deploy/lens/README.md b/deploy/lens/README.md index d5b9409e583..379913b3b7f 100644 --- a/deploy/lens/README.md +++ b/deploy/lens/README.md @@ -6,21 +6,20 @@ Agent exporters send traces directly to Lens. LiteLLM sends its optional request ## New local installation -Install Docker with Compose and Git, then build the gateway and Lens from one checkout: +Install Docker with Compose, Python 3.10 or later, and Git. Clone LiteLLM, select a published release that includes Lens, and start the existing Compose stack: ```bash git clone https://github.com/BerriAI/litellm.git cd litellm -export LITELLM_RELEASE_TAG="sha-$(git rev-parse HEAD)" -export LITELLM_MASTER_KEY="sk-$(openssl rand -hex 24)" -export LITELLM_LENS_SERVICE_TOKEN="$(openssl rand -hex 32)" -export OPENAI_API_KEY='' -docker compose -f docker/docker-compose.tracing.yml up -d --build +python3 deploy/lens/configure.py --version +docker compose --env-file deploy/lens/.env -f deploy/lens/stack.yaml up -d --wait ``` -Save the generated keys privately and reuse them when restarting or upgrading. This stack binds to localhost and uses development database passwords; use your normal secrets, TLS, backups, and ingress for a hosted deployment +The configuration command generates your keys and database passwords once, saves them in `deploy/lens/.env` with owner-only permissions, and preserves them on subsequent runs. Back up this file alongside your database volumes. Both images use the selected release; there is no local image build -Open `http://localhost:4002/ui/` and sign in as `admin` with `LITELLM_MASTER_KEY`. Under **Lens > Traces > Set up tracing**, generate a tracing key and copy the ingestion URL. Local exporters use `http://localhost:4318`. Model calls keep their existing LiteLLM URL and model key +Open `http://localhost:4000/ui/` and sign in as `admin` using `LITELLM_MASTER_KEY` from the saved file. Open **Lens**, select your framework, generate a tracing key, and copy the displayed configuration. The trace endpoint is already filled in. Keep your agent's existing model credentials; the tracing key only authorizes trace uploads + +PostgreSQL and ClickHouse use persistent Docker volumes and have no host ports. The dashboard and trace listener bind to localhost. Use your normal TLS and ingress for a hosted deployment. Stop the stack with `docker compose --env-file deploy/lens/.env -f deploy/lens/stack.yaml down`; omit `-v` to retain data Under **Lens > Investigations > Connect worker**, choose an analysis model and monthly budget. The deployed service connects automatically after you save these settings. There is no worker command or second token to copy @@ -68,14 +67,30 @@ Lens does not need provider credentials, PostgreSQL credentials, a GPU, or the L ### Kubernetes with Helm -Both `helm/litellm` and `helm/litellm-helm` support the Lens service. Keep your existing release, namespace, values, and database configuration. Create two Secrets through your normal secret manager: `litellm-lens-service` with key `service-token`, and `litellm-lens-clickhouse` with key `url` +Both `helm/litellm` and `helm/litellm-helm` support Lens. Keep your existing chart, release name, namespace, and values. Add: + +```yaml +lensWorker: + enabled: true +``` + +Then run your usual Helm deployment command using the matching published chart. The chart supplies the matching Lens image, generates the shared service secret, starts a single ClickHouse instance with a persistent volume, and connects the services. Your cluster needs a default storage class, or set `lensWorker.clickhouse.storageClassName`. Bundled storage defaults to 20 GiB; set `lensWorker.clickhouse.storage` before installation to choose another size + +When your chart manages an ingress with one hostname, the chart fills in the public tracing address and routes `/lens-ingest` directly to Lens. TLS is detected from `ingress.tls` or an ALB certificate annotation. With custom ingress, multiple hostnames, or TLS terminated elsewhere, set the address explicitly: + +```yaml +lensWorker: + enabled: true + publicUrl: https:///lens-ingest +``` + +For a dedicated trace hostname, configure `lensWorker.ingress.enabled`, `host`, `className`, and `tls`. Its hostname supplies the public address unless you override `publicUrl`. Internal Lens routes stay private + +To use an existing ClickHouse database and secrets managed by your platform, keep these overrides: ```yaml lensWorker: enabled: true - image: - repository: - digest: sha256: serviceTokenSecret: name: litellm-lens-service key: service-token @@ -84,21 +99,13 @@ lensWorker: key: url clickhouseDatabase: litellm retentionDays: 14 - publicUrl: https:///lens-ingest ``` -Set `clickhouseDatabase` and `retentionDays` to your existing database and retention before upgrading +Supplying `clickhouseSecret.name` uses that database and disables bundled storage. Keep your database name and retention policy. For GitOps tools that render Helm without cluster access, supply both existing secrets so rendering cannot regenerate credentials -When the chart's main ingress is enabled, it routes `/lens-ingest` directly to Lens. With a custom ingress, add that route yourself. For a dedicated hostname, use `lensWorker.ingress.enabled`, `host`, `className`, and `tls`, and set `publicUrl` to that hostname. The chart connects LiteLLM to Lens internally and gives both services the shared secret +Normal Helm upgrades reuse the generated credentials. Secrets are retained on uninstall, and the ClickHouse volume is retained by Kubernetes. Back them up together. Treat changing the database, storage class, or secret reference as an infrastructure change, not a routine version update -Update your existing component image overrides to matching builds, then use the chart from that checkout: - -```bash -helm upgrade --install litellm ./helm/litellm \ - --namespace litellm -f values.yaml --wait -``` - -Use `./helm/litellm-helm` if that is your existing chart. `lensWorker.replicaCount` scales ingestion and investigations. Each replica needs access to the same ClickHouse and gateway. Credentials refresh every 30 seconds; a newly created key may briefly receive a retryable 429. Revocations propagate on refresh, and a replica stops accepting traces when its credential snapshot reaches 90 seconds +After deployment, open **Lens**. If it was already open, click **Check setup**. The setup section moves to your framework and tracing key when Lens is reachable and storage is ready. Investigation setup asks for the analysis model and budget; the installed service connects automatically ## Upgrade @@ -106,13 +113,9 @@ Upgrade LiteLLM and Lens from the same source commit and release identity. For a Keep the same databases, encryption keys, shared service secret, and public ingestion URL. Pause scheduled investigations and finish or cancel active runs, update both images through your usual deployment process, then check ingestion and run an investigation before resuming schedules. Do not run `docker compose down -v` -When upgrading from the Python worker, replace it with the Rust Lens service, move the existing ClickHouse connection to Lens, and configure the service URLs and secret on LiteLLM. Existing trace data remains in the same ClickHouse database; findings and settings remain in PostgreSQL. Stop the old worker. Generate dedicated tracing keys and change agent exporters to the ingestion URL. A virtual model key no longer authorizes uploads; the old gateway upload endpoints return 410 with setup guidance - -If you retain an explicit `LENS_WORKER_TOKEN`, it remains an optional investigation credential. Normal setup uses the shared service connection and registers one managed worker identity. Configure the analysis model and billing key in the dashboard; provider keys stay on LiteLLM - ## Development -`make lens-dev` starts LiteLLM, the Rust Lens service, and the hot-reload dashboard. Set `LENS_DEV_PROXY_PORT` and `LENS_DEV_UI_PORT` to change the local ports. For containers, pass the same release identity to both builds. Unversioned or incompatible workers are refused before claiming work +`make lens-dev` starts LiteLLM, Lens, and the hot-reload dashboard. Set `LENS_DEV_PROXY_PORT` and `LENS_DEV_UI_PORT` to change the local ports. For containers, pass the same release identity to both builds. Unversioned or incompatible workers are refused before claiming work ## Configure a lens diff --git a/deploy/lens/config.yaml b/deploy/lens/config.yaml index 43bfe32ec26..f9eb15865a7 100644 --- a/deploy/lens/config.yaml +++ b/deploy/lens/config.yaml @@ -1,3 +1,5 @@ +model_list: [] + general_settings: master_key: os.environ/LITELLM_MASTER_KEY tracing: diff --git a/deploy/lens/configure.py b/deploy/lens/configure.py new file mode 100644 index 00000000000..866d986c570 --- /dev/null +++ b/deploy/lens/configure.py @@ -0,0 +1,73 @@ +from __future__ import annotations + +import argparse +import os +import re +import secrets +import shlex +import sys +from pathlib import Path +from typing import Final + +SECRET_NAMES: Final = ( + "LITELLM_MASTER_KEY", + "LITELLM_SALT_KEY", + "LITELLM_LENS_SERVICE_TOKEN", + "POSTGRES_PASSWORD", + "CLICKHOUSE_PASSWORD", +) + + +def environment_content(path: Path) -> str: + if not path.exists(): + return "".join( + f"{name}={'sk-' if name.endswith('KEY') else ''}{secrets.token_hex(32)}\n" for name in SECRET_NAMES + ) + saved: Final = path.read_text().splitlines() + values: Final = dict(line.split("=", 1) for line in saved if "=" in line) + if any(not values.get(name) for name in SECRET_NAMES): + raise ValueError(f"{path} is incomplete. Restore your saved credentials before continuing") + return "\n".join(line for line in saved if not line.startswith("LITELLM_VERSION=")) + "\n" + + +def configure(path: Path, version: str) -> None: + release: Final = version.removeprefix("v") + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+(?:[-.][a-zA-Z0-9.-]+)?", release): + raise ValueError("Use a published release version, such as 1.82.0 or 1.82.0-nightly") + if path.is_symlink(): + raise ValueError(f"Refusing to replace a symlink: {path}") + existing: Final = path.exists() + content: Final = environment_content(path) + temporary: Final = path.with_name(f".{path.name}.{secrets.token_hex(8)}") + descriptor: Final = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) + try: + with os.fdopen(descriptor, "w") as output: + output.write(content + f"LITELLM_VERSION={release}\n") + if existing: + os.replace(temporary, path) + else: + os.link(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def main() -> None: + parser: Final = argparse.ArgumentParser(description="Create or update the configuration for the Lens Compose stack") + parser.add_argument("--version", required=True, help="Published LiteLLM release; Lens uses the matching version") + parser.add_argument("--env-file", type=Path, default=Path(__file__).with_name(".env")) + arguments: Final = parser.parse_args() + try: + configure(arguments.env_file, arguments.version) + except (OSError, ValueError) as error: + parser.exit(1, f"Could not configure Lens: {error}\n") + sys.stdout.write( + f"Saved {arguments.env_file}. Existing keys and database passwords are preserved\n" + f"Start with: docker compose --env-file {shlex.quote(str(arguments.env_file))} " + "-f deploy/lens/stack.yaml up -d --wait\n" + "Open http://localhost:4000/ui/ and sign in as admin with LITELLM_MASTER_KEY from the saved file\n" + "Back up this file with your database volumes. Do not commit it\n" + ) + + +if __name__ == "__main__": + main() diff --git a/deploy/lens/test_configure.py b/deploy/lens/test_configure.py new file mode 100644 index 00000000000..dc61871ab74 --- /dev/null +++ b/deploy/lens/test_configure.py @@ -0,0 +1,52 @@ +import tempfile +import unittest +from pathlib import Path + +from configure import SECRET_NAMES, configure + + +class ComposeConfigurationTests(unittest.TestCase): + def test_restart_and_upgrade_preserve_private_credentials_and_custom_settings(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / ".env" + configure(path, "v1.2.3") + original = dict(line.split("=", 1) for line in path.read_text().splitlines()) + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + self.assertEqual(len({original[name] for name in SECRET_NAMES}), len(SECRET_NAMES)) + self.assertTrue(all(len(original[name]) >= 64 for name in SECRET_NAMES)) + with path.open("a") as output: + output.write("LITELLM_LENS_PUBLIC_URL=https://traces.example/prefix\n") + configure(path, "1.2.3") + configure(path, "v1.2.4-nightly") + updated = dict(line.split("=", 1) for line in path.read_text().splitlines()) + self.assertEqual({name: updated[name] for name in SECRET_NAMES}, {name: original[name] for name in SECRET_NAMES}) + self.assertEqual(updated["LITELLM_VERSION"], "1.2.4-nightly") + self.assertEqual(updated["LITELLM_LENS_PUBLIC_URL"], "https://traces.example/prefix") + self.assertEqual(path.stat().st_mode & 0o777, 0o600) + + def test_incomplete_configuration_is_never_replaced_with_new_database_passwords(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / ".env" + original = "POSTGRES_PASSWORD=existing\n" + path.write_text(original) + with self.assertRaisesRegex(ValueError, "incomplete"): + configure(path, "1.2.3") + self.assertEqual(path.read_text(), original) + + def test_invalid_release_and_symlink_leave_existing_files_untouched(self) -> None: + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / ".env" + target = Path(directory) / "saved" + target.write_text("preserve") + path.symlink_to(target) + with self.assertRaisesRegex(ValueError, "symlink"): + configure(path, "1.2.3") + self.assertEqual(target.read_text(), "preserve") + path.unlink() + with self.assertRaisesRegex(ValueError, "published release"): + configure(path, "1.2.3\nPOSTGRES_PASSWORD=replaced") + self.assertFalse(path.exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/helm/litellm-helm/templates/_helpers.tpl b/helm/litellm-helm/templates/_helpers.tpl index 5e5b47b586a..6bf4023b353 100644 --- a/helm/litellm-helm/templates/_helpers.tpl +++ b/helm/litellm-helm/templates/_helpers.tpl @@ -371,3 +371,30 @@ shutdown drain window. {{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}} {{- toYaml $labels -}} {{- end -}} + +{{- define "litellm.lensWorker.serviceTokenSecretName" -}} +{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}} +{{- end -}} + +{{- define "litellm.lensWorker.bundledClickhouse" -}} +{{- if and .Values.lensWorker.enabled .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}} +{{- end -}} + +{{- define "litellm.lensWorker.publicUrl" -}} +{{- if .Values.lensWorker.publicUrl -}} +{{- .Values.lensWorker.publicUrl -}} +{{- else if .Values.lensWorker.ingress.enabled -}} +{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} +{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}} +{{- else if and .Values.ingress.enabled (eq (len .Values.ingress.hosts) 1) -}} +{{- $host := required "ingress.hosts[0].host is required" (first .Values.ingress.hosts).host -}} +{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} +{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) $host -}} +{{- else -}} +{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}} +{{- end -}} +{{- end -}} + +{{- define "litellm.lensWorker.clickhouseName" -}} +{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}} +{{- end -}} diff --git a/helm/litellm-helm/templates/deployment.yaml b/helm/litellm-helm/templates/deployment.yaml index 4aac75fba8c..bfca52a927b 100644 --- a/helm/litellm-helm/templates/deployment.yaml +++ b/helm/litellm-helm/templates/deployment.yaml @@ -60,11 +60,11 @@ spec: - name: LITELLM_LENS_URL value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }} - name: LITELLM_LENS_PUBLIC_URL - value: {{ required "lensWorker.publicUrl is required" .Values.lensWorker.publicUrl | quote }} + value: {{ include "litellm.lensWorker.publicUrl" . | quote }} - name: LITELLM_LENS_SERVICE_TOKEN valueFrom: secretKeyRef: - name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }} + name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }} key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }} {{- end }} {{- include "litellm.proxyEnv" . | nindent 12 }} diff --git a/helm/litellm-helm/templates/lens/clickhouse.yaml b/helm/litellm-helm/templates/lens/clickhouse.yaml new file mode 100644 index 00000000000..e046f86caf2 --- /dev/null +++ b/helm/litellm-helm/templates/lens/clickhouse.yaml @@ -0,0 +1,98 @@ +{{- if include "litellm.lensWorker.bundledClickhouse" . }} +{{- $name := include "litellm.lensWorker.clickhouseName" . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} +spec: + clusterIP: None + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: lens-clickhouse + ports: + - name: http + port: 8123 + targetPort: http +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} +spec: + serviceName: {{ $name }} + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: lens-clickhouse + template: + metadata: + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: lens-clickhouse + spec: + automountServiceAccountToken: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + fsGroup: 101 + seccompProfile: + type: RuntimeDefault + containers: + - name: clickhouse + image: {{ .Values.lensWorker.clickhouse.image | quote }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + env: + - name: CLICKHOUSE_USER + value: default + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $name }} + key: password + - name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT + value: "1" + ports: + - name: http + containerPort: 8123 + startupProbe: + httpGet: + path: /ping + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /ping + port: http + livenessProbe: + httpGet: + path: /ping + port: http + timeoutSeconds: 3 + resources: + {{- toYaml .Values.lensWorker.clickhouse.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /var/lib/clickhouse + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + {{- if ne .Values.lensWorker.clickhouse.storageClassName nil }} + storageClassName: {{ .Values.lensWorker.clickhouse.storageClassName | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.lensWorker.clickhouse.storage | quote }} +{{- end }} diff --git a/helm/litellm-helm/templates/lens/deployment.yaml b/helm/litellm-helm/templates/lens/deployment.yaml index dee141acde8..b1693dfd9ff 100644 --- a/helm/litellm-helm/templates/lens/deployment.yaml +++ b/helm/litellm-helm/templates/lens/deployment.yaml @@ -45,13 +45,23 @@ spec: - name: LITELLM_LENS_SERVICE_TOKEN valueFrom: secretKeyRef: - name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }} + name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }} key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }} + {{- if include "litellm.lensWorker.bundledClickhouse" . }} + - name: CLICKHOUSE_HOST + value: {{ include "litellm.lensWorker.clickhouseName" . | quote }} + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "litellm.lensWorker.clickhouseName" . | quote }} + key: password + {{- else }} - name: CLICKHOUSE_URL valueFrom: secretKeyRef: name: {{ required "lensWorker.clickhouseSecret.name is required" .Values.lensWorker.clickhouseSecret.name | quote }} key: {{ .Values.lensWorker.clickhouseSecret.key | quote }} + {{- end }} - name: CLICKHOUSE_DATABASE value: {{ .Values.lensWorker.clickhouseDatabase | quote }} - name: AGENT_TRACING_RETENTION_DAYS diff --git a/helm/litellm-helm/templates/lens/secrets.yaml b/helm/litellm-helm/templates/lens/secrets.yaml new file mode 100644 index 00000000000..3809167c1da --- /dev/null +++ b/helm/litellm-helm/templates/lens/secrets.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.lensWorker.enabled (not .Values.lensWorker.serviceTokenSecret.name) }} +{{- $name := include "litellm.lensWorker.serviceTokenSecretName" . }} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + {{ .Values.lensWorker.serviceTokenSecret.key }}: {{ if $existing }}{{ required "Saved Lens service secret is missing its key" (index $existing.data .Values.lensWorker.serviceTokenSecret.key) | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }} +{{- end }} +{{- if include "litellm.lensWorker.bundledClickhouse" . }} +{{- $name := include "litellm.lensWorker.clickhouseName" . }} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + password: {{ if $existing }}{{ required "Saved Lens ClickHouse secret is missing its password" (index $existing.data "password") | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }} +{{- end }} diff --git a/helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml b/helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml new file mode 100644 index 00000000000..22dcc7e9f9c --- /dev/null +++ b/helm/litellm-helm/tests/lens_endpoint_defaults_tests.yaml @@ -0,0 +1,84 @@ +suite: Lens endpoint defaults +templates: + - deployment.yaml + - configmap-litellm.yaml +set: + lensWorker.enabled: true + ingress.enabled: true + ingress.hosts: [{host: gateway.example, paths: [{path: /, pathType: Prefix}]}] + ingress.tls: + - hosts: [gateway.example] + secretName: tls +tests: + - it: derives the trace endpoint from the deployment hostname and TLS + template: deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://gateway.example/lens-ingest + - it: preserves an explicitly configured public address + set: + lensWorker.publicUrl: https://custom.example/prefix + template: deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://custom.example/prefix + - it: uses the dedicated Lens ingress when configured + set: + lensWorker.ingress.enabled: true + lensWorker.ingress.host: traces.example + lensWorker.ingress.tls: + - hosts: [traces.example] + secretName: traces-tls + template: deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://traces.example + - it: uses HTTPS for a dedicated ALB ingress with certificate annotations + set: + lensWorker.ingress.enabled: true + lensWorker.ingress.host: traces.example + lensWorker.ingress.className: alb + lensWorker.ingress.tls: [] + lensWorker.ingress.annotations: + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]' + alb.ingress.kubernetes.io/ssl-redirect: "443" + template: deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://traces.example + - it: uses HTTP for a dedicated ingress without its own TLS + set: + lensWorker.ingress.enabled: true + lensWorker.ingress.host: traces.example + lensWorker.ingress.tls: [] + lensWorker.ingress.annotations: {} + template: deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: http://traces.example + - it: uses HTTP when ingress has no TLS + set: + ingress.tls: [] + template: deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: http://gateway.example/lens-ingest diff --git a/helm/litellm-helm/tests/lens_saved_secrets_tests.yaml b/helm/litellm-helm/tests/lens_saved_secrets_tests.yaml new file mode 100644 index 00000000000..1af9bc77594 --- /dev/null +++ b/helm/litellm-helm/tests/lens_saved_secrets_tests.yaml @@ -0,0 +1,45 @@ +suite: Lens preserves credentials across Helm upgrades +templates: + - lens/secrets.yaml +set: + fullnameOverride: lens-test + lensWorker.enabled: true +release: + namespace: lens + name: lens-test + upgrade: true +kubernetesProvider: + scheme: + v1/Secret: + gvr: + version: v1 + resource: secrets + namespaced: true + objects: + - apiVersion: v1 + kind: Secret + metadata: + name: lens-test-lens-service + namespace: lens + data: + service-token: c2F2ZWQtc2VydmljZS10b2tlbg== + - apiVersion: v1 + kind: Secret + metadata: + name: lens-test-lens-clickhouse + namespace: lens + data: + password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= +tests: + - it: reuses the service credential instead of breaking running services + documentIndex: 0 + asserts: + - equal: + path: data.service-token + value: c2F2ZWQtc2VydmljZS10b2tlbg== + - it: reuses the database password instead of locking out stored traces + documentIndex: 1 + asserts: + - equal: + path: data.password + value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= diff --git a/helm/litellm-helm/tests/lens_service_tests.yaml b/helm/litellm-helm/tests/lens_service_tests.yaml index 197f447f5f3..483db4d42c2 100644 --- a/helm/litellm-helm/tests/lens_service_tests.yaml +++ b/helm/litellm-helm/tests/lens_service_tests.yaml @@ -117,7 +117,7 @@ tests: lensWorker.clickhouseSecret.name: lens-storage asserts: - failedTemplate: - errorMessage: lensWorker.publicUrl is required + errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname - it: omits Lens connection settings when disabled in deployment.yaml template: deployment.yaml asserts: diff --git a/helm/litellm-helm/tests/lens_setup_tests.yaml b/helm/litellm-helm/tests/lens_setup_tests.yaml new file mode 100644 index 00000000000..25b2da62743 --- /dev/null +++ b/helm/litellm-helm/tests/lens_setup_tests.yaml @@ -0,0 +1,160 @@ +suite: Lens managed setup +set: + fullnameOverride: lens-test + lensWorker.enabled: true + lensWorker.publicUrl: https://traces.example +release: + namespace: lens + name: lens-test +templates: + - lens/deployment.yaml + - lens/clickhouse.yaml + - lens/secrets.yaml +tests: + - it: generates both private credentials for a new installation + template: lens/secrets.yaml + asserts: + - hasDocuments: + count: 2 + - matchRegex: + path: data.service-token + pattern: '^[A-Za-z0-9+/]{86}==$' + documentIndex: 0 + - matchRegex: + path: data.password + pattern: '^[A-Za-z0-9+/]{86}==$' + documentIndex: 1 + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - it: connects Lens to its private bundled storage + template: lens/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_HOST + value: lens-test-lens-clickhouse + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: lens-test-lens-clickhouse + key: password + - it: stores traces on a persistent volume with the chosen storage class + template: lens/clickhouse.yaml + documentIndex: 1 + set: + lensWorker.clickhouse.storage: 40Gi + lensWorker.clickhouse.storageClassName: fast + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec + value: + accessModes: [ReadWriteOnce] + storageClassName: fast + resources: + requests: + storage: 40Gi + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: lens-test-lens-clickhouse + key: password + - it: keeps an existing external database instead of creating a new one + template: lens/clickhouse.yaml + set: + lensWorker.clickhouseSecret.name: external-clickhouse + asserts: + - hasDocuments: + count: 0 + - it: leaves supplied secrets under their existing manager + template: lens/secrets.yaml + set: + lensWorker.clickhouseSecret.name: external-clickhouse + lensWorker.serviceTokenSecret.name: external-service + asserts: + - hasDocuments: + count: 0 + - it: creates no credentials or database when Lens is disabled + templates: + - lens/secrets.yaml + - lens/clickhouse.yaml + set: + lensWorker.enabled: false + asserts: + - hasDocuments: + count: 0 + - it: requires an external database when bundled storage is explicitly disabled + template: lens/deployment.yaml + set: + lensWorker.clickhouse.enabled: false + asserts: + - failedTemplate: + errorMessage: lensWorker.clickhouseSecret.name is required + - it: keeps storage names valid and uses the same name for the connection + set: + fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + asserts: + - matchRegex: + path: metadata.name + pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' + template: lens/clickhouse.yaml + documentIndex: 0 + - matchRegex: + path: metadata.name + pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' + template: lens/clickhouse.yaml + documentIndex: 1 + - equal: + path: metadata.name + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + template: lens/clickhouse.yaml + documentIndex: 0 + - equal: + path: spec.serviceName + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + template: lens/clickhouse.yaml + documentIndex: 1 + - equal: + path: metadata.name + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + template: lens/secrets.yaml + documentIndex: 1 + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_HOST + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + template: lens/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + key: password + template: lens/deployment.yaml + + - it: allows cold storage startup and inherits registry credentials + template: lens/clickhouse.yaml + documentIndex: 1 + set: + imagePullSecrets: [{name: registry-auth}] + asserts: + - equal: + path: spec.template.spec.imagePullSecrets + value: [{name: registry-auth}] + - equal: + path: spec.template.spec.containers[0].startupProbe + value: + httpGet: {path: /ping, port: http} + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 60 diff --git a/helm/litellm-helm/values.yaml b/helm/litellm-helm/values.yaml index 821557fd116..57929738cd5 100644 --- a/helm/litellm-helm/values.yaml +++ b/helm/litellm-helm/values.yaml @@ -667,6 +667,17 @@ lensWorker: serviceTokenSecret: name: "" key: service-token + clickhouse: + enabled: true + image: clickhouse/clickhouse-server:26.9.6.6 + storage: 20Gi + storageClassName: null + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + memory: 2Gi clickhouseDatabase: litellm retentionDays: 14 clickhouseSecret: diff --git a/helm/litellm/templates/_helpers.tpl b/helm/litellm/templates/_helpers.tpl index 9c76e2da748..2728d0271ec 100644 --- a/helm/litellm/templates/_helpers.tpl +++ b/helm/litellm/templates/_helpers.tpl @@ -520,11 +520,11 @@ shutdown drain window. - name: LITELLM_LENS_URL value: {{ printf "http://%s-lens-worker:%v" (include "litellm.fullname" .) .Values.lensWorker.service.port | quote }} - name: LITELLM_LENS_PUBLIC_URL - value: {{ required "lensWorker.publicUrl is required" .Values.lensWorker.publicUrl | quote }} + value: {{ include "litellm.lensWorker.publicUrl" . | quote }} - name: LITELLM_LENS_SERVICE_TOKEN valueFrom: secretKeyRef: - name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }} + name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }} key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }} {{- end }} {{- end -}} @@ -534,3 +534,29 @@ shutdown drain window. {{- $_ := set $labels "app.kubernetes.io/name" (printf "%s-lens-worker" (include "litellm.name" . | trunc 51 | trimSuffix "-")) -}} {{- toYaml $labels -}} {{- end -}} + +{{- define "litellm.lensWorker.serviceTokenSecretName" -}} +{{- .Values.lensWorker.serviceTokenSecret.name | default (printf "%s-lens-service" (include "litellm.fullname" .)) -}} +{{- end -}} + +{{- define "litellm.lensWorker.bundledClickhouse" -}} +{{- if and .Values.lensWorker.enabled .Values.lensWorker.clickhouse.enabled (not .Values.lensWorker.clickhouseSecret.name) -}}true{{- end -}} +{{- end -}} + +{{- define "litellm.lensWorker.publicUrl" -}} +{{- if .Values.lensWorker.publicUrl -}} +{{- .Values.lensWorker.publicUrl -}} +{{- else if .Values.lensWorker.ingress.enabled -}} +{{- $tls := or (not (empty .Values.lensWorker.ingress.tls)) (hasKey .Values.lensWorker.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} +{{- printf "%s://%s" (ternary "https" "http" $tls) (required "lensWorker.ingress.host is required" .Values.lensWorker.ingress.host) -}} +{{- else if and .Values.ingress.enabled .Values.ingress.host -}} +{{- $tls := or (not (empty .Values.ingress.tls)) (hasKey .Values.ingress.annotations "alb.ingress.kubernetes.io/certificate-arn") -}} +{{- printf "%s://%s/lens-ingest" (ternary "https" "http" $tls) .Values.ingress.host -}} +{{- else -}} +{{- fail "lensWorker.publicUrl is required when there is no single ingress hostname" -}} +{{- end -}} +{{- end -}} + +{{- define "litellm.lensWorker.clickhouseName" -}} +{{- printf "%s-lens-clickhouse" (include "litellm.fullname" . | trunc 47 | trimSuffix "-") -}} +{{- end -}} diff --git a/helm/litellm/templates/lens/clickhouse.yaml b/helm/litellm/templates/lens/clickhouse.yaml new file mode 100644 index 00000000000..e046f86caf2 --- /dev/null +++ b/helm/litellm/templates/lens/clickhouse.yaml @@ -0,0 +1,98 @@ +{{- if include "litellm.lensWorker.bundledClickhouse" . }} +{{- $name := include "litellm.lensWorker.clickhouseName" . }} +apiVersion: v1 +kind: Service +metadata: + name: {{ $name }} +spec: + clusterIP: None + selector: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: lens-clickhouse + ports: + - name: http + port: 8123 + targetPort: http +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $name }} +spec: + serviceName: {{ $name }} + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: lens-clickhouse + template: + metadata: + labels: + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: lens-clickhouse + spec: + automountServiceAccountToken: false + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + securityContext: + runAsNonRoot: true + runAsUser: 101 + runAsGroup: 101 + fsGroup: 101 + seccompProfile: + type: RuntimeDefault + containers: + - name: clickhouse + image: {{ .Values.lensWorker.clickhouse.image | quote }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: [ALL] + env: + - name: CLICKHOUSE_USER + value: default + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $name }} + key: password + - name: CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT + value: "1" + ports: + - name: http + containerPort: 8123 + startupProbe: + httpGet: + path: /ping + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 60 + readinessProbe: + httpGet: + path: /ping + port: http + livenessProbe: + httpGet: + path: /ping + port: http + timeoutSeconds: 3 + resources: + {{- toYaml .Values.lensWorker.clickhouse.resources | nindent 12 }} + volumeMounts: + - name: data + mountPath: /var/lib/clickhouse + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: [ReadWriteOnce] + {{- if ne .Values.lensWorker.clickhouse.storageClassName nil }} + storageClassName: {{ .Values.lensWorker.clickhouse.storageClassName | quote }} + {{- end }} + resources: + requests: + storage: {{ .Values.lensWorker.clickhouse.storage | quote }} +{{- end }} diff --git a/helm/litellm/templates/lens/deployment.yaml b/helm/litellm/templates/lens/deployment.yaml index 93772a7a650..2ba13609188 100644 --- a/helm/litellm/templates/lens/deployment.yaml +++ b/helm/litellm/templates/lens/deployment.yaml @@ -45,13 +45,23 @@ spec: - name: LITELLM_LENS_SERVICE_TOKEN valueFrom: secretKeyRef: - name: {{ required "lensWorker.serviceTokenSecret.name is required" .Values.lensWorker.serviceTokenSecret.name | quote }} + name: {{ include "litellm.lensWorker.serviceTokenSecretName" . | quote }} key: {{ .Values.lensWorker.serviceTokenSecret.key | quote }} + {{- if include "litellm.lensWorker.bundledClickhouse" . }} + - name: CLICKHOUSE_HOST + value: {{ include "litellm.lensWorker.clickhouseName" . | quote }} + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "litellm.lensWorker.clickhouseName" . | quote }} + key: password + {{- else }} - name: CLICKHOUSE_URL valueFrom: secretKeyRef: name: {{ required "lensWorker.clickhouseSecret.name is required" .Values.lensWorker.clickhouseSecret.name | quote }} key: {{ .Values.lensWorker.clickhouseSecret.key | quote }} + {{- end }} - name: CLICKHOUSE_DATABASE value: {{ .Values.lensWorker.clickhouseDatabase | quote }} - name: AGENT_TRACING_RETENTION_DAYS diff --git a/helm/litellm/templates/lens/secrets.yaml b/helm/litellm/templates/lens/secrets.yaml new file mode 100644 index 00000000000..3809167c1da --- /dev/null +++ b/helm/litellm/templates/lens/secrets.yaml @@ -0,0 +1,27 @@ +{{- if and .Values.lensWorker.enabled (not .Values.lensWorker.serviceTokenSecret.name) }} +{{- $name := include "litellm.lensWorker.serviceTokenSecretName" . }} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + {{ .Values.lensWorker.serviceTokenSecret.key }}: {{ if $existing }}{{ required "Saved Lens service secret is missing its key" (index $existing.data .Values.lensWorker.serviceTokenSecret.key) | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }} +{{- end }} +{{- if include "litellm.lensWorker.bundledClickhouse" . }} +{{- $name := include "litellm.lensWorker.clickhouseName" . }} +{{- $existing := lookup "v1" "Secret" .Release.Namespace $name }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + annotations: + helm.sh/resource-policy: keep +type: Opaque +data: + password: {{ if $existing }}{{ required "Saved Lens ClickHouse secret is missing its password" (index $existing.data "password") | quote }}{{ else }}{{ randAlphaNum 64 | b64enc | quote }}{{ end }} +{{- end }} diff --git a/helm/litellm/tests/lens_endpoint_defaults_tests.yaml b/helm/litellm/tests/lens_endpoint_defaults_tests.yaml new file mode 100644 index 00000000000..2d98dcf471e --- /dev/null +++ b/helm/litellm/tests/lens_endpoint_defaults_tests.yaml @@ -0,0 +1,86 @@ +suite: Lens endpoint defaults +templates: + - gateway/deployment.yaml + - gateway/configmap.yaml +set: + lensWorker.enabled: true + ingress.enabled: true + ingress.host: gateway.example + ingress.tls: + - hosts: [gateway.example] + secretName: tls +tests: + - it: derives the trace endpoint from the deployment hostname and TLS + template: gateway/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://gateway.example/lens-ingest + - it: preserves an explicitly configured public address + set: + lensWorker.publicUrl: https://custom.example/prefix + template: gateway/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://custom.example/prefix + - it: uses the dedicated Lens ingress when configured + set: + lensWorker.ingress.enabled: true + lensWorker.ingress.host: traces.example + lensWorker.ingress.tls: + - hosts: [traces.example] + secretName: traces-tls + template: gateway/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://traces.example + - it: uses HTTPS for a dedicated ALB ingress with certificate annotations + set: + lensWorker.ingress.enabled: true + lensWorker.ingress.host: traces.example + lensWorker.ingress.className: alb + lensWorker.ingress.tls: [] + lensWorker.ingress.annotations: + alb.ingress.kubernetes.io/certificate-arn: arn:aws:acm:us-west-2:123456789012:certificate/test + alb.ingress.kubernetes.io/listen-ports: '[{"HTTP":80},{"HTTPS":443}]' + alb.ingress.kubernetes.io/ssl-redirect: "443" + template: gateway/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: https://traces.example + - it: uses HTTP for a dedicated ingress without its own TLS + set: + lensWorker.ingress.enabled: true + lensWorker.ingress.host: traces.example + lensWorker.ingress.tls: [] + lensWorker.ingress.annotations: {} + template: gateway/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: http://traces.example + - it: uses HTTP when ingress has no TLS + set: + ingress.tls: [] + template: gateway/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_PUBLIC_URL + value: http://gateway.example/lens-ingest +values: + - ./values/required.yaml diff --git a/helm/litellm/tests/lens_saved_secrets_tests.yaml b/helm/litellm/tests/lens_saved_secrets_tests.yaml new file mode 100644 index 00000000000..294d9adfe96 --- /dev/null +++ b/helm/litellm/tests/lens_saved_secrets_tests.yaml @@ -0,0 +1,47 @@ +suite: Lens preserves credentials across Helm upgrades +templates: + - lens/secrets.yaml +set: + fullnameOverride: lens-test + lensWorker.enabled: true +release: + namespace: lens + name: lens-test + upgrade: true +kubernetesProvider: + scheme: + v1/Secret: + gvr: + version: v1 + resource: secrets + namespaced: true + objects: + - apiVersion: v1 + kind: Secret + metadata: + name: lens-test-lens-service + namespace: lens + data: + service-token: c2F2ZWQtc2VydmljZS10b2tlbg== + - apiVersion: v1 + kind: Secret + metadata: + name: lens-test-lens-clickhouse + namespace: lens + data: + password: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= +tests: + - it: reuses the service credential instead of breaking running services + documentIndex: 0 + asserts: + - equal: + path: data.service-token + value: c2F2ZWQtc2VydmljZS10b2tlbg== + - it: reuses the database password instead of locking out stored traces + documentIndex: 1 + asserts: + - equal: + path: data.password + value: c2F2ZWQtZGF0YWJhc2UtcGFzc3dvcmQ= +values: + - ./values/required.yaml diff --git a/helm/litellm/tests/lens_service_tests.yaml b/helm/litellm/tests/lens_service_tests.yaml index c5504025572..81ad93bbeee 100644 --- a/helm/litellm/tests/lens_service_tests.yaml +++ b/helm/litellm/tests/lens_service_tests.yaml @@ -136,7 +136,7 @@ tests: lensWorker.clickhouseSecret.name: lens-storage asserts: - failedTemplate: - errorMessage: lensWorker.publicUrl is required + errorMessage: lensWorker.publicUrl is required when there is no single ingress hostname - it: omits Lens connection settings when disabled in gateway/deployment.yaml template: gateway/deployment.yaml asserts: diff --git a/helm/litellm/tests/lens_setup_tests.yaml b/helm/litellm/tests/lens_setup_tests.yaml new file mode 100644 index 00000000000..d052ae81c78 --- /dev/null +++ b/helm/litellm/tests/lens_setup_tests.yaml @@ -0,0 +1,162 @@ +suite: Lens managed setup +set: + fullnameOverride: lens-test + lensWorker.enabled: true + lensWorker.publicUrl: https://traces.example +release: + namespace: lens + name: lens-test +templates: + - lens/deployment.yaml + - lens/clickhouse.yaml + - lens/secrets.yaml +tests: + - it: generates both private credentials for a new installation + template: lens/secrets.yaml + asserts: + - hasDocuments: + count: 2 + - matchRegex: + path: data.service-token + pattern: '^[A-Za-z0-9+/]{86}==$' + documentIndex: 0 + - matchRegex: + path: data.password + pattern: '^[A-Za-z0-9+/]{86}==$' + documentIndex: 1 + - equal: + path: metadata.annotations["helm.sh/resource-policy"] + value: keep + - it: connects Lens to its private bundled storage + template: lens/deployment.yaml + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_HOST + value: lens-test-lens-clickhouse + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: lens-test-lens-clickhouse + key: password + - it: stores traces on a persistent volume with the chosen storage class + template: lens/clickhouse.yaml + documentIndex: 1 + set: + lensWorker.clickhouse.storage: 40Gi + lensWorker.clickhouse.storageClassName: fast + asserts: + - equal: + path: spec.volumeClaimTemplates[0].spec + value: + accessModes: [ReadWriteOnce] + storageClassName: fast + resources: + requests: + storage: 40Gi + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: lens-test-lens-clickhouse + key: password + - it: keeps an existing external database instead of creating a new one + template: lens/clickhouse.yaml + set: + lensWorker.clickhouseSecret.name: external-clickhouse + asserts: + - hasDocuments: + count: 0 + - it: leaves supplied secrets under their existing manager + template: lens/secrets.yaml + set: + lensWorker.clickhouseSecret.name: external-clickhouse + lensWorker.serviceTokenSecret.name: external-service + asserts: + - hasDocuments: + count: 0 + - it: creates no credentials or database when Lens is disabled + templates: + - lens/secrets.yaml + - lens/clickhouse.yaml + set: + lensWorker.enabled: false + asserts: + - hasDocuments: + count: 0 + - it: requires an external database when bundled storage is explicitly disabled + template: lens/deployment.yaml + set: + lensWorker.clickhouse.enabled: false + asserts: + - failedTemplate: + errorMessage: lensWorker.clickhouseSecret.name is required + - it: keeps storage names valid and uses the same name for the connection + set: + fullnameOverride: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa + asserts: + - matchRegex: + path: metadata.name + pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' + template: lens/clickhouse.yaml + documentIndex: 0 + - matchRegex: + path: metadata.name + pattern: '^[a-z]([-a-z0-9]{0,61}[a-z0-9])?$' + template: lens/clickhouse.yaml + documentIndex: 1 + - equal: + path: metadata.name + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + template: lens/clickhouse.yaml + documentIndex: 0 + - equal: + path: spec.serviceName + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + template: lens/clickhouse.yaml + documentIndex: 1 + - equal: + path: metadata.name + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + template: lens/secrets.yaml + documentIndex: 1 + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_HOST + value: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + template: lens/deployment.yaml + - contains: + path: spec.template.spec.containers[0].env + content: + name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-lens-clickhouse + key: password + template: lens/deployment.yaml + - it: allows cold storage startup and inherits registry credentials + template: lens/clickhouse.yaml + documentIndex: 1 + set: + imagePullSecrets: [{name: registry-auth}] + asserts: + - equal: + path: spec.template.spec.imagePullSecrets + value: [{name: registry-auth}] + - equal: + path: spec.template.spec.containers[0].startupProbe + value: + httpGet: {path: /ping, port: http} + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 60 + +values: + - ./values/required.yaml diff --git a/helm/litellm/tests/lens_worker_tests.yaml b/helm/litellm/tests/lens_worker_tests.yaml index b93797dae18..b935734b4b3 100644 --- a/helm/litellm/tests/lens_worker_tests.yaml +++ b/helm/litellm/tests/lens_worker_tests.yaml @@ -77,13 +77,20 @@ tests: asserts: - hasDocuments: count: 0 - - it: requires a shared service secret when enabled + - it: uses the managed service secret when none is supplied template: lens/deployment.yaml set: lensWorker.enabled: true + lensWorker.publicUrl: https://traces.example asserts: - - failedTemplate: - errorMessage: lensWorker.serviceTokenSecret.name is required + - contains: + path: spec.template.spec.containers[0].env + content: + name: LITELLM_LENS_SERVICE_TOKEN + valueFrom: + secretKeyRef: + name: RELEASE-NAME-litellm-lens-service + key: service-token - it: uses the chart release and a secret without granting Kubernetes access template: lens/deployment.yaml chart: diff --git a/helm/litellm/values.yaml b/helm/litellm/values.yaml index 3fd3245d166..5851c5a6cb1 100644 --- a/helm/litellm/values.yaml +++ b/helm/litellm/values.yaml @@ -644,6 +644,17 @@ lensWorker: serviceTokenSecret: name: "" key: service-token + clickhouse: + enabled: true + image: clickhouse/clickhouse-server:26.9.6.6 + storage: 20Gi + storageClassName: null + resources: + requests: + cpu: 100m + memory: 512Mi + limits: + memory: 2Gi clickhouseDatabase: litellm retentionDays: 14 clickhouseSecret: diff --git a/litellm/proxy/lens/endpoints.py b/litellm/proxy/lens/endpoints.py index 746fba26316..8e83a1ad6f9 100644 --- a/litellm/proxy/lens/endpoints.py +++ b/litellm/proxy/lens/endpoints.py @@ -194,16 +194,37 @@ async def service_connection(auth: Auth) -> ServiceConnection: public_url: Final = os.environ.get("LITELLM_LENS_PUBLIC_URL", "").rstrip("/") try: connection: Final = LensConnection.from_env() + except ValueError: + return ServiceConnection( + url=public_url, + connected=False, + status=ServiceStatus(), + configured=bool(os.environ.get("LITELLM_LENS_URL")), + release=release_tag(), + ) + try: client: Final = connection.control_client() async with client.stream( "GET", connection.endpoint("/internal/status"), headers=connection.headers, timeout=2 ) as response: if response.status_code == 200: status: Final = ServiceStatus.model_validate_json(await bounded_response(response, 16 * 1024)) - return ServiceConnection(url=public_url, connected=True, status=status) + return ServiceConnection( + url=public_url, + connected=True, + status=status, + configured=True, + release=release_tag(), + ) except (ValueError, RuntimeError, httpx.HTTPError): pass - return ServiceConnection(url=public_url, connected=False, status=ServiceStatus()) + return ServiceConnection( + url=public_url, + connected=False, + status=ServiceStatus(), + configured=True, + release=release_tag(), + ) async def credential_snapshot() -> IngestionSnapshot: diff --git a/litellm/proxy/lens/ingestion.py b/litellm/proxy/lens/ingestion.py index 50cb4d82998..c7e34353b38 100644 --- a/litellm/proxy/lens/ingestion.py +++ b/litellm/proxy/lens/ingestion.py @@ -59,6 +59,8 @@ class ServiceConnection(Record): url: str connected: bool status: ServiceStatus + configured: bool = False + release: str = "" @dataclass(frozen=True, slots=True) diff --git a/tests/unit/proxy/lens/test_endpoints.py b/tests/unit/proxy/lens/test_endpoints.py index 5b088462aed..fee53320802 100644 --- a/tests/unit/proxy/lens/test_endpoints.py +++ b/tests/unit/proxy/lens/test_endpoints.py @@ -1048,6 +1048,7 @@ async def test_service_status_uses_internal_auth_and_only_advertises_the_public_ result: Final = await service_connection(UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER)) assert result.url == "https://traces.example/lens-ingest" assert result.connected is connected + assert result.configured is True assert result.status.storage_ready is connected assert route.calls[0].request.headers["Authorization"] == "Bearer " + "x" * 32 assert "private storage details" not in result.model_dump_json() @@ -1057,6 +1058,23 @@ async def test_service_status_uses_internal_auth_and_only_advertises_the_public_ assert denied.value.status_code == 403 +@pytest.mark.asyncio +@pytest.mark.parametrize("url,configured", (("", False), ("http://lens", True))) +async def test_service_setup_distinguishes_missing_installation_from_incomplete_configuration( + monkeypatch: pytest.MonkeyPatch, url: str, configured: bool +) -> None: + from litellm.proxy.lens.endpoints import service_connection + + monkeypatch.setenv("LITELLM_LENS_URL", url) + monkeypatch.delenv("LITELLM_LENS_SERVICE_TOKEN", raising=False) + monkeypatch.setenv("LITELLM_RELEASE_TAG", "v1.2.3") + result: Final = await service_connection(UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER)) + assert result.configured is configured + assert result.connected is False + assert result.release == "v1.2.3" + assert result.status.storage_ready is False + + @pytest.mark.asyncio async def test_credential_snapshot_excludes_expired_keys_and_disables_caching(monkeypatch: pytest.MonkeyPatch) -> None: from unittest.mock import AsyncMock diff --git a/ui/litellm-dashboard/eslint-suppressions.json b/ui/litellm-dashboard/eslint-suppressions.json index 280d3cc4115..1e8e6ee0e39 100644 --- a/ui/litellm-dashboard/eslint-suppressions.json +++ b/ui/litellm-dashboard/eslint-suppressions.json @@ -1116,11 +1116,6 @@ "count": 1 } }, - "src/app/(dashboard)/usage/_components/components/EntityUsage/EntityUsage.tsx": { - "local/no-complex-jsx-arrow": { - "count": 2 - } - }, "src/app/(dashboard)/usage/_components/components/UsageAIChatPanel.tsx": { "no-nested-ternary": { "count": 1 @@ -1130,17 +1125,11 @@ } }, "src/app/(dashboard)/usage/_components/components/UsagePageView.tsx": { - "local/no-complex-jsx-arrow": { - "count": 2 - }, - "max-lines": { - "count": 1 - }, "react-hooks/purity": { "count": 1 }, "react-hooks/set-state-in-effect": { - "count": 2 + "count": 1 } }, "src/app/(dashboard)/users/_components/BulkEditUsers.tsx": { diff --git a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx index 65b7a94711b..1a81235b361 100644 --- a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx @@ -1,7 +1,7 @@ import { act, fireEvent, screen, within, waitFor } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { renderWithProviders, testQueryClient } from "@/../tests/test-utils"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import { readRequest, requestPath } from "@/../tests/lens-test-utils"; import { LensWorkspace } from "./LensWorkspace"; import { createLensDemoData } from "./data/demo/fixtures"; @@ -19,16 +19,19 @@ const worker = () => ({ scope: data.lenses[0].scope, }); -function serve({ enabled = false, traces = false, requests = false, connected = false } = {}) { +function serve({ enabled = false, traces = false, requests = false, connected = false, storageReady = true } = {}) { list.mockResolvedValue({ lenses: [], workers: connected ? [worker()] : [], tracing_enabled: enabled }); network.mockImplementation(async (input, init) => { const { path, method, body, query } = await readRequest(input, init); - if (path === "/lens/service") - return Response.json({ + if (path === "/lens/service") { + const service = { url: "https://traces.test", - connected: true, - status: { storage_ready: true, credentials_ready: true }, - }); + configured: enabled, + connected: enabled, + status: { storage_ready: storageReady, credentials_ready: true }, + }; + return Response.json(service); + } if (path === "/v1/traces") return enabled ? Response.json({ data: traces ? [data.runs[0].trace.summary] : [] }) @@ -143,6 +146,19 @@ describe("Lens introduction", () => { }); describe("Lens setup journey", () => { + it("waits for storage readiness before completing installation", async () => { + serve({ enabled: true, storageReady: false }); + const user = userEvent.setup(); + renderWorkspace(); + const installation = await screen.findByRole("region", { name: /Install Lens/ }); + expect(await within(installation).findByText(/trace storage is unavailable/)).toBeVisible(); + expect(screen.queryByText("Trace storage is connected")).not.toBeInTheDocument(); + expect(screen.queryByRole("combobox", { name: "Your agent framework" })).not.toBeInTheDocument(); + serve({ enabled: true }); + await user.click(within(installation).getByRole("button", { name: "Check setup" })); + expect(await screen.findByRole("combobox", { name: "Your agent framework" })).toBeVisible(); + }); + it.each(["/lens", "/lens/activity/available"])( "keeps recorded traces visible while %s is pending", async (pendingPath) => { @@ -192,6 +208,41 @@ describe("Lens setup journey", () => { await connectWorkerFromSettings(user); }); + it("continues to agent setup when a background service check detects the installation", async () => { + const user = userEvent.setup(); + renderWorkspace({ searchParams: "?setup=lens" }); + const intro = within(await screen.findByRole("region", { name: "Get started with Lens" })); + expect(await intro.findByRole("button", { name: "Check setup" })).toBeVisible(); + serve({ enabled: true }); + await act(() => testQueryClient.refetchQueries({ queryKey: ["lens-service"] })); + await user.click(await intro.findByRole("button", { name: "Continue to your agent" })); + expect(await intro.findByRole("combobox", { name: "Your agent framework" })).toBeVisible(); + expect(intro.getByRole("button", { name: "Generate tracing key" })).toBeEnabled(); + expect(intro.getByRole("button", { name: "Copy tracing configuration" })).toBeVisible(); + await chooseSelectOption(user, intro.getByRole("combobox", { name: "Your agent framework" }), "LangGraph"); + const normal = network.getMockImplementation()!; + network.mockImplementation((input, init) => + requestPath(input) === "/lens/tracing/keys" + ? Promise.resolve(Response.json({ key: "sk-tracing-setup", active: true })) + : normal(input, init), + ); + await user.click(intro.getByRole("button", { name: "Generate tracing key" })); + expect(await intro.findByText("Your tracing key")).toBeVisible(); + serve({ enabled: true, storageReady: false }); + await act(() => testQueryClient.refetchQueries({ queryKey: ["lens-service"] })); + const installation = within(await intro.findByRole("region", { name: /Install Lens/ })); + expect(await installation.findByText(/trace storage is unavailable/)).toBeVisible(); + serve({ enabled: true }); + await act(() => testQueryClient.refetchQueries({ queryKey: ["lens-service"] })); + const agent = within(await intro.findByRole("region", { name: /Send your first trace/ })); + expect(await agent.findByRole("combobox", { name: "Your agent framework" })).toHaveTextContent("LangGraph"); + expect(agent.getByText("Your tracing key")).toBeVisible(); + expect(agent.queryByRole("button", { name: "Generate tracing key" })).not.toBeInTheDocument(); + serve({ enabled: true, traces: true }); + await user.click(intro.getByRole("button", { name: "Check for traces" })); + expect(await intro.findByRole("button", { name: "Continue to worker" })).toBeEnabled(); + }); + it("resumes setup from the URL and leaves only when the user chooses traces", async () => { serve({ enabled: true, traces: true }); const user = userEvent.setup(); @@ -222,7 +273,7 @@ describe("Lens setup journey", () => { const intro = within(await screen.findByRole("region", { name: "Get started with Lens" })); expect(await intro.findByRole("heading", { name: "Before you start" })).toBeVisible(); expect(intro.getByRole("button", { name: "Connect worker" })).toBeEnabled(); - await user.click(intro.getByRole("button", { name: /Enable tracing on the gateway/ })); + await user.click(intro.getByRole("button", { name: /Install Lens/ })); expect(intro.getByRole("button", { name: "Continue with request logs" })).toBeEnabled(); await user.click(intro.getByRole("button", { name: /Send your first trace/ })); await user.click(intro.getByRole("button", { name: "Continue with request logs" })); diff --git a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx index 4b81efa552e..0225bfb19e7 100644 --- a/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/investigations/InvestigationsView.integration.test.tsx @@ -445,7 +445,7 @@ it("guides a first-time administrator into worker connection and lens setup", as expect.objectContaining({ authorization: "Bearer test" }), ); expect(guide.queryByRole("button", { name: /Send your first trace/ })).not.toBeInTheDocument(); - expect(guide.queryByRole("button", { name: /Enable tracing on the gateway/ })).not.toBeInTheDocument(); + expect(guide.queryByRole("button", { name: /Install Lens/ })).not.toBeInTheDocument(); expect(guide.getByRole("button", { name: /Connect a worker/ })).toHaveAttribute("aria-expanded", "true"); expect(guide.queryByRole("button", { name: "View traces" })).not.toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); @@ -602,7 +602,9 @@ it.each([false, true])( const user = userEvent.setup(); renderWithProviders(); const guide = within(await screen.findByRole("region", { name: "Get Lens running" })); - expect(guide.getByRole("button", { name: /Send your first trace/ })).toHaveAttribute("aria-expanded", "true"); + await waitFor(() => + expect(guide.getByRole("button", { name: /Send your first trace/ })).toHaveAttribute("aria-expanded", "true"), + ); expect(await guide.findByRole("button", { name: "Check for traces" })).toBeVisible(); await user.click(guide.getByRole("button", { name: /Connect a worker/ })); expect(guide.getByRole("button", { name: "Connect worker" })).toBeDisabled(); diff --git a/ui/litellm-dashboard/src/components/lens/onboarding/LensGettingStarted.tsx b/ui/litellm-dashboard/src/components/lens/onboarding/LensGettingStarted.tsx index 542dcad188c..ceb20f4db38 100644 --- a/ui/litellm-dashboard/src/components/lens/onboarding/LensGettingStarted.tsx +++ b/ui/litellm-dashboard/src/components/lens/onboarding/LensGettingStarted.tsx @@ -9,9 +9,9 @@ import { OnboardingSetup } from "./OnboardingSetup"; const PREREQUISITES = [ { title: "LiteLLM gateway", detail: "Access to its configuration" }, - { title: "ClickHouse", detail: "Self-hosted or managed trace storage" }, - { title: "A server with Docker", detail: "To run the analysis worker" }, - { title: "An analysis model", detail: "Available through your gateway" }, + { title: "Trace storage", detail: "Included, or use your own ClickHouse" }, + { title: "Docker or Kubernetes", detail: "Use your existing deployment" }, + { title: "An analysis model", detail: "For investigations, after tracing is connected" }, ] as const; export interface LensGettingStartedProps { diff --git a/ui/litellm-dashboard/src/components/lens/onboarding/OnboardingSteps.tsx b/ui/litellm-dashboard/src/components/lens/onboarding/OnboardingSteps.tsx index 5e48ab68df7..8bd63f06764 100644 --- a/ui/litellm-dashboard/src/components/lens/onboarding/OnboardingSteps.tsx +++ b/ui/litellm-dashboard/src/components/lens/onboarding/OnboardingSteps.tsx @@ -3,7 +3,7 @@ import { useId, useRef, useState, type ReactNode } from "react"; import { ArrowRight, ChevronDown } from "lucide-react"; import { Button } from "@/components/ui/button"; -import { TracingSetupFields } from "@/components/lens/onboarding/tracing/TracingSetupCard"; +import { TracingSetupFields, useLensService } from "@/components/lens/onboarding/tracing/TracingSetupCard"; import { cn } from "@/lib/cva.config"; import { useLensAccessToken } from "../data/LensServices"; import type { LensReadiness } from "../hooks/useLensReadiness"; @@ -11,17 +11,17 @@ import { initialSetupStep } from "../model/readiness"; import { StepIndicator, type StepState } from "../ui/StepIndicator"; import { useOnboarding } from "./OnboardingContext"; -type StepProps = { state: LensReadiness; goTo: (step: number) => void }; +type StepProps = { state: LensReadiness; goTo: (step: number) => void; storageReady: boolean }; function useLocked() { const { readOnly, canInvestigate } = useOnboarding(); return readOnly || !canInvestigate; } -function StorageStep({ state, goTo }: StepProps) { +function StorageStep({ state, goTo, storageReady }: StepProps) { const { readOnly, openTrace } = useOnboarding(); const accessToken = useLensAccessToken(); - if (!state.tracingEnabled) + if (!storageReady) return ( <> -

Connect trace storage in step 1 before sending a trace.

- - - ); return ( <> ); diff --git a/ui/litellm-dashboard/src/components/lens/onboarding/tracing/TracingSetupCard.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/onboarding/tracing/TracingSetupCard.integration.test.tsx index 16a0a4fe946..a5857078419 100644 --- a/ui/litellm-dashboard/src/components/lens/onboarding/tracing/TracingSetupCard.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/onboarding/tracing/TracingSetupCard.integration.test.tsx @@ -1,7 +1,7 @@ -import { screen } from "@testing-library/react"; +import { act, screen } from "@testing-library/react"; import userEvent from "@testing-library/user-event"; import { beforeEach, describe, expect, it, vi } from "vitest"; -import { chooseSelectOption, renderWithProviders } from "@/../tests/test-utils"; +import { chooseSelectOption, renderWithProviders, testQueryClient } from "@/../tests/test-utils"; import { copyToClipboard } from "@/utils/dataUtils"; import { agentTraceCall, apiClient } from "../../../networking"; import { @@ -51,14 +51,18 @@ const renderCard = async ( const network = vi.fn(); beforeEach(() => { + testQueryClient.clear(); vi.clearAllMocks(); vi.stubGlobal("fetch", network); network.mockResolvedValue(Response.json({})); - vi.mocked(apiClient.get).mockResolvedValue({ + const readyService = { url: "https://traces.test", + configured: true, + release: "v1.2.3", connected: true, status: { storage_ready: true, credentials_ready: true }, - }); + }; + vi.mocked(apiClient.get).mockResolvedValue(readyService); vi.mocked(apiClient.post).mockResolvedValue({ key: SECRET, active: true }); }); @@ -72,7 +76,6 @@ describe("TracingSetupCard", () => { expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); expect(network).not.toHaveBeenCalled(); expect(card).not.toHaveTextContent("store: clickhouse"); - await user.click(screen.getByText("Set up manually")); expect(screen.getByText(/^export LITELLM_TRACING_KEY=/)).toBeVisible(); expect(card).not.toHaveTextContent(/langsmith/i); }); @@ -114,7 +117,6 @@ describe("TracingSetupCard", () => { normalizeWhitespace: false, }); - await user.click(screen.getByText("Set up manually")); expect(screen.getByText(/^pip install opentelemetry-distro/)).toHaveTextContent( "crewai openinference-instrumentation-crewai", ); @@ -128,7 +130,6 @@ describe("TracingSetupCard", () => { const user = userEvent.setup(); const { card } = await renderCard(); await chooseSelectOption(user, screen.getByRole("combobox", { name: "Your agent framework" }), "Vercel AI SDK"); - await user.click(screen.getByText("Set up manually")); expect(screen.queryByRole("combobox", { name: "Model" })).not.toBeInTheDocument(); expect(card).toHaveTextContent("npm install ai @ai-sdk/otel"); expect(card).toHaveTextContent('const AGENT_NAME = "research_agent"'); @@ -145,7 +146,6 @@ describe("TracingSetupCard", () => { vi.mocked(apiClient.post).mockResolvedValue({ key: SECRET }); const { card } = await renderCard(); await chooseSelectOption(user, screen.getByRole("combobox", { name: "Your agent framework" }), "Hermes"); - await user.click(screen.getByText("Set up manually")); expect(screen.queryByRole("combobox", { name: "Model" })).not.toBeInTheDocument(); expect(card).toHaveTextContent("Keep your existing model settings"); await user.click(screen.getByRole("button", { name: "Generate tracing key" })); @@ -175,7 +175,6 @@ describe("TracingSetupCard", () => { const user = userEvent.setup(); vi.mocked(apiClient.post).mockResolvedValue({ key: SECRET }); const { card } = await renderCard(); - await user.click(screen.getByText("Set up manually")); await user.click(screen.getByRole("button", { name: "Generate tracing key" })); expect(await screen.findByText("Your tracing key")).toBeVisible(); @@ -226,18 +225,92 @@ describe("TracingSetupCard", () => { expect(screen.queryByRole("button", { name: /View trace/ })).not.toBeInTheDocument(); }); - it("guides proxy setup before agent setup and allows checking readiness", async () => { + it("shows matching-version installation instructions without changing the landing design", async () => { const user = userEvent.setup(); const onCheck = vi.fn(); + const missingService = { + configured: false, + connected: false, + release: "v1.2.3", + url: "", + status: {}, + }; + vi.mocked(apiClient.get).mockResolvedValue(missingService); const { card } = await renderCard({ detail: "Agent tracing is not enabled", onCheck }); - expect(screen.getByRole("heading", { name: "Enable tracing" })).toBeVisible(); - expect(card).toHaveTextContent("LITELLM_LENS_URL"); - expect(card).toHaveTextContent("LITELLM_LENS_SERVICE_TOKEN"); + expect(await screen.findByText("Install Lens")).toBeVisible(); + expect(card).toHaveTextContent("Use Lens v1.2.3 to match this LiteLLM deployment"); + expect(screen.getByRole("link", { name: "Helm setup" })).toHaveAttribute( + "href", + "https://docs.litellm.ai/docs/proxy/lens/deployment#using-helm", + ); + expect(screen.getByRole("link", { name: "Docker setup" })).toHaveAttribute( + "href", + "https://docs.litellm.ai/docs/proxy/lens/deployment#using-docker", + ); expect(screen.queryByRole("combobox", { name: "Your agent framework" })).not.toBeInTheDocument(); - expect(screen.queryByRole("button", { name: "Send a test trace" })).not.toBeInTheDocument(); + const readyService = { + configured: true, + connected: true, + url: "https://traces.test", + status: { storage_ready: true }, + }; + vi.mocked(apiClient.get).mockResolvedValue(readyService); await user.click(screen.getByRole("button", { name: "Check setup" })); expect(onCheck).toHaveBeenCalledOnce(); - expect(screen.getByText(/Tracing is still unavailable/)).toBeVisible(); + expect(await screen.findByRole("combobox", { name: "Your agent framework" })).toBeVisible(); + }); + + it.each([ + [false, false, "Lens is configured, but LiteLLM cannot reach it"], + [true, false, "Lens is connected, but its trace storage is unavailable"], + ])( + "explains a configured service failure without recommending reinstallation", + async (connected, storageReady, message) => { + const service = { + configured: true, + connected, + url: "https://traces.test", + status: { storage_ready: storageReady }, + }; + vi.mocked(apiClient.get).mockResolvedValue(service); + await renderCard({ detail: "Service unavailable" }); + expect(await screen.findByText(message, { exact: false })).toBeVisible(); + expect(screen.queryByText("Install Lens")).not.toBeInTheDocument(); + expect(screen.queryByRole("button", { name: "Generate tracing key" })).not.toBeInTheDocument(); + }, + ); + + it("preserves the framework and uncopied key across a background service outage", async () => { + const user = userEvent.setup(); + await renderCard(); + await chooseSelectOption(user, screen.getByRole("combobox", { name: "Your agent framework" }), "LangGraph"); + await user.click(screen.getByRole("button", { name: "Generate tracing key" })); + await screen.findByText("Your tracing key"); + const ready = testQueryClient.getQueryData(["lens-service", "sk-admin"]); + const unavailable = { + configured: true, + connected: false, + url: "https://traces.test", + status: { storage_ready: false }, + }; + act(() => testQueryClient.setQueryData(["lens-service", "sk-admin"], unavailable)); + expect(await screen.findByText(/Lens is configured, but LiteLLM cannot reach it/)).toBeVisible(); + act(() => testQueryClient.setQueryData(["lens-service", "sk-admin"], ready)); + expect(await screen.findByRole("combobox", { name: "Your agent framework" })).toHaveTextContent("LangGraph"); + expect(screen.getByText("Your tracing key")).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Copy tracing configuration" })); + expect(copyToClipboard).toHaveBeenLastCalledWith(tracingEnvSnippet("https://traces.test", SECRET)); + expect(apiClient.post).toHaveBeenCalledOnce(); + }); + + it("shows the copyable configuration without another disclosure and includes the generated key", async () => { + const user = userEvent.setup(); + await renderCard(); + expect(screen.getByText(/^export LITELLM_TRACING_KEY=/)).toBeVisible(); + await user.click(screen.getByRole("button", { name: "Generate tracing key" })); + await screen.findByText("Your tracing key"); + await user.click(screen.getByRole("button", { name: "Copy tracing configuration" })); + expect(copyToClipboard).toHaveBeenLastCalledWith(tracingEnvSnippet("https://traces.test", SECRET)); }); }); diff --git a/ui/litellm-dashboard/src/components/lens/onboarding/tracing/TracingSetupCard.tsx b/ui/litellm-dashboard/src/components/lens/onboarding/tracing/TracingSetupCard.tsx index 299954ada99..2961eeb1700 100644 --- a/ui/litellm-dashboard/src/components/lens/onboarding/tracing/TracingSetupCard.tsx +++ b/ui/litellm-dashboard/src/components/lens/onboarding/tracing/TracingSetupCard.tsx @@ -23,6 +23,7 @@ import type { TraceSummary } from "../../traces/types"; const COPIED_RESET_MS = 1500; const DOCS_URL = "https://docs.litellm.ai/docs/proxy/lens"; +const DEPLOYMENT_URL = `${DOCS_URL}/deployment`; const EXAMPLE_MODEL = "openai/gpt-6.1-sol"; const SAMPLE_TRACE_POLL_MS = 1000; export const TRACING_KEY_REQUEST = { name: "Agent tracing" } as const; @@ -77,22 +78,18 @@ export const otlpEndpoints = (proxyUrl: string): readonly (readonly [string, str ["Protocol", "OTLP/HTTP (protobuf or JSON)", false], ]; -export const PROXY_CONFIG_SNIPPET = [ - 'export LITELLM_LENS_URL="http://lens-worker:4318"', - 'export LITELLM_LENS_PUBLIC_URL="https://traces.example.com"', - 'export LITELLM_LENS_SERVICE_TOKEN=""', -].join("\n"); - function CodeBlock({ code, display = code, tabs, wrap = false, + copyLabel = "Copy", }: { code: string; display?: string; tabs?: React.ReactNode; wrap?: boolean; + copyLabel?: string; }) { const [copied, setCopied] = useState(false); useTimeout(() => setCopied(false), copied ? COPIED_RESET_MS : null); @@ -104,7 +101,7 @@ function CodeBlock({ - + + ); } @@ -498,7 +538,7 @@ function CodingAgentSetup({

Run the setup command in your agent’s project. It uses LITELLM_TRACING_KEY{" "} - for traces and keeps your model key separate . + for traces and keeps your model key separate.

+
+ ); + if (!connection.data.connected || !connection.data.status.storage_ready || !connection.data.url) + return ; + return ( - ) : ( - ); } export function TracingSetupCard(props: TracingSetupProps) { - const enabled = props.detail === null; + const connection = useLensService(props.accessToken); + const enabled = Boolean(connection.data?.connected && connection.data.status.storage_ready && connection.data.url); return (
@@ -746,7 +812,7 @@ export function TracingSetupCard(props: TracingSetupProps) { ) : ( {enabled ? "Send your agent’s runs to LiteLLM to see its inputs, outputs, and tool calls." - : "Tracing needs a Lens service with ClickHouse access and a connection from LiteLLM."} + : "Connect Lens to start recording your agent’s runs."}

diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx index 5362605a057..eee113f6f0c 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx @@ -34,6 +34,24 @@ vi.mock("../detail/run/RunView", () => ({ import { agentTraceListCall, apiClient } from "../../../networking"; +const readyService = { + configured: true, + url: "https://traces.test", + connected: true, + status: { storage_ready: true, credentials_ready: true }, +}; +const missingService = { + configured: false, + url: "", + connected: false, + release: "v1.2.3", + status: { storage_ready: false, credentials_ready: false }, +}; +const serveService = (ready: boolean) => { + const service = ready ? readyService : missingService; + vi.mocked(apiClient.get).mockImplementation(async (path) => (path === "/lens/service" ? service : { data: [] })); +}; + const runs = (traceList as TracePage).data as TraceSummary[]; const lastUrl = (onUrlUpdate: ReturnType) => @@ -77,11 +95,7 @@ describe("AgentTracesSection", () => { setupIntersectionMocking(vi.fn); testQueryClient.clear(); vi.mocked(agentTraceListCall).mockReset(); - vi.mocked(apiClient.get).mockImplementation(async (path) => - path === "/lens/service" - ? { url: "https://traces.test", connected: true, status: { storage_ready: true, credentials_ready: true } } - : { data: [] }, - ); + serveService(true); vi.mocked(apiClient.post).mockImplementation(async (_path, options) => { const body = options?.body as { traces: { trace_id: string; trace_ref?: string }[] }; return body.traces.map((trace) => ({ ...trace, finding_count: null })); @@ -158,21 +172,18 @@ describe("AgentTracesSection", () => { expect(agentTraceListCall).toHaveBeenCalledOnce(); }); - it("renders the setup snippet when the proxy answers 501", async () => { + it("links to installation when the proxy has no Lens configured", async () => { + serveService(false); vi.mocked(agentTraceListCall).mockRejectedValue( new ApiError("Agent tracing is not enabled", 501, { detail: "Agent tracing is not enabled" }), ); renderSection(); - const card = await screen.findByTestId("tracing-setup-card"); - expect(card).toHaveTextContent("Tracing is not enabled"); - expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); - expect(card).toHaveTextContent("LITELLM_LENS_URL"); - expect(card).toHaveTextContent("LITELLM_LENS_SERVICE_TOKEN"); + expect(await screen.findByText("Install Lens")).toBeVisible(); + expect(screen.getByRole("link", { name: "Helm setup" })).toBeVisible(); + expect(screen.getByRole("link", { name: "Docker setup" })).toBeVisible(); expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); - expect(card).not.toHaveTextContent(/langsmith/i); - expect(card).toHaveTextContent("Lens service setup"); - expect(card).toHaveTextContent("Run the Lens service"); + expect(screen.queryByRole("button", { name: "Generate tracing key" })).not.toBeInTheDocument(); }); it("shows the waiting guide when tracing is on but no runs have arrived", async () => { @@ -180,7 +191,7 @@ describe("AgentTracesSection", () => { renderSection(); const card = await screen.findByTestId("tracing-setup-card"); - expect(card).toHaveTextContent("Connect your agent"); + expect(await screen.findByRole("heading", { name: "Connect your agent" })).toBeVisible(); expect(await screen.findByText("Waiting for your first trace")).toBeInTheDocument(); expect(screen.queryByRole("button", { name: "Preview sample" })).not.toBeInTheDocument(); expect(card).not.toHaveTextContent("store: clickhouse"); @@ -188,7 +199,7 @@ describe("AgentTracesSection", () => { it("keeps the trace list available when traces exist outside the current time window", async () => { vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); - vi.mocked(apiClient.get).mockResolvedValue(traceList); + vi.mocked(apiClient.get).mockImplementation(async (path) => (path === "/lens/service" ? readyService : traceList)); renderSection(); expect(await screen.findByText("No runs in this time range")).toBeVisible(); expect(screen.queryByTestId("tracing-setup-card")).not.toBeInTheDocument(); @@ -199,10 +210,12 @@ describe("AgentTracesSection", () => { }); it("checks proxy readiness, waits for an agent, and confirms receipt using actual query results", async () => { + serveService(false); vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); renderSection(); const checkSetup = await screen.findByRole("button", { name: "Check setup" }); vi.mocked(agentTraceListCall).mockResolvedValue({ ...(traceList as TracePage), data: [] }); + serveService(true); fireEvent.click(checkSetup); expect(await screen.findByRole("heading", { name: "Connect your agent" })).toBeVisible(); expect(await screen.findByText("Waiting for your first trace")).toBeVisible(); @@ -215,6 +228,7 @@ describe("AgentTracesSection", () => { }); it("keeps setup visible while checking and explains when tracing is still disabled", async () => { + serveService(false); const failure = new ApiError("Tracing is not enabled", 501, {}); vi.mocked(agentTraceListCall).mockRejectedValue(failure); renderSection(); @@ -228,17 +242,19 @@ describe("AgentTracesSection", () => { ); fireEvent.click(checkSetup); expect(await screen.findByRole("button", { name: "Checking…" })).toBeDisabled(); - expect(screen.getByRole("heading", { name: "Enable tracing" })).toBeVisible(); + expect(screen.getByRole("heading", { name: "Set up Lens" })).toBeVisible(); await act(async () => rejectCheck(failure)); - expect(await screen.findByText(/Tracing is still unavailable/)).toBeVisible(); + expect(await screen.findByText("Install Lens")).toBeVisible(); expect(screen.getByRole("button", { name: "Check setup" })).toBeEnabled(); }); it("keeps received traces and the open drawer visible during subsequent fetches", async () => { + serveService(false); vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Tracing is not enabled", 501, {})); renderSection(); const checkSetup = await screen.findByRole("button", { name: "Check setup" }); vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); + serveService(true); fireEvent.click(checkSetup); const rows = await screen.findAllByTestId("agent-trace-row"); fireEvent.click(rows[0]); @@ -269,7 +285,7 @@ describe("AgentTracesSection", () => { expect(screen.getByText("No runs in this time range")).toBeVisible(); expect(screen.queryByText(/Could not load runs/)).not.toBeInTheDocument(); - vi.mocked(apiClient.get).mockResolvedValue(traceList); + vi.mocked(apiClient.get).mockImplementation(async (path) => (path === "/lens/service" ? readyService : traceList)); fireEvent.click(screen.getByRole("button", { name: "Retry trace check" })); await waitFor(() => expect(screen.queryByRole("alert")).not.toBeInTheDocument()); expect(screen.getByText("No runs in this time range")).toBeVisible(); @@ -277,12 +293,13 @@ describe("AgentTracesSection", () => { }); it("treats a proxy without the trace routes (404) like tracing being off", async () => { + serveService(false); vi.mocked(agentTraceListCall).mockRejectedValue(new ApiError("Not Found", 404, { detail: "Not Found" })); renderSection(); const card = await screen.findByTestId("tracing-setup-card"); - expect(card).toHaveTextContent("Tracing is not enabled"); - expect(card).toHaveTextContent("LITELLM_LENS_SERVICE_TOKEN"); + expect(await screen.findByText("Install Lens")).toBeVisible(); + expect(card).toHaveTextContent("Use Lens v1.2.3"); }); it("lists uninvestigated runs without presenting tool errors as failures", async () => { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5e957efe8dc..17d1e494fe6 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -45534,8 +45534,18 @@ export interface components { }; /** ServiceConnection */ ServiceConnection: { + /** + * Configured + * @default false + */ + configured: boolean; /** Connected */ connected: boolean; + /** + * Release + * @default + */ + release: string; status: components["schemas"]["ServiceStatus"]; /** Url */ url: string; From 02b8b5cc8036a6a2d507e7f8e31a1757285d9f25 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:24:50 -0700 Subject: [PATCH 03/65] fix(proxy): traceparent/baggage fallback must not override caller metadata (#43688) * fix(proxy): traceparent/baggage fallback must not override caller metadata The W3C traceparent/baggage fallback in add_litellm_metadata_from_request_headers documents itself as last-resort: Lower priority than everything above - only fires when neither the explicit litellm headers nor the Anthropic-metadata path found anything But it guards on the top-level `litellm_trace_id` / `litellm_session_id` body keys and never checks `metadata`, which is the documented way callers set trace_id / session_id (`metadata: {"trace_id": ...}` on /chat/completions). A caller that explicitly sets metadata.trace_id has it silently replaced by the header value, so the implementation contradicts its own stated precedence. This is not a corner case on managed platforms: GCP's front end injects a traceparent into every inbound request, so the fallback fires on traffic whose caller never sent the header. The request still returns 200 and the trace still reaches the logging backend, just under an id the caller never chose, so any caller correlating by its own id silently fails to find its trace. Guard both fallbacks on the caller's request-body metadata as well. Deliberately narrow: - x-litellm-trace-id still outranks the body (documented priority #1) - a request that steers neither field still adopts traceparent/baggage exactly as before - steering is per-field: setting only trace_id still lets session_id come from baggage - litellm_metadata is checked too, for LITELLM_METADATA_ROUTES (/v1/responses, /v1/messages, batches, files) Also corrects the comment, which understated the guard. 4 new tests; each fails without the source change. The existing traceparent and baggage tests are unchanged and still pass. * fix(proxy): check only the active metadata container and ignore empty values Addresses review. The first version treated any value in either `metadata` or `litellm_metadata` as caller steering. On LITELLM_METADATA_ROUTES the body `metadata` is provider-facing and is only promoted later, so a session_id there suppressed baggage before apply_missing_session_id_policy ran, and a request with a usable baggage session id got a 400 under `missing_session_id: reject`. An empty session_id did the same, since the policy treats "" as absent Now only the active metadata container counts, and only a truthy value, which matches how apply_missing_session_id_policy decides a session id is present. The helper takes a typed `object` rather than a bare dict Adds regression tests for both cases, including the end to end reject path, and drops the test docstrings * fix(proxy): count promoted caller trace ids on litellm_metadata routes Addresses review. On LITELLM_METADATA_ROUTES the proxy promotes the caller's trace control fields (trace_id, session_id, ...) from `metadata` into `litellm_metadata`, but only after the header fallback runs. Checking only `litellm_metadata` let traceparent and baggage fill those fields first, and the promotion then skipped them because they were already set, so the trace was recorded under the header ids The check now also reads `metadata` for fields in LITELLM_TRACE_CONTROL_METADATA_FIELDS on those routes, and apply_missing_session_id_policy uses the same check, so `reject` no longer refuses a request whose session id is about to be promoted The earlier test asserting that a session_id in `metadata` must not block baggage on /v1/responses had the premise wrong, since that value is promoted. It is replaced by tests that assert the promoted caller ids win on /v1/responses and /v1/messages, with no policy, reject, and generate * fix(proxy): an empty trace field in litellm_metadata shadows the promoted one Addresses review. Promotion copies a trace control field from `metadata` into `litellm_metadata` only when the key is absent, so a key that is present but empty in `litellm_metadata` wins and the `metadata` value is never used. The check still looked at `metadata` in that case, which let `missing_session_id: reject` accept a request that ends up with an empty session id, and let an empty trace_id or session_id block the headers The check now follows the same rule as promotion. If the active container has the key, its value decides. Only when the key is absent does the promoted `metadata` value count. This makes the conflicting-bucket cases behave exactly as on main * fix(proxy): stay within the LIT006 cast budget The type-discipline gate failed: this branch added 4 unsuppressed `cast()` calls and LIT006 was already at its limit. Each cast now carries a `# cast-ok` reason. The two in the helper follow an `isinstance` check that proves the Mapping. The two at the call sites stay because the method's `data` parameter is a bare `dict`, and dropping those casts adds basedpyright unknown-type errors instead. No behavior change * test(proxy): cover caller metadata precedence over W3C trace headers Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): generated session uses promoted caller trace id Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): move test context into assertion messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): audit cells for w3c fallback precedence Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): tighten audit chaos cells Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(test): drop stray blank lines Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): ignore body-less model-info probes in langfuse precedence upstreams Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): keep root trace/session fields on equal ids and usable caller values The _caller_trace_field gating added for caller-metadata precedence was presence-aware, which regressed three pre-call behaviors versus main: - Equal ids in W3C headers and body metadata suppressed the traceparent/baggage branch entirely, leaving the root litellm_trace_id/litellm_session_id unset so router fallbacks minted a fresh uuid4 per request. The header branch now also fires when the caller value equals the header-derived id, so equal ids stamp the root fields exactly like main. - A truthy but non-string metadata value (e.g. session_id 4815162342) counted as caller-supplied and suppressed the baggage fallback, so downstream str-only consumers (code interpreter sandbox reuse) got a new sandbox per turn. _caller_trace_field now counts only non-empty string values, and the generate policy falls through to generation when the caller value is not usable. - On litellm_metadata routes a usable caller session id suppressed the missing_session_id policies while never landing on the root field, so the root session stayed unset. The policy now promotes the caller's usable session id to litellm_session_id instead of leaving it stranded. * test(integration): non-string caller ids fall back to W3C like empty ids The audit cell compared a headers leg against a headerless leg for equality, which only held while a truthy non-string id suppressed the W3C fallback on both legs. With unusable values ignored again, the headers leg resolves to the header ids while the headerless leg cannot, so assert the headers leg's concrete outcome instead, mirroring the empty/null cell. * style(proxy): trim the session promotion comment to the non-obvious why --------- Co-authored-by: Filipe Andujar Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/litellm_pre_call_utils.py | 67 +- .../observability/test_langfuse_delivery.py | 1541 ++++++++++++++++- .../unit/proxy/test_litellm_pre_call_utils.py | 264 +++ 3 files changed, 1857 insertions(+), 15 deletions(-) diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 03a93016177..46cdf328af6 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -154,6 +154,27 @@ def _session_id_from_baggage(baggage: str) -> str | None: return None +def _caller_trace_field(data: Mapping[str, object], metadata_variable_name: str, field: str) -> str | None: + """The caller's value for a trace-control field, counted only when it is a + usable id: a non-empty string. An explicitly empty/unusable value on the + active metadata container still shadows the promoted requester value, but + neither ever counts as "the caller supplied this field" on its own, so a + numeric session id or an empty string cannot suppress the W3C header + fallback or satisfy a missing-session-id policy.""" + active: Final = data.get(metadata_variable_name) + if isinstance(active, Mapping) and field in active: + active_map: Final = cast(Mapping[str, object], active) # cast-ok: isinstance above, free-form JSON values + active_value: Final = active_map[field] + return active_value if isinstance(active_value, str) and active_value else None + promoted: Final = metadata_variable_name == "litellm_metadata" and field in LITELLM_TRACE_CONTROL_METADATA_FIELDS + requester: Final = data.get("metadata") + if not promoted or not isinstance(requester, Mapping): + return None + requester_map: Final = cast(Mapping[str, object], requester) # cast-ok: isinstance above, free-form JSON values + requester_value: Final = requester_map.get(field) + return requester_value if isinstance(requester_value, str) and requester_value else None + + def _stampable_key_hash(user_api_key_dict: UserAPIKeyAuth) -> str | None: """Only proxy-validated keys are stamped, proven by the unforgeable via_virtual_key marker AND a known non-secret shape: the sha256 hex digest @@ -839,11 +860,22 @@ def apply_missing_session_id_policy( ): metadata["session_id"] = body_session_id return - if data.get("litellm_session_id") or metadata.get("session_id"): + caller_session_id: Final = _caller_trace_field(data, _metadata_variable_name, "session_id") + if caller_session_id is not None: + # Consumers that read the root field (router fallbacks, spend logs, + # sandbox reuse) otherwise see no session and mint a uuid4 per request. + if not data.get("litellm_session_id"): + data["litellm_session_id"] = caller_session_id # rebind-ok: data is an out-param + return + if data.get("litellm_session_id"): return match policy: case "generate": - session_id: Final = str(data.get("litellm_trace_id") or metadata.get("trace_id") or uuid.uuid4()) + session_id: Final = str( + data.get("litellm_trace_id") + or _caller_trace_field(data, _metadata_variable_name, "trace_id") + or uuid.uuid4() + ) data["litellm_session_id"] = session_id # rebind-ok: data is an out-param data.setdefault("litellm_trace_id", session_id) metadata["session_id"] = session_id @@ -1596,16 +1628,28 @@ class LiteLLMProxyRequestSetup: # Last-resort fallback: the W3C standards for trace/session propagation # (https://www.w3.org/TR/trace-context/, https://www.w3.org/TR/baggage/). # Lower priority than everything above - only fires when neither the - # explicit litellm headers nor the Anthropic-metadata path found - # anything - but lets a caller's existing traceparent/baggage headers - # (from real OTel instrumentation) correlate with litellm's own logs - # instead of generating an unrelated trace_id. + # explicit litellm headers, the Anthropic-metadata path, nor the + # caller's own request metadata set the field to a DIFFERENT usable id + # - but lets a caller's existing traceparent/baggage headers (from + # real OTel instrumentation) correlate with litellm's own logs instead + # of generating an unrelated trace_id. normalized_headers: Final = MappingProxyType({k.lower(): v for k, v in headers.items() if isinstance(k, str)}) if "litellm_trace_id" not in data: traceparent: Final = normalized_headers.get("traceparent") if isinstance(traceparent, str): trace_id_from_traceparent: Final = _trace_id_from_traceparent(traceparent) - if trace_id_from_traceparent: + # The caller's metadata wins over the header fallback unless + # both carry the same id: stamping the root field then claims + # nothing the caller did not already ask for, and keeps the + # W3C-correlated root trace id instead of a generated uuid4. + caller_trace_id: Final = _caller_trace_field( + cast(Mapping[str, object], data), # cast-ok: request body is a str-keyed JSON object + _metadata_variable_name, + "trace_id", + ) + if trace_id_from_traceparent and ( + caller_trace_id is None or caller_trace_id == trace_id_from_traceparent + ): metadata_from_headers["trace_id"] = trace_id_from_traceparent data["litellm_trace_id"] = trace_id_from_traceparent # rebind-ok: data is an out-param verbose_proxy_logger.debug( @@ -1615,7 +1659,14 @@ class LiteLLMProxyRequestSetup: baggage: Final = normalized_headers.get("baggage") if isinstance(baggage, str): session_id_from_baggage: Final = _session_id_from_baggage(baggage) - if session_id_from_baggage: + caller_session_id: Final = _caller_trace_field( + cast(Mapping[str, object], data), # cast-ok: request body is a str-keyed JSON object + _metadata_variable_name, + "session_id", + ) + if session_id_from_baggage and ( + caller_session_id is None or caller_session_id == session_id_from_baggage + ): metadata_from_headers["session_id"] = session_id_from_baggage data["litellm_session_id"] = session_id_from_baggage # rebind-ok: data is an out-param verbose_proxy_logger.debug("Extracted session_id from W3C baggage header") diff --git a/tests/integration/observability/test_langfuse_delivery.py b/tests/integration/observability/test_langfuse_delivery.py index 13be5a95887..00eee082c33 100644 --- a/tests/integration/observability/test_langfuse_delivery.py +++ b/tests/integration/observability/test_langfuse_delivery.py @@ -1,15 +1,33 @@ +import asyncio import base64 import json +import signal +import threading import time import uuid -from collections.abc import Sequence +from collections.abc import Iterator, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass from pathlib import Path from typing import Final +import anthropic +import httpx +import openai +import psutil +import pytest import yaml -from integration._support.client import Gateway, eventually, object_value, string_value +from _s3_v2_support import _chat_stream_frames, _responses_stream_frames +from integration._support.client import ( + Gateway, + Scenario, + eventually, + gateway_from_environment, + object_value, + string_value, +) from integration._support.database import read_rows, scratch_database -from integration._support.process import owned_proxy +from integration._support.process import group_members, owned_proxy, owned_proxy_process from integration._support.wire import Reply, Request, Wire, wire_server from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest from opentelemetry.proto.common.v1.common_pb2 import KeyValue @@ -49,7 +67,20 @@ def _completion(text: str) -> Reply: def _projects() -> Reply: - return Reply(body=json.dumps({"data": [{"id": "integration-project", "name": "integration"}]}).encode()) + return Reply( + body=json.dumps( + { + "data": [ + { + "id": "integration-project", + "name": "integration", + "organization": {"id": "integration-org", "name": "integration"}, + "metadata": {}, + } + ] + } + ).encode() + ) def _text_prompt(name: str) -> Reply: @@ -68,15 +99,20 @@ def _text_prompt(name: str) -> Reply: ) -def _langfuse_config(tmp_path: Path) -> Path: +def _langfuse_config(tmp_path: Path, general_settings: Mapping[str, JsonValue] | None = None) -> Path: config: Final = _PROXY_CONFIG.validate_python(yaml.safe_load(STOCK_CONFIG.read_text())) settings: Final = { **_SETTINGS.validate_python(config["litellm_settings"]), "success_callback": ["langfuse"], "failure_callback": ["langfuse"], } - path: Final = tmp_path / "langfuse.yaml" - path.write_text(yaml.safe_dump({**config, "litellm_settings": settings})) + general: Final = { + **_SETTINGS.validate_python(config["general_settings"]), + **(general_settings or {}), + } + name: Final = "langfuse.yaml" if general_settings is None else "langfuse-merged.yaml" + path: Final = tmp_path / name + path.write_text(yaml.safe_dump({**config, "litellm_settings": settings, "general_settings": general})) return path @@ -367,3 +403,1494 @@ def test_prompt_fetch_encodes_the_name_retries_a_5xx_once_and_keeps_langfuse_hea assert leak not in json.dumps(dict(failure.headers)) assert "set-cookie" not in failure.headers and "x-upstream-internal" not in failure.headers assert sum(1 for target in seen_prompt_gets if target.startswith(PROMPTS_PATH + missing_prompt)) == 1 + + +def _responses_result(identity: str) -> Reply: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": "msg_" + identity, + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": "integration answer", "annotations": []}], + } + ], + "usage": {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + } + ).encode() + ) + + +def _trace_body(kind: str, model: str, marker: str, metadata: Mapping[str, JsonValue] | None) -> dict[str, JsonValue]: + metadata_field: Final[dict[str, JsonValue]] = {} if metadata is None else {"metadata": dict(metadata)} + if kind == "responses": + return {"model": model, "input": marker + "-question", **metadata_field} + if kind == "messages": + return { + "model": model, + "max_tokens": 16, + "messages": [{"role": "user", "content": marker + "-question"}], + **metadata_field, + } + return { + "model": model, + "messages": [{"role": "user", "content": marker + "-question"}], + "cache": {"no-cache": True}, + **metadata_field, + } + + +def _w3c_headers(header_trace: str, baggage_session: str | None) -> dict[str, str]: + baggage_field: Final = {} if baggage_session is None else {"baggage": f"session.id={baggage_session}"} + return {"traceparent": f"00-{header_trace}-00f067aa0ba902b7-01", **baggage_field} + + +def _await_span(received: list[Request], destination: Wire, call_id: str) -> Span: + def exported() -> tuple[Span, ...]: + received.extend(destination.drain()) + return tuple( + span + for span in _spans(received) + if _attribute(span.attributes, "langfuse.observation.metadata.litellm_call_id") == call_id + ) + + spans: Final = eventually(exported, lambda values: len(values) == 1, seconds=20) + return spans[0] + + +@pytest.mark.parametrize( + ("endpoint", "kind", "metadata_mode", "expected_trace", "expected_session", "expected_target"), + ( + pytest.param( + "/v1/chat/completions", "chat", "both", "caller", "caller", "/v1/chat/completions", id="chat_caller_ids" + ), + pytest.param( + "/v1/responses", "responses", "both", "caller", "caller", "/v1/responses", id="responses_caller_ids" + ), + pytest.param( + "/v1/messages", "messages", "both", "caller", "caller", "/v1/responses", id="messages_caller_ids" + ), + pytest.param( + "/v1/chat/completions", "chat", "none", "header", "baggage", "/v1/chat/completions", id="chat_header_ids" + ), + pytest.param( + "/v1/chat/completions", + "chat", + "trace", + "caller", + "baggage", + "/v1/chat/completions", + id="chat_caller_trace_header_session", + ), + pytest.param( + "/v1/chat/completions", + "chat", + "empty_session", + "header", + "baggage", + "/v1/chat/completions", + id="chat_empty_session_header_ids", + ), + ), +) +def test_langfuse_trace_and_session_prefer_caller_metadata_over_w3c_headers( + gateway: Gateway, + tmp_path: Path, + endpoint: str, + kind: str, + metadata_mode: str, + expected_trace: str, + expected_session: str, + expected_target: str, +) -> None: + marker: Final = "w3c" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"my-session-id-{marker}" + metadata: Final = { + "both": {"trace_id": caller_trace, "session_id": caller_session}, + "trace": {"trace_id": caller_trace}, + "empty_session": {"session_id": ""}, + "none": None, + }[metadata_mode] + expected_trace_value: Final = {"caller": caller_trace, "header": header_trace}[expected_trace] + expected_session_value: Final = {"caller": caller_session, "baggage": baggage_session}[expected_session] + upstream_targets: Final[list[str]] = [] # mutable-ok: records which upstream endpoint each call hit + + def upstream(request: Request) -> Reply: + if not request.body: + return Reply(status=404) + assert request.headers["authorization"] == f"Bearer {provider_secret}" + upstream_targets.append(request.target) + if request.target == "/v1/responses": + return _responses_result("resp-" + marker) + assert request.target == "/v1/chat/completions", request.target + return _completion(marker + "-answer") + + def langfuse(request: Request) -> Reply: + if request.method == "GET" and request.target.startswith(PROJECTS_PATH): + return _projects() + return Reply(body=b"", content_type="application/x-protobuf") + + with ( + wire_server(upstream) as provider, + wire_server(langfuse) as destination, + owned_proxy( + gateway, tmp_path, _langfuse_environment(destination), config=_langfuse_config(tmp_path) + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, metadata), + headers=_w3c_headers(header_trace, baggage_session), + ) + assert response.status_code == 200, response.text + assert upstream_targets == [expected_target], upstream_targets + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + span: Final = _await_span(received, destination, response.headers["x-litellm-call-id"]) + assert span.trace_id.hex() == expected_trace_value + assert _attribute(span.attributes, "session.id") == expected_session_value + + +def test_missing_session_id_reject_accepts_caller_metadata_and_baggage_fallback( + gateway: Gateway, tmp_path: Path +) -> None: + marker: Final = "reject" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + upstream_targets: Final[list[str]] = [] # mutable-ok: records which upstream endpoint each call hit + + def upstream(request: Request) -> Reply: + if not request.body: + return Reply(status=404) + assert request.headers["authorization"] == f"Bearer {provider_secret}" + upstream_targets.append(request.target) + if request.target == "/v1/responses": + return _responses_result("resp-" + marker) + assert request.target == "/v1/chat/completions", request.target + return _completion(marker + "-answer") + + def langfuse(request: Request) -> Reply: + if request.method == "GET" and request.target.startswith(PROJECTS_PATH): + return _projects() + return Reply(body=b"", content_type="application/x-protobuf") + + with ( + wire_server(upstream) as provider, + wire_server(langfuse) as destination, + owned_proxy( + gateway, + tmp_path, + _langfuse_environment(destination), + config=_langfuse_config(tmp_path, {"missing_session_id": "reject"}), + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + + caller_session: Final = f"my-session-id-{marker}-r1" + header_trace: Final = uuid.uuid4().hex + first: Final = candidate.request( + "POST", + "/v1/responses", + {"model": model, "input": marker + "-r1", "metadata": {"session_id": caller_session}}, + headers=_w3c_headers(header_trace, None), + ) + assert first.status_code == 200, first.text + first_span: Final = _await_span(received, destination, first.headers["x-litellm-call-id"]) + assert _attribute(first_span.attributes, "session.id") == caller_session + + baggage_session: Final = "baggage-" + marker + "-r2" + second: Final = candidate.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": marker + "-r2"}], + "metadata": {"session_id": ""}, + }, + headers=_w3c_headers(uuid.uuid4().hex, baggage_session), + ) + assert second.status_code == 200, second.text + second_span: Final = _await_span(received, destination, second.headers["x-litellm-call-id"]) + assert _attribute(second_span.attributes, "session.id") == baggage_session + + third: Final = candidate.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": marker + "-r3"}]}, + headers=_w3c_headers(uuid.uuid4().hex, None), + ) + assert third.status_code == 400, third.text + assert upstream_targets == ["/v1/responses", "/v1/chat/completions"], upstream_targets + + +@pytest.mark.parametrize( + ("endpoint", "kind", "expected_target"), + ( + pytest.param("/v1/responses", "responses", "/v1/responses", id="responses_caller_trace"), + pytest.param("/v1/messages", "messages", "/v1/responses", id="messages_caller_trace"), + ), +) +def test_missing_session_id_generate_derives_session_from_caller_trace( + gateway: Gateway, tmp_path: Path, endpoint: str, kind: str, expected_target: str +) -> None: + marker: Final = "gen" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + caller_trace: Final = uuid.uuid4().hex + upstream_targets: Final[list[str]] = [] # mutable-ok: records which upstream endpoint each call hit + + def upstream(request: Request) -> Reply: + if not request.body: + return Reply(status=404) + assert request.headers["authorization"] == f"Bearer {provider_secret}" + upstream_targets.append(request.target) + if request.target == "/v1/responses": + return _responses_result("resp-" + marker) + assert request.target == "/v1/chat/completions", request.target + return _completion(marker + "-answer") + + def langfuse(request: Request) -> Reply: + if request.method == "GET" and request.target.startswith(PROJECTS_PATH): + return _projects() + return Reply(body=b"", content_type="application/x-protobuf") + + with ( + wire_server(upstream) as provider, + wire_server(langfuse) as destination, + owned_proxy( + gateway, + tmp_path, + _langfuse_environment(destination), + config=_langfuse_config(tmp_path, {"missing_session_id": "generate"}), + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, {"trace_id": caller_trace}), + headers=_w3c_headers(uuid.uuid4().hex, None), + ) + assert response.status_code == 200, response.text + assert upstream_targets == [expected_target], upstream_targets + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + span: Final = _await_span(received, destination, response.headers["x-litellm-call-id"]) + assert _attribute(span.attributes, "session.id") == caller_trace + + +_AUDIT_ENDPOINTS: Final = ( + pytest.param("/v1/chat/completions", "chat", "/v1/chat/completions", id="chat"), + pytest.param("/v1/responses", "responses", "/v1/responses", id="responses"), + pytest.param("/v1/messages", "messages", "/v1/responses", id="messages"), +) + + +def _audit_upstream(provider_secret: str, marker: str, targets: list[str]): + def upstream(request: Request) -> Reply: + if not request.body: + return Reply(status=404) + assert request.headers["authorization"] == f"Bearer {provider_secret}" + targets.append(request.target) + body: Final = TypeAdapter(dict[str, JsonValue]).validate_json(request.body) + index: Final = len(targets) + if request.target == "/v1/responses": + if body.get("stream"): + return Reply( + content_type="text/event-stream", chunks=_responses_stream_frames(f"resp-{marker}-{index}") + ) + return _responses_result(f"resp-{marker}-{index}") + assert request.target == "/v1/chat/completions", request.target + if body.get("stream"): + return Reply( + content_type="text/event-stream", chunks=_chat_stream_frames(f"chatcmpl-{marker}-{index}") + ) + return _completion(f"{marker}-{index}-answer") + + return upstream + + +def _audit_sink(): + def langfuse(request: Request) -> Reply: + if request.method == "GET" and request.target.startswith(PROJECTS_PATH): + return _projects() + return Reply(body=b"", content_type="application/x-protobuf") + + return langfuse + + +@dataclass(frozen=True, slots=True) +class _AuditRig: + candidate: Gateway + scenario: Scenario + destination: Wire + + +@pytest.fixture(scope="module") +def audit_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_AuditRig]: + directory: Final = tmp_path_factory.mktemp("audit") + with ( + gateway_from_environment() as gateway, + wire_server(_audit_sink()) as destination, + owned_proxy( + gateway, directory, _langfuse_environment(destination), config=_langfuse_config(directory) + ) as candidate, + candidate.scenario() as scenario, + ): + yield _AuditRig(candidate, scenario, destination) + + +@pytest.fixture(scope="module") +def audit_generate_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_AuditRig]: + directory: Final = tmp_path_factory.mktemp("audit-generate") + with ( + gateway_from_environment() as gateway, + wire_server(_audit_sink()) as destination, + owned_proxy( + gateway, + directory, + _langfuse_environment(destination), + config=_langfuse_config(directory, {"missing_session_id": "generate"}), + ) as candidate, + candidate.scenario() as scenario, + ): + yield _AuditRig(candidate, scenario, destination) + + +@pytest.fixture(scope="module") +def audit_reject_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_AuditRig]: + directory: Final = tmp_path_factory.mktemp("audit-reject") + with ( + gateway_from_environment() as gateway, + wire_server(_audit_sink()) as destination, + owned_proxy( + gateway, + directory, + _langfuse_environment(destination), + config=_langfuse_config(directory, {"missing_session_id": "reject"}), + ) as candidate, + candidate.scenario() as scenario, + ): + yield _AuditRig(candidate, scenario, destination) + + +@pytest.fixture(scope="module") +def audit_omit_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_AuditRig]: + directory: Final = tmp_path_factory.mktemp("audit-omit") + with ( + gateway_from_environment() as gateway, + wire_server(_audit_sink()) as destination, + owned_proxy( + gateway, + directory, + _langfuse_environment(destination), + config=_langfuse_config(directory, {"missing_session_id": "omit"}), + ) as candidate, + candidate.scenario() as scenario, + ): + yield _AuditRig(candidate, scenario, destination) + + +@pytest.fixture(scope="module") +def audit_otel_rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_AuditRig]: + directory: Final = tmp_path_factory.mktemp("audit-otel") + + def otlp(request: Request) -> Reply: + return Reply() + + with ( + gateway_from_environment() as gateway, + wire_server(otlp) as destination, + owned_proxy( + gateway, + directory, + {"LITELLM_OTEL_V2": "1", "OTEL_BSP_SCHEDULE_DELAY": "300"}, + config=_otel_config(directory, destination.url), + ) as candidate, + candidate.scenario() as scenario, + ): + yield _AuditRig(candidate, scenario, destination) + + +def _await_spend_row(call_id: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT session_id, status, metadata, cache_hit FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s', + (call_id,), + ), + lambda values: len(values) == 1, + seconds=240, + ) + return rows[0] + + +def _assert_call( + response: httpx.Response, + received: list[Request], + destination: Wire, + targets: list[str], + expected_target: str, + expected_trace: str | None, + expected_session: str | None, +) -> None: + assert targets == [expected_target], targets + _assert_span_spend(response, received, destination, expected_trace, expected_session) + + +def _assert_span_spend( + response: httpx.Response, + received: list[Request], + destination: Wire, + expected_trace: str | None, + expected_session: str | None, +) -> None: + call_id: Final = response.headers["x-litellm-call-id"] + span: Final = _await_span(received, destination, call_id) + if expected_trace is not None: + assert span.trace_id.hex() == expected_trace, f"call {call_id}: trace id" + assert _attribute(span.attributes, "session.id") == expected_session, f"call {call_id}: session.id" + row: Final = _await_spend_row(call_id) + assert row["session_id"] == expected_session, f"call {call_id}: spend session {row}" + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +@pytest.mark.parametrize("metadata_mode", ("both", "none", "trace", "session")) +def test_audit_caller_metadata_wins_over_w3c_per_field( + audit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str, metadata_mode: str +) -> None: + marker: Final = "audit" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"my-session-id-{marker}" + metadata: Final = { + "both": {"trace_id": caller_trace, "session_id": caller_session}, + "trace": {"trace_id": caller_trace}, + "session": {"session_id": caller_session}, + "none": None, + }[metadata_mode] + expected_trace: Final = caller_trace if metadata_mode in ("both", "trace") else header_trace + expected_session: Final = caller_session if metadata_mode in ("both", "session") else baggage_session + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, metadata), + headers=_w3c_headers(header_trace, baggage_session), + ) + assert response.status_code == 200, response.text + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + _assert_call( + response, received, audit_rig.destination, targets, expected_target, expected_trace, expected_session + ) + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +def test_audit_caller_metadata_wins_on_streamed_calls( + audit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str +) -> None: + marker: Final = "auditstream" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"my-session-id-{marker}" + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_rig.candidate.request( + "POST", + endpoint, + { + **_trace_body(kind, model, marker, {"trace_id": caller_trace, "session_id": caller_session}), + "stream": True, + }, + headers=_w3c_headers(header_trace, baggage_session), + ) + assert response.status_code == 200, response.text + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + _assert_call(response, received, audit_rig.destination, targets, expected_target, caller_trace, caller_session) + + +def test_audit_caller_metadata_wins_through_official_sdk_clients(audit_rig: _AuditRig) -> None: + marker: Final = "auditsdk" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + candidate: Final = audit_rig.candidate + destination: Final = audit_rig.destination + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + base_url: Final = str(candidate.client.base_url).rstrip("/") + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + + def drive(client_kind: str) -> tuple[str, str, httpx.Headers]: + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"{client_kind}-session-{marker}" + body_metadata: Final = { + "trace_id": caller_trace, + "session_id": caller_session, + "generation_name": marker + "-" + client_kind, + } + headers: Final = _w3c_headers(uuid.uuid4().hex, "baggage-" + marker + "-" + client_kind) + if client_kind == "chat_openai_sync": + return caller_trace, caller_session, ( + openai.OpenAI(base_url=f"{base_url}/v1", api_key=candidate.key) + .chat.completions.with_raw_response.create( + model=model, + messages=[{"role": "user", "content": marker + "-chat"}], + extra_body={"metadata": body_metadata, "cache": {"no-cache": True}}, + extra_headers=headers, + ) + .headers + ) + if client_kind == "responses_openai_async": + + async def responses_call() -> httpx.Headers: + answer: Final = await openai.AsyncOpenAI( + base_url=f"{base_url}/v1", api_key=candidate.key + ).responses.with_raw_response.create( + model=model, + input=marker + "-responses", + extra_body={"metadata": body_metadata, "cache": {"no-cache": True}}, + extra_headers=headers, + ) + return answer.headers + + return caller_trace, caller_session, asyncio.run(responses_call()) + if client_kind == "messages_anthropic_sync": + return caller_trace, caller_session, ( + anthropic.Anthropic(base_url=base_url, api_key=candidate.key) + .messages.with_raw_response.create( + model=model, + max_tokens=16, + messages=[{"role": "user", "content": marker + "-messages"}], + extra_body={"metadata": body_metadata}, + extra_headers=headers, + ) + .headers + ) + + async def messages_call() -> httpx.Headers: + answer: Final = await anthropic.AsyncAnthropic( + base_url=base_url, api_key=candidate.key + ).messages.with_raw_response.create( + model=model, + max_tokens=16, + messages=[{"role": "user", "content": marker + "-messages-async"}], + extra_body={"metadata": body_metadata}, + extra_headers=headers, + ) + return answer.headers + + return caller_trace, caller_session, asyncio.run(messages_call()) + + expected: Final = { + call_headers["x-litellm-call-id"]: (caller_trace, caller_session, client_kind) + for client_kind, (caller_trace, caller_session, call_headers) in ( + (kind, drive(kind)) + for kind in ( + "chat_openai_sync", + "responses_openai_async", + "messages_anthropic_sync", + "messages_anthropic_async", + ) + ) + } + + def spans_named() -> tuple[Span, ...]: + received.extend(destination.drain()) + return tuple( + span + for span in _spans(received) + if _attribute(span.attributes, "langfuse.observation.metadata.litellm_call_id") in expected + ) + + spans: Final = eventually(spans_named, lambda values: len(values) == len(expected), seconds=60) + for span in spans: + call_id: Final = str( + _attribute(span.attributes, "langfuse.observation.metadata.litellm_call_id") + ) + caller_trace, caller_session, client_kind = expected[call_id] + assert span.trace_id.hex() == caller_trace, f"{client_kind}: trace id" + assert _attribute(span.attributes, "session.id") == caller_session, f"{client_kind}: session.id" + row: Final = _await_spend_row(call_id) + assert row["session_id"] == caller_session, f"{client_kind} {call_id}: spend session {row}" + + +def _otel_config(tmp_path: Path, sink_url: str) -> Path: + config: Final = _PROXY_CONFIG.validate_python(yaml.safe_load(STOCK_CONFIG.read_text())) + settings: Final = {**_SETTINGS.validate_python(config["litellm_settings"]), "callbacks": ["otel"]} + path: Final = tmp_path / "otel.yaml" + path.write_text( + yaml.safe_dump( + { + **config, + "litellm_settings": settings, + "callback_settings": { + "otel": {"exporter": "http/json", "endpoint": sink_url, "mapper_names": ["genai"]} + }, + } + ) + ) + return path + + +_OTEL_SPAN: Final = TypeAdapter(dict[str, JsonValue]) + + +def _otel_spans(batches: Sequence[Request]) -> tuple[dict[str, JsonValue], ...]: + spans: list[dict[str, JsonValue]] = [] # mutable-ok: flattens nested OTLP batches into a tuple + for batch in batches: + if not batch.target.endswith("/v1/traces"): + continue + payload: Final = TypeAdapter(JsonValue).validate_json(batch.body) + envelopes: Final = payload if isinstance(payload, list) else [payload] + for envelope in envelopes: + for resource in TypeAdapter(list[JsonValue]).validate_python( + object_value(envelope)["resourceSpans"] + ): + for scope in object_value(resource)["scopeSpans"]: + spans.extend(TypeAdapter(list[JsonValue]).validate_python(object_value(scope)["spans"])) + return tuple(_OTEL_SPAN.validate_python(span) for span in spans) + + +def _otel_attribute(span: Mapping[str, JsonValue], key: str) -> str | None: + for attribute in TypeAdapter(list[JsonValue]).validate_python(span.get("attributes") or []): + entry: Final = object_value(attribute) + if entry["key"] == key: + value: Final = object_value(entry["value"]) + raw: Final = value.get("stringValue") + return str(raw) if raw is not None else None + return None + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS[:2]) +def test_audit_otel_span_carries_caller_ids( + audit_otel_rig: _AuditRig, endpoint: str, kind: str, expected_target: str +) -> None: + marker: Final = "auditotel" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"my-session-id-{marker}" + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + destination: Final = audit_otel_rig.destination + model: Final = audit_otel_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_otel_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, {"trace_id": caller_trace, "session_id": caller_session}), + headers=_w3c_headers(header_trace, baggage_session), + ) + assert response.status_code == 200, response.text + assert targets == [expected_target], targets + response_id: Final = string_value(object_value(response.json())["id"]) + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + + def exported() -> tuple[dict[str, JsonValue], ...]: + received.extend(destination.drain()) + return tuple( + span + for span in _otel_spans(received) + if _otel_attribute(span, "gen_ai.response.id") == response_id + ) + + spans: Final = eventually(exported, lambda values: len(values) == 1, seconds=60) + span: Final = spans[0] + print( + f"H7 record: otel span trace={span['traceId']} header={header_trace} " + f"session.id={_otel_attribute(span, 'session.id')} " + f"gen_ai.conversation.id={_otel_attribute(span, 'gen_ai.conversation.id')}" + ) + assert span["traceId"] == header_trace, ( + f"otel span for {response_id}: trace id must be the ambient W3C header trace" + ) + assert _otel_attribute(span, "gen_ai.conversation.id") == caller_session, ( + f"otel span conversation id for {response_id}" + ) + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +@pytest.mark.parametrize( + ("case", "metadata_mode", "headers_mode", "expected_session"), + ( + pytest.param("g1", "trace", "traceparent", "caller_trace", id="g1_caller_trace_and_header"), + pytest.param("g3", "none", "traceparent", "header_trace", id="g3_header_only"), + pytest.param("g5", "session", "traceparent", "caller_session", id="g5_caller_session"), + ), +) +def test_audit_generate_policy_derives_session_from_caller_trace( + audit_generate_rig: _AuditRig, + endpoint: str, + kind: str, + expected_target: str, + case: str, + metadata_mode: str, + headers_mode: str, + expected_session: str, +) -> None: + marker: Final = "auditgen" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"my-session-id-{marker}" + metadata: Final = {"trace": {"trace_id": caller_trace}, "session": {"session_id": caller_session}, "none": None}[ + metadata_mode + ] + expected: Final = {"caller_trace": caller_trace, "header_trace": header_trace, "caller_session": caller_session}[ + expected_session + ] + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_generate_rig.scenario.model( + api_base=provider.url + "/v1", api_key=provider_secret + ) + response: Final = audit_generate_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, metadata), + headers={"traceparent": f"00-{header_trace}-00f067aa0ba902b7-01"}, + ) + assert response.status_code == 200, response.text + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + call_id: Final = response.headers["x-litellm-call-id"] + assert targets == [expected_target], targets + span: Final = _await_span(received, audit_generate_rig.destination, call_id) + assert _attribute(span.attributes, "session.id") == expected, f"call {call_id}: session.id" + row: Final = _await_spend_row(call_id) + assert row["session_id"] == expected, f"call {call_id}: spend session {row}" + + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +def test_audit_generate_policy_is_stable_across_repeated_caller_trace( + audit_generate_rig: _AuditRig, endpoint: str, kind: str, expected_target: str +) -> None: + marker: Final = "auditgen2" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + caller_trace: Final = uuid.uuid4().hex + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_generate_rig.scenario.model( + api_base=provider.url + "/v1", api_key=provider_secret + ) + responses: Final = tuple( + audit_generate_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, f"{marker}-{attempt}", {"trace_id": caller_trace}), + ) + for attempt in ("first", "second") + ) + sessions: Final[list[str | None]] = [] # mutable-ok: collects the two observed sessions in order + for attempt, response in zip(("first", "second"), responses): + assert response.status_code == 200, f"{attempt}: {response.text}" + row: Final = _await_spend_row(response.headers["x-litellm-call-id"]) + sessions.append(string_value(row["session_id"])) + assert sessions == [caller_trace, caller_trace], ( + f"generate must derive both sessions from the caller trace {caller_trace}" + ) + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +def test_audit_generate_policy_fresh_session_without_any_ids( + audit_generate_rig: _AuditRig, endpoint: str, kind: str, expected_target: str +) -> None: + marker: Final = "auditgen4" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_generate_rig.scenario.model( + api_base=provider.url + "/v1", api_key=provider_secret + ) + sessions: Final[list[str | None]] = [] # mutable-ok: collects the two observed sessions in order + for attempt in ("first", "second"): + response: Final = audit_generate_rig.candidate.request( + "POST", endpoint, _trace_body(kind, model, f"{marker}-{attempt}", None) + ) + assert response.status_code == 200, response.text + session: Final = _await_spend_row(response.headers["x-litellm-call-id"])["session_id"] + assert session, f"{attempt}: generated session id must be non-empty" + sessions.append(string_value(session)) + assert sessions[0] != sessions[1], f"two id-less calls must not share a session: {sessions}" + assert targets and set(targets) == {expected_target}, targets + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +@pytest.mark.parametrize( + ("case", "metadata", "baggage", "expected_status", "expected_session"), + ( + pytest.param( + "r1", "caller_session", None, 200, "caller_session", id="r1_caller_session_no_baggage" + ), + pytest.param("r2", "empty_session", "baggage", 200, "baggage", id="r2_empty_session_baggage"), + pytest.param("r3", "none", None, 400, None, id="r3_nothing_rejected"), + ), +) +def test_audit_reject_policy( + audit_reject_rig: _AuditRig, + endpoint: str, + kind: str, + expected_target: str, + case: str, + metadata: str, + baggage: str, + expected_status: int, + expected_session: str | None, +) -> None: + marker: Final = "auditreject" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + baggage_session: Final = "baggage-" + marker + caller_session: Final = f"my-session-id-{marker}" + body_metadata: Final = { + "caller_session": {"session_id": caller_session}, + "empty_session": {"session_id": ""}, + "none": None, + }[metadata] + expected: Final = {"caller_session": caller_session, "baggage": baggage_session}[expected_session] if expected_session else None + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_reject_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_reject_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, body_metadata), + headers=_w3c_headers(uuid.uuid4().hex, baggage_session if baggage else None), + ) + assert response.status_code == expected_status, response.text + if expected_status != 200: + assert targets == [], targets + rejected_rows: Final = eventually( + lambda: read_rows( + 'SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s', + (response.headers["x-litellm-call-id"],), + ), + lambda values: len(values) == 1, + seconds=240, + ) + assert rejected_rows[0]["status"] == "failure", ( + f"rejected call {response.headers['x-litellm-call-id']} must write exactly one failure spend row: {rejected_rows}" + ) + return + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + _assert_call(response, received, audit_reject_rig.destination, targets, expected_target, None, expected) + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +def test_audit_omit_policy_records_no_session(audit_omit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str) -> None: + marker: Final = "auditomit" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_omit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_omit_rig.candidate.request( + "POST", endpoint, _trace_body(kind, model, marker, None) + ) + assert response.status_code == 200, response.text + call_id: Final = response.headers["x-litellm-call-id"] + assert targets == [expected_target], targets + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + span: Final = _await_span(received, audit_omit_rig.destination, call_id) + row: Final = _await_spend_row(call_id) + span_session: Final = _attribute(span.attributes, "session.id") + assert row["session_id"] == span_session, ( + f"call {call_id}: spend session {row['session_id']!r} must match span session {span_session!r}" + ) + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +@pytest.mark.parametrize("bad_value", (123, ["x"]), ids=["int", "list"]) +def test_audit_non_string_caller_ids_fall_back_to_w3c( + audit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str, bad_value: JsonValue +) -> None: + marker: Final = "auditbad" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, {"trace_id": bad_value, "session_id": bad_value}), + headers=_w3c_headers(header_trace, baggage_session), + ) + assert response.status_code == 200, response.text + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + _assert_call(response, received, audit_rig.destination, targets, expected_target, header_trace, baggage_session) + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +@pytest.mark.parametrize( + "metadata", + ({"trace_id": "", "session_id": ""}, {"trace_id": None, "session_id": None}), + ids=["empty", "null"], +) +def test_audit_empty_and_null_caller_ids_fall_back_to_w3c( + audit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str, metadata: dict[str, object] +) -> None: + marker: Final = "auditempty" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, dict(metadata)), + headers=_w3c_headers(header_trace, baggage_session), + ) + assert response.status_code == 200, response.text + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + _assert_call(response, received, audit_rig.destination, targets, expected_target, header_trace, baggage_session) + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS[:2]) +def test_audit_five_kilobyte_caller_ids_win_verbatim( + audit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str +) -> None: + marker: Final = "auditbig" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + caller_trace: Final = "T" * 5120 + caller_session: Final = "S" * 5120 + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, {"trace_id": caller_trace, "session_id": caller_session}), + headers=_w3c_headers(header_trace, baggage_session), + ) + assert response.status_code == 200, response.text + call_id: Final = response.headers["x-litellm-call-id"] + assert targets == [expected_target], targets + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + span: Final = _await_span(received, audit_rig.destination, call_id) + assert span.trace_id.hex() != header_trace, f"call {call_id}: caller trace must beat the W3C header" + assert _attribute(span.attributes, "session.id") == caller_session, f"call {call_id}: session.id" + assert _await_spend_row(call_id)["session_id"] == caller_session, f"call {call_id}: spend session" + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS) +def test_audit_identical_requests_twice_log_per_call( + audit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str +) -> None: + marker: Final = "auditdup" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"my-session-id-{marker}" + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + for attempt in ("first", "second"): + response: Final = audit_rig.candidate.request( + "POST", + endpoint, + _trace_body( + kind, model, marker, {"trace_id": caller_trace, "session_id": caller_session} + ), + headers=_w3c_headers(header_trace, baggage_session), + ) + assert response.status_code == 200, f"{attempt}: {response.text}" + call_id: Final = response.headers["x-litellm-call-id"] + span: Final = _await_span(received, audit_rig.destination, call_id) + assert span.trace_id.hex() == caller_trace, f"{attempt} {call_id}: trace id" + assert _attribute(span.attributes, "session.id") == caller_session, f"{attempt} {call_id}: session.id" + assert _await_spend_row(call_id)["session_id"] == caller_session, f"{attempt} {call_id}: spend" + assert targets and set(targets) == {expected_target}, targets + + +def test_audit_malformed_w3c_headers_are_ignored(audit_rig: _AuditRig) -> None: + marker: Final = "auditmal" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_rig.candidate.request( + "POST", + "/v1/chat/completions", + _trace_body("chat", model, marker, None), + headers={"traceparent": "00-zz-00f067aa0ba902b7-01", "baggage": "not-a-session-key"}, + ) + assert response.status_code == 200, response.text + assert targets == ["/v1/chat/completions"], targets + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + span: Final = _await_span(received, audit_rig.destination, response.headers["x-litellm-call-id"]) + assert len(span.trace_id.hex()) == 32 and "zz" not in span.trace_id.hex(), span.trace_id.hex() + row: Final = _await_spend_row(response.headers["x-litellm-call-id"]) + span_session: Final = _attribute(span.attributes, "session.id") + assert span_session in (None, row["session_id"]), ( + f"span session {span_session!r} diverges from spend session {row['session_id']!r}" + ) + assert row["session_id"], row + + +def test_audit_unauthenticated_call_leaves_no_spend_row(audit_rig: _AuditRig) -> None: + marker: Final = "auditunauth" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + candidate: Final = audit_rig.candidate + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + denied: Final = candidate.request( + "POST", + "/v1/chat/completions", + _trace_body( + "chat", + model, + marker, + {"trace_id": uuid.uuid4().hex, "session_id": f"my-session-id-{marker}"}, + ), + headers=_w3c_headers(uuid.uuid4().hex, "baggage-" + marker), + key="sk-wrong-key", + ) + assert denied.status_code == 401, denied.text + assert targets == [], targets + control: Final = candidate.request( + "POST", "/v1/chat/completions", _trace_body("chat", model, marker + "-control", None) + ) + assert control.status_code == 200, control.text + _await_spend_row(control.headers["x-litellm-call-id"]) + assert ( + read_rows( + 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE litellm_call_id=%s', + (denied.headers.get("x-litellm-call-id") or "",), + ) + == [] + ), "an unauthenticated call must not write a spend row" + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS[:2]) +@pytest.mark.parametrize("upstream_status", (500, 401), ids=["upstream_500", "upstream_401"]) +def test_audit_upstream_error_still_logs_caller_session( + audit_rig: _AuditRig, + endpoint: str, + kind: str, + expected_target: str, + upstream_status: int, +) -> None: + marker: Final = "auditerr" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"my-session-id-{marker}" + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + def upstream(request: Request) -> Reply: + if request.body: + targets.append(request.target) + return Reply(status=upstream_status, body=b'{"error": {"message": "scripted upstream failure"}}') + + with wire_server(upstream) as provider: + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + response: Final = audit_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, {"trace_id": caller_trace, "session_id": caller_session}), + headers=_w3c_headers(uuid.uuid4().hex, "baggage-" + marker), + ) + assert response.status_code == upstream_status, response.text + assert targets and set(targets) == {expected_target}, targets + call_id: Final = response.headers["x-litellm-call-id"] + row: Final = _await_spend_row(call_id) + assert row["session_id"] == caller_session, f"call {call_id}: failure spend session {row}" + + +def test_audit_sink_rejection_does_not_break_the_caller(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "auditreject" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + attempts: Final[list[int]] = [] # mutable-ok: sink status sequence counter + + def langfuse(request: Request) -> Reply: + if request.method == "GET" and request.target.startswith(PROJECTS_PATH): + return _projects() + if request.target == TRACES_PATH: + attempts.append(1) + if len(attempts) == 1: + return Reply(status=403) + if len(attempts) == 2: + return Reply(status=404) + return Reply(body=b"", content_type="application/x-protobuf") + + with ( + wire_server(_audit_upstream(provider_secret, marker, targets)) as provider, + wire_server(langfuse) as destination, + owned_proxy( + gateway, tmp_path, _langfuse_environment(destination), config=_langfuse_config(tmp_path) + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + for attempt in ("first", "second", "third"): + response: Final = candidate.request( + "POST", + "/v1/chat/completions", + _trace_body( + "chat", + model, + f"{marker}-{attempt}", + {"trace_id": uuid.uuid4().hex, "session_id": f"my-session-id-{marker}-{attempt}"}, + ), + ) + assert response.status_code == 200, f"{attempt}: {response.text}" + _await_spend_row(response.headers["x-litellm-call-id"]) + assert targets == ["/v1/chat/completions"] * 3, targets + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS[:2]) +def test_audit_string_metadata_body_does_not_crash(audit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str) -> None: + marker: Final = "auditstr" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + candidate: Final = audit_rig.candidate + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + body: Final = {**_trace_body(kind, model, marker, None), "metadata": "x"} + response: Final = candidate.request( + "POST", endpoint, body, headers=_w3c_headers(uuid.uuid4().hex, "baggage-" + marker) + ) + assert response.status_code < 500, f"metadata string must not crash the proxy: {response.status_code} {response.text}" + follow_up: Final = candidate.request( + "POST", "/v1/chat/completions", _trace_body("chat", model, marker + "-follow", None) + ) + assert follow_up.status_code == 200, follow_up.text + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS[:2]) +def test_audit_key_metadata_session_still_honoured(audit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str) -> None: + marker: Final = "auditkey" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + key_session: Final = f"key-session-{marker}" + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + token: Final = audit_rig.scenario.key(metadata={"session_id": key_session}) + response: Final = audit_rig.candidate.request( + "POST", + endpoint, + _trace_body(kind, model, marker, None), + headers=_w3c_headers(uuid.uuid4().hex, "baggage-" + marker), + key=token, + ) + assert response.status_code == 200, response.text + call_id: Final = response.headers["x-litellm-call-id"] + assert targets == [expected_target], targets + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + span: Final = _await_span(received, audit_rig.destination, call_id) + row: Final = _await_spend_row(call_id) + assert _attribute(span.attributes, "session.id") == row["session_id"], ( + f"call {call_id}: spend session {row['session_id']!r} must match span session" + ) + + +@pytest.mark.parametrize(("endpoint", "kind", "expected_target"), _AUDIT_ENDPOINTS[:2]) +@pytest.mark.parametrize("metadata_mode", ("both", "none"), ids=["caller_ids", "no_metadata"]) +def test_audit_cache_hit_call_keeps_winning_ids( + audit_rig: _AuditRig, endpoint: str, kind: str, expected_target: str, metadata_mode: str +) -> None: + marker: Final = "auditcache" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"my-session-id-{marker}" + metadata: Final = ( + {"trace_id": caller_trace, "session_id": caller_session} if metadata_mode == "both" else None + ) + expected_trace: Final = caller_trace if metadata_mode == "both" else header_trace + expected_session: Final = caller_session if metadata_mode == "both" else baggage_session + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + candidate: Final = audit_rig.candidate + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + body: Final = {key: value for key, value in _trace_body(kind, model, marker, metadata).items() if key != "cache"} + headers: Final = _w3c_headers(header_trace, baggage_session) + first: Final = candidate.request("POST", endpoint, body, headers=headers) + assert first.status_code == 200, first.text + second: Final = candidate.request("POST", endpoint, body, headers=headers) + assert second.status_code == 200, second.text + assert second.headers.get("x-litellm-cache-key"), ( + f"second identical call must be a cache hit: {dict(second.headers)}" + ) + call_id: Final = second.headers["x-litellm-call-id"] + assert targets == [expected_target], targets + span: Final = _await_span(received, audit_rig.destination, call_id) + if metadata_mode == "both": + assert span.trace_id.hex() == expected_trace, f"cache hit {call_id}: trace id" + assert _attribute(span.attributes, "session.id") == expected_session, ( + f"cache hit {call_id}: session.id" + ) + else: + span_session: Final = _attribute(span.attributes, "session.id") + print( + f"E2 record: cache-hit span trace={span.trace_id.hex()} session={span_session!r} " + f"header trace={header_trace} baggage session={baggage_session!r}" + ) + assert len(span.trace_id.hex()) == 32, span.trace_id.hex() + assert _await_spend_row(call_id)["session_id"] == expected_session, f"cache hit {call_id}: spend" + + +def test_audit_concurrent_requests_each_keep_their_caller_ids(audit_rig: _AuditRig) -> None: + marker: Final = "auditconc" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with wire_server(_audit_upstream(provider_secret, marker, targets)) as provider: + candidate: Final = audit_rig.candidate + model: Final = audit_rig.scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + jobs: Final = tuple( + (index, endpoint, kind, uuid.uuid4().hex, f"my-session-id-{marker}-{index}") + for index, (endpoint, kind, _) in tuple( + enumerate(tuple(row.values for row in _AUDIT_ENDPOINTS) * 4) + )[:10] + ) + + def fire(job: tuple[int, str, str, str, str]) -> tuple[str, str, httpx.Response]: + index, endpoint, kind, caller_trace, caller_session = job + response: Final = candidate.request( + "POST", + endpoint, + _trace_body( + kind, model, f"{marker}-{index}", {"trace_id": caller_trace, "session_id": caller_session} + ), + headers=_w3c_headers(header_trace, baggage_session), + ) + return caller_trace, caller_session, response + + with ThreadPoolExecutor(max_workers=10) as pool: + answered: Final = tuple(pool.map(fire, jobs)) + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + assert len(targets) == 10, targets + for caller_trace, caller_session, response in answered: + assert response.status_code == 200, response.text + _assert_span_spend(response, received, audit_rig.destination, caller_trace, caller_session) + + +def test_audit_sink_outage_mid_burst_loses_no_spend_rows(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "auditoutage" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + outage: Final = threading.Event() + + def langfuse(request: Request) -> Reply: + if outage.is_set(): + return Reply(status=503) + if request.method == "GET" and request.target.startswith(PROJECTS_PATH): + return _projects() + return Reply(body=b"", content_type="application/x-protobuf") + + with ( + wire_server(_audit_upstream(provider_secret, marker, targets)) as provider, + wire_server(langfuse) as destination, + owned_proxy( + gateway, tmp_path, _langfuse_environment(destination), config=_langfuse_config(tmp_path) + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + jobs: Final = tuple( + (index, endpoint, kind, uuid.uuid4().hex, f"my-session-id-{marker}-{index}") + for index, (endpoint, kind, _) in enumerate(tuple(row.values for row in _AUDIT_ENDPOINTS) * 10) + ) + + def fire(job: tuple[int, str, str, str, str]) -> tuple[str, str, httpx.Response]: + index, endpoint, kind, caller_trace, caller_session = job + stream: Final = index % 3 == 0 + response: Final = candidate.request( + "POST", + endpoint, + { + **_trace_body( + kind, + model, + f"{marker}-{index}", + {"trace_id": caller_trace, "session_id": caller_session}, + ), + "stream": stream, + }, + headers=_w3c_headers(header_trace, baggage_session), + ) + return caller_trace, caller_session, response + + outage.set() + with ThreadPoolExecutor(max_workers=30) as pool: + first_wave: Final = tuple(pool.map(fire, jobs[:10])) + unhealthy: Final = candidate.request("GET", "/health/services?service=langfuse") + assert unhealthy.status_code != 200 or "unhealthy" in unhealthy.text, ( + f"langfuse must report unhealthy while the sink 503s: {unhealthy.status_code} {unhealthy.text}" + ) + outage.clear() + healthy: Final = candidate.request("GET", "/health/services?service=langfuse") + assert healthy.status_code == 200, healthy.text + with ThreadPoolExecutor(max_workers=30) as pool: + answered: Final = first_wave + tuple(pool.map(fire, jobs[10:])) + for _, _, response in answered: + assert response.status_code == 200, response.text + expected_by_call: Final = { + response.headers["x-litellm-call-id"]: caller_session + for caller_session, response in ((session, res) for _, session, res in answered) + } + call_ids: Final = sorted(expected_by_call) + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + delivered: Final = eventually( + lambda: ( + received.extend(destination.drain()) or tuple( + span + for span in _spans(received) + if _attribute(span.attributes, "langfuse.observation.metadata.litellm_call_id") + in expected_by_call + ) + ), + lambda spans: len(spans) >= len(answered), + seconds=60, + ) + spans_per_call: Final = { + call_id: sum( + 1 + for span in delivered + if _attribute(span.attributes, "langfuse.observation.metadata.litellm_call_id") == call_id + ) + for call_id in call_ids + } + assert sorted(spans_per_call.values()) == [1] * len(answered), ( + f"each call id must arrive on exactly one span: {spans_per_call}" + ) + for span in delivered: + span_call: Final = str( + _attribute(span.attributes, "langfuse.observation.metadata.litellm_call_id") + ) + assert _attribute(span.attributes, "session.id") == expected_by_call[span_call], ( + f"call {span_call}: session.id" + ) + spend_rows: Final = eventually( + lambda: read_rows( + 'SELECT session_id, litellm_call_id FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = ANY(%s)', + (call_ids,), + ), + lambda values: len(values) == len(answered), + seconds=150, + ) + for row in spend_rows: + row_call: Final = string_value(row["litellm_call_id"]) + assert row["session_id"] == expected_by_call[row_call], f"call {row_call}: spend {row}" + + +def test_audit_surviving_worker_keeps_serving_after_kill(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "auditworker" + uuid.uuid4().hex + provider_secret: Final = "synthetic-provider-secret-" + marker + header_trace: Final = uuid.uuid4().hex + baggage_session: Final = "baggage-" + marker + targets: Final[list[str]] = [] # mutable-ok: the upstream records every hit + + with ( + wire_server(_audit_upstream(provider_secret, marker, targets)) as provider, + wire_server(_audit_sink()) as destination, + owned_proxy_process( + gateway, + tmp_path, + _langfuse_environment(destination), + config=_langfuse_config(tmp_path), + workers=2, + ) as owned, + owned.gateway.scenario() as scenario, + ): + candidate: Final = owned.gateway + model: Final = scenario.model(api_base=provider.url + "/v1", api_key=provider_secret) + workers: Final = tuple( + member for member in group_members(owned.process.pid) if member.pid != owned.process.pid + ) + assert workers, "expected worker processes under the owned proxy" + workers[0].send_signal(signal.SIGKILL) + psutil.wait_procs([workers[0]], timeout=5) + + def fire(index: int) -> tuple[str, httpx.Response]: + caller_trace: Final = uuid.uuid4().hex + caller_session: Final = f"my-session-id-{marker}-{index}" + response: Final = candidate.request( + "POST", + "/v1/chat/completions", + _trace_body( + "chat", + model, + f"{marker}-{index}", + {"trace_id": caller_trace, "session_id": caller_session}, + ), + headers=_w3c_headers(header_trace, baggage_session), + ) + return caller_session, response + + with ThreadPoolExecutor(max_workers=10) as pool: + answered: Final = tuple(pool.map(fire, range(10))) + received: Final[list[Request]] = [] # mutable-ok: drain() consumes the queue, later polls keep earlier ones + for caller_session, response in answered: + assert response.status_code == 200, response.text + call_id: Final = response.headers["x-litellm-call-id"] + span: Final = _await_span(received, destination, call_id) + assert _attribute(span.attributes, "session.id") == caller_session, f"{call_id}: session.id" + expected_by_call: Final = { + response.headers["x-litellm-call-id"]: caller_session + for caller_session, response in answered + } + spend_rows: Final = eventually( + lambda: read_rows( + 'SELECT session_id, litellm_call_id FROM "LiteLLM_SpendLogs" WHERE litellm_call_id = ANY(%s)', + (list(expected_by_call),), + ), + lambda values: len(values) == len(answered), + seconds=150, + ) + for row in spend_rows: + row_call: Final = string_value(row["litellm_call_id"]) + assert row["session_id"] == expected_by_call[row_call], f"call {row_call}: spend {row}" diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index 86492537fd3..c6c473ae71e 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -3674,6 +3674,106 @@ def test_add_litellm_metadata_from_request_headers_explicit_trace_id_beats_trace assert data["litellm_session_id"] == "explicit-trace-id-value" +def test_add_litellm_metadata_from_request_headers_body_trace_id_beats_traceparent(): + headers = {"traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"} + data = {"metadata": {"trace_id": "caller-chosen-trace-id"}} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=data, _metadata_variable_name="metadata" + ) + assert data["metadata"]["trace_id"] == "caller-chosen-trace-id" + assert "litellm_trace_id" not in data + + +def test_add_litellm_metadata_from_request_headers_body_session_id_beats_baggage(): + headers = {"baggage": "session.id=baggage-session-42"} + data = {"metadata": {"session_id": "caller-chosen-session-id"}} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=data, _metadata_variable_name="metadata" + ) + assert data["metadata"]["session_id"] == "caller-chosen-session-id" + assert "litellm_session_id" not in data + + +def test_add_litellm_metadata_from_request_headers_body_steering_is_per_field(): + headers = { + "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + "baggage": "session.id=baggage-session-42", + } + data = {"metadata": {"trace_id": "caller-chosen-trace-id"}} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=data, _metadata_variable_name="metadata" + ) + assert data["metadata"]["trace_id"] == "caller-chosen-trace-id" + assert data["litellm_session_id"] == "baggage-session-42" + + +def test_add_litellm_metadata_from_request_headers_litellm_metadata_steering_honoured(): + headers = {"traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"} + data = {"litellm_metadata": {"trace_id": "caller-chosen-trace-id"}} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=data, _metadata_variable_name="litellm_metadata" + ) + assert data["litellm_metadata"]["trace_id"] == "caller-chosen-trace-id" + assert "litellm_trace_id" not in data + + +@pytest.mark.parametrize("empty_session_id", ["", None]) +def test_add_litellm_metadata_from_request_headers_empty_body_session_id_falls_back_to_baggage( + empty_session_id: str | None, +): + headers = {"baggage": "session.id=baggage-session-42"} + data = {"metadata": {"session_id": empty_session_id}} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=data, _metadata_variable_name="metadata" + ) + assert data["litellm_session_id"] == "baggage-session-42" + assert data["metadata"]["session_id"] == "baggage-session-42" + + +@pytest.mark.parametrize("field", ["trace_id", "session_id"]) +def test_add_litellm_metadata_from_request_headers_promoted_metadata_beats_headers(field: str): + headers = { + "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + "baggage": "session.id=baggage-session-42", + } + data = {"metadata": {field: "caller-chosen"}, "litellm_metadata": {}} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=data, _metadata_variable_name="litellm_metadata" + ) + assert field not in data["litellm_metadata"] + assert f"litellm_{field}" not in data + + +def test_add_litellm_metadata_from_request_headers_equal_ids_still_stamp_root_fields(): + headers = { + "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + "baggage": "session.id=matching-session-42", + } + data = { + "metadata": {"trace_id": "4bf92f3577b34da6a3ce929d0e0e4736", "session_id": "matching-session-42"}, + } + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=data, _metadata_variable_name="metadata" + ) + assert data["litellm_trace_id"] == "4bf92f3577b34da6a3ce929d0e0e4736" + assert data["litellm_session_id"] == "matching-session-42" + assert data["metadata"]["trace_id"] == "4bf92f3577b34da6a3ce929d0e0e4736" + assert data["metadata"]["session_id"] == "matching-session-42" + + +@pytest.mark.parametrize("non_string_session_id", [4815162342, True, {"session": "nested"}]) +def test_add_litellm_metadata_from_request_headers_non_string_body_session_id_falls_back_to_baggage( + non_string_session_id: object, +): + headers = {"baggage": "session.id=header-session-42"} + data = {"metadata": {"session_id": non_string_session_id}} + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=data, _metadata_variable_name="metadata" + ) + assert data["litellm_session_id"] == "header-session-42" + assert data["metadata"]["session_id"] == "header-session-42" + + def _otel_span_with_trace_id(trace_id: int) -> NonRecordingSpan: return NonRecordingSpan(SpanContext(trace_id=trace_id, span_id=0x00F067AA0BA902B7, is_remote=False)) @@ -8222,6 +8322,85 @@ async def test_missing_session_id_omit_keeps_client_supplied_session_id(): assert _spend_log_session_id(updated) == "client-session-1" +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("path", "client_body"), + [ + ("/v1/chat/completions", {"model": "gpt-4o", "messages": [], "metadata": {"session_id": ""}}), + ("/v1/responses", {"model": "gpt-4o", "input": "hi"}), + ], +) +async def test_missing_session_id_reject_accepts_baggage_session_id(path: str, client_body: dict[str, object]): + request = _request_for(path) + request.headers = {"baggage": "session.id=baggage-session-42"} + + updated = await add_litellm_data_to_request( + data=client_body, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + + assert updated["litellm_session_id"] == "baggage-session-42" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/messages"]) +@pytest.mark.parametrize("policy", [None, "reject", "generate"]) +async def test_promoted_caller_trace_ids_beat_traceparent_and_baggage(path: str, policy: str | None): + request = _request_for(path) + request.headers = { + "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + "baggage": "session.id=baggage-session-42", + } + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "metadata": {"trace_id": "caller-trace", "session_id": "caller-session"}}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": policy} if policy else {}, + ) + + assert updated["litellm_metadata"]["trace_id"] == "caller-trace" + assert updated["litellm_metadata"]["session_id"] == "caller-session" + + +@pytest.mark.asyncio +async def test_missing_session_id_reject_ignores_requester_session_id_shadowed_by_empty_litellm_metadata(): + with pytest.raises(ProxyException) as exc_info: + await add_litellm_data_to_request( + data={ + "model": "gpt-4o", + "input": "hi", + "metadata": {"session_id": "caller-session"}, + "litellm_metadata": {"session_id": ""}, + }, + request=_request_for("/v1/responses"), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "reject"}, + ) + assert exc_info.value.code == "400" + + +def test_add_litellm_metadata_from_request_headers_empty_litellm_metadata_field_falls_back_to_headers(): + headers = { + "traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01", + "baggage": "session.id=baggage-session-42", + } + data = { + "metadata": {"trace_id": "caller-trace", "session_id": "caller-session"}, + "litellm_metadata": {"trace_id": "", "session_id": ""}, + } + LiteLLMProxyRequestSetup.add_litellm_metadata_from_request_headers( + headers=headers, data=data, _metadata_variable_name="litellm_metadata" + ) + assert data["litellm_trace_id"] == "4bf92f3577b34da6a3ce929d0e0e4736" + assert data["litellm_session_id"] == "baggage-session-42" + + @pytest.mark.asyncio @pytest.mark.parametrize( "client_body", @@ -8341,6 +8520,91 @@ async def test_missing_session_id_generate_reuses_traceparent_trace_id(): assert _spend_log_session_id(updated) == "4bf92f3577b34da6a3ce929d0e0e4736" +@pytest.mark.asyncio +@pytest.mark.parametrize("path", ["/v1/responses", "/v1/messages"]) +@pytest.mark.parametrize( + "headers", + [ + {"traceparent": "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"}, + {}, + ], + ids=["with_traceparent", "no_traceparent"], +) +async def test_missing_session_id_generate_reuses_promoted_caller_trace_id(path: str, headers: dict[str, str]): + request = _request_for(path) + request.headers = headers + + updated = await add_litellm_data_to_request( + data={"model": "gpt-4o", "input": "hi", "metadata": {"trace_id": "caller-trace"}}, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "generate"}, + ) + + caller_trace_msg: Final = "generate must derive the session from the caller's un-promoted metadata.trace_id" + assert updated["litellm_session_id"] == "caller-trace", caller_trace_msg + assert updated["litellm_metadata"]["session_id"] == "caller-trace", caller_trace_msg + assert updated["litellm_metadata"][SESSION_ID_GENERATED_METADATA_KEY] is True, ( + "the derived session id must still be marked generated" + ) + assert _spend_log_session_id(updated, "litellm_metadata") == "caller-trace", ( + "spend log and callback session ids must agree on the caller trace id" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("policy", ["generate", "reject"]) +async def test_missing_session_id_policy_promotes_caller_session_to_root_field(policy: str): + """A caller-supplied usable session id satisfies the missing_session_id policies on + litellm_metadata routes (where it is not yet the managed metadata field) and must also + land on the root ``litellm_session_id`` field: consumers that read the root field + (router fallbacks, spend logs, sandbox reuse) otherwise mint a fresh uuid4 per request.""" + request = _request_for("/v1/responses") + + updated = await add_litellm_data_to_request( + data={ + "model": "gpt-4o", + "input": "hi", + "litellm_trace_id": "root-trace-42", + "metadata": {"session_id": "caller-session-42"}, + }, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": policy}, + ) + + assert updated["litellm_trace_id"] == "root-trace-42" + assert updated["litellm_session_id"] == "caller-session-42" + assert updated["litellm_metadata"]["session_id"] == "caller-session-42" + assert SESSION_ID_GENERATED_METADATA_KEY not in updated["litellm_metadata"] + + +@pytest.mark.asyncio +async def test_missing_session_id_generate_ignores_non_string_caller_session_id(): + """A non-string session id is not a usable session: the generate policy must fall through + to generation instead of letting an unusable value strand the root session field.""" + request = _request_for("/v1/responses") + + updated = await add_litellm_data_to_request( + data={ + "model": "gpt-4o", + "input": "hi", + "litellm_trace_id": "root-trace-42", + "metadata": {"session_id": 4815162342}, + }, + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings={"missing_session_id": "generate"}, + ) + + assert updated["litellm_session_id"] == "root-trace-42" + assert updated["litellm_metadata"]["session_id"] == "root-trace-42" + assert updated["litellm_metadata"][SESSION_ID_GENERATED_METADATA_KEY] is True + + @pytest.mark.asyncio @pytest.mark.parametrize("policy", ["generate", "reject"]) async def test_missing_session_id_policy_keeps_client_supplied_session_id(policy: str): From efe0715831a5cc6ce94f43b6d6b2e69da2d5e4ed Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Wed, 7 Oct 2026 19:25:28 -0700 Subject: [PATCH 04/65] fix(proxy): accept router-wide default_litellm_params in required-param validation (#44483) * fix(proxy): accept router-wide default_litellm_params in required-param validation The required-body-param check added in #43787 only consulted request data and deployment-level litellm_params, so a param supplied solely by router_settings.default_litellm_params (e.g. max_tokens for /v1/messages, documents for /rerank) was rejected with a 400 at route entry even though the router would have injected it during dispatch. Consult the router-wide defaults in the same place, treating None-valued defaults as absent to mirror the setdefault merge. * fix(proxy): consult only the defaults the dispatching router will merge Review follow-up: router-wide defaults are now consulted only where dispatch actually applies them. user_config requests dispatch on their own throwaway Router, so they consult that config's defaults instead of the global router's; search, managed-agent and eval routes, and model-less direct dispatch never pass through the router's defaults merge, so they keep the route-entry 400. Also adds rerank router-default coverage (unit + e2e) and drops the redundant source comment. --- litellm/proxy/route_llm_request.py | 30 +++- .../test_missing_body_param_status.py | 93 ++++++++++ tests/unit/proxy/test_route_llm_request.py | 170 ++++++++++++++++++ 3 files changed, 292 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 30dfb5fb8a6..22b04062534 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -245,11 +245,13 @@ def _find_missing_required_body_param( if not missing_present_params: return None candidate_litellm_params: Final = _candidate_deployment_litellm_params(data, llm_router) + router_default_litellm_params: Final = _router_default_litellm_params(route_type, data, llm_router) missing_param: Final = next( ( param for param in missing_present_params - if not any(deployment_params.get(param) is not None for deployment_params in candidate_litellm_params) + if router_default_litellm_params.get(param) is None + and not any(deployment_params.get(param) is not None for deployment_params in candidate_litellm_params) ), None, ) @@ -258,6 +260,32 @@ def _find_missing_required_body_param( return MissingBodyParam(name=missing_param, model_deployments_loaded=bool(candidate_litellm_params)) +_ROUTE_TYPES_WITHOUT_ROUTER_DEFAULTS_MERGE: Final[frozenset[str]] = frozenset( + {"asearch", "acreate_agent", "acreate_eval", "acreate_run"} +) + + +def _router_default_litellm_params( + route_type: str, + data: Mapping[str, object], + llm_router: LitellmRouter | None, +) -> Mapping[str, object]: + # Mirror exactly the defaults the dispatching router will merge at dispatch time: + # user_config requests dispatch on their own throwaway Router, and the listed route + # types (plus model-less direct dispatch) never pass through the router's merge. + user_config: Final[Mapping[str, object] | None] = ( + data.get("user_config") if isinstance(data.get("user_config"), Mapping) else None + ) + if user_config is not None: + defaults: Final[Mapping[str, object] | None] = user_config.get("default_litellm_params") + return defaults if isinstance(defaults, Mapping) else {} + model_name: Final = data.get("model") + if route_type in _ROUTE_TYPES_WITHOUT_ROUTER_DEFAULTS_MERGE or not isinstance(model_name, str) or not model_name: + return {} + router_defaults: Final[Mapping[str, object] | None] = getattr(llm_router, "default_litellm_params", None) + return router_defaults if isinstance(router_defaults, Mapping) else {} + + def _candidate_deployment_litellm_params( data: Mapping[str, object], llm_router: LitellmRouter | None, diff --git a/tests/integration/compatibility/test_missing_body_param_status.py b/tests/integration/compatibility/test_missing_body_param_status.py index 3e76ebaeda5..facf8007910 100644 --- a/tests/integration/compatibility/test_missing_body_param_status.py +++ b/tests/integration/compatibility/test_missing_body_param_status.py @@ -1025,6 +1025,99 @@ def test_anthropic_messages_uses_deployment_max_tokens_default(gateway: Gateway) assert outbound.get("max_tokens") == 32, provider_requests +def test_anthropic_messages_uses_router_wide_max_tokens_default(gateway: Gateway, tmp_path: Path) -> None: + with gateway.scenario() as scenario: + identity: Final = f"router-default-{uuid.uuid4().hex}" + handle: Final = register_scenario(identity, _response("anthropic_messages")) + scenario.cleanups.callback(delete_scenario, handle) + config: Final = tmp_path / "router-default.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "router-default-anthropic", + "litellm_params": { + "model": "anthropic/claude-haiku-4-5", + "api_base": handle.api_base(), + "api_key": identity, + }, + } + ], + "router_settings": {"default_litellm_params": {"max_tokens": 32}}, + } + ), + encoding="utf-8", + ) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + candidate: Final = Gateway(owned.gateway.client, owned.gateway.key, gateway.upstream_url) + response: Final = _post( + candidate, + "/v1/messages", + {"model": "router-default-anthropic", "messages": [{"role": "user", "content": "router default"}]}, + ) + assert response.status_code == 200, response.text + _assert_scripted_response("anthropic_messages", JSON_OBJECT.validate_python(response.json())) + observations: Final = _Observations(gateway.upstream_url) + captured: Final = eventually( + observations.read, + lambda _items: any(item.get("method") == "POST" for item in observations.for_scenario(identity)), + seconds=20, + ) + provider_requests: Final = tuple( + item for item in observations.for_scenario(identity) if item.get("method") == "POST" + ) + assert len(provider_requests) == 1, captured + outbound: Final = object_value(provider_requests[0]["body"]) + assert outbound.get("max_tokens") == 32, provider_requests + + +def test_rerank_uses_router_wide_documents_default(gateway: Gateway, tmp_path: Path) -> None: + with gateway.scenario() as scenario: + identity: Final = f"router-default-rerank-{uuid.uuid4().hex}" + handle: Final = register_scenario(identity, _response("arerank")) + scenario.cleanups.callback(delete_scenario, handle) + config: Final = tmp_path / "router-default-rerank.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": "router-default-rerank", + "litellm_params": { + "model": "cohere/rerank-v4.0", + "api_base": handle.api_base(), + "api_key": identity, + }, + } + ], + "router_settings": {"default_litellm_params": {"documents": ["router default document"]}}, + } + ), + encoding="utf-8", + ) + with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: + candidate: Final = Gateway(owned.gateway.client, owned.gateway.key, gateway.upstream_url) + response: Final = _post( + candidate, + "/rerank", + {"model": "router-default-rerank", "query": "which document?"}, + ) + assert response.status_code == 200, response.text + observations: Final = _Observations(gateway.upstream_url) + captured: Final = eventually( + observations.read, + lambda _items: any(item.get("method") == "POST" for item in observations.for_scenario(identity)), + seconds=20, + ) + provider_requests: Final = tuple( + item for item in observations.for_scenario(identity) if item.get("method") == "POST" + ) + assert len(provider_requests) == 1, captured + outbound: Final = object_value(provider_requests[0]["body"]) + assert outbound.get("documents") == ["router default document"], provider_requests + + def test_anthropic_messages_explicit_null_reaches_upstream(gateway: Gateway) -> None: with gateway.scenario() as scenario: model, identity, _handle = _register( diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index 3517f1412d7..a891e1079d8 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -1,6 +1,7 @@ import pytest +from types import SimpleNamespace from typing import Final from unittest.mock import MagicMock @@ -1200,6 +1201,175 @@ def test_required_present_body_param_without_router_default_still_raises() -> No assert exc_info.value.param == "max_tokens" +@pytest.mark.parametrize( + "route_type, data, param, default", + [ + ("anthropic_messages", {"model": "claude-router-default", "messages": []}, "max_tokens", 16), + ("arerank", {"model": "rerank-router-default", "query": "hi"}, "documents", ["router default document"]), + ], +) +def test_required_present_body_param_uses_router_wide_default( + route_type: str, data: dict[str, object], param: str, default: object +) -> None: + import litellm + from litellm.proxy.route_llm_request import raise_if_required_body_param_missing + + router = litellm.Router( + model_list=[ + { + "model_name": str(data["model"]), + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + }, + } + ], + default_litellm_params={param: default}, + ) + + raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=router) + + +@pytest.mark.parametrize( + "route_type, data, param", + [ + ("anthropic_messages", {"model": "claude", "messages": []}, "max_tokens"), + ("arerank", {"model": "rerank-model", "query": "hi"}, "documents"), + ], +) +def test_required_present_body_param_with_none_router_wide_default_still_raises( + route_type: str, data: dict[str, object], param: str +) -> None: + import litellm + from litellm.proxy.route_llm_request import ( + ProxyMissingRequiredParamError, + raise_if_required_body_param_missing, + ) + + router = litellm.Router( + model_list=[ + { + "model_name": str(data["model"]), + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + }, + } + ], + default_litellm_params={param: None}, + ) + + with pytest.raises(ProxyMissingRequiredParamError) as exc_info: + raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=router) + + assert exc_info.value.param == param + + +@pytest.mark.parametrize( + "route_type, data, param", + [ + ("asearch", {"model": "search-model"}, "query"), + ("acreate_eval", {"model": "eval-model"}, "data_source_config"), + ("acreate_run", {"model": "eval-model"}, "data_source"), + ("acreate_agent", {"model": "agent-model"}, "name"), + ("avector_store_search", {}, "query"), + ], +) +def test_required_present_body_param_ignores_router_wide_default_when_dispatch_skips_router( + route_type: str, data: dict[str, object], param: str +) -> None: + import litellm + from litellm.proxy.route_llm_request import ( + ProxyMissingRequiredParamError, + raise_if_required_body_param_missing, + ) + + router = litellm.Router( + model_list=[ + { + "model_name": str(data.get("model") or "any-model"), + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + }, + } + ], + default_litellm_params={param: "supplied-by-router-default"}, + ) + + with pytest.raises(ProxyMissingRequiredParamError) as exc_info: + raise_if_required_body_param_missing(route_type=route_type, data=data, llm_router=router) + + assert exc_info.value.param == param + + +def test_required_present_body_param_uses_user_config_router_defaults() -> None: + from litellm.proxy.route_llm_request import raise_if_required_body_param_missing + + raise_if_required_body_param_missing( + route_type="anthropic_messages", + data={ + "model": "claude-user-config", + "messages": [], + "user_config": {"default_litellm_params": {"max_tokens": 16}}, + }, + llm_router=None, + ) + + +def test_required_present_body_param_ignores_global_router_defaults_for_user_config_requests() -> None: + import litellm + from litellm.proxy.route_llm_request import ( + ProxyMissingRequiredParamError, + raise_if_required_body_param_missing, + ) + + router = litellm.Router( + model_list=[ + { + "model_name": "claude-user-config", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "test-key", + }, + } + ], + default_litellm_params={"max_tokens": 16}, + ) + + with pytest.raises(ProxyMissingRequiredParamError) as exc_info: + raise_if_required_body_param_missing( + route_type="anthropic_messages", + data={"model": "claude-user-config", "messages": [], "user_config": {}}, + llm_router=router, + ) + + assert exc_info.value.param == "max_tokens" + + +@pytest.mark.parametrize( + "llm_router", + [ + pytest.param(MagicMock(), id="magicmock-router"), + pytest.param(SimpleNamespace(get_model_list=lambda **_kwargs: ()), id="router-without-defaults-attr"), + ], +) +def test_required_present_body_param_ignores_non_mapping_router_defaults(llm_router: object) -> None: + from litellm.proxy.route_llm_request import ( + ProxyMissingRequiredParamError, + raise_if_required_body_param_missing, + ) + + with pytest.raises(ProxyMissingRequiredParamError) as exc_info: + raise_if_required_body_param_missing( + route_type="arerank", + data={"model": "rerank-model", "query": "hi"}, + llm_router=llm_router, # pyright: ignore[reportArgumentType] # deliberately duck-typed routers + ) + + assert exc_info.value.param == "documents" + + @pytest.mark.parametrize( "route_type, data, param", [ From 4f49f8817239053e084f6751f5b45fe1606d710f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:26:30 -0700 Subject: [PATCH 05/65] fix(proxy): keep idle responses websockets open until a configurable session limit (#44433) * add config-driven bound to /responses websocket connection * test(proxy): cover responses websocket session limit and idle first frame Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(ci): sync API schema and Ruff formatting Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): close responses websocket client before upstream cleanup at session limit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): assert recorded websocket outcomes and simplify session reaping Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): move responses websocket session limit timing coverage to integration Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): run provider-close checks off the event loop Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(integration): cover handshake auth, subprotocol, DB override and worker kill for the responses websocket session limit Adds the audit cells for the configurable session limit: a rejected bearer fails the handshake with 403, a requested Sec-WebSocket-Protocol is echoed back, a DB override through /config/field/update caps new sessions on every worker without a restart while sessions accepted before it keep their limit, deleting the override restores the default, out-of-range and wrong-type updates answer 400 and leave the stored row alone, and SIGKILLing the worker holding idle sockets drops only its sockets while the proxy keeps serving * test(integration): open override sockets until both proxy workers hold one uvicorn workers share one listening socket and a worker that wakes first accepts a whole simultaneous burst, so eight sockets opened at once can all land on one worker. The DB override cell now opens its sockets one at a time, alternating the two paths, until at least eight are open and every worker holds one, bounded at forty --------- Co-authored-by: Mrinal Chanshetty Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/proxy/_types.py | 9 + .../proxy/response_api_endpoints/endpoints.py | 108 +- .../test_responses_websocket_session_limit.py | 1479 +++++++++++++++++ .../response_api_endpoints/test_endpoints.py | 45 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 + 5 files changed, 1609 insertions(+), 38 deletions(-) create mode 100644 tests/integration/providers/test_responses_websocket_session_limit.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 7f92573fde7..d4229c825b6 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -2738,6 +2738,9 @@ class ScheduledJobStaggerSettings(LiteLLMPydanticObjectBase): ) +DEFAULT_RESPONSES_WEBSOCKET_SESSION_LIMIT_SECONDS: Final[float] = 3600.0 + + class ConfigGeneralSettings(LiteLLMPydanticObjectBase): """ Documents all the fields supported by `general_settings` in config.yaml @@ -3048,6 +3051,12 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): default=None, description="Default upstream request timeout in seconds for native and custom pass-through endpoints that use pass_through_request. Defaults to 600 when unset.", ) + responses_websocket_session_limit_seconds: float = Field( + default=DEFAULT_RESPONSES_WEBSOCKET_SESSION_LIMIT_SECONDS, + ge=60, + le=7200, + description="Maximum lifetime in seconds of a Responses API WebSocket session, measured from connection accept and covering the idle wait for the first response.create frame. Defaults to 3600, matching OpenAI's documented 60-minute WebSocket connection limit. Must be between 60 and 7200 seconds.", + ) pass_through_endpoints: list[PassThroughGenericEndpoint] | None = Field( default=None, description="Set-up pass-through endpoints for provider-specific endpoints. Docs - https://docs.litellm.ai/docs/proxy/pass_through", diff --git a/litellm/proxy/response_api_endpoints/endpoints.py b/litellm/proxy/response_api_endpoints/endpoints.py index 17ea2754821..7f9fcca369b 100644 --- a/litellm/proxy/response_api_endpoints/endpoints.py +++ b/litellm/proxy/response_api_endpoints/endpoints.py @@ -12,7 +12,7 @@ from fastapi import APIRouter, Depends, HTTPException, Request, Response from fastapi.responses import JSONResponse from openai.types.responses import ResponseItemList from openai.types.responses.response_create_params import ResponseInputParam -from pydantic import ConfigDict, ValidationError +from pydantic import ConfigDict, TypeAdapter, ValidationError from starlette.websockets import WebSocket, WebSocketDisconnect from typing_extensions import ReadOnly, TypedDict @@ -50,6 +50,7 @@ if TYPE_CHECKING: from litellm.router import Router router: Final = APIRouter() +_RESPONSES_WS_CONFIG_VALUE_ADAPTER: Final[TypeAdapter[object | None]] = TypeAdapter(object | None) _ResponseDocSchemas: TypeAlias = dict[int | str, dict[str, object]] # fastapi's responses kwarg @@ -1316,6 +1317,21 @@ async def cancel_response( ) +def _resolve_responses_ws_session_limit_seconds() -> float: + from litellm.proxy.proxy_server import general_settings + + field: Final = "responses_websocket_session_limit_seconds" + raw: Final = _RESPONSES_WS_CONFIG_VALUE_ADAPTER.validate_python(general_settings.get(field)) + try: + return ConfigGeneralSettings.model_validate( + {} if raw is None else {field: raw} + ).responses_websocket_session_limit_seconds + except ValidationError as e: + default: Final = DEFAULT_RESPONSES_WEBSOCKET_SESSION_LIMIT_SECONDS + verbose_proxy_logger.warning("invalid general_settings.%s=%r (%s); using default %ss", field, raw, e, default) + return default + + async def _read_ws_model_from_first_frame( websocket: WebSocket, query_model: str | None = None, @@ -1323,12 +1339,10 @@ async def _read_ws_model_from_first_frame( """Read the first WS frame and return (model, raw_message), or None on error. Sends an appropriate error frame and closes the socket before returning None. + The session-duration deadline is enforced by the caller, not here. """ try: - first_message: Final = await asyncio.wait_for(websocket.receive_text(), timeout=30) - except asyncio.TimeoutError: - await websocket.close(code=1008, reason="Timed out waiting for first message") - return None + first_message: Final = await websocket.receive_text() except WebSocketDisconnect: return None except Exception: @@ -1474,25 +1488,11 @@ async def _enforce_responses_ws_first_frame_model_auth( ) -@router.websocket("/v1/responses") -@router.websocket("/responses") -async def responses_websocket_endpoint( +async def _responses_websocket_session( websocket: WebSocket, - model: str | None = fastapi.Query(None, description="The model to use for the responses WebSocket session."), - user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), -): - """ - Responses API WebSocket mode endpoint. - - Keeps a persistent WebSocket connection for response.create events, - enabling lower-latency agentic workflows with many tool-call round trips. - - Follows the OpenAI split: the bearer token is validated at connection time - (before accept); the model is resolved either from the ?model= query param - or from the first response.create frame, whichever is present. - - See: https://developers.openai.com/api/docs/guides/websocket-mode/ - """ + model: str | None, + user_api_key_dict: UserAPIKeyAuth, +) -> None: from litellm.proxy.proxy_server import ( general_settings, llm_router, @@ -1507,16 +1507,6 @@ async def responses_websocket_endpoint( ) from litellm.proxy.route_llm_request import route_request - # Accept the WebSocket handshake. Key was already validated by the Depends - # above; we can safely accept regardless of whether ?model= was supplied. - requested_protocols: Final = [ - p.strip() for p in (websocket.headers.get("sec-websocket-protocol") or "").split(",") if p.strip() - ] - accept_kwargs: Final[dict] = {} - if requested_protocols: - accept_kwargs["subprotocol"] = requested_protocols[0] - await websocket.accept(**accept_kwargs) - result: Final = await _read_ws_model_from_first_frame(websocket, query_model=model) if result is None: return @@ -1621,3 +1611,55 @@ async def responses_websocket_endpoint( request_data=routed_data, ) await websocket.close(code=1011, reason="Internal server error") + + +@router.websocket("/v1/responses") +@router.websocket("/responses") +async def responses_websocket_endpoint( + websocket: WebSocket, + model: str | None = fastapi.Query(None, description="The model to use for the responses WebSocket session."), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth_websocket), +): + """ + Responses API WebSocket mode endpoint. + + Keeps a persistent WebSocket connection for response.create events, + enabling lower-latency agentic workflows with many tool-call round trips. + + Follows the OpenAI split: the bearer token is validated at connection time + (before accept); the model is resolved either from the ?model= query param + or from the first response.create frame, whichever is present. + + The session is bounded by a lifetime measured from accept, configured via + general_settings.responses_websocket_session_limit_seconds (60-7200, + default 3600). There is no separate first-frame deadline, so + pre-established connections may sit idle until their first response.create. + + See: https://developers.openai.com/api/docs/guides/websocket-mode/ + """ + # Accept the WebSocket handshake. Key was already validated by the Depends + # above; we can safely accept regardless of whether ?model= was supplied. + requested_protocols: Final = [ + p.strip() for p in (websocket.headers.get("sec-websocket-protocol") or "").split(",") if p.strip() + ] + accept_kwargs: Final[dict] = {} + if requested_protocols: + accept_kwargs["subprotocol"] = requested_protocols[0] + await websocket.accept(**accept_kwargs) + + limit_seconds: Final = _resolve_responses_ws_session_limit_seconds() + session_task: Final = asyncio.ensure_future( + _responses_websocket_session(websocket=websocket, model=model, user_api_key_dict=user_api_key_dict) + ) + try: + await asyncio.wait_for(asyncio.shield(session_task), timeout=limit_seconds) + except asyncio.TimeoutError: + verbose_proxy_logger.info("Responses WebSocket closed: session duration limit reached") + session_task.cancel() + with contextlib.suppress(Exception): + await websocket.close(code=1000, reason="Session duration limit reached") + finally: + if not session_task.done(): + session_task.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await session_task diff --git a/tests/integration/providers/test_responses_websocket_session_limit.py b/tests/integration/providers/test_responses_websocket_session_limit.py new file mode 100644 index 00000000000..32b01d8387a --- /dev/null +++ b/tests/integration/providers/test_responses_websocket_session_limit.py @@ -0,0 +1,1479 @@ +from __future__ import annotations + +import asyncio +import itertools +import json +import re +import ssl +import threading +import time +import uuid +from collections.abc import Generator, Iterator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from queue import Empty, SimpleQueue +from types import MappingProxyType +from typing import Final, TypeVar +from urllib.parse import parse_qs, urlsplit + +import httpx +import psutil +import pytest +import websockets +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.database import read_rows, scratch_database +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy, owned_proxy_process +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.upstream import ( + JsonResponse, + RoutedResponse, + delete_scenario, + register_scenario, +) +from pydantic import JsonValue, TypeAdapter, ValidationError +from websockets.asyncio.server import ServerConnection, serve +from websockets.exceptions import ConnectionClosed, InvalidStatus +from websockets.typing import Subprotocol + +OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 180 +pytestmark: Final = pytest.mark.timeout(max(360.0, OWNED_PROXY_CELL_SECONDS)) + +PROVIDER_MODEL: Final = "ws-peer-model" +STALL_PROVIDER_MODEL: Final = "ws-stall-peer-model" +DEAF_PROVIDER_MODEL: Final = "ws-deaf-peer-model" +DEAF_RESPONSE_DELAY_SECONDS: Final = 10 +DEAF_READ_PAUSE_SECONDS: Final = 20 +PEER_TEXT: Final = "responses websocket peer" +TERMINAL: Final = frozenset({"response.completed", "response.failed", "error"}) +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +SESSION_LIMIT_FIELD: Final = "responses_websocket_session_limit_seconds" +LIMIT_CLOSE_REASON: Final = "Session duration limit reached" +RELOAD_INTERVAL_SECONDS: Final = 3 +WORKER_SYNC_SECONDS: Final = RELOAD_INTERVAL_SECONDS + 7 +CAP_SECONDS: Final = 60 +RESTORED_IDLE_SECONDS: Final = 75 +CAPPED_PATHS: Final = ("/v1/responses", "/responses") +CAPPED_SOCKETS_MINIMUM: Final = 8 +CAPPED_SOCKETS_MAXIMUM: Final = 40 +KILLED_POOL_SIZE: Final = 8 +SUBPROTOCOLS: Final = (Subprotocol("litellm-responses-first"), Subprotocol("litellm-responses-second")) +STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +WORKER_PIDS: Final = TypeAdapter(tuple[int, ...]) +CLIENT_ADDRESS: Final = TypeAdapter(tuple[str, int]) +T = TypeVar("T") + + +@dataclass(frozen=True, slots=True) +class PeerConnection: + path: str + frames: SimpleQueue[dict[str, JsonValue]] + closed: SimpleQueue[float] + + +@dataclass(frozen=True, slots=True) +class ResponsesPeer: + url: str + connections: SimpleQueue[PeerConnection] + connections_by_model: Mapping[str, SimpleQueue[PeerConnection]] + + +@dataclass(frozen=True, slots=True) +class PeerSnapshot: + path: str + frames: tuple[dict[str, JsonValue], ...] + closed: bool + + +@dataclass(frozen=True, slots=True) +class ReceiveResult: + timed_out: bool + closed: bool + frame: dict[str, JsonValue] | None + close_code: int | None + close_reason: str | None + + +@dataclass(frozen=True, slots=True) +class SessionResult: + text: str + idle: ReceiveResult + events: tuple[dict[str, JsonValue], ...] + error: str | None + + +@dataclass(frozen=True, slots=True) +class HandshakeResult: + status_code: int | None + error: str | None + + +@dataclass(frozen=True, slots=True) +class SubprotocolResult: + text: str + negotiated: str | None + events: tuple[dict[str, JsonValue], ...] + error: str | None + + +@dataclass(frozen=True, slots=True) +class DefaultResults: + pool: tuple[SessionResult, ...] + query: SessionResult + rejected: HandshakeResult + subprotocol: SubprotocolResult + provider: tuple[PeerSnapshot, ...] + + +@dataclass(frozen=True, slots=True) +class AuthResult: + idle: ReceiveResult + frame: dict[str, JsonValue] | None + close_code: int | None + close_reason: str | None + error: str | None + + +@dataclass(frozen=True, slots=True) +class AuthResults: + immediate: AuthResult + delayed: AuthResult + provider_connections: int + + +@dataclass(frozen=True, slots=True) +class CloseResult: + outcome: ReceiveResult + elapsed: float + + +@dataclass(frozen=True, slots=True) +class ActiveResult: + turn: tuple[dict[str, JsonValue], ...] + turn_error: str | None + close: CloseResult + provider_closed: bool + + +@dataclass(frozen=True, slots=True) +class MidResult: + created: bool + turn_error: str | None + close: CloseResult + provider_closed: bool + fresh_completed: bool + fresh_error: str | None + + +@dataclass(frozen=True, slots=True) +class CapResults: + idle: CloseResult + active: ActiveResult + mid: MidResult + deaf: MidResult + provider: tuple[PeerSnapshot, ...] + + +@dataclass(frozen=True, slots=True) +class BurstResult: + path: str + status_code: int | None + body: dict[str, JsonValue] | None + error: str | None + + +@dataclass(frozen=True, slots=True) +class InvalidResult: + session: SessionResult + warning_found: bool + + +@dataclass(frozen=True, slots=True) +class UpdateResult: + status_code: int + body: str + + +@dataclass(frozen=True, slots=True) +class CappedSocket: + path: str + worker_pid: int | None + close: CloseResult + + +@dataclass(frozen=True, slots=True) +class OpenedSocket: + path: str + worker_pid: int | None + connection: websockets.ClientConnection + started: float + + +@dataclass(frozen=True, slots=True) +class HeldResult: + held_seconds: float + events: tuple[dict[str, JsonValue], ...] + error: str | None + + +@dataclass(frozen=True, slots=True) +class OverrideResults: + workers: frozenset[int] + updates: tuple[UpdateResult, ...] + stored_after_updates: JsonValue + capped: tuple[CappedSocket, ...] + earlier: HeldResult + delete: UpdateResult + stored_after_delete: dict[str, JsonValue] | None + restored: SessionResult + + +@dataclass(frozen=True, slots=True) +class TurnOutcome: + events: tuple[dict[str, JsonValue], ...] + error: str | None + + +@dataclass(frozen=True, slots=True) +class KillResults: + held_by: Mapping[int, int] + victim: int + victim_closes: tuple[ReceiveResult, ...] + survivor_turns: tuple[TurnOutcome, ...] + health_status: int + fresh_completed: bool + + +@dataclass(frozen=True, slots=True) +class ChaosResults: + burst: tuple[BurstResult, ...] + health_status: int + websocket_completed: bool + kill: KillResults + + +def _object(value: JsonValue) -> dict[str, JsonValue]: + assert isinstance(value, dict), value + return value + + +def _list(value: JsonValue) -> list[JsonValue]: + assert isinstance(value, list), value + return value + + +def _string(value: JsonValue) -> str: + assert isinstance(value, str), value + return value + + +def _drain(queue: SimpleQueue[T]) -> tuple[T, ...]: + return tuple(queue.get_nowait() for _ in range(queue.qsize())) + + +def _events(response_id: str, model: str, text: str, *, stall: bool = False) -> tuple[dict[str, JsonValue], ...]: + message: Final[dict[str, JsonValue]] = { + "type": "message", + "id": f"msg_{response_id}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": text, "annotations": []}], + } + response: Final[dict[str, JsonValue]] = { + "id": response_id, + "object": "response", + "created_at": 1700000000, + "model": model, + } + created: Final[dict[str, JsonValue]] = { + "type": "response.created", + "response": {**response, "status": "in_progress", "output": []}, + } + if stall: + return (created,) + return ( + created, + { + "type": "response.output_text.delta", + "item_id": f"msg_{response_id}", + "output_index": 0, + "content_index": 0, + "delta": text, + }, + { + "type": "response.completed", + "response": {**response, "status": "completed", "output": [message]}, + }, + ) + + +def _path_model(path: str) -> str | None: + values: Final = parse_qs(urlsplit(path).query).get("model") + return values[0] if values else None + + +async def _peer_handler(connection: ServerConnection, peer: ResponsesPeer) -> None: + path: Final = connection.request.path if connection.request is not None else "" + record: Final = PeerConnection(path, SimpleQueue(), SimpleQueue()) + peer.connections.put(record) + model: Final = _path_model(path) + if model is not None and model in peer.connections_by_model: + peer.connections_by_model[model].put(record) + turns: Final = itertools.count(1) + try: + async for raw in connection: + await _peer_frame(raw, connection, record, turns) + finally: + record.closed.put(time.monotonic()) + + +async def _peer_frame( + raw: str | bytes, + connection: ServerConnection, + record: PeerConnection, + turns: itertools.count[int], +) -> None: + frame: Final = JSON_OBJECT.validate_json(raw) + record.frames.put(frame) + if frame.get("type") != "response.create": + return + model: Final = _string(frame.get("model", "")) + stall: Final = model in (STALL_PROVIDER_MODEL, DEAF_PROVIDER_MODEL) + if model == DEAF_PROVIDER_MODEL: + await asyncio.sleep(DEAF_RESPONSE_DELAY_SECONDS) + for event in _events(f"resp_peer_{next(turns)}", model, PEER_TEXT, stall=stall): + await connection.send(json.dumps(event)) + if model == DEAF_PROVIDER_MODEL: + transport: Final = connection.transport + assert transport is not None + transport.pause_reading() + try: + await asyncio.sleep(DEAF_READ_PAUSE_SECONDS) + finally: + transport.resume_reading() + + +async def _serve_peer( + tls: ssl.SSLContext, + peer: ResponsesPeer, + ports: SimpleQueue[int], + stop: asyncio.Event, +) -> None: + async with serve(lambda connection: _peer_handler(connection, peer), "127.0.0.1", 0, ssl=tls) as server: + address: object = next(iter(server.sockets)).getsockname() # pyright: ignore[reportAny] # socket.getsockname is typed Any + port: Final = TypeAdapter(tuple[str, int]).validate_python(address)[1] + ports.put(port) + await stop.wait() + + +@contextmanager +def responses_peer(cert: tuple[Path, Path]) -> Generator[ResponsesPeer, None, None]: + loop: Final = asyncio.new_event_loop() + stop: Final = asyncio.Event() + ports: Final = SimpleQueue[int]() + connections_by_model: Final[Mapping[str, SimpleQueue[PeerConnection]]] = MappingProxyType( + {model: SimpleQueue[PeerConnection]() for model in (PROVIDER_MODEL, STALL_PROVIDER_MODEL, DEAF_PROVIDER_MODEL)} + ) + peer: Final = ResponsesPeer("", SimpleQueue(), connections_by_model) + thread: Final = threading.Thread( + target=loop.run_until_complete, + args=(_serve_peer(server_context(*cert), peer, ports, stop),), + daemon=True, + ) + thread.start() + try: + port: Final = ports.get(timeout=10) + yield ResponsesPeer(f"https://127.0.0.1:{port}/v1", peer.connections, peer.connections_by_model) + finally: + loop.call_soon_threadsafe(stop.set) + thread.join(timeout=10) + loop.close() + + +def _proxy_ws_url(candidate: Gateway, path: str) -> str: + base: Final = str(candidate.client.base_url).rstrip("/").replace("http://", "ws://", 1) + return f"{base}{path}" + + +def _create(model: str, text: str, *, include_model: bool = True) -> str: + body: Final[dict[str, JsonValue]] = { + "type": "response.create", + "input": [{"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]}], + **({"model": model} if include_model else {}), + } + return json.dumps(body) + + +async def _receive(connection: websockets.ClientConnection, timeout: float) -> ReceiveResult: + try: + raw: Final = await asyncio.wait_for(connection.recv(), timeout=timeout) + except asyncio.TimeoutError: + return ReceiveResult(True, False, None, None, None) + except ConnectionClosed as error: + received: Final = error.rcvd + return ReceiveResult( + False, + True, + None, + received.code if received is not None else None, + received.reason if received is not None else None, + ) + return ReceiveResult(False, False, JSON_OBJECT.validate_json(raw), None, None) + + +async def _wait_for_close(connection: websockets.ClientConnection, timeout: float) -> ReceiveResult: + started: Final = time.monotonic() + result: Final = await _receive(connection, timeout) + if result.closed or result.timed_out: + return result + remaining: Final = timeout - (time.monotonic() - started) + if remaining <= 0: + return ReceiveResult(True, False, None, None, None) + return await _wait_for_close(connection, remaining) + + +async def _until_terminal( + connection: websockets.ClientConnection, + received: tuple[dict[str, JsonValue], ...] = (), +) -> tuple[dict[str, JsonValue], ...]: + event: Final = JSON_OBJECT.validate_json(await asyncio.wait_for(connection.recv(), timeout=20)) + collected: Final = (*received, event) + if event.get("type") in TERMINAL or len(collected) >= 50: + return collected + return await _until_terminal(connection, collected) + + +async def _turn(connection: websockets.ClientConnection, frame: str) -> tuple[dict[str, JsonValue], ...]: + await connection.send(frame) + return await _until_terminal(connection) + + +async def _idle_turn( + proxy: str, + key: str, + model: str, + path: str, + text: str, + *, + query_model: bool = False, + idle_seconds: float = 35, +) -> SessionResult: + query: Final = f"?model={model}" if query_model else "" + try: + async with websockets.connect( + f"{proxy}{path}{query}", + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) as connection: + idle: Final = await _receive(connection, idle_seconds) + try: + events: Final = await _turn(connection, _create(model, text, include_model=not query_model)) + except (ConnectionClosed, asyncio.TimeoutError) as error: + return SessionResult(text, idle, (), f"{type(error).__name__}: {error}") + return SessionResult(text, idle, events, None) + except (ConnectionClosed, asyncio.TimeoutError) as error: + return SessionResult(text, ReceiveResult(False, True, None, None, None), (), f"{type(error).__name__}: {error}") + + +async def _pool_workload(candidate: Gateway, key: str, model: str) -> tuple[SessionResult, ...]: + proxy: Final = _proxy_ws_url(candidate, "") + paths: Final = ("/v1/responses",) * 4 + ("/responses",) * 4 + return tuple( + await asyncio.gather(*tuple(_idle_turn(proxy, key, model, path, f"pool-{uuid.uuid4().hex}") for path in paths)) + ) + + +async def _rejected_handshake(proxy: str) -> HandshakeResult: + try: + async with websockets.connect( + f"{proxy}/v1/responses", + additional_headers={"Authorization": f"Bearer sk-rejected-{uuid.uuid4().hex}"}, + open_timeout=10, + ): + return HandshakeResult(None, "handshake accepted") + except InvalidStatus as error: + return HandshakeResult(error.response.status_code, None) + except (ConnectionClosed, asyncio.TimeoutError, OSError) as error: + return HandshakeResult(None, f"{type(error).__name__}: {error}") + + +async def _subprotocol_turn(proxy: str, key: str, model: str) -> SubprotocolResult: + text: Final = f"subprotocol-{uuid.uuid4().hex}" + try: + async with websockets.connect( + f"{proxy}/v1/responses", + additional_headers={"Authorization": f"Bearer {key}"}, + subprotocols=SUBPROTOCOLS, + open_timeout=10, + ) as connection: + events, error = await _turn_result(connection, _create(model, text)) + return SubprotocolResult(text, connection.subprotocol, events, error) + except (ConnectionClosed, asyncio.TimeoutError, InvalidStatus) as error: + return SubprotocolResult(text, None, (), f"{type(error).__name__}: {error}") + + +async def _default_workload( + candidate: Gateway, key: str, model: str +) -> tuple[tuple[SessionResult, ...], SessionResult, HandshakeResult, SubprotocolResult]: + proxy: Final = _proxy_ws_url(candidate, "") + pool, query, rejected, negotiated = await asyncio.gather( + _pool_workload(candidate, key, model), + _idle_turn(proxy, key, model, "/v1/responses", f"query-{uuid.uuid4().hex}", query_model=True), + _rejected_handshake(proxy), + _subprotocol_turn(proxy, key, model), + ) + return pool, query, rejected, negotiated + + +def _snapshot(record: PeerConnection) -> PeerSnapshot: + return PeerSnapshot(record.path, _drain(record.frames), record.closed.qsize() > 0) + + +def _snapshots(peer: ResponsesPeer, count: int) -> tuple[PeerSnapshot, ...]: + eventually(lambda: peer.connections.qsize(), lambda size: size >= count, seconds=15, return_last_on_timeout=True) + records: Final = _drain(peer.connections) + eventually( + lambda: tuple(record.closed.qsize() for record in records), + lambda counts: all(count > 0 for count in counts), + seconds=10, + return_last_on_timeout=True, + ) + return tuple(_snapshot(record) for record in records) + + +def _available_snapshots(peer: ResponsesPeer) -> tuple[PeerSnapshot, ...]: + eventually(lambda: peer.connections.qsize(), lambda count: count >= 1, seconds=10) + records: Final = _drain(peer.connections) + eventually( + lambda: tuple(record.closed.qsize() for record in records), + lambda counts: all(count > 0 for count in counts), + seconds=10, + ) + return tuple(_snapshot(record) for record in records) + + +def _frame_text(frame: dict[str, JsonValue]) -> str: + input_value: Final = _list(frame["input"]) + message: Final = _object(input_value[0]) + content: Final = _list(message["content"]) + item: Final = _object(content[0]) + return _string(item["text"]) + + +def _completed_text(events: tuple[dict[str, JsonValue], ...]) -> str: + assert events, events + completed: Final = events[-1] + assert completed.get("type") == "response.completed", completed + response: Final = _object(completed["response"]) + output: Final = _list(response["output"]) + message: Final = _object(output[0]) + content: Final = _list(message["content"]) + item: Final = _object(content[0]) + return _string(item["text"]) + + +def _auth_close(error: ConnectionClosed) -> tuple[int | None, str | None]: + received: Final = error.rcvd + return ( + received.code if received is not None else None, + received.reason if received is not None else None, + ) + + +async def _auth_attempt(proxy: str, key: str, model: str, *, delay: bool) -> AuthResult: + try: + async with websockets.connect( + f"{proxy}/v1/responses", + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) as connection: + idle: Final = await _receive(connection, 35) if delay else ReceiveResult(False, False, None, None, None) + try: + await connection.send(_create(model, f"auth-{uuid.uuid4().hex}")) + frame: Final = JSON_OBJECT.validate_json(await asyncio.wait_for(connection.recv(), timeout=20)) + try: + await asyncio.wait_for(connection.recv(), timeout=20) + except ConnectionClosed as error: + code, reason = _auth_close(error) + return AuthResult(idle, frame, code, reason, None) + return AuthResult(idle, frame, None, None, "connection remained open after rejection") + except (ConnectionClosed, asyncio.TimeoutError) as error: + code, reason = _auth_close(error) if isinstance(error, ConnectionClosed) else (None, None) + return AuthResult(idle, None, code, reason, f"{type(error).__name__}: {error}") + except (ConnectionClosed, asyncio.TimeoutError) as error: + return AuthResult( + ReceiveResult(False, True, None, None, None), + None, + None, + None, + f"{type(error).__name__}: {error}", + ) + + +async def _auth_workload(candidate: Gateway, key: str, model: str, peer: ResponsesPeer) -> AuthResults: + proxy: Final = _proxy_ws_url(candidate, "") + immediate, delayed = await asyncio.gather( + _auth_attempt(proxy, key, model, delay=False), + _auth_attempt(proxy, key, model, delay=True), + ) + return AuthResults(immediate, delayed, peer.connections.qsize()) + + +async def _close_at_limit( + candidate: Gateway, + key: str, + model: str, + *, + delay: float | None = None, +) -> CloseResult: + started: Final = time.monotonic() + outcome: Final = await _close_session(candidate, key, model, delay=delay) + return CloseResult(outcome, time.monotonic() - started) + + +async def _close_session( + candidate: Gateway, + key: str, + model: str, + *, + delay: float | None, +) -> ReceiveResult: + started: Final = time.monotonic() + try: + async with websockets.connect( + _proxy_ws_url(candidate, "/v1/responses"), + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) as connection: + if delay is not None: + await asyncio.sleep(max(0, started + delay - time.monotonic())) + return await _wait_for_close(connection, 75) + except (ConnectionClosed, asyncio.TimeoutError) as error: + code, reason = _auth_close(error) if isinstance(error, ConnectionClosed) else (None, None) + return ReceiveResult(False, True, None, code, reason) + + +async def _turn_result( + connection: websockets.ClientConnection, + frame: str, +) -> tuple[tuple[dict[str, JsonValue], ...], str | None]: + try: + return await _turn(connection, frame), None + except (ConnectionClosed, asyncio.TimeoutError) as error: + return (), f"{type(error).__name__}: {error}" + + +async def _active_session(candidate: Gateway, key: str, model: str, peer: ResponsesPeer) -> ActiveResult: + started: Final = time.monotonic() + try: + async with websockets.connect( + _proxy_ws_url(candidate, "/v1/responses"), + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) as connection: + await asyncio.sleep(max(0, started + 5 - time.monotonic())) + turn, turn_error = await _turn_result(connection, _create(model, f"active-{uuid.uuid4().hex}")) + remaining: Final = max(0, 75 - (time.monotonic() - started)) + close: Final = CloseResult(await _wait_for_close(connection, remaining), time.monotonic() - started) + provider_closed: Final = await asyncio.to_thread(_provider_closed_within, peer, PROVIDER_MODEL, 5) + return ActiveResult(turn, turn_error, close, provider_closed) + except (ConnectionClosed, asyncio.TimeoutError) as error: + code, reason = _auth_close(error) if isinstance(error, ConnectionClosed) else (None, None) + return ActiveResult( + (), + f"{type(error).__name__}: {error}", + CloseResult(ReceiveResult(False, True, None, code, reason), time.monotonic() - started), + False, + ) + + +def _provider_closed_within(peer: ResponsesPeer, model: str, seconds: float) -> bool: + deadline: Final = time.monotonic() + seconds + try: + record: Final = peer.connections_by_model[model].get(timeout=seconds) + except Empty: + return False + try: + eventually( + lambda: record.closed.qsize(), + lambda count: count >= 1, + seconds=max(0, deadline - time.monotonic()), + ) + except AssertionError: + return False + return True + + +async def _stall_turn(connection: websockets.ClientConnection, model: str) -> tuple[bool, str | None]: + try: + await connection.send(_create(model, f"stall-{uuid.uuid4().hex}")) + first: Final = JSON_OBJECT.validate_json(await asyncio.wait_for(connection.recv(), timeout=20)) + return first.get("type") == "response.created", None + except (ConnectionClosed, asyncio.TimeoutError) as error: + return False, f"{type(error).__name__}: {error}" + + +async def _mid_session( + candidate: Gateway, + key: str, + stall_model: str, +) -> tuple[bool, str | None, CloseResult]: + started: Final = time.monotonic() + try: + async with websockets.connect( + _proxy_ws_url(candidate, "/v1/responses"), + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) as connection: + await asyncio.sleep(max(0, started + 40 - time.monotonic())) + created, turn_error = await _stall_turn(connection, stall_model) + remaining: Final = max(0, 75 - (time.monotonic() - started)) + close: Final = CloseResult(await _wait_for_close(connection, remaining), time.monotonic() - started) + return created, turn_error, close + except (ConnectionClosed, asyncio.TimeoutError) as error: + code, reason = _auth_close(error) if isinstance(error, ConnectionClosed) else (None, None) + return ( + False, + f"{type(error).__name__}: {error}", + CloseResult( + ReceiveResult(False, True, None, code, reason), + time.monotonic() - started, + ), + ) + + +async def _fresh_session(candidate: Gateway, key: str, normal_model: str) -> tuple[bool, str | None]: + try: + async with websockets.connect( + _proxy_ws_url(candidate, "/v1/responses"), + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) as connection: + events: Final = await _turn(connection, _create(normal_model, f"fresh-{uuid.uuid4().hex}")) + return _completed_text(events) == PEER_TEXT, None + except (ConnectionClosed, asyncio.TimeoutError) as error: + return False, f"{type(error).__name__}: {error}" + + +async def _mid_response( + candidate: Gateway, + key: str, + normal_model: str, + stall_model: str, + peer: ResponsesPeer, +) -> MidResult: + created, turn_error, close = await _mid_session(candidate, key, stall_model) + provider_closed: Final = await asyncio.to_thread(_provider_closed_within, peer, STALL_PROVIDER_MODEL, 5) + fresh_completed, fresh_error = await _fresh_session(candidate, key, normal_model) + return MidResult(created, turn_error, close, provider_closed, fresh_completed, fresh_error) + + +async def _deaf_response( + candidate: Gateway, + key: str, + normal_model: str, + deaf_model: str, + peer: ResponsesPeer, +) -> MidResult: + created, turn_error, close = await _mid_session(candidate, key, deaf_model) + provider_closed: Final = await asyncio.to_thread(_provider_closed_within, peer, DEAF_PROVIDER_MODEL, 30) + fresh_completed, fresh_error = await _fresh_session(candidate, key, normal_model) + return MidResult(created, turn_error, close, provider_closed, fresh_completed, fresh_error) + + +async def _cap_workload( + candidate: Gateway, + key: str, + normal_model: str, + stall_model: str, + deaf_model: str, + peer: ResponsesPeer, +) -> tuple[CloseResult, ActiveResult, MidResult, MidResult]: + idle, active, mid, deaf = await asyncio.gather( + _close_at_limit(candidate, key, normal_model), + _active_session(candidate, key, normal_model, peer), + _mid_response(candidate, key, normal_model, stall_model, peer), + _deaf_response(candidate, key, normal_model, deaf_model, peer), + ) + return idle, active, mid, deaf + + +def _session_config(path: Path, seconds: int) -> Path: + source: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" + content: Final = source.read_text() + updated: Final = content.replace( + "general_settings:\n", + f"general_settings:\n responses_websocket_session_limit_seconds: {seconds}\n", + 1, + ) + path.write_text(updated) + return path + + +def _worker_pids(log: Path) -> frozenset[int]: + return frozenset( + eventually( + lambda: WORKER_PIDS.validate_python(STARTED_WORKER.findall(log.read_text())), + lambda pids: len(pids) == 2, + seconds=30, + ) + ) + + +def _client_port(connection: websockets.ClientConnection) -> int: + return CLIENT_ADDRESS.validate_python(connection.transport.get_extra_info("sockname"))[1] + + +def _holding_worker(workers: frozenset[int], client_port: int) -> int | None: + def holds(pid: int) -> bool: + return any( + connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == client_port + for connection in psutil.Process(pid).net_connections(kind="tcp") + ) + + return next((pid for pid in sorted(workers) if holds(pid)), None) + + +def _settled_on_every_worker(written_at: float) -> None: + eventually( + lambda: time.monotonic() - written_at, + lambda elapsed: elapsed >= WORKER_SYNC_SECONDS, + seconds=WORKER_SYNC_SECONDS + 5, + ) + + +def _config_field(candidate: Gateway, action: str, body: Mapping[str, JsonValue]) -> UpdateResult: + response: Final = candidate.request("POST", f"/config/field/{action}", body) + return UpdateResult(response.status_code, response.text) + + +def _update_limit(candidate: Gateway, value: JsonValue) -> UpdateResult: + return _config_field( + candidate, + "update", + {"field_name": SESSION_LIMIT_FIELD, "field_value": value, "config_type": "general_settings"}, + ) + + +def _delete_limit(candidate: Gateway) -> UpdateResult: + return _config_field(candidate, "delete", {"field_name": SESSION_LIMIT_FIELD, "config_type": "general_settings"}) + + +def _general_settings_row(database_url: str | None) -> dict[str, JsonValue] | None: + rows: Final = read_rows( + 'SELECT param_value FROM "LiteLLM_Config" WHERE param_name = %s', + ("general_settings",), + database_url=database_url, + ) + return _object(rows[0]["param_value"]) if rows else None + + +def _stored_limit(database_url: str) -> JsonValue: + row: Final = _general_settings_row(database_url) + return None if row is None else row.get(SESSION_LIMIT_FIELD) + + +def _health_status(candidate: Gateway) -> int: + try: + return candidate.request("GET", "/health/liveliness").status_code + except httpx.HTTPError: + return 0 + + +async def _update_sequence(candidate: Gateway, values: tuple[int, ...]) -> tuple[UpdateResult, ...]: + if not values: + return () + first: Final = await asyncio.to_thread(_update_limit, candidate, values[0]) + return (first, *await _update_sequence(candidate, values[1:])) + + +async def _open_capped_socket(proxy: str, key: str, path: str, workers: frozenset[int]) -> OpenedSocket: + started: Final = time.monotonic() + connection: Final = await websockets.connect( + f"{proxy}{path}", + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) + holder: Final = await asyncio.to_thread(_holding_worker, workers, _client_port(connection)) + return OpenedSocket(path, holder, connection, started) + + +async def _sockets_on_every_worker( + proxy: str, key: str, workers: frozenset[int], opened: tuple[OpenedSocket, ...] = () +) -> tuple[OpenedSocket, ...]: + holders: Final = frozenset(socket.worker_pid for socket in opened) + enough: Final = len(opened) >= CAPPED_SOCKETS_MINIMUM + if enough and (holders >= workers or len(opened) >= CAPPED_SOCKETS_MAXIMUM): + return opened + path: Final = CAPPED_PATHS[len(opened) % len(CAPPED_PATHS)] + next_socket: Final = await _open_capped_socket(proxy, key, path, workers) + return await _sockets_on_every_worker(proxy, key, workers, (*opened, next_socket)) + + +async def _capped_close(socket: OpenedSocket) -> CappedSocket: + try: + outcome: Final = await _wait_for_close(socket.connection, CAP_SECONDS + 15) + return CappedSocket(socket.path, socket.worker_pid, CloseResult(outcome, time.monotonic() - socket.started)) + finally: + await socket.connection.close() + + +async def _held_turn(connection: websockets.ClientConnection, model: str, accepted_at: float) -> HeldResult: + events, error = await _turn_result(connection, _create(model, f"earlier-{uuid.uuid4().hex}")) + return HeldResult(time.monotonic() - accepted_at, events, error) + + +async def _override_workload(owned: OwnedProxy, key: str, model: str, database_url: str) -> OverrideResults: + candidate: Final = owned.gateway + proxy: Final = _proxy_ws_url(candidate, "") + workers: Final = await asyncio.to_thread(_worker_pids, owned.log) + async with websockets.connect( + f"{proxy}/v1/responses", + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) as earlier: + accepted_at: Final = time.monotonic() + updates: Final = await _update_sequence(candidate, (7200, CAP_SECONDS, CAP_SECONDS)) + written_at: Final = time.monotonic() + stored_after_updates: Final = await asyncio.to_thread(_stored_limit, database_url) + await asyncio.to_thread(_settled_on_every_worker, written_at) + opened: Final = await _sockets_on_every_worker(proxy, key, workers) + capped: Final = await asyncio.gather(*tuple(_capped_close(socket) for socket in opened)) + earlier_result: Final = await _held_turn(earlier, model, accepted_at) + delete: Final = await asyncio.to_thread(_delete_limit, candidate) + deleted_at: Final = time.monotonic() + stored_after_delete: Final = await asyncio.to_thread(_general_settings_row, database_url) + await asyncio.to_thread(_settled_on_every_worker, deleted_at) + restored: Final = await _idle_turn( + proxy, + key, + model, + "/v1/responses", + f"restored-{uuid.uuid4().hex}", + idle_seconds=RESTORED_IDLE_SECONDS, + ) + return OverrideResults( + workers, + updates, + stored_after_updates, + tuple(capped), + earlier_result, + delete, + stored_after_delete, + restored, + ) + + +async def _kill_workload(owned: OwnedProxy, key: str, model: str) -> KillResults: + candidate: Final = owned.gateway + workers: Final = await asyncio.to_thread(_worker_pids, owned.log) + connections: Final = await asyncio.gather( + *tuple( + websockets.connect( + _proxy_ws_url(candidate, "/v1/responses"), + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) + for _ in range(KILLED_POOL_SIZE) + ) + ) + holders: Final = await asyncio.gather( + *tuple(asyncio.to_thread(_holding_worker, workers, _client_port(connection)) for connection in connections) + ) + held_by: Final = MappingProxyType({pid: holders.count(pid) for pid in sorted(workers)}) + victim: Final = max(sorted(workers), key=held_by.__getitem__) + psutil.Process(victim).kill() + victims: Final = tuple( + connection for connection, holder in zip(connections, holders, strict=True) if holder == victim + ) + survivors: Final = tuple( + connection for connection, holder in zip(connections, holders, strict=True) if holder != victim + ) + victim_closes, survivor_turns = await asyncio.gather( + asyncio.gather(*tuple(_wait_for_close(connection, 30) for connection in victims)), + asyncio.gather( + *tuple(_turn_result(connection, _create(model, f"survivor-{uuid.uuid4().hex}")) for connection in survivors) + ), + ) + health: Final = await asyncio.to_thread( + eventually, + lambda: _health_status(candidate), + lambda status_code: status_code == 200, + 30, + ) + fresh: Final = await _chaos_turn(candidate, key, model) + await asyncio.gather(*tuple(connection.close() for connection in survivors)) + return KillResults( + held_by, + victim, + tuple(victim_closes), + tuple(TurnOutcome(events, error) for events, error in survivor_turns), + health, + fresh, + ) + + +def _responses_body(model: str) -> JsonResponse: + message: Final[dict[str, JsonValue]] = { + "type": "message", + "id": "msg_$UNIQUE_ID", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "responses-$UNIQUE_ID", "annotations": []}], + } + return JsonResponse( + content_type="application/json", + body={ + "id": "$UNIQUE_ID", + "object": "response", + "created_at": 1700000000, + "status": "completed", + "model": model, + "output": [message], + "usage": {"input_tokens": 1, "output_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _chat_body(model: str) -> JsonResponse: + return JsonResponse( + content_type="application/json", + body={ + "id": "$UNIQUE_ID", + "object": "chat.completion", + "created": 1700000000, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "chat-$UNIQUE_ID"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + ) + + +def _responses_spec(model: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/responses", {"model": model, "input": f"responses-{uuid.uuid4().hex}"} + + +def _chat_spec(model: str) -> tuple[str, dict[str, JsonValue]]: + return "/v1/chat/completions", { + "model": model, + "messages": [{"role": "user", "content": f"chat-{uuid.uuid4().hex}"}], + } + + +async def _burst_request( + client: httpx.AsyncClient, + key: str, + path: str, + body: dict[str, JsonValue], +) -> BurstResult: + try: + response: Final = await client.post(path, json=body, headers={"Authorization": f"Bearer {key}"}) + try: + parsed: Final = JSON_OBJECT.validate_json(response.content) + except ValidationError as error: + return BurstResult(path, response.status_code, None, str(error)) + return BurstResult(path, response.status_code, parsed, None) + except httpx.HTTPError as error: + return BurstResult(path, None, None, str(error)) + + +async def _chaos_workload( + owned: OwnedProxy, + key: str, + scripted_model: str, + peer_model: str, +) -> ChaosResults: + candidate: Final = owned.gateway + base: Final = str(candidate.client.base_url) + specs: Final = tuple( + _responses_spec(scripted_model) if index % 2 == 0 else _chat_spec(scripted_model) for index in range(20) + ) + connections: Final = await asyncio.gather( + *tuple( + websockets.connect( + _proxy_ws_url(candidate, "/v1/responses"), + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) + for _ in range(30) + ) + ) + async with httpx.AsyncClient(base_url=base, timeout=30, trust_env=False) as client: + results: Final = await asyncio.gather(*tuple(_burst_request(client, key, path, body) for path, body in specs)) + tuple(connection.transport.abort() for connection in connections) + health: Final = candidate.request("GET", "/health/liveliness").status_code + websocket_completed: Final = await _chaos_turn(candidate, key, peer_model) + kill: Final = await _kill_workload(owned, key, peer_model) + return ChaosResults(tuple(results), health, websocket_completed, kill) + + +async def _chaos_turn(candidate: Gateway, key: str, model: str) -> bool: + async with websockets.connect( + _proxy_ws_url(candidate, "/v1/responses"), + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) as connection: + events: Final = await _turn(connection, _create(model, f"chaos-peer-{uuid.uuid4().hex}")) + return _completed_text(events) == PEER_TEXT + + +@pytest.fixture(scope="module") +def cert(tmp_path_factory: pytest.TempPathFactory) -> tuple[Path, Path]: + return write_self_signed_cert(tmp_path_factory.mktemp("responses-ws-session-limit")) + + +@pytest.fixture(scope="module") +def default_results( + cert: tuple[Path, Path], + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[DefaultResults]: + with responses_peer(cert) as peer, gateway_from_environment() as base: + with ( + owned_proxy( + base, + tmp_path_factory.mktemp("responses-ws-default"), + {"SSL_CERT_FILE": str(cert[0])}, + workers=2, + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(model=f"openai/{PROVIDER_MODEL}", api_base=peer.url) + key: Final = scenario.key(models=[model]) + pool, query, rejected, negotiated = asyncio.run(_default_workload(candidate, key, model)) + provider: Final = _snapshots(peer, 10) + yield DefaultResults(pool, query, rejected, negotiated, provider) + + +@pytest.fixture(scope="module") +def cap_results( + cert: tuple[Path, Path], + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[CapResults]: + with responses_peer(cert) as peer, gateway_from_environment() as base: + config: Final = _session_config(tmp_path_factory.mktemp("responses-ws-cap") / "config.yaml", 60) + with ( + owned_proxy( + base, + tmp_path_factory.mktemp("responses-ws-cap-proxy"), + {"SSL_CERT_FILE": str(cert[0])}, + config=config, + workers=2, + ) as candidate, + candidate.scenario() as scenario, + ): + normal: Final = scenario.model(model=f"openai/{PROVIDER_MODEL}", api_base=peer.url) + stall: Final = scenario.model(model=f"openai/{STALL_PROVIDER_MODEL}", api_base=peer.url) + deaf: Final = scenario.model(model=f"openai/{DEAF_PROVIDER_MODEL}", api_base=peer.url) + key: Final = scenario.key(models=[normal, stall, deaf]) + idle, active, mid, deaf_result = asyncio.run(_cap_workload(candidate, key, normal, stall, deaf, peer)) + provider: Final = _available_snapshots(peer) + yield CapResults(idle, active, mid, deaf_result, provider) + + +@pytest.fixture(scope="module") +def invalid_results( + cert: tuple[Path, Path], + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[InvalidResult]: + with responses_peer(cert) as peer, gateway_from_environment() as base: + config: Final = _session_config(tmp_path_factory.mktemp("responses-ws-invalid") / "config.yaml", 30) + with owned_proxy_process( + base, + tmp_path_factory.mktemp("responses-ws-invalid-proxy"), + {"SSL_CERT_FILE": str(cert[0])}, + config=config, + workers=2, + ) as owned: + with owned.gateway.scenario() as scenario: + model: Final = scenario.model(model=f"openai/{PROVIDER_MODEL}", api_base=peer.url) + key: Final = scenario.key(models=[model]) + session: Final = asyncio.run( + _idle_turn( + _proxy_ws_url(owned.gateway, ""), + key, + model, + "/v1/responses", + f"invalid-{uuid.uuid4().hex}", + ) + ) + warning: Final = "invalid general_settings.responses_websocket_session_limit_seconds=30" + yield InvalidResult(session, warning in owned.log.read_text()) + + +@pytest.fixture(scope="module") +def chaos_results( + cert: tuple[Path, Path], + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[ChaosResults]: + with responses_peer(cert) as peer, gateway_from_environment() as base: + with ( + owned_proxy_process( + base, + tmp_path_factory.mktemp("responses-ws-chaos"), + {"SSL_CERT_FILE": str(cert[0])}, + workers=2, + ) as owned, + owned.gateway.scenario() as scenario, + ): + scenario_id: Final = f"responses-chaos-{uuid.uuid4().hex}" + scripted: Final = register_scenario( + scenario_id, + RoutedResponse( + content_type="application/x-routed", + routes={ + "POST /responses": _responses_body("scripted-responses-model"), + "POST /chat/completions": _chat_body("scripted-chat-model"), + }, + ), + control_url=owned.gateway.upstream_url, + ) + scenario.cleanups.callback(delete_scenario, scripted) + scripted_model: Final = scenario.model( + model="openai/scripted-responses-model", + api_base=scripted.api_base(), + ) + peer_model: Final = scenario.model(model=f"openai/{PROVIDER_MODEL}", api_base=peer.url) + key: Final = scenario.key(models=[scripted_model, peer_model]) + yield asyncio.run(_chaos_workload(owned, key, scripted_model, peer_model)) + + +@pytest.fixture(scope="module") +def override_results( + cert: tuple[Path, Path], + tmp_path_factory: pytest.TempPathFactory, +) -> Iterator[OverrideResults]: + with responses_peer(cert) as peer, gateway_from_environment() as base, scratch_database() as database_url: + with owned_proxy_process( + base, + tmp_path_factory.mktemp("responses-ws-override"), + { + "SSL_CERT_FILE": str(cert[0]), + "DATABASE_URL": database_url, + "PROXY_CONFIG_RELOAD_INTERVAL_SECONDS": str(RELOAD_INTERVAL_SECONDS), + }, + remove_environment=("DATABASE_URL_READ_REPLICA",), + workers=2, + ) as owned: + with owned.gateway.scenario() as scenario: + model: Final = scenario.model(model=f"openai/{PROVIDER_MODEL}", api_base=peer.url) + key: Final = scenario.key(models=[model]) + yield asyncio.run(_override_workload(owned, key, model, database_url)) + + +def test_default_config_idle_pool_stays_open_and_completes(default_results: DefaultResults) -> None: + assert all(result.idle.timed_out for result in default_results.pool), default_results.pool + assert all(result.error is None for result in default_results.pool), default_results.pool + assert all(_completed_text(result.events) == PEER_TEXT for result in default_results.pool) + texts: Final = frozenset(result.text for result in default_results.pool) + provider: Final = tuple(snapshot for snapshot in default_results.provider if snapshot.frames) + assert len(provider) == 10, default_results.provider + assert all(snapshot.path == f"/v1/responses?model={PROVIDER_MODEL}" for snapshot in provider) + frames: Final = tuple(snapshot.frames[0] for snapshot in provider) + expected_texts: Final = texts | {default_results.query.text, default_results.subprotocol.text} + assert frozenset(_frame_text(frame) for frame in frames) == expected_texts + assert all(frame.get("model") == PROVIDER_MODEL for frame in frames) + + +def test_default_config_query_model_socket_stays_open_and_completes(default_results: DefaultResults) -> None: + assert default_results.query.idle.timed_out, default_results.query + assert default_results.query.error is None, default_results.query + assert _completed_text(default_results.query.events) == PEER_TEXT + assert any( + default_results.query.text == _frame_text(snapshot.frames[0]) + for snapshot in default_results.provider + if snapshot.frames + ) + + +def test_delayed_first_frame_model_auth_matches_immediate_rejection( + cert: tuple[Path, Path], + tmp_path_factory: pytest.TempPathFactory, +) -> None: + with responses_peer(cert) as peer, gateway_from_environment() as base: + with ( + owned_proxy( + base, + tmp_path_factory.mktemp("responses-ws-auth"), + {"SSL_CERT_FILE": str(cert[0])}, + workers=2, + ) as candidate, + candidate.scenario() as scenario, + ): + model: Final = scenario.model(model=f"openai/{PROVIDER_MODEL}", api_base=peer.url) + key: Final = scenario.key(models=["not-authorized-model"]) + results: Final = asyncio.run(_auth_workload(candidate, key, model, peer)) + assert results.immediate.error is None, results.immediate + assert results.delayed.idle.timed_out, results.delayed + assert results.delayed.error is None, results.delayed + assert results.immediate.frame is not None, results.immediate + assert results.immediate.frame.get("type") == "error", results.immediate + rejection: Final = _object(results.immediate.frame["error"]) + assert rejection.get("type") == "invalid_request_error", results.immediate + assert results.immediate.close_code == 1008, results.immediate + assert results.immediate.close_reason == "Pre-call error", results.immediate + assert results.immediate.frame == results.delayed.frame + assert results.immediate.close_code == results.delayed.close_code + assert results.immediate.close_reason == results.delayed.close_reason + assert results.provider_connections == 0, results + + +def test_session_cap_closes_never_started_socket(cap_results: CapResults) -> None: + assert cap_results.idle.outcome.closed, cap_results.idle + assert cap_results.idle.outcome.close_code == 1000, cap_results.idle + assert cap_results.idle.outcome.close_reason == "Session duration limit reached", cap_results.idle + assert 59 <= cap_results.idle.elapsed <= 75, cap_results.idle + + +def test_session_cap_closes_active_socket_and_provider(cap_results: CapResults) -> None: + assert cap_results.active.turn_error is None, cap_results.active + assert _completed_text(cap_results.active.turn) == PEER_TEXT + assert cap_results.active.close.outcome.closed, cap_results.active + assert cap_results.active.close.outcome.close_code == 1000, cap_results.active + assert cap_results.active.close.outcome.close_reason == "Session duration limit reached", cap_results.active + assert 59 <= cap_results.active.close.elapsed <= 75, cap_results.active + assert cap_results.active.provider_closed, cap_results.active + assert all(snapshot.closed for snapshot in cap_results.provider), cap_results.provider + + +def test_session_cap_closes_mid_response_and_allows_new_session(cap_results: CapResults) -> None: + assert cap_results.mid.created, cap_results.mid + assert cap_results.mid.turn_error is None, cap_results.mid + assert cap_results.mid.close.outcome.closed, cap_results.mid + assert cap_results.mid.close.outcome.close_code == 1000, cap_results.mid + assert cap_results.mid.close.outcome.close_reason == "Session duration limit reached", cap_results.mid + assert 59 <= cap_results.mid.close.elapsed <= 75, cap_results.mid + assert cap_results.mid.provider_closed, cap_results.mid + assert cap_results.mid.fresh_error is None, cap_results.mid + assert cap_results.mid.fresh_completed, cap_results.mid + + +def test_session_cap_closes_client_promptly_when_provider_ignores_close(cap_results: CapResults) -> None: + assert cap_results.deaf.created, cap_results.deaf + assert cap_results.deaf.turn_error is None, cap_results.deaf + assert cap_results.deaf.close.outcome.closed, cap_results.deaf + assert cap_results.deaf.close.outcome.close_code == 1000, cap_results.deaf + assert cap_results.deaf.close.outcome.close_reason == "Session duration limit reached", cap_results.deaf + assert 59 <= cap_results.deaf.close.elapsed <= 63, cap_results.deaf + assert cap_results.deaf.provider_closed, cap_results.deaf + assert cap_results.deaf.fresh_error is None, cap_results.deaf + assert cap_results.deaf.fresh_completed, cap_results.deaf + + +def test_invalid_session_cap_falls_back_to_default(invalid_results: InvalidResult) -> None: + assert invalid_results.session.idle.timed_out, invalid_results.session + assert invalid_results.session.error is None, invalid_results.session + assert _completed_text(invalid_results.session.events) == PEER_TEXT + assert invalid_results.warning_found + + +def _matches_scripted_body(result: BurstResult) -> bool: + if result.body is None: + return False + if result.path == "/v1/responses": + output: Final = _list(result.body["output"]) + responses_message: Final = _object(output[0]) + message_id: Final = _string(responses_message["id"]) + if not message_id.startswith("msg_"): + return False + content: Final = _list(responses_message["content"]) + return _string(_object(content[0])["text"]) == f"responses-{message_id.removeprefix('msg_')}" + identity: Final = _string(result.body["id"]) + choices: Final = _list(result.body["choices"]) + chat_message: Final = _object(_object(choices[0])["message"]) + return _string(chat_message["content"]) == f"chat-{identity}" + + +def test_aborted_idle_pool_preserves_http_health_and_new_websocket(chaos_results: ChaosResults) -> None: + results: Final = chaos_results.burst + assert len(results) == 20 + assert all(result.status_code == 200 and result.error is None for result in results), results + identities: Final = tuple(_string(_object(result.body)["id"]) for result in results if result.body is not None) + assert len(set(identities)) == 20, identities + assert all(_matches_scripted_body(result) for result in results), results + assert chaos_results.health_status == 200 + assert chaos_results.websocket_completed + + +def test_rejected_key_fails_the_handshake_with_403(default_results: DefaultResults) -> None: + assert default_results.rejected == HandshakeResult(403, None), default_results.rejected + + +def test_requested_subprotocol_is_accepted_and_the_turn_completes(default_results: DefaultResults) -> None: + negotiated: Final = default_results.subprotocol + assert negotiated.error is None, negotiated + assert negotiated.negotiated == SUBPROTOCOLS[0], negotiated + assert _completed_text(negotiated.events) == PEER_TEXT + + +def test_db_override_caps_new_sessions_on_every_worker_without_restart(override_results: OverrideResults) -> None: + assert len(override_results.workers) == 2, override_results.workers + capped: Final = override_results.capped + assert CAPPED_SOCKETS_MINIMUM <= len(capped) <= CAPPED_SOCKETS_MAXIMUM, capped + assert tuple(socket.path for socket in capped) == tuple( + CAPPED_PATHS[i % len(CAPPED_PATHS)] for i in range(len(capped)) + ) + assert all(socket.close.outcome.closed and socket.close.outcome.close_code == 1000 for socket in capped), capped + assert all(socket.close.outcome.close_reason == LIMIT_CLOSE_REASON for socket in capped), capped + assert all(CAP_SECONDS - 1 <= socket.close.elapsed <= CAP_SECONDS + 15 for socket in capped), capped + assert frozenset(socket.worker_pid for socket in capped) == override_results.workers, capped + + +def test_db_override_leaves_sessions_accepted_before_it_alone(override_results: OverrideResults) -> None: + earlier: Final = override_results.earlier + assert earlier.error is None, earlier + assert earlier.held_seconds > CAP_SECONDS, earlier + assert _completed_text(earlier.events) == PEER_TEXT + + +def test_db_override_accepts_the_range_bounds_and_repeats(override_results: OverrideResults) -> None: + updates: Final = override_results.updates + assert tuple(update.status_code for update in updates) == (200, 200, 200), updates + assert override_results.stored_after_updates == CAP_SECONDS + + +def test_deleting_the_db_override_restores_the_default(override_results: OverrideResults) -> None: + assert override_results.delete.status_code == 200, override_results.delete + assert override_results.stored_after_delete is not None + assert SESSION_LIMIT_FIELD not in override_results.stored_after_delete + restored: Final = override_results.restored + assert restored.idle.timed_out, restored + assert restored.error is None, restored + assert _completed_text(restored.events) == PEER_TEXT + + +@pytest.mark.parametrize( + ("value", "kind"), + ( + pytest.param(30, "int", id="below-minimum"), + pytest.param(7201, "int", id="above-maximum"), + pytest.param("abc", "str", id="text"), + pytest.param("", "str", id="empty-string"), + pytest.param("x" * 5000, "str", id="five-kilobyte-string"), + pytest.param([], "list", id="list"), + pytest.param({}, "dict", id="object"), + pytest.param(None, "NoneType", id="null"), + ), +) +def test_out_of_range_or_wrong_type_update_is_refused(gateway: Gateway, value: JsonValue, kind: str) -> None: + before: Final = _general_settings_row(None) + refused: Final = _update_limit(gateway, value) + assert refused.status_code == 400, refused + assert json.loads(refused.body) == {"detail": {"error": f"Invalid type of field value= passed in."}} + assert _general_settings_row(None) == before + + +def test_killing_one_worker_drops_only_its_sockets_and_the_proxy_keeps_serving(chaos_results: ChaosResults) -> None: + kill: Final = chaos_results.kill + assert sum(kill.held_by.values()) == KILLED_POOL_SIZE, kill.held_by + assert len(kill.victim_closes) == kill.held_by[kill.victim] >= 1, kill + assert all(close.closed and close.close_code is None for close in kill.victim_closes), kill.victim_closes + assert all(turn.error is None for turn in kill.survivor_turns), kill.survivor_turns + assert all(_completed_text(turn.events) == PEER_TEXT for turn in kill.survivor_turns), kill.survivor_turns + assert kill.health_status == 200 + assert kill.fresh_completed diff --git a/tests/unit/proxy/response_api_endpoints/test_endpoints.py b/tests/unit/proxy/response_api_endpoints/test_endpoints.py index a8aff3ffff0..5dac1d25572 100644 --- a/tests/unit/proxy/response_api_endpoints/test_endpoints.py +++ b/tests/unit/proxy/response_api_endpoints/test_endpoints.py @@ -728,11 +728,17 @@ class TestResponsesWSFirstFrameModelAuth: async def fake_llm_call(): return None + authenticated_models: Final[list[str]] = [] + + async def record_model_auth(*, model: str, **_kwargs: object) -> None: + authenticated_models.append(model) + with ( patch( "litellm.proxy.response_api_endpoints.endpoints._enforce_responses_ws_first_frame_model_auth", new_callable=AsyncMock, - ) as mock_model_auth, + side_effect=record_model_auth, + ), patch( "litellm.proxy.response_api_endpoints.endpoints.ProxyBaseLLMRequestProcessing", return_value=processor, @@ -749,7 +755,7 @@ class TestResponsesWSFirstFrameModelAuth: user_api_key_dict=MagicMock(), ) - mock_model_auth.assert_awaited_once() + assert authenticated_models == ["gpt-4o-mini"] @pytest.mark.asyncio @pytest.mark.parametrize("nested", [False, True]) @@ -957,6 +963,7 @@ class TestResponsesWSFirstFrameModelAuth: request = Request({"type": "http", "method": "POST", "path": "/v1/responses", "headers": []}) user_api_key_dict = MagicMock() llm_router = MagicMock() + empty_settings: Final[dict[str, object]] = {} with ( patch( @@ -973,7 +980,7 @@ class TestResponsesWSFirstFrameModelAuth: ), patch("litellm.proxy.proxy_server.master_key", "sk-test"), patch("litellm.proxy.proxy_server.user_custom_auth", None), - patch("litellm.proxy.proxy_server.general_settings", {}), + patch("litellm.proxy.proxy_server.general_settings", empty_settings), ): await _enforce_responses_ws_first_frame_model_auth( request=request, @@ -1000,7 +1007,7 @@ class TestResponsesWSFirstFrameModelAuth: class TestReadWSModelFromFirstFrameErrors: @pytest.mark.asyncio - async def test_timeout_closes_without_error_frame(self): + async def test_transport_error_first_frame_closes_with_internal_error(self): import asyncio from litellm.proxy.response_api_endpoints.endpoints import ( @@ -1016,7 +1023,7 @@ class TestReadWSModelFromFirstFrameErrors: assert result is None ws.send_text.assert_not_awaited() - ws.close.assert_awaited_once_with(code=1008, reason="Timed out waiting for first message") + ws.close.assert_awaited_once_with(code=1011, reason="Internal server error") @pytest.mark.asyncio async def test_invalid_json_sends_error_and_closes(self): @@ -1108,6 +1115,34 @@ class TestReadWSModelFromFirstFrameErrors: ws.close.assert_not_awaited() + +@pytest.mark.parametrize( + "configured,expected", + [ + (None, 3600.0), + (60, 60.0), + (1200, 1200.0), + (7200, 7200.0), + (59, 3600.0), + (0, 3600.0), + (9000, 3600.0), + ("not-a-number", 3600.0), + ], +) +def test_responses_ws_session_limit_resolution(monkeypatch, configured, expected): + from litellm.proxy.proxy_server import general_settings + from litellm.proxy.response_api_endpoints.endpoints import ( + _resolve_responses_ws_session_limit_seconds, + ) + + if configured is None: + monkeypatch.delitem(general_settings, "responses_websocket_session_limit_seconds", raising=False) + else: + monkeypatch.setitem(general_settings, "responses_websocket_session_limit_seconds", configured) + + assert _resolve_responses_ws_session_limit_seconds() == expected + + class TestManagedResponsesSameProvider: def _handler(self, model, custom_llm_provider=None): from litellm.responses.streaming_iterator import ( diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 17d1e494fe6..5060fc3575e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30330,6 +30330,12 @@ export interface components { * @description When set to True, rejects requests that contain client-side 'metadata.tags' to prevent users from influencing budgets by sending different tags. Tags can only be inherited from the API key metadata. */ reject_clientside_metadata_tags?: boolean | null; + /** + * Responses Websocket Session Limit Seconds + * @description Maximum lifetime in seconds of a Responses API WebSocket session, measured from connection accept and covering the idle wait for the first response.create frame. Defaults to 3600, matching OpenAI's documented 60-minute WebSocket connection limit. Must be between 60 and 7200 seconds. + * @default 3600 + */ + responses_websocket_session_limit_seconds: number; /** @description Spreads the proxy's scheduled background jobs (spend flushes, budget resets, config reloads, exports) across a window instead of firing them together on every replica. On by default; set to tune the window, pin a job, or turn it off. */ scheduled_job_stagger?: components["schemas"]["ScheduledJobStaggerSettings"] | null; /** From e4d9a591dddbf09daad8c4e4bbe5220067288fbe Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:26:50 -0700 Subject: [PATCH 06/65] fix(lens-ui): open traces on the top-level step and stop showing model names as input (#45254) * fix(lens-ui): open a trace on its top-level step instead of the first failed one Co-Authored-By: Claude Opus 5.5 * test(lens-ui): pin that failed runs open on the root step Co-Authored-By: Claude Opus 5.5 * refactor(lens-ui): drop the unused firstErrorSpan helper Co-Authored-By: Claude Opus 5.5 * test(lens-ui): stop testing the removed firstErrorSpan helper Co-Authored-By: Claude Opus 5.5 * fix(lens-ui): stop showing the model name as a run's input when none was recorded Co-Authored-By: Claude Opus 5.5 * test(lens-ui): cover the trace list row with no recorded input Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../lens/traces/detail/run/RunView.test.tsx | 58 ++++++------------- .../lens/traces/detail/run/useRunTree.ts | 26 ++------- .../traces/list/AgentTracesTable.test.tsx | 20 +++++++ .../lens/traces/list/AgentTracesTable.tsx | 11 ++-- .../src/components/lens/traces/utils.test.ts | 10 +--- .../src/components/lens/traces/utils.ts | 10 ---- 6 files changed, 50 insertions(+), 85 deletions(-) diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.test.tsx index f6bc52e1fe9..f25517e2909 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.test.tsx @@ -113,19 +113,12 @@ describe("RunView", () => { expect(screen.getByRole("banner")).toHaveTextContent(`Step errors ${swarm.summary.error_count}`); }); - it("opens a failed run on its first failed span", async () => { - renderRun(swarm); - - const pane = await screen.findByTestId("detail-pane"); - const { selectedId } = initialRunSelection(swarm); - expect(selectedId).not.toBe(rootSpanId(swarm)); - expect(pane).toHaveAttribute("data-row-id", selectedId); - expect(swarm.spans.find((s) => s.span_id === selectedId)?.status).toBe("error"); - }); - - it("opens a healthy run on the root span", async () => { - renderRun(research); - expect(await screen.findByTestId("detail-pane")).toHaveAttribute("data-row-id", rootSpanId(research)); + it.each([ + ["failed", swarm], + ["healthy", research], + ])("opens a %s run on the root span", async (_label, trace) => { + renderRun(trace); + expect(await screen.findByTestId("detail-pane")).toHaveAttribute("data-row-id", rootSpanId(trace)); }); it("inside the drawer moves spans with the arrow keys and leaves J / K to switch runs", async () => { @@ -191,7 +184,7 @@ describe("RunView", () => { expect(screen.getByRole("heading", { level: 1 })).toHaveTextContent(traceDisplayName(swarm.summary)), ); expect(screen.getByTestId("run-view")).toHaveAttribute("aria-busy", "false"); - expect(screen.getByTestId("detail-pane")).toHaveAttribute("data-row-id", initialRunSelection(swarm).selectedId); + expect(screen.getByTestId("detail-pane")).toHaveAttribute("data-row-id", rootSpanId(swarm)); }); it("moves the selection with J / K and closes the detail pane with Esc", async () => { @@ -712,36 +705,19 @@ describe("initialRunSelection", () => { return { ...base, ...defaults, ...over }; }; - it("lands on a visible failure, never on a framework span the tree hides", () => { - const hiddenFields: Partial = { span_id: "mw", type: "framework", status: "error", start_offset_ms: 1 }; - const toolFields: Partial = { span_id: "tool", type: "tool", status: "error", start_offset_ms: 5 }; - const hiddenFailure = child(hiddenFields); - const toolFailure = child(toolFields); - const trace = { ...research, spans: [base, hiddenFailure, toolFailure] }; - expect(initialRunSelection(trace).selectedId).toBe("tool"); - }); - - it("folds other agent branches while revealing the failed step", () => { - const first = child({ span_id: "first", type: "agent" }); - const second = child({ span_id: "second", type: "agent" }); - const failureFields: Partial = { span_id: "failed", parent_span_id: "second", type: "tool", status: "error" }; - const failure = child(failureFields); - const { selectedId, state } = initialRunSelection({ ...research, spans: [base, first, second, failure] }); - expect(selectedId).toBe("failed"); - expect(state.collapsedSpanIds.has("first")).toBe(true); - expect(state.collapsedSpanIds.has("second")).toBe(false); - }); - - it("falls back to the nearest visible ancestor when only a hidden span failed", () => { - const agent = child({ span_id: "agent", type: "agent", name: "researcher" }); - const hiddenFields: Partial = { span_id: "mw", parent_span_id: "agent", type: "framework", status: "error" }; - const hiddenFailure = child(hiddenFields); - const trace = { ...research, spans: [base, agent, hiddenFailure] }; - expect(initialRunSelection(trace).selectedId).toBe("agent"); + it("opens on the root and folds nested agent branches even when a step failed", () => { + const agentFields: Partial = { span_id: "agent", type: "agent" }; + const failureFields: Partial = { span_id: "failed", parent_span_id: "agent", type: "tool", status: "error" }; + const { selectedId, state } = initialRunSelection({ + ...research, + spans: [base, child(agentFields), child(failureFields)], + }); + expect(selectedId).toBe(base.span_id); + expect(state.collapsedSpanIds.has("agent")).toBe(true); }); }); -it("opens a cited span instead of the default failed span", () => { +it("opens a cited span instead of the root", () => { const cited = research.spans.find((span) => span.parent_span_id !== null)!; expect(initialRunSelection(research, cited.span_id).selectedId).toBe(cited.span_id); expect(initialRunSelection(research, "missing")).toEqual(initialRunSelection(research)); diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/run/useRunTree.ts b/ui/litellm-dashboard/src/components/lens/traces/detail/run/useRunTree.ts index d9eaf9a18de..2b956be765d 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/run/useRunTree.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/run/useRunTree.ts @@ -3,15 +3,7 @@ import { useCallback, useMemo, useState } from "react"; import type { RunSelection } from "../../routing"; import type { SpanTreeState, TreeRow } from "../../tree"; import type { Trace } from "../../types"; -import { - buildTreeRows, - findTraceSteps, - firstErrorSpan, - GROUP_PAGE_SIZE, - isFrameworkSpan, - nearestVisibleSpanId, - revealSpanInState, -} from "../../utils"; +import { buildTreeRows, findTraceSteps, GROUP_PAGE_SIZE, nearestVisibleSpanId, revealSpanInState } from "../../utils"; const INITIAL_STATE: SpanTreeState = { hideFramework: true, @@ -20,7 +12,7 @@ const INITIAL_STATE: SpanTreeState = { groupRevealCounts: {}, }; -/** First failed span if the run has errors (with its tree path opened), otherwise the root agent. */ +/** The linked step when one is given, otherwise the root agent with nested agent branches folded. */ export function initialRunSelection( trace: Trace, initialSpanId?: string, @@ -30,22 +22,14 @@ export function initialRunSelection( const state = revealSpanInState(trace.spans, { ...INITIAL_STATE, hideFramework: false }, selectedId); return { selectedId, state }; } - const initialState = { + const root = trace.spans.find((span) => span.parent_span_id === null); + const state = { ...INITIAL_STATE, collapsedSpanIds: new Set( trace.spans.filter((span) => span.type === "agent" && span.parent_span_id !== null).map((span) => span.span_id), ), }; - const failed = firstErrorSpan(trace.spans); - if (!failed || failed.parent_span_id === null) { - const root = trace.spans.find((s) => s.parent_span_id === null); - return { selectedId: root?.span_id ?? "", state: initialState }; - } - const visibleFailure = trace.spans - .filter((s) => s.status === "error" && s.parent_span_id !== null && !isFrameworkSpan(s)) - .sort((a, b) => a.start_offset_ms - b.start_offset_ms)[0]; - const selectedId = visibleFailure?.span_id ?? nearestVisibleSpanId(trace.spans, failed.span_id, true); - return { selectedId, state: revealSpanInState(trace.spans, initialState, selectedId) }; + return { selectedId: root?.span_id ?? "", state }; } const toggle = (set: ReadonlySet, id: string): Set => { diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.test.tsx index b2a8111656f..d6c75700f62 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.test.tsx @@ -183,6 +183,26 @@ describe("AgentTracesTable input cell", () => { ); expect(screen.getByText("CURRENT USER REQUEST: - add feedback to Lens")).toBeInTheDocument(); }); + + it("says no input was recorded instead of passing the run's model name off as its input", () => { + const template = (traceList as TracePage).data[0] as TraceSummary; + const run = { ...template, name: "chat openai/gpt-6-astra", input_preview: "" }; + renderWithProviders( + inList( + , + ), + ); + expect(screen.getByText("No input recorded")).toBeInTheDocument(); + expect(screen.queryByText(run.name)).not.toBeInTheDocument(); + }); }); describe("AgentTracesTable column picker", () => { diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.tsx index fa4051a34a4..cb39eea63e0 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.tsx @@ -20,7 +20,7 @@ import { SignalPills } from "../ui/SignalPills"; import { FrameworkLogo, traceFramework } from "../ui/TraceFramework"; import type { TraceSummary } from "../types"; import { traceRefOf } from "../routing"; -import { fmtMs, previewText, traceDisplayName, traceAgentNames } from "../utils"; +import { fmtMs, previewText, traceAgentNames } from "../utils"; interface AgentTracesTableProps { traces: TraceSummary[]; @@ -103,11 +103,14 @@ function AgentCell({ run }: { run: TraceSummary }) { } function InputCell({ run }: { run: TraceSummary }) { + const input = singleLine(previewText(run.input_preview)); return (
- - {singleLine(previewText(run.input_preview)) || traceDisplayName(run)} - + {input ? ( + {input} + ) : ( + No input recorded + )} {run.resolution_limited && ( { describe("revealSpanInState", () => { it("opens the path to a span nested in a folded group so the view can land on it", () => { - const failed = firstErrorSpan(swarm.spans) as Span; + const failed = swarm.spans.find((s) => s.status === "error" && s.parent_span_id !== null) as Span; const state = revealSpanInState(swarm.spans, STATE, failed.span_id); const rows = buildTreeRows(swarm.spans, state); expect(rows.some((r) => r.id === failed.span_id)).toBe(true); @@ -345,13 +344,6 @@ describe("errorSource", () => { }); describe("payload helpers", () => { - it("finds the earliest failing non-root span", () => { - const failed = firstErrorSpan(swarm.spans); - expect(failed?.status).toBe("error"); - expect(failed?.parent_span_id).not.toBeNull(); - expect(firstErrorSpan(deepAgent.spans)).toBeNull(); - }); - it("parses llm message payloads and rejects non-message JSON", () => { expect(parseMessages('[{"role":"user","content":"hi"}]')).toEqual([{ role: "user", content: "hi" }]); expect(parseMessages('{"file_path":"/tmp/x"}')).toBeNull(); diff --git a/ui/litellm-dashboard/src/components/lens/traces/utils.ts b/ui/litellm-dashboard/src/components/lens/traces/utils.ts index 63ae58486cc..d4e5abf1cfc 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/utils.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/utils.ts @@ -254,16 +254,6 @@ export function nearestVisibleSpanId(spans: readonly Span[], spanId: string, hid return current?.span_id ?? spanId; } -/* ------------------------------------------------------------------ */ -/* Trace-level rollups */ -/* ------------------------------------------------------------------ */ - -/** Earliest failing non-root span (the root just echoes its children), else the root. */ -export function firstErrorSpan(spans: readonly Span[]): Span | null { - const failed = spans.filter((s) => s.status === "error").sort(byStart); - return failed.find((s) => s.parent_span_id !== null) ?? failed[0] ?? null; -} - /* ------------------------------------------------------------------ */ /* Span detail payloads */ /* ------------------------------------------------------------------ */ From 0e09e991468d174a528a8c3e7a90a3616d4bbb27 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 19:45:45 -0700 Subject: [PATCH 07/65] refactor(decisions): dispatch /v1/decisions through provider configs and the shared HTTP handler (#45130) * refactor(decisions): dispatch /v1/decisions through provider configs and the shared HTTP handler Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): map OpenRouter connection failures to APIConnectionError and drop explanatory docstrings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(decisions): keep APIConnectionError for unreachable deployments on every provider `_map_upstream_exception` now maps a status `_handle_error` synthesized for a non-HTTP failure to `APIConnectionError` before `exception_type` runs, so a Perplexity (openai-compatible) deployment that cannot be reached no longer answers `InternalServerError` where main answers `APIConnectionError`. Audit cells: the unreachable-deployment integration cell is parametrized over the five providers, OpenRouter connection failures are pinned on chat, stream, embeddings, decisions and the SDK, a deployment whose provider has no decisions config gets the 400 naming every supported provider, and a unit cell pins the mapper alone. The first docstring sentence of the five translation bases is dropped. --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/__init__.py | 15 + litellm/_lazy_imports_registry.py | 13 + litellm/decisions/main.py | 310 +++++++----------- .../exception_mapping_utils.py | 5 +- litellm/llms/base_llm/decisions/__init__.py | 4 +- .../llms/base_llm/decisions/transformation.py | 121 +++++-- .../cloudflare/decisions/transformation.py | 36 +- litellm/llms/custom_httpx/llm_http_handler.py | 100 ++++++ .../openrouter/decisions/transformation.py | 16 +- .../perplexity/decisions/transformation.py | 16 +- .../decisions/transformation.py | 14 +- .../llms/typesafe/decisions/transformation.py | 15 +- litellm/types/decisions.py | 3 + litellm/utils.py | 15 + .../providers/test_decisions_wire.py | 86 ++++- tests/integration/proxy_config.yaml | 24 ++ .../translation/decisions/__init__.py | 0 .../translation/decisions/bases/__init__.py | 0 .../translation/decisions/bases/cloudflare.py | 84 +++++ .../translation/decisions/bases/openrouter.py | 83 +++++ .../translation/decisions/bases/perplexity.py | 79 +++++ .../decisions/bases/strands_decider.py | 35 ++ .../translation/decisions/bases/typesafe.py | 79 +++++ .../translation/decisions/basic/__init__.py | 0 .../basic/test_decisions_basic_cloudflare.py | 11 + .../basic/test_decisions_basic_openrouter.py | 11 + .../basic/test_decisions_basic_perplexity.py | 11 + .../test_decisions_basic_strands_decider.py | 11 + .../basic/test_decisions_basic_typesafe.py | 11 + tests/unit/decisions/test_main.py | 92 +++++- .../test_exception_mapping_utils.py | 28 ++ 31 files changed, 1034 insertions(+), 294 deletions(-) create mode 100644 tests/integration/translation/decisions/__init__.py create mode 100644 tests/integration/translation/decisions/bases/__init__.py create mode 100644 tests/integration/translation/decisions/bases/cloudflare.py create mode 100644 tests/integration/translation/decisions/bases/openrouter.py create mode 100644 tests/integration/translation/decisions/bases/perplexity.py create mode 100644 tests/integration/translation/decisions/bases/strands_decider.py create mode 100644 tests/integration/translation/decisions/bases/typesafe.py create mode 100644 tests/integration/translation/decisions/basic/__init__.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py create mode 100644 tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py diff --git a/litellm/__init__.py b/litellm/__init__.py index c0dc61e2911..4cc1399ee9e 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1667,6 +1667,21 @@ if TYPE_CHECKING: from .llms.hosted_vllm.rerank.transformation import ( HostedVLLMRerankConfig as HostedVLLMRerankConfig, ) + from .llms.perplexity.decisions.transformation import ( + PerplexityDecisionsConfig as PerplexityDecisionsConfig, + ) + from .llms.typesafe.decisions.transformation import ( + TypeSafeDecisionsConfig as TypeSafeDecisionsConfig, + ) + from .llms.openrouter.decisions.transformation import ( + OpenRouterDecisionsConfig as OpenRouterDecisionsConfig, + ) + from .llms.cloudflare.decisions.transformation import ( + CloudflareDecisionsConfig as CloudflareDecisionsConfig, + ) + from .llms.strands_decider.decisions.transformation import ( + StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig, + ) from .llms.nvidia_nim.rerank.transformation import ( NvidiaNimRerankConfig as NvidiaNimRerankConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 78f5e4ef04b..f95679684b8 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -156,6 +156,11 @@ LLM_CONFIG_NAMES: Final = ( "ScalewayRerankConfig", "DeepinfraRerankConfig", "HostedVLLMRerankConfig", + "PerplexityDecisionsConfig", + "TypeSafeDecisionsConfig", + "OpenRouterDecisionsConfig", + "CloudflareDecisionsConfig", + "StrandsDeciderDecisionsConfig", "NvidiaNimRerankConfig", "NvidiaNimRankingConfig", "VertexAIRerankConfig", @@ -707,6 +712,14 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.hosted_vllm.rerank.transformation", "HostedVLLMRerankConfig", ), + "PerplexityDecisionsConfig": (".llms.perplexity.decisions.transformation", "PerplexityDecisionsConfig"), + "TypeSafeDecisionsConfig": (".llms.typesafe.decisions.transformation", "TypeSafeDecisionsConfig"), + "OpenRouterDecisionsConfig": (".llms.openrouter.decisions.transformation", "OpenRouterDecisionsConfig"), + "CloudflareDecisionsConfig": (".llms.cloudflare.decisions.transformation", "CloudflareDecisionsConfig"), + "StrandsDeciderDecisionsConfig": ( + ".llms.strands_decider.decisions.transformation", + "StrandsDeciderDecisionsConfig", + ), "NvidiaNimRerankConfig": ( ".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig", diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index 8304c52a886..ecb2669db9c 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,6 +1,5 @@ from collections.abc import Mapping from dataclasses import dataclass, field -from types import MappingProxyType from typing import Final import httpx @@ -8,107 +7,87 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -from litellm.llms.base_llm.decisions.transformation import DecisionsProviderConfig -from litellm.llms.cloudflare.decisions.transformation import CLOUDFLARE_DECISIONS_ENDPOINT -from litellm.llms.custom_httpx.http_handler import get_async_httpx_client, get_httpx_client -from litellm.llms.openrouter.decisions.transformation import OPENROUTER_DECISIONS_ENDPOINT -from litellm.llms.perplexity.decisions.transformation import PERPLEXITY_DECISIONS_ENDPOINT -from litellm.llms.strands_decider.decisions.transformation import STRANDS_DECIDER_DECISIONS_ENDPOINT -from litellm.llms.typesafe.decisions.transformation import TYPESAFE_DECISIONS_ENDPOINT -from litellm.secret_managers.main import get_secret_str +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler from litellm.types.decisions import ( DecisionQuestion, DecisionsJSON, DecisionsRequest, DecisionsResponse, ) -from litellm.utils import client - -DECISIONS_ENDPOINTS: Final[Mapping[str, DecisionsProviderConfig]] = MappingProxyType( - { - "perplexity": PERPLEXITY_DECISIONS_ENDPOINT, - "typesafe": TYPESAFE_DECISIONS_ENDPOINT, - "openrouter": OPENROUTER_DECISIONS_ENDPOINT, - "cloudflare": CLOUDFLARE_DECISIONS_ENDPOINT, - "strands_decider": STRANDS_DECIDER_DECISIONS_ENDPOINT, - } -) +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager, client _DECISIONS_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest) -_DECISIONS_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) -_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) +_HANDLER: Final = BaseLLMHTTPHandler() @dataclass(frozen=True, slots=True, repr=False) -class _PreparedDecisionsRequest: - config: DecisionsProviderConfig - provider: str - upstream_model: str - url: str +class _DecisionsCall: + model: str + custom_llm_provider: str + provider_config: BaseDecisionsConfig + request: DecisionsRequest + api_base: str api_key: str | None = field(repr=False) - headers: Mapping[str, str] = field(repr=False) - body: Mapping[str, object] = field(repr=False) + logging_obj: LiteLLMLoggingObj | None + headers: Mapping[str, str] + timeout: float | httpx.Timeout | None -def _resolve_provider_model(model: str, custom_llm_provider: str | None) -> tuple[str, str]: - provider: Final = model.partition("/")[0] if custom_llm_provider is None else custom_llm_provider - if provider not in DECISIONS_ENDPOINTS: - supported: Final = ", ".join(DECISIONS_ENDPOINTS) - raise litellm.BadRequestError( - message=f"Unknown Decisions provider '{provider}'. Supported providers: {supported}", - model=model, - llm_provider=provider, - ) - upstream_model: Final = model.removeprefix(f"{provider}/") - if not upstream_model: - raise litellm.BadRequestError( - message="A model name is required for the Decisions API", - model=model, - llm_provider=provider, - ) - return provider, upstream_model - - -def _resolve_api_key( - *, - provider: str, - model: str, - endpoint: DecisionsProviderConfig, - api_key: str | None, -) -> str | None: - if api_key is not None: - return api_key - - server_api_key: Final = next( - (key for key in (get_secret_str(name) for name in endpoint.api_key_env) if key), - None, +def _supported_providers() -> tuple[str, ...]: + return tuple( + provider.value + for provider in LlmProviders + if ProviderConfigManager.get_provider_decisions_config(model="", provider=provider) is not None ) - if server_api_key is None: - if not endpoint.api_key_required: - return None - raise litellm.AuthenticationError( - message=f"Missing API key for Decisions provider '{provider}'", + + +def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfig: + provider: Final = next((member for member in LlmProviders if member.value == custom_llm_provider), None) + provider_config: Final = ( + None if provider is None else ProviderConfigManager.get_provider_decisions_config(model, provider) + ) + if provider_config is None: + supported: Final = ", ".join(_supported_providers()) + raise litellm.BadRequestError( + message=f"Unknown Decisions provider '{custom_llm_provider}'. Supported providers: {supported}", model=model, - llm_provider=provider, + llm_provider=custom_llm_provider, ) - - return server_api_key + return provider_config -def _prepare_request( +def _prepare_call( *, model: str, state: DecisionsJSON, questions: Mapping[str, DecisionQuestion | Mapping[str, object]], api_key: str | None, api_base: str | None, + timeout: float | httpx.Timeout | None, custom_llm_provider: str | None, extra_headers: Mapping[str, str] | None, -) -> _PreparedDecisionsRequest: - provider, upstream_model = _resolve_provider_model(model, custom_llm_provider) + kwargs: Mapping[str, object], +) -> _DecisionsCall: + upstream_model, provider, dynamic_api_key, dynamic_api_base = litellm.get_llm_provider( + model=model, + custom_llm_provider=custom_llm_provider, + api_base=api_base, + api_key=api_key, + ) + provider_config: Final = _provider_config(upstream_model, provider) + canonical_model: Final = provider_config.canonical_model(upstream_model) + if not upstream_model: + raise litellm.BadRequestError( + message="A model name is required for the Decisions API", + model=model, + llm_provider=provider, + ) try: - validated_request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python( - {"model": model, "state": state, "questions": questions} + request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python( + {"model": canonical_model, "state": state, "questions": questions} ) except ValidationError as error: raise litellm.BadRequestError( @@ -117,109 +96,56 @@ def _prepare_request( llm_provider=provider, ) from error - endpoint: Final = DECISIONS_ENDPOINTS[provider] - env_api_base: Final = get_secret_str(endpoint.api_base_env) - default_api_base: Final = endpoint.default_api_base() - resolved_api_base: Final = api_base or env_api_base or default_api_base + resolved_api_base: Final = provider_config.resolve_api_base(dynamic_api_base or api_base) if resolved_api_base is None: raise litellm.BadRequestError( - message=endpoint.missing_api_base_message(provider), + message=provider_config.missing_api_base_message(provider), + model=model, + llm_provider=provider, + ) + resolved_api_key: Final = provider_config.resolve_api_key(dynamic_api_key or api_key) + if resolved_api_key is None and provider_config.api_key_required: + raise litellm.AuthenticationError( + message=f"Missing API key for Decisions provider '{provider}'", model=model, llm_provider=provider, ) - resolved_api_key: Final = _resolve_api_key( - provider=provider, - model=model, - endpoint=endpoint, - api_key=api_key, - ) - - canonical_model: Final = endpoint.canonical_model(upstream_model) - outbound_headers: Final = MappingProxyType( - { - **{ - name: value - for name, value in (extra_headers or {}).items() - if name.lower() not in {"authorization", "content-type"} - }, - **({"Authorization": f"Bearer {resolved_api_key}"} if resolved_api_key is not None else {}), - "Content-Type": "application/json", - } - ) - body: Final = MappingProxyType( - { - "model": endpoint.request_model(canonical_model), - "state": validated_request.state, - "questions": { - name: question.model_dump(mode="json", exclude_none=True) - for name, question in validated_request.questions.items() - }, - } - ) - return _PreparedDecisionsRequest( - config=endpoint, - provider=provider, - upstream_model=canonical_model, - url=endpoint.endpoint_url(resolved_api_base, canonical_model), - api_key=resolved_api_key, - headers=outbound_headers, - body=body, - ) - - -def _log_request( - prepared: _PreparedDecisionsRequest, - kwargs: Mapping[str, object], -) -> LiteLLMLoggingObj | None: logging_obj: Final = kwargs.get("litellm_logging_obj") - if not isinstance(logging_obj, LiteLLMLoggingObj): - return None - logging_obj.update_from_kwargs( - kwargs=dict(kwargs), - model=prepared.upstream_model, - litellm_params={ - "litellm_call_id": kwargs.get("litellm_call_id"), - "api_base": prepared.url, - }, - custom_llm_provider=prepared.provider, + if isinstance(logging_obj, LiteLLMLoggingObj): + logging_obj.update_from_kwargs( + kwargs=dict(kwargs), + model=canonical_model, + litellm_params={ + "litellm_call_id": kwargs.get("litellm_call_id"), + "api_base": provider_config.get_complete_url(resolved_api_base, canonical_model), + }, + custom_llm_provider=provider, + ) + return _DecisionsCall( + model=canonical_model, + custom_llm_provider=provider, + provider_config=provider_config, + request=request, + api_base=resolved_api_base, + api_key=resolved_api_key, + logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, + headers=extra_headers or {}, + timeout=timeout, ) - request_body: Final = dict(prepared.body) - request_headers: Final = dict(prepared.headers) - logging_obj.pre_call( - input=request_body, - api_key=prepared.api_key, - model=prepared.upstream_model, - additional_args={ - "api_base": prepared.url, - "complete_input_dict": request_body, - "headers": request_headers, - }, - ) - return logging_obj -def _parse_response( - response: httpx.Response, - prepared: _PreparedDecisionsRequest, -) -> DecisionsResponse: - response.raise_for_status() - payload: Final[object] = _DECISIONS_PAYLOAD_ADAPTER.validate_json(response.content) - result: Final = _DECISIONS_RESPONSE_ADAPTER.validate_python(prepared.config.unwrap_response(payload)) - result.hidden_params.update( - { - "model": f"{prepared.provider}/{prepared.upstream_model}", - "custom_llm_provider": prepared.provider, - "provider_response_model": f"{prepared.provider}/{prepared.upstream_model}", - } - ) - return result - - -def _map_upstream_exception(error: Exception, prepared: _PreparedDecisionsRequest) -> Exception: +def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception: + if isinstance(error, BaseLLMException) and error.status_code_is_synthesized: + provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception" + return litellm.APIConnectionError( + message=f"{provider_label} - {error.message}", + llm_provider=call.custom_llm_provider, + model=f"{call.custom_llm_provider}/{call.model}", + ) return litellm.exception_type( - model=f"{prepared.provider}/{prepared.upstream_model}", - custom_llm_provider=prepared.provider, + model=f"{call.custom_llm_provider}/{call.model}", + custom_llm_provider=call.custom_llm_provider, original_exception=error, ) @@ -236,28 +162,31 @@ async def adecisions( extra_headers: Mapping[str, str] | None = None, **kwargs: object, ) -> DecisionsResponse: - prepared: Final = _prepare_request( + call: Final = _prepare_call( model=model, state=state, questions=questions, api_key=api_key, api_base=api_base, + timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + kwargs=kwargs, ) - logging_obj: Final = _log_request(prepared, kwargs) try: - handler: Final = get_async_httpx_client(llm_provider=prepared.provider) - response: Final = await handler.post( - prepared.url, - json=dict(prepared.body), - headers=dict(prepared.headers), - timeout=timeout, - logging_obj=logging_obj, + return await _HANDLER.adecisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.request, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, ) - return _parse_response(response=response, prepared=prepared) except Exception as error: - raise _map_upstream_exception(error, prepared) from error + raise _map_upstream_exception(error, call) from error @client @@ -272,28 +201,31 @@ def decisions( extra_headers: Mapping[str, str] | None = None, **kwargs: object, ) -> DecisionsResponse: - prepared: Final = _prepare_request( + call: Final = _prepare_call( model=model, state=state, questions=questions, api_key=api_key, api_base=api_base, + timeout=timeout, custom_llm_provider=custom_llm_provider, extra_headers=extra_headers, + kwargs=kwargs, ) - logging_obj: Final = _log_request(prepared, kwargs) try: - handler: Final = get_httpx_client() - response: Final = handler.post( - prepared.url, - json=dict(prepared.body), - headers=dict(prepared.headers), - timeout=timeout, - logging_obj=logging_obj, + return _HANDLER.decisions( + model=call.model, + custom_llm_provider=call.custom_llm_provider, + logging_obj=call.logging_obj, + provider_config=call.provider_config, + request=call.request, + api_base=call.api_base, + api_key=call.api_key, + headers=call.headers, + timeout=call.timeout, ) - return _parse_response(response=response, prepared=prepared) except Exception as error: - raise _map_upstream_exception(error, prepared) from error + raise _map_upstream_exception(error, call) from error -__all__ = ["DECISIONS_ENDPOINTS", "adecisions", "decisions"] +__all__ = ["adecisions", "decisions"] diff --git a/litellm/litellm_core_utils/exception_mapping_utils.py b/litellm/litellm_core_utils/exception_mapping_utils.py index 7fb28fb418b..b5afa962cdc 100644 --- a/litellm/litellm_core_utils/exception_mapping_utils.py +++ b/litellm/litellm_core_utils/exception_mapping_utils.py @@ -2147,7 +2147,10 @@ def _map_openrouter_exception( exception_provider: str, extra_information: str, ) -> None: - if hasattr(original_exception, "status_code"): + received_status: Final = hasattr(original_exception, "status_code") and not getattr( + original_exception, "status_code_is_synthesized", False + ) + if received_status: if original_exception.status_code == 400: raise BadRequestError( message=f"{exception_provider} - {error_str}", diff --git a/litellm/llms/base_llm/decisions/__init__.py b/litellm/llms/base_llm/decisions/__init__.py index c18ac9b00f2..313e57e9762 100644 --- a/litellm/llms/base_llm/decisions/__init__.py +++ b/litellm/llms/base_llm/decisions/__init__.py @@ -1,3 +1,3 @@ -from .transformation import DecisionsProviderConfig, JevCompatibleDecisionsEndpoint +from .transformation import BaseDecisionsConfig -__all__ = ["DecisionsProviderConfig", "JevCompatibleDecisionsEndpoint"] +__all__ = ["BaseDecisionsConfig"] diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index d4fcea24793..36739555cea 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,20 +1,40 @@ -from dataclasses import dataclass -from typing import Protocol +from abc import ABC +from collections.abc import Mapping +from typing import Final + +import httpx +from pydantic import TypeAdapter, ValidationError + +from litellm.llms.base_llm.chat.transformation import BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.decisions import DecisionsRequest, DecisionsResponse + +PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) +_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) +_RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) -@dataclass(frozen=True, slots=True) -class JevCompatibleDecisionsEndpoint: - default_api_base_value: str | None - path: str - api_key_env: tuple[str, ...] - api_base_env: str +class BaseDecisionsConfig(ABC): + path: str = "/v1/systemone" + api_key_env: tuple[str, ...] = () + api_base_env: tuple[str, ...] = () api_key_required: bool = True - def default_api_base(self) -> str | None: - return self.default_api_base_value + def get_default_api_base(self) -> str | None: + return None - def missing_api_base_message(self, provider: str) -> str: - return f"api_base is required for Decisions provider '{provider}'" + def missing_api_base_message(self, custom_llm_provider: str) -> str: + return f"api_base is required for Decisions provider '{custom_llm_provider}'" + + def resolve_api_base(self, api_base: str | None) -> str | None: + return api_base or self._first_secret(self.api_base_env) or self.get_default_api_base() + + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key or self._first_secret(self.api_key_env) + + @staticmethod + def _first_secret(names: tuple[str, ...]) -> str | None: + return next((value for value in (get_secret_str(name) for name in names) if value), None) def canonical_model(self, model: str) -> str: return model @@ -22,31 +42,66 @@ class JevCompatibleDecisionsEndpoint: def request_model(self, model: str) -> str: return model - def endpoint_url(self, api_base: str, model: str) -> str: + def validate_environment(self, headers: Mapping[str, str], model: str, api_key: str | None) -> dict[str, str]: + return { + **{name: value for name, value in headers.items() if name.lower() not in _RESERVED_HEADERS}, + **({"Authorization": f"Bearer {api_key}"} if api_key is not None else {}), + "Content-Type": "application/json", + } + + def get_complete_url(self, api_base: str, model: str) -> str: return f"{api_base.rstrip('/').removesuffix('/v1')}{self.path}" + def transform_decisions_request( + self, + model: str, + request: DecisionsRequest, + custom_llm_provider: str, + ) -> dict[str, object]: + return { + "model": self.request_model(model), + "state": request.state, + "questions": { + name: question.model_dump(mode="json", exclude_none=True) + for name, question in request.questions.items() + }, + } + def unwrap_response(self, payload: object) -> object: return payload + def transform_decisions_response( + self, + model: str, + custom_llm_provider: str, + raw_response: httpx.Response, + request: DecisionsRequest, + ) -> DecisionsResponse: + payload: Final[object] = PAYLOAD_ADAPTER.validate_json(raw_response.content) + try: + response: Final = _RESPONSE_ADAPTER.validate_python(self.unwrap_response(payload)) + except ValidationError as error: + raise BaseLLMException( + status_code=500, + message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}", + ) from error + self.set_hidden_params(response, model, custom_llm_provider) + return response -class DecisionsProviderConfig(Protocol): - @property - def api_key_env(self) -> tuple[str, ...]: ... + @staticmethod + def set_hidden_params(response: DecisionsResponse, model: str, custom_llm_provider: str) -> None: + response.set_hidden_params( + { + "model": f"{custom_llm_provider}/{model}", + "custom_llm_provider": custom_llm_provider, + "provider_response_model": f"{custom_llm_provider}/{model}", + } + ) - @property - def api_base_env(self) -> str: ... - - @property - def api_key_required(self) -> bool: ... - - def default_api_base(self) -> str | None: ... - - def missing_api_base_message(self, provider: str) -> str: ... - - def canonical_model(self, model: str) -> str: ... - - def request_model(self, model: str) -> str: ... - - def endpoint_url(self, api_base: str, model: str) -> str: ... - - def unwrap_response(self, payload: object) -> object: ... + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, + ) -> BaseLLMException: + return BaseLLMException(status_code=status_code, message=error_message, headers=headers) diff --git a/litellm/llms/cloudflare/decisions/transformation.py b/litellm/llms/cloudflare/decisions/transformation.py index 6e8b2999778..fa8c68ad3a4 100644 --- a/litellm/llms/cloudflare/decisions/transformation.py +++ b/litellm/llms/cloudflare/decisions/transformation.py @@ -1,9 +1,9 @@ from collections.abc import Mapping -from dataclasses import dataclass from typing import Final -from pydantic import TypeAdapter +from pydantic import TypeAdapter, ValidationError +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.secret_managers.main import ( get_secret_str, normalize_nonempty_secret_str, @@ -12,19 +12,17 @@ from litellm.secret_managers.main import ( _RESPONSE_MAPPING_ADAPTER: Final[TypeAdapter[Mapping[str, object]]] = TypeAdapter(Mapping[str, object]) -@dataclass(frozen=True, slots=True) -class CloudflareDecisionsEndpoint: - api_key_env: tuple[str, ...] = ("CLOUDFLARE_API_KEY",) - api_base_env: str = "CLOUDFLARE_API_BASE" - api_key_required: bool = True +class CloudflareDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("CLOUDFLARE_API_KEY",) + api_base_env = ("CLOUDFLARE_API_BASE",) - def default_api_base(self) -> str | None: + def get_default_api_base(self) -> str | None: account_id: Final = normalize_nonempty_secret_str(get_secret_str("CLOUDFLARE_ACCOUNT_ID")) if account_id is None: return None return f"https://api.cloudflare.com/client/v4/accounts/{account_id}/ai/run" - def missing_api_base_message(self, provider: str) -> str: + def missing_api_base_message(self, custom_llm_provider: str) -> str: return "Missing CLOUDFLARE_ACCOUNT_ID - set CLOUDFLARE_ACCOUNT_ID or pass api_base explicitly" def canonical_model(self, model: str) -> str: @@ -35,7 +33,7 @@ class CloudflareDecisionsEndpoint: def request_model(self, model: str) -> str: return model.rsplit("/", maxsplit=1)[-1] - def endpoint_url(self, api_base: str, model: str) -> str: + def get_complete_url(self, api_base: str, model: str) -> str: normalized_api_base: Final = api_base.rstrip("/") if normalized_api_base.endswith("/ai/v1"): return f"{normalized_api_base.removesuffix('/ai/v1')}/ai/run/{model}" @@ -44,15 +42,13 @@ class CloudflareDecisionsEndpoint: return f"{normalized_api_base}/ai/run/{model}" def unwrap_response(self, payload: object) -> object: - if not isinstance(payload, Mapping): + try: + response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) + except ValidationError: return payload - response_mapping: Final = _RESPONSE_MAPPING_ADAPTER.validate_python(payload) if "answers" in response_mapping: - return payload - result: Final = response_mapping.get("result") - if isinstance(result, Mapping): - return result - return payload - - -CLOUDFLARE_DECISIONS_ENDPOINT: Final[CloudflareDecisionsEndpoint] = CloudflareDecisionsEndpoint() + return response_mapping + try: + return _RESPONSE_MAPPING_ADAPTER.validate_python(response_mapping.get("result")) + except ValidationError: + return response_mapping diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index b57ce8ea2e8..de271cb09ba 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -70,6 +70,7 @@ from litellm.llms.base_llm.base_model_iterator import ( from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException from litellm.llms.base_llm.containers.transformation import BaseContainerConfig +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.evals.transformation import BaseEvalsAPIConfig from litellm.llms.base_llm.files.transformation import ( @@ -122,6 +123,7 @@ from litellm.types.containers.main import ( ContainerObject, DeleteContainerResult, ) +from litellm.types.decisions import DecisionsRequest, DecisionsResponse from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig from litellm.types.integrations.custom_logger import ( NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, @@ -1481,6 +1483,103 @@ class BaseLLMHTTPHandler: request_data=request_data, ) + def _prepare_decisions_request( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsRequest, + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + ) -> tuple[str, dict[str, str], dict[str, object]]: + outbound_headers: Final = provider_config.validate_environment(headers=headers, model=model, api_key=api_key) + url: Final = provider_config.get_complete_url(api_base=api_base, model=model) + data: Final = provider_config.transform_decisions_request( + model=model, request=request, custom_llm_provider=custom_llm_provider + ) + if logging_obj is not None: + logging_obj.pre_call( + input=data, + api_key=api_key, + model=model, + additional_args={"api_base": url, "complete_input_dict": data, "headers": outbound_headers}, + ) + return url, outbound_headers, data + + def decisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsRequest, + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + client: HTTPHandler | None = None, + ) -> DecisionsResponse: + url, outbound_headers, data = self._prepare_decisions_request( + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + provider_config=provider_config, + request=request, + api_base=api_base, + api_key=api_key, + headers=headers, + ) + sync_httpx_client: Final = client if client is not None else get_httpx_client() + try: + response: Final = sync_httpx_client.post( + url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + + async def adecisions( + self, + model: str, + custom_llm_provider: str, + logging_obj: LiteLLMLoggingObj | None, + provider_config: BaseDecisionsConfig, + request: DecisionsRequest, + api_base: str, + api_key: str | None, + headers: Mapping[str, str], + timeout: float | httpx.Timeout | None, + client: AsyncHTTPHandler | None = None, + ) -> DecisionsResponse: + url, outbound_headers, data = self._prepare_decisions_request( + model=model, + custom_llm_provider=custom_llm_provider, + logging_obj=logging_obj, + provider_config=provider_config, + request=request, + api_base=api_base, + api_key=api_key, + headers=headers, + ) + async_httpx_client: Final = ( + client + if client is not None + else get_async_httpx_client(llm_provider=litellm.LlmProviders(custom_llm_provider)) + ) + try: + response: Final = await async_httpx_client.post( + url, json=data, headers=outbound_headers, timeout=timeout, logging_obj=logging_obj + ) + except httpx.HTTPError as e: + raise self._handle_error(e=e, provider_config=provider_config) + return provider_config.transform_decisions_response( + model=model, custom_llm_provider=custom_llm_provider, raw_response=response, request=request + ) + def _prepare_audio_transcription_request( self, model: str, @@ -6156,6 +6255,7 @@ class BaseLLMHTTPHandler: e: Exception, provider_config: Union[ BaseConfig, + BaseDecisionsConfig, BaseRerankConfig, BaseResponsesAPIConfig, BaseImageEditConfig, diff --git a/litellm/llms/openrouter/decisions/transformation.py b/litellm/llms/openrouter/decisions/transformation.py index 7a7466b1239..53ddce8b402 100644 --- a/litellm/llms/openrouter/decisions/transformation.py +++ b/litellm/llms/openrouter/decisions/transformation.py @@ -1,10 +1,10 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -OPENROUTER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://openrouter.ai/api", - path="/alpha/decisions", - api_key_env=("OPENROUTER_API_KEY",), - api_base_env="OPENROUTER_API_BASE", -) +class OpenRouterDecisionsConfig(BaseDecisionsConfig): + path = "/alpha/decisions" + api_key_env = ("OPENROUTER_API_KEY",) + api_base_env = ("OPENROUTER_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://openrouter.ai/api" diff --git a/litellm/llms/perplexity/decisions/transformation.py b/litellm/llms/perplexity/decisions/transformation.py index 69a4753f4a3..11e2a38f06e 100644 --- a/litellm/llms/perplexity/decisions/transformation.py +++ b/litellm/llms/perplexity/decisions/transformation.py @@ -1,10 +1,10 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -PERPLEXITY_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://api.perplexity.ai", - path="/v1/decisions", - api_key_env=("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY"), - api_base_env="PERPLEXITY_API_BASE", -) +class PerplexityDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + api_key_env = ("PERPLEXITYAI_API_KEY", "PERPLEXITY_API_KEY") + api_base_env = ("PERPLEXITY_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://api.perplexity.ai" diff --git a/litellm/llms/strands_decider/decisions/transformation.py b/litellm/llms/strands_decider/decisions/transformation.py index 265afb2b148..9ef966e18f2 100644 --- a/litellm/llms/strands_decider/decisions/transformation.py +++ b/litellm/llms/strands_decider/decisions/transformation.py @@ -1,11 +1,7 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -STRANDS_DECIDER_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value=None, - path="/v1/systemone", - api_key_env=("STRANDS_DECIDER_API_KEY",), - api_base_env="STRANDS_DECIDER_API_BASE", - api_key_required=False, -) +class StrandsDeciderDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("STRANDS_DECIDER_API_KEY",) + api_base_env = ("STRANDS_DECIDER_API_BASE",) + api_key_required = False diff --git a/litellm/llms/typesafe/decisions/transformation.py b/litellm/llms/typesafe/decisions/transformation.py index 17fb24ce443..e71c526a840 100644 --- a/litellm/llms/typesafe/decisions/transformation.py +++ b/litellm/llms/typesafe/decisions/transformation.py @@ -1,10 +1,9 @@ -from typing import Final +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig -from litellm.llms.base_llm.decisions.transformation import JevCompatibleDecisionsEndpoint -TYPESAFE_DECISIONS_ENDPOINT: Final[JevCompatibleDecisionsEndpoint] = JevCompatibleDecisionsEndpoint( - default_api_base_value="https://api.typesafe.ai", - path="/v1/systemone", - api_key_env=("TYPESAFE_API_KEY",), - api_base_env="TYPESAFE_API_BASE", -) +class TypeSafeDecisionsConfig(BaseDecisionsConfig): + api_key_env = ("TYPESAFE_API_KEY",) + api_base_env = ("TYPESAFE_API_BASE",) + + def get_default_api_base(self) -> str | None: + return "https://api.typesafe.ai" diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 2745e00b6c6..8bc535ffebe 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -125,3 +125,6 @@ class DecisionsResponse(LiteLLMPydanticObjectBase): @property def hidden_params(self) -> dict[str, object]: # mutable-ok: API requires mutation return self._hidden_params + + def set_hidden_params(self, params: Mapping[str, object]) -> None: + self._hidden_params.update(params) diff --git a/litellm/utils.py b/litellm/utils.py index 72ce14912a9..7207c6bacaa 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -379,6 +379,7 @@ if TYPE_CHECKING: # Type stubs for lazy-loaded config classes and types from litellm.llms.base_llm.batches.transformation import BaseBatchesConfig from litellm.llms.base_llm.containers.transformation import BaseContainerConfig + from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig from litellm.llms.base_llm.files.transformation import BaseFilesConfig from litellm.llms.base_llm.google_genai.transformation import ( @@ -8961,6 +8962,20 @@ class ProviderConfigManager: return get_dashscope_family_rerank_config(provider.value) return litellm.CohereRerankConfig() + @staticmethod + def get_provider_decisions_config(model: str, provider: LlmProviders) -> BaseDecisionsConfig | None: + if provider == LlmProviders.PERPLEXITY: + return litellm.PerplexityDecisionsConfig() + if provider == LlmProviders.TYPESAFE: + return litellm.TypeSafeDecisionsConfig() + if provider == LlmProviders.OPENROUTER: + return litellm.OpenRouterDecisionsConfig() + if provider == LlmProviders.CLOUDFLARE: + return litellm.CloudflareDecisionsConfig() + if provider == LlmProviders.STRANDS_DECIDER: + return litellm.StrandsDeciderDecisionsConfig() + return None + @staticmethod def get_provider_anthropic_messages_config( model: str, diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 9878559cb07..548a0d27f89 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -25,6 +25,7 @@ _PASS_THROUGH_MODEL: Final = "gpt-6-luna" _PASS_THROUGH_AUTHORIZATION: Final = "Bearer customer-held-upstream-key" _PASS_THROUGH_NEIGHBOUR: Final = "decisions-beside-a-pass-through" _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_QUESTION_MAPPINGS: Final = TypeAdapter(dict[str, dict[str, object]]) _USAGE: Final[dict[str, JsonValue]] = {"input_tokens": 367, "output_tokens": 3} _STATE: Final[dict[str, JsonValue]] = {"ticket": "The export job hangs at 99%", "component": "billing"} _QUESTIONS: Final[dict[str, JsonValue]] = { @@ -32,6 +33,7 @@ _QUESTIONS: Final[dict[str, JsonValue]] = { "severity": {"type": "choice", "criteria": {"low": "cosmetic", "high": "blocks users"}, "weight": 2}, "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, } +_SDK_QUESTIONS: Final = _QUESTION_MAPPINGS.validate_python(_QUESTIONS) _ANSWERS: Final[dict[str, JsonValue]] = { "defect": {"type": "noul", "noul": 0.93}, "severity": {"type": "choice", "choice": "high", "confidence": 0.8, "probabilities": {"low": 0.2, "high": 0.8}}, @@ -103,6 +105,11 @@ _PROVIDERS: Final = ( ), ) _PERPLEXITY: Final = _PROVIDERS[0] +_OPENROUTER: Final = _PROVIDERS[2] +_OPENROUTER_CHAT_MODEL: Final = "openrouter/openai/gpt-5-mini" +_UNSUPPORTED_PROVIDER_MODEL: Final = "openai/gpt-6-luna" +_CONNECTION_ERROR: Final = "litellm.APIConnectionError" +_GENERIC_API_ERROR: Final = "litellm.APIError" _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( ("missing questions", {"state": _STATE}), ("missing state", {"questions": _QUESTIONS}), @@ -177,6 +184,12 @@ def _spend_row(call_id: str) -> dict[str, JsonValue]: return rows[0] +def _assert_connection_error(response: httpx.Response) -> None: + assert 500 <= response.status_code < 600, response.text + assert _CONNECTION_ERROR in response.text, response.text + assert _GENERIC_API_ERROR not in response.text, response.text + + def _free_closed_port() -> int: with socket.socket() as probe: probe.bind(("127.0.0.1", 0)) @@ -257,10 +270,10 @@ async def test_sdk_sync_and_async_clients_send_the_same_request(gateway: Gateway with gateway.scenario() as scenario: handle: Final = _register(scenario, _answer_body(provider)) synchronous: Final = litellm.decisions( - model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) asynchronous: Final = await litellm.adecisions( - model=provider.model, state=_STATE, questions=_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY + model=provider.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=handle.api_base(), api_key=_API_KEY ) for response in (synchronous, asynchronous): assert response.model_dump(mode="json") == { @@ -441,16 +454,75 @@ def test_upstream_success_without_answers_is_a_gateway_side_server_error(gateway assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" -def test_unreachable_upstream_fails_only_its_own_deployment(gateway: Gateway) -> None: +@pytest.mark.parametrize("provider", _PROVIDERS, ids=lambda provider: provider.name) +def test_unreachable_upstream_fails_only_its_own_deployment(gateway: Gateway, provider: _Provider) -> None: with gateway.scenario() as scenario: - handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) - healthy: Final = _deployment(scenario, handle, _PERPLEXITY) + handle: Final = _register(scenario, _answer_body(provider)) + healthy: Final = _deployment(scenario, handle, provider) dead: Final = scenario.model( - model=_PERPLEXITY.model, api_base=f"http://127.0.0.1:{_free_closed_port()}", api_key=_API_KEY + model=provider.model, api_base=f"http://127.0.0.1:{_free_closed_port()}", api_key=provider.api_key ) failed: Final = _decide(gateway, dead, num_retries=0) - assert 500 <= failed.status_code < 600, failed.text + _assert_connection_error(failed) assert _spend_row(failed.headers["x-litellm-call-id"])["status"] == "failure" served: Final = _decide(gateway, healthy) assert served.status_code == 200, served.text assert len(_upstream_calls(gateway, handle)) == 1 + + +def test_an_unreachable_openrouter_deployment_reports_a_connection_error_on_chat_embeddings_and_decisions( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + dead_api_base: Final = f"http://127.0.0.1:{_free_closed_port()}" + chat_model: Final = scenario.model(model=_OPENROUTER_CHAT_MODEL, api_base=dead_api_base, api_key=_API_KEY) + decisions_model: Final = scenario.model(model=_OPENROUTER.model, api_base=dead_api_base, api_key=_API_KEY) + chat: Final = _chat(gateway, chat_model, num_retries=0) + streamed: Final = _chat(gateway, chat_model, num_retries=0, stream=True) + embeddings: Final = gateway.request( + "POST", "/v1/embeddings", {"model": chat_model, "input": "hi", "num_retries": 0} + ) + decisions: Final = _decide(gateway, decisions_model, num_retries=0) + for response in (chat, streamed, embeddings, decisions): + _assert_connection_error(response) + for response in (chat, decisions): + assert _spend_row(response.headers["x-litellm-call-id"])["status"] == "failure" + + +async def test_sdk_openrouter_connection_failures_raise_a_connection_error(gateway: Gateway) -> None: + dead_api_base: Final = f"http://127.0.0.1:{_free_closed_port()}" + with pytest.raises(litellm.APIConnectionError): + litellm.completion( + model=_OPENROUTER_CHAT_MODEL, + messages=[{"role": "user", "content": "hi"}], + api_base=dead_api_base, + api_key=_API_KEY, + ) + with pytest.raises(litellm.APIConnectionError): + await litellm.acompletion( + model=_OPENROUTER_CHAT_MODEL, + messages=[{"role": "user", "content": "hi"}], + api_base=dead_api_base, + api_key=_API_KEY, + ) + with pytest.raises(litellm.APIConnectionError): + litellm.decisions( + model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY + ) + with pytest.raises(litellm.APIConnectionError): + await litellm.adecisions( + model=_OPENROUTER.model, state=_STATE, questions=_SDK_QUESTIONS, api_base=dead_api_base, api_key=_API_KEY + ) + + +def test_a_deployment_whose_provider_has_no_decisions_support_is_refused_naming_every_supported_provider( + gateway: Gateway, +) -> None: + with gateway.scenario() as scenario: + handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) + model: Final = scenario.model(model=_UNSUPPORTED_PROVIDER_MODEL, api_base=handle.api_base(), api_key=_API_KEY) + response: Final = _decide(gateway, model) + assert response.status_code == 400, response.text + for provider in _PROVIDERS: + assert provider.name in response.text, response.text + assert _upstream_calls(gateway, handle) == [] diff --git a/tests/integration/proxy_config.yaml b/tests/integration/proxy_config.yaml index 6924b641f13..4c027254838 100644 --- a/tests/integration/proxy_config.yaml +++ b/tests/integration/proxy_config.yaml @@ -322,6 +322,30 @@ model_list: api_base: http://127.0.0.1:8191 api_key: synthetic-bedrock-mantle-key aws_region_name: us-east-1 + - model_name: perplexity/pplx-decider-v1-27b + litellm_params: + model: perplexity/pplx-decider-v1-27b + api_base: http://127.0.0.1:8191 + api_key: synthetic-perplexity-key + - model_name: typesafe/jev-1.13.0 + litellm_params: + model: typesafe/jev-1.13.0 + api_base: http://127.0.0.1:8191 + api_key: synthetic-typesafe-key + - model_name: openrouter/typesafe/jev-1.13 + litellm_params: + model: openrouter/typesafe/jev-1.13 + api_base: http://127.0.0.1:8191 + api_key: synthetic-openrouter-key + - model_name: cloudflare/@cf/cloudflare/clef + litellm_params: + model: cloudflare/@cf/cloudflare/clef + api_base: http://127.0.0.1:8191 + api_key: synthetic-cloudflare-key + - model_name: strands_decider/strands-decider-2B-hobson-v19 + litellm_params: + model: strands_decider/strands-decider-2B-hobson-v19 + api_base: http://127.0.0.1:8191 general_settings: master_key: os.environ/LITELLM_MASTER_KEY database_url: os.environ/DATABASE_URL diff --git a/tests/integration/translation/decisions/__init__.py b/tests/integration/translation/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/decisions/bases/__init__.py b/tests/integration/translation/decisions/bases/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/decisions/bases/cloudflare.py b/tests/integration/translation/decisions/bases/cloudflare.py new file mode 100644 index 00000000000..1668dc215a8 --- /dev/null +++ b/tests/integration/translation/decisions/bases/cloudflare.py @@ -0,0 +1,84 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://developers.cloudflare.com/workers-ai (POST /ai/run/@cf/cloudflare/clef, reply wrapped in result). Mock reply captured live on 2026-10-06. +""" +CLEF_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "cloudflare/@cf/cloudflare/clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/ai/run/@cf/cloudflare/clef", + expected_provider_headers={"authorization": "Bearer synthetic-cloudflare-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "clef", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "result": { + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, + "success": True, + "errors": [], + "messages": [], + }, + expected_litellm_response={ + "model": "clef", + "answers": { + "defect": {"type": "noul", "noul": 0.9345}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.8067, + "probabilities": {"low": 0.0509, "high": 0.9491}, + }, + "confidence": { + "type": "score", + "score": 0.9036, + "confidence": 0.6515, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.0964, "1": 0.9036}, + }, + }, + "usage": {"input_tokens": 290, "output_tokens": 0}, + }, +) diff --git a/tests/integration/translation/decisions/bases/openrouter.py b/tests/integration/translation/decisions/bases/openrouter.py new file mode 100644 index 00000000000..f6217bcf818 --- /dev/null +++ b/tests/integration/translation/decisions/bases/openrouter.py @@ -0,0 +1,83 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://openrouter.ai/docs (POST /api/alpha/decisions). Mock reply captured live on 2026-10-06. +""" +TYPESAFE_JEV_1_13_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "openrouter/typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/alpha/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-openrouter-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "typesafe/jev-1.13", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, + expected_litellm_response={ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "defect": {"type": "noul", "noul": 0.81}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.5, + "confidence": 0, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.5, "1": 0.5}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62, "cost": 1.5834e-05}, + "id": "gen-dec-1791323839-GA15kY0nt34oiJ7srfki", + "provider": "TypeSafe", + }, +) diff --git a/tests/integration/translation/decisions/bases/perplexity.py b/tests/integration/translation/decisions/bases/perplexity.py new file mode 100644 index 00000000000..71ef7a85220 --- /dev/null +++ b/tests/integration/translation/decisions/bases/perplexity.py @@ -0,0 +1,79 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://docs.perplexity.ai (POST /v1/decisions). Mock reply captured live on 2026-10-06. +""" +PPLX_DECIDER_V1_27B_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "perplexity/pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/decisions", + expected_provider_headers={"authorization": "Bearer synthetic-perplexity-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "pplx-decider-v1-27b", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, + expected_litellm_response={ + "model": "pplx-decider-v1-27b", + "answers": { + "defect": {"type": "noul", "noul": 0.9989100737587077}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.9964631215356778, + "probabilities": {"low": 0.0017684392321610232, "high": 0.9982315607678389}, + }, + "confidence": { + "type": "score", + "score": 0.07367392327139817, + "confidence": 0.8526521534572037, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.9263260767286018, "1": 0.07367392327139817}, + }, + }, + "usage": {"input_tokens": 318, "output_tokens": 3}, + }, +) diff --git a/tests/integration/translation/decisions/bases/strands_decider.py b/tests/integration/translation/decisions/bases/strands_decider.py new file mode 100644 index 00000000000..83d7ae5e33a --- /dev/null +++ b/tests/integration/translation/decisions/bases/strands_decider.py @@ -0,0 +1,35 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://github.com/strands-agents/decider (POST /v1/systemone, no auth). Mock reply captured live on 2026-10-06. +""" +STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "strands_decider/strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"content-type": "application/json"}, + expected_provider_request={ + "model": "strands-decider-2B-hobson-v19", + "state": "Help! My payouts have been failing for 3 days!", + "questions": {"is_urgent": {"type": "noul", "instructions": "Does this convey urgency?"}}, + }, + mock_provider_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, + expected_litellm_response={ + "model": "strands-decider-2B-hobson-v19", + "answers": {"is_urgent": {"type": "noul", "noul": 0.8277}}, + "usage": {"input_tokens": 86, "output_tokens": 1}, + "latency_ms": 140.03, + }, +) diff --git a/tests/integration/translation/decisions/bases/typesafe.py b/tests/integration/translation/decisions/bases/typesafe.py new file mode 100644 index 00000000000..2fb384a94ae --- /dev/null +++ b/tests/integration/translation/decisions/bases/typesafe.py @@ -0,0 +1,79 @@ +from typing import Final + +from integration.translation.case import TranslationTestCase + +"""Provider request and reply shape from https://docs.typesafe.ai (POST /v1/systemone). Mock reply captured live on 2026-10-06. +""" +JEV_1_13_0_TEST_CASE: Final = TranslationTestCase( + scenario="basic", + litellm_endpoint="/v1/decisions", + litellm_request={ + "model": "typesafe/jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + "cache": {"no-cache": True}, + }, + expected_provider_endpoint="/v1/systemone", + expected_provider_headers={"authorization": "Bearer synthetic-typesafe-key", "content-type": "application/json"}, + expected_provider_request={ + "model": "jev-1.13.0", + "state": "Ticket (billing): The export job hangs at 99% and never finishes", + "questions": { + "defect": {"type": "noul", "instructions": "Is this a defect?"}, + "severity": { + "type": "choice", + "instructions": "How severe is it?", + "criteria": {"low": "cosmetic", "high": "blocks users"}, + }, + "confidence": {"type": "score", "instructions": "How sure are you?", "criteria": ["unsure", "sure"]}, + }, + }, + mock_provider_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, + expected_litellm_response={ + "model": "jev-1.13.0", + "answers": { + "defect": {"type": "noul", "noul": 0.78}, + "severity": { + "type": "choice", + "choice": "high", + "confidence": 0.99, + "probabilities": {"low": 0.01, "high": 0.99}, + }, + "confidence": { + "type": "score", + "score": 0.51, + "confidence": 0.03, + "legend": {"0": "unsure", "1": "sure"}, + "probabilities": {"0": 0.49, "1": 0.51}, + }, + }, + "usage": {"input_tokens": 377, "output_tokens": 62}, + }, +) diff --git a/tests/integration/translation/decisions/basic/__init__.py b/tests/integration/translation/decisions/basic/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py new file mode 100644 index 00000000000..79fc0574b3b --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_cloudflare.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.cloudflare import CLEF_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [CLEF_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_cloudflare(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py new file mode 100644 index 00000000000..a74cb23f6b4 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_openrouter.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.openrouter import TYPESAFE_JEV_1_13_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [TYPESAFE_JEV_1_13_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_openrouter(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py new file mode 100644 index 00000000000..e341d24ba53 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_perplexity.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.perplexity import PPLX_DECIDER_V1_27B_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [PPLX_DECIDER_V1_27B_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_perplexity(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py new file mode 100644 index 00000000000..cae7ca70c78 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_strands_decider.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.strands_decider import STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_strands_decider(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py new file mode 100644 index 00000000000..302636e6fb4 --- /dev/null +++ b/tests/integration/translation/decisions/basic/test_decisions_basic_typesafe.py @@ -0,0 +1,11 @@ +import pytest +from integration._support.client import Gateway +from integration._support.provider import SharedProvider +from integration.translation.case import TranslationTestCase +from integration.translation.decisions.bases.typesafe import JEV_1_13_0_TEST_CASE +from integration.translation.runner import assert_translation + + +@pytest.mark.parametrize("case", [JEV_1_13_0_TEST_CASE], ids=lambda case: case.id) +def test_decisions_basic_typesafe(case: TranslationTestCase, gateway: Gateway, provider: SharedProvider) -> None: + assert_translation(case, gateway, provider) diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index 684248b3561..e5964f972b9 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -6,6 +6,7 @@ from collections.abc import Mapping from types import MappingProxyType from typing import Final +import httpx import pytest import respx @@ -252,10 +253,7 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: answers={}, usage=DecisionsUsage(input_tokens=_INPUT_TOKENS, output_tokens=_OUTPUT_TOKENS), ) - response._hidden_params = { - "model": "perplexity/pplx-decider-v1-27b", - "custom_llm_provider": "perplexity", - } + response.set_hidden_params({"model": "perplexity/pplx-decider-v1-27b", "custom_llm_provider": "perplexity"}) cost: Final = litellm.completion_cost(completion_response=response) perplexity_cost: Final = litellm.model_cost["perplexity/pplx-decider-v1-27b"] @@ -308,7 +306,7 @@ async def test_decisions_cost_is_in_standard_logging_object(respx_mock: respx.Mo @pytest.mark.asyncio async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: - with pytest.raises(litellm.BadRequestError, match="Supported providers"): + with pytest.raises(litellm.BadRequestError, match="LLM Provider NOT provided"): await litellm.adecisions( model="unknown/jev-1.13", state="review", @@ -320,17 +318,79 @@ async def test_unknown_provider_is_rejected_before_http(respx_mock: respx.MockRo @pytest.mark.asyncio -async def test_empty_custom_provider_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: - with pytest.raises(litellm.BadRequestError, match="Supported providers"): +async def test_provider_without_decisions_support_is_rejected_before_http(respx_mock: respx.MockRouter) -> None: + with pytest.raises(litellm.BadRequestError, match=r"Unknown Decisions provider 'anthropic'\. Supported providers"): + await litellm.adecisions( + model="anthropic/claude-sonnet-4-5", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_empty_custom_provider_falls_back_to_the_model_prefix(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + custom_llm_provider="", + ) + + assert route.called + assert json.loads(respx_mock.calls[0].request.content)["model"] == "pplx-decider-v1-27b" + assert response._hidden_params["custom_llm_provider"] == "perplexity" + + +@pytest.mark.asyncio +async def test_upstream_reply_without_answers_is_a_server_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json={"model": "pplx-decider-v1-27b", "usage": {"input_tokens": 10, "output_tokens": 0}} + ) + + with pytest.raises(litellm.InternalServerError, match="unexpected response"): await litellm.adecisions( model="perplexity/pplx-decider-v1-27b", state="review", questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, api_key="caller-key", - custom_llm_provider="", ) - assert len(respx_mock.calls) == 0 + +@pytest.mark.asyncio +async def test_router_sends_the_openrouter_deployment_key_when_the_provider_is_already_resolved( + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.delenv("OPENROUTER_API_KEY", raising=False) + router: Final = litellm.Router( + model_list=[ + { + "model_name": "jev", + "litellm_params": { + "model": "openrouter/typesafe/jev-1.13", + "api_key": "deployment-key", + "api_base": "https://egress.example/openrouter", + }, + } + ] + ) + upstream: Final = respx_mock.post("https://egress.example/openrouter/alpha/decisions").respond(json=_RESPONSE) + + await router.adecisions( + model="jev", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert upstream.called + assert respx_mock.calls[0].request.headers["authorization"] == "Bearer deployment-key" + assert json.loads(respx_mock.calls[0].request.content)["model"] == "typesafe/jev-1.13" @pytest.mark.asyncio @@ -361,6 +421,20 @@ def test_upstream_bad_request_maps_to_litellm_error(respx_mock: respx.MockRouter ) +def test_unreachable_upstream_maps_to_a_connection_error(respx_mock: respx.MockRouter) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").mock( + side_effect=httpx.ConnectError("Cannot connect to host api.perplexity.ai:443") + ) + + with pytest.raises(litellm.APIConnectionError, match="PerplexityException - Cannot connect to host"): + litellm.decisions( + model="perplexity/pplx-decider-v1-27b", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + api_key="caller-key", + ) + + def test_server_key_is_sent_to_an_explicit_api_base( monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter, diff --git a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py index 486070781ac..7e7f1b536f8 100644 --- a/tests/unit/litellm_core_utils/test_exception_mapping_utils.py +++ b/tests/unit/litellm_core_utils/test_exception_mapping_utils.py @@ -1231,6 +1231,34 @@ def test_branchless_provider_transport_error_maps_to_api_connection_error(): ) +def test_openrouter_transport_error_maps_to_api_connection_error(): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + original_exception = BaseLLMException(status_code=500, message="[Errno 111] Connection refused") + original_exception.status_code_is_synthesized = True + + with pytest.raises(litellm.APIConnectionError): + exception_type( + model="typesafe/jev-1.13", + original_exception=original_exception, + custom_llm_provider="openrouter", + ) + + +def test_openrouter_upstream_500_still_maps_to_api_error(): + from litellm.llms.base_llm.chat.transformation import BaseLLMException + + original_exception = BaseLLMException(status_code=500, message="upstream exploded") + + with pytest.raises(litellm.APIError) as excinfo: + exception_type( + model="typesafe/jev-1.13", + original_exception=original_exception, + custom_llm_provider="openrouter", + ) + assert excinfo.value.status_code == 500 + + def test_branchless_provider_upstream_500_still_maps_to_internal_server_error(): from litellm.llms.base_llm.chat.transformation import BaseLLMException From b85756104f8c29e0ac5c3788d633d87e6a04f8d6 Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:01:55 -0700 Subject: [PATCH 08/65] feat(lens): show agent, user and slack thread first in the run header (#45261) * feat(lens): add who started the run to RunSource * feat(lens): carry agent.source.user on trace span rows * feat(lens): resolve the run source user from the span row * feat(lens): read agent.source.user in trace_spans query * feat(lens): read agent.source.user in trace_page_spans query * feat(lens): read agent.source.user in trace_span_batch query * feat(lens): read agent.source.user in trace_list_span_batch query * feat(lens): decode the source user from clickhouse span rows * test(lens): add source user to trace cache test rows * test(lens): add source user to trace cache read fixtures * test(lens): add source user to trace cache snapshot fixtures * test(lens): add source user to capture fixture rows * test(lens): round-trip source user on span row contract * test(lens): cover run source user resolution * chore(lens): regenerate python trace types with source user * chore(lens): regenerate trace json schema with source user * chore(lens): regenerate trace page json schema with source user * chore(ui): add source user to api types * feat(lens): add run user chip and slack thread chip * feat(lens): put agent, user and thread on the first row of the run header * test(lens): cover the run header identity row and compact totals * feat(lens): start demo runs from a slack thread --- litellm-rust/crates/traces-cache/src/cache.rs | 1 + .../crates/traces-cache/tests/read.rs | 1 + .../crates/traces-cache/tests/snapshots.rs | 1 + .../query/trace_list_span_batch.sql | 2 +- .../query/trace_page_spans.sql | 2 +- .../query/trace_span_batch.sql | 2 +- .../traces-clickhouse/query/trace_spans.sql | 2 +- .../traces-clickhouse/src/query/named.rs | 4 +- litellm-rust/crates/traces/src/query/named.rs | 2 + .../crates/traces/src/resolve/view.rs | 1 + litellm-rust/crates/traces/src/view.rs | 4 + litellm-rust/crates/traces/tests/captures.rs | 2 + .../crates/traces/tests/query/named.rs | 2 +- litellm-rust/crates/traces/tests/resolve.rs | 11 + litellm/rust_bridge/trace/generated/types.py | 1 + .../trace_codegen/schemas/traces/Trace.json | 5 + .../schemas/traces/TracePage.json | 5 + .../src/components/lens/data/demo/fixtures.ts | 10 + .../lens/traces/detail/run/RunHeader.tsx | 255 +++++++++++------- .../lens/traces/detail/run/RunView.test.tsx | 52 +++- .../components/lens/traces/ui/RunSource.tsx | 43 +-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 2 + 22 files changed, 279 insertions(+), 131 deletions(-) diff --git a/litellm-rust/crates/traces-cache/src/cache.rs b/litellm-rust/crates/traces-cache/src/cache.rs index 287160df96b..eda054e6c7d 100644 --- a/litellm-rust/crates/traces-cache/src/cache.rs +++ b/litellm-rust/crates/traces-cache/src/cache.rs @@ -328,6 +328,7 @@ mod tests { source_type: String::new(), source_url: String::new(), source_title: String::new(), + source_user: String::new(), team_id: String::new(), api_key_hash: String::new(), user_id: String::new(), diff --git a/litellm-rust/crates/traces-cache/tests/read.rs b/litellm-rust/crates/traces-cache/tests/read.rs index ade70b71800..22293db98e8 100644 --- a/litellm-rust/crates/traces-cache/tests/read.rs +++ b/litellm-rust/crates/traces-cache/tests/read.rs @@ -289,6 +289,7 @@ fn span(index: usize) -> TraceSpansRow { source_type: String::new(), source_url: String::new(), source_title: String::new(), + source_user: String::new(), team_id: "team".into(), api_key_hash: "key".into(), user_id: "user".into(), diff --git a/litellm-rust/crates/traces-cache/tests/snapshots.rs b/litellm-rust/crates/traces-cache/tests/snapshots.rs index bebff0bb1aa..4ad4e2b7545 100644 --- a/litellm-rust/crates/traces-cache/tests/snapshots.rs +++ b/litellm-rust/crates/traces-cache/tests/snapshots.rs @@ -42,6 +42,7 @@ fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> Trac source_type: String::new(), source_url: String::new(), source_title: String::new(), + source_user: String::new(), team_id: String::new(), api_key_hash: String::new(), user_id: String::new(), diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql index b764005024e..327483d7a45 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_list_span_batch.sql @@ -14,7 +14,7 @@ SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS orig coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), '')) AS tool_call_id, o.SpanAttributes['agent.source.type'] AS source_type, o.SpanAttributes['agent.source.url'] AS source_url, - o.SpanAttributes['agent.source.title'] AS source_title, + o.SpanAttributes['agent.source.title'] AS source_title, o.SpanAttributes['agent.source.user'] AS source_user, o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash FROM otel_traces AS o WHERE o.Timestamp >= fromUnixTimestamp64Milli({start_ms:Int64}) diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql index 5af30920df9..dd8cdee7e80 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_page_spans.sql @@ -13,7 +13,7 @@ SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS orig coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), '')) AS tool_call_id, o.SpanAttributes['agent.source.type'] AS source_type, o.SpanAttributes['agent.source.url'] AS source_url, - o.SpanAttributes['agent.source.title'] AS source_title, + o.SpanAttributes['agent.source.title'] AS source_title, o.SpanAttributes['agent.source.user'] AS source_user, o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash FROM otel_traces AS o WHERE o.Timestamp >= fromUnixTimestamp64Milli({start_ms:Int64}) diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql index c7d50a44544..85345cb05c1 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_span_batch.sql @@ -14,7 +14,7 @@ SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS orig coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), '')) AS tool_call_id, o.SpanAttributes['agent.source.type'] AS source_type, o.SpanAttributes['agent.source.url'] AS source_url, - o.SpanAttributes['agent.source.title'] AS source_title, + o.SpanAttributes['agent.source.title'] AS source_title, o.SpanAttributes['agent.source.user'] AS source_user, o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash FROM otel_traces AS o WHERE o.TraceId = {trace_id:String} diff --git a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql index 974a6d050a4..a2327e57660 100644 --- a/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql +++ b/litellm-rust/crates/traces-clickhouse/query/trace_spans.sql @@ -13,7 +13,7 @@ SELECT o.TraceId AS trace_id, o.SpanAttributes['lens.original_trace_id'] AS orig coalesce(nullIf(o.SpanAttributes['gen_ai.tool.call.id'], ''), nullIf(o.SpanAttributes['tool.id'], ''), '')) AS tool_call_id, o.SpanAttributes['agent.source.type'] AS source_type, o.SpanAttributes['agent.source.url'] AS source_url, - o.SpanAttributes['agent.source.title'] AS source_title, + o.SpanAttributes['agent.source.title'] AS source_title, o.SpanAttributes['agent.source.user'] AS source_user, o.UserId AS user_id, o.TeamId AS team_id, o.ApiKeyHash AS api_key_hash FROM otel_traces AS o WHERE o.TraceId = {trace_id:String} diff --git a/litellm-rust/crates/traces-clickhouse/src/query/named.rs b/litellm-rust/crates/traces-clickhouse/src/query/named.rs index 1b98ad39912..b7de28646e4 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/named.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/named.rs @@ -134,6 +134,8 @@ struct TraceSpansRowEncoding { pub source_url: String, #[serde(default)] pub source_title: String, + #[serde(default)] + pub source_user: String, pub team_id: String, pub api_key_hash: String, pub user_id: String, @@ -355,7 +357,7 @@ mod tests { quoted, ); round_trip::( - json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "source_type": "slack", "source_url": "https://acme.slack.com/archives/C1/p1", "source_title": "thread", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "claude-agent-sdk", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "source_type": "slack", "source_url": "https://acme.slack.com/archives/C1/p1", "source_title": "thread", "source_user": "tin@berri.ai", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), quoted, ); round_trip::( diff --git a/litellm-rust/crates/traces/src/query/named.rs b/litellm-rust/crates/traces/src/query/named.rs index 03459645ca9..dfa7ac2f2cd 100644 --- a/litellm-rust/crates/traces/src/query/named.rs +++ b/litellm-rust/crates/traces/src/query/named.rs @@ -116,6 +116,8 @@ pub struct TraceSpansRow { pub source_url: String, #[serde(default)] pub source_title: String, + #[serde(default)] + pub source_user: String, pub team_id: String, pub api_key_hash: String, pub user_id: String, diff --git a/litellm-rust/crates/traces/src/resolve/view.rs b/litellm-rust/crates/traces/src/resolve/view.rs index 97f83c255f8..62c27676605 100644 --- a/litellm-rust/crates/traces/src/resolve/view.rs +++ b/litellm-rust/crates/traces/src/resolve/view.rs @@ -146,6 +146,7 @@ fn source(row: &TraceSpansRow) -> Option { .unwrap_or(RunSourceType::Custom), url: row.source_url.clone(), title: row.source_title.clone(), + user: row.source_user.clone(), }) } diff --git a/litellm-rust/crates/traces/src/view.rs b/litellm-rust/crates/traces/src/view.rs index 1613d96d232..729ab5ef1c3 100644 --- a/litellm-rust/crates/traces/src/view.rs +++ b/litellm-rust/crates/traces/src/view.rs @@ -88,6 +88,10 @@ pub struct RunSource { pub kind: RunSourceType, pub url: String, pub title: String, + /// Who started the conversation, e.g. the Slack user's email. + #[serde(default, skip_serializing_if = "String::is_empty")] + #[cfg_attr(feature = "schema", schemars(extend("x-python-optional" = true)))] + pub user: String, } #[macro_rules_attribute::apply(response_type)] diff --git a/litellm-rust/crates/traces/tests/captures.rs b/litellm-rust/crates/traces/tests/captures.rs index 922e28f97ac..a81bca609ea 100644 --- a/litellm-rust/crates/traces/tests/captures.rs +++ b/litellm-rust/crates/traces/tests/captures.rs @@ -210,6 +210,7 @@ fn trace_span(span: DecodedSpan) -> TraceSpansRow { source_type: String::new(), source_url: String::new(), source_title: String::new(), + source_user: String::new(), team_id: "fixture-team".into(), api_key_hash: "fixture-key".into(), user_id: "fixture-user".into(), @@ -297,6 +298,7 @@ fn unrelated_transport(call: &TraceSpansRow) -> TraceSpansRow { source_type: String::new(), source_url: String::new(), source_title: String::new(), + source_user: String::new(), team_id: call.team_id.clone(), api_key_hash: call.api_key_hash.clone(), user_id: call.user_id.clone(), diff --git a/litellm-rust/crates/traces/tests/query/named.rs b/litellm-rust/crates/traces/tests/query/named.rs index 25cfd7083a5..af33ee3fe85 100644 --- a/litellm-rust/crates/traces/tests/query/named.rs +++ b/litellm-rust/crates/traces/tests/query/named.rs @@ -53,7 +53,7 @@ fn result_contracts_preserve_public_field_names() { json!({"trace_id": "trace", "trace_ref": "ref", "team_id": "team", "api_key_hash": "key", "user_id": "user", "name": "agent", "service": "service", "input_preview": "input", "status": "STATUS_CODE_OK", "start_ms": -1, "duration_ms": 20, "span_count": u64::MAX, "agent_count": 1, "agent_invocations": 2, "agent_names": ["agent"], "frameworks": ["framework"], "llm_calls": 3, "tool_calls": 4, "input_tokens": 5, "output_tokens": 6, "models": ["model"], "error_count": 0, "request_ids": ["request"]}), ); round_trip::( - json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "source_type": "slack", "source_url": "https://acme.slack.com/archives/C1/p1", "source_title": "thread", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), + json!({"trace_id": "trace", "original_trace_id": "original", "span_id": "span", "parent_span_id": "parent", "name": "agent", "type": "agent", "wrapper_candidate": 1, "agent": "agent", "framework": "framework", "status": "STATUS_CODE_ERROR", "status_message": "error", "error_truncated": 1, "start_ns": -1, "duration_ns": u64::MAX, "service": "service", "input_preview": "input", "model": "model", "input_tokens": u32::MAX, "output_tokens": 6, "litellm_request_id": "request", "call_keys": ["provider_response:request"], "call_evidence": "complete", "tool_call_id": "call", "source_type": "slack", "source_url": "https://acme.slack.com/archives/C1/p1", "source_title": "thread", "source_user": "tin@berri.ai", "team_id": "team", "api_key_hash": "key", "user_id": "user"}), ); round_trip::( json!({"span_id": "span", "input": "input", "output": "output", "attributes": {"count": "42"}}), diff --git a/litellm-rust/crates/traces/tests/resolve.rs b/litellm-rust/crates/traces/tests/resolve.rs index baaa4c338cf..73f691cb2a6 100644 --- a/litellm-rust/crates/traces/tests/resolve.rs +++ b/litellm-rust/crates/traces/tests/resolve.rs @@ -36,6 +36,7 @@ fn row(span_id: &str, parent: &str, name: &str, kind: &str, agent: &str) -> Trac source_type: String::new(), source_url: String::new(), source_title: String::new(), + source_user: String::new(), team_id: "team".into(), api_key_hash: "key".into(), user_id: String::new(), @@ -227,6 +228,16 @@ fn summary_source_type_picks_the_app(#[case] source_type: &str, #[case] expected assert_eq!(source.map(|source| source.kind), Some(expected)); } +#[rstest] +#[case::set("tin@berri.ai")] +#[case::missing("")] +fn summary_source_carries_who_started_it(#[case] user: &str) { + let mut root = sourced(row("root", "", "agent", "agent", "agent"), THREAD, "t"); + root.source_user = user.into(); + let source = resolve_trace("t", "", &[root], &[]).unwrap().summary.source; + assert_eq!(source.map(|source| source.user), Some(user.to_owned())); +} + #[rstest] fn spans_are_offset_from_the_trace_start() { let trace = resolve_trace("t1", "", &deep_agent(1), &[]).unwrap(); diff --git a/litellm/rust_bridge/trace/generated/types.py b/litellm/rust_bridge/trace/generated/types.py index 4234cc5e6fa..12f38da6e28 100644 --- a/litellm/rust_bridge/trace/generated/types.py +++ b/litellm/rust_bridge/trace/generated/types.py @@ -109,6 +109,7 @@ class RunSource(typing_extensions.TypedDict): type: ReadOnly[RunSourceType] url: ReadOnly[str] title: ReadOnly[str] + user: ReadOnly[NotRequired[str]] class Span(typing_extensions.TypedDict): diff --git a/scripts/trace_codegen/schemas/traces/Trace.json b/scripts/trace_codegen/schemas/traces/Trace.json index 3fcbf654101..b8b893607dc 100644 --- a/scripts/trace_codegen/schemas/traces/Trace.json +++ b/scripts/trace_codegen/schemas/traces/Trace.json @@ -71,6 +71,11 @@ }, "url": { "type": "string" + }, + "user": { + "description": "Who started the conversation, e.g. the Slack user's email.", + "type": "string", + "x-python-optional": true } }, "required": [ diff --git a/scripts/trace_codegen/schemas/traces/TracePage.json b/scripts/trace_codegen/schemas/traces/TracePage.json index 9d437bd60a1..47f48a676de 100644 --- a/scripts/trace_codegen/schemas/traces/TracePage.json +++ b/scripts/trace_codegen/schemas/traces/TracePage.json @@ -11,6 +11,11 @@ }, "url": { "type": "string" + }, + "user": { + "description": "Who started the conversation, e.g. the Slack user's email.", + "type": "string", + "x-python-optional": true } }, "required": [ diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts index f33f86cd280..c90853dfbad 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/fixtures.ts @@ -5,6 +5,15 @@ import { scenarios, type Scenario } from "./scenarios"; const executionId = (traceId: string) => btoa(JSON.stringify(["traces", "", traceId])); const iso = (time: number) => new Date(time).toISOString(); +const DEMO_ASKERS = ["maya@acme.dev", "jordan@acme.dev", "priya@acme.dev"]; + +/** Demo runs start from a Slack thread so the run header shows who asked and where. */ +const demoSource = (scene: Scenario, index: number): NonNullable => ({ + type: "slack", + url: `https://acme.slack.com/archives/C0DEMO/p${1_700_000_000_000 + index}`, + title: scene.question, + user: DEMO_ASKERS[index % DEMO_ASKERS.length], +}); function makeTrace(scene: Scenario, index: number, now: number) { const traceId = (index + 1).toString(16).padStart(32, "0"); @@ -79,6 +88,7 @@ function makeTrace(scene: Scenario, index: number, now: number) { status: scene.failed ? "error" : "ok", tool_calls: toolCount, trace_id: traceId, + source: demoSource(scene, index), }, agents: [ { diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx index cfe8674a6c5..29b6051ddeb 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunHeader.tsx @@ -1,58 +1,55 @@ "use client"; -import { ArrowLeft, Check, Copy, Link, ListTree, MessagesSquare, RefreshCw } from "lucide-react"; +import { + ArrowLeft, + Check, + ChevronDown, + Copy, + DatabaseZap, + Link, + ListTree, + MessagesSquare, + RefreshCw, +} from "lucide-react"; import { useState } from "react"; import { useTimeout } from "usehooks-ts"; import { Button } from "@/components/ui/button"; +import { ButtonGroup } from "@/components/ui/button-group"; +import { + DropdownMenu, + DropdownMenuContent, + DropdownMenuItem, + DropdownMenuTrigger, +} from "@/components/ui/dropdown-menu"; import { TabsList, TabsTrigger } from "@/components/ui/tabs"; import { cn } from "@/lib/cva.config"; import { copyToClipboard } from "@/utils/dataUtils"; -import { AddToDatasetButton } from "../../../datasets/AddToDatasetDialog"; +import { AddToDatasetDialog, useCanAddToDataset } from "../../../datasets/AddToDatasetDialog"; import type { TraceHandoff } from "../../api"; import { runCost } from "../../list/AgentTracesTable"; import { traceRefOf, traceShareUrl } from "../../routing"; import { IdChip } from "../../ui/IdChip"; -import { RunSourceLink } from "../../ui/RunSource"; +import { RunSourceLink, RunUser } from "../../ui/RunSource"; import { SpanIcon } from "../../ui/SpanIcon"; import { FrameworkLogo, traceFramework } from "../../ui/TraceFramework"; import type { SignalFlag, Trace } from "../../types"; import { SignalPills } from "../../ui/SignalPills"; import { fmtMs, fmtTok, traceAgentNames, traceDisplayName } from "../../utils"; -interface CopyButtonProps { - label: string; - icon: typeof Copy; - text: () => string; - toast: string; +type Summary = Trace["summary"]; + +function statusLabel(failed: boolean, errors: number): string { + if (errors === 0) return failed ? "Errors recorded" : "Recorded"; + const stepErrors = `${errors.toLocaleString()} step ${errors === 1 ? "error" : "errors"}`; + return failed ? stepErrors : `Recorded · ${stepErrors}`; } -function CopyButton({ label, icon: Icon, text, toast }: CopyButtonProps) { - const [copied, setCopied] = useState(false); - useTimeout(() => setCopied(false), copied ? 1600 : null); - return ( - - ); -} - -function Stat({ label, value, error = false }: { label: string; value: string; error?: boolean }) { - return ( - - {label} {value} - - ); -} - -function StatusPill({ failed }: { failed: boolean }) { +/** One pill for how the run went, so a failure is stated once instead of as a pill, a stat and a red icon. */ +function StatusPill({ summary }: { summary: Summary }) { + const failed = summary.status === "error"; + const label = statusLabel(failed, summary.error_count); return ( - {failed ? "Errors recorded" : "Recorded"} + {label} ); } -function RunIcon({ summary, failed }: { summary: Trace["summary"]; failed: boolean }) { +/** Duration, steps, tokens and cost as one quiet line; cost is left out when nothing was priced. */ +function Totals({ summary }: { summary: Summary }) { + const cost = runCost(summary); + const parts = [ + fmtMs(summary.duration_ms), + `${summary.span_count.toLocaleString()} steps`, + `${fmtTok(summary.input_tokens + summary.output_tokens)} tokens`, + cost && [cost.label, cost.partial?.long].filter(Boolean).join(" · "), + ].filter(Boolean); + return {parts.join(" · ")}; +} + +function RunIcon({ summary }: { summary: Summary }) { const framework = traceFramework(summary); - if (!framework) return ; + if (!framework) return ; return ( setCopied(false), copied ? 1600 : null); + return ( + <> + + + + + } + > + + + + + void copyToClipboard(traceShareUrl(traceRefOf(summary), window.location), "Trace link copied") + } + > + + Copy link + + {canAddToDataset && ( + setAddingToDataset(true)}> + + Add run to dataset + + )} + + + + {addingToDataset && ( + setAddingToDataset(false)} + /> + )} + + ); +} + interface RunHeaderProps { trace: Trace; handoff: TraceHandoff; @@ -95,7 +159,7 @@ interface RunHeaderProps { signals?: readonly SignalFlag[]; } -/** Run identity, view switch and totals in two tight rows. */ +/** Row one says which agent ran, for whom and from where; row two says how it went. */ export function RunHeader({ trace, handoff, @@ -109,73 +173,66 @@ export function RunHeader({ signals = [], }: RunHeaderProps) { const { summary } = trace; - const failed = summary.status === "error"; - const cost = runCost(summary); return ( -
-
-
- {!embedded && ( - - )} -

{traceDisplayName(summary)}

- - -
-
- - - - Steps - - - - Thread - - -
- - - +
+ {!embedded && ( + + )} + +

{traceDisplayName(summary)}

+ {summary.source?.user && } + {summary.source && } + +
+
+ Live + + +
-
- +
+ {signals.length > 0 && } - {summary.source && } - - - - - {summary.error_count > 0 && } + + + + + Steps + + + + Thread + +
); diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.test.tsx index f25517e2909..b4107143487 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.test.tsx @@ -78,8 +78,8 @@ describe("RunView", () => { "title", research.summary.trace_id, ); - expect(header).toHaveTextContent("Duration 40.20s"); - expect(header).toHaveTextContent(`Steps ${research.summary.span_count}`); + expect(header).toHaveTextContent("40.20s"); + expect(header).toHaveTextContent(`${research.summary.span_count} steps`); expect(header).toHaveTextContent("Recorded"); expect(header).not.toHaveTextContent("Completed"); }); @@ -97,6 +97,37 @@ describe("RunView", () => { expect(within(header).queryByTestId("span-icon")).not.toBeInTheDocument(); }); + it("puts who started the run and its Slack thread next to the agent name", async () => { + const thread = "https://acme.slack.com/archives/C1/p1"; + renderRun({ + ...research, + summary: { + ...research.summary, + spend: null, + source: { type: "slack", url: thread, title: "why is the deploy failing?", user: "tin@berri.ai" }, + }, + }); + + const header = await screen.findByRole("banner"); + expect(within(header).getByTestId("run-user")).toHaveTextContent("tin@berri.ai"); + expect(within(header).getByRole("link", { name: "Open Slack thread" })).toHaveAttribute("href", thread); + expect(header).not.toHaveTextContent("Not reported"); + }); + + it("leaves the user out when the source does not say who started it", async () => { + renderRun({ + ...research, + summary: { + ...research.summary, + source: { type: "slack", url: "https://acme.slack.com/archives/C1/p1", title: "" }, + }, + }); + + const header = await screen.findByRole("banner"); + expect(within(header).queryByTestId("run-user")).not.toBeInTheDocument(); + expect(within(header).getByRole("link", { name: "Open Slack thread" })).toBeInTheDocument(); + }); + it("keeps the generic agent icon when the trace has no known SDK", async () => { renderRun({ ...research, summary: { ...research.summary, frameworks: ["some-other-sdk"] } }); @@ -110,7 +141,7 @@ describe("RunView", () => { const tree = await screen.findByRole("tree", { name: "Spans in time order" }); expect(tree).toHaveTextContent("researcher×12"); - expect(screen.getByRole("banner")).toHaveTextContent(`Step errors ${swarm.summary.error_count}`); + expect(screen.getByRole("banner")).toHaveTextContent(`${swarm.summary.error_count} step errors`); }); it.each([ @@ -395,7 +426,7 @@ describe("RunView", () => { expect(await screen.findByTestId("detail-pane")).toHaveAttribute("data-row-id", rootSpanId(research)); fireEvent.click(screen.getByRole("button", { name: "Refresh run" })); expect(await screen.findByText("newly received step")).toBeVisible(); - expect(screen.getByRole("banner")).toHaveTextContent(`Steps ${research.summary.span_count + 1}`); + expect(screen.getByRole("banner")).toHaveTextContent(`${research.summary.span_count + 1} steps`); expect(screen.getByTestId("detail-pane")).toHaveAttribute("data-row-id", rootSpanId(research)); }); @@ -413,11 +444,11 @@ describe("RunView", () => { await act(async () => { await vi.advanceTimersByTimeAsync(100); }); - expect(screen.getByRole("banner")).toHaveTextContent(`Steps ${research.summary.span_count}`); + expect(screen.getByRole("banner")).toHaveTextContent(`${research.summary.span_count} steps`); await act(async () => { await vi.advanceTimersByTimeAsync(30_100); }); - expect(screen.getByRole("banner")).toHaveTextContent(`Steps ${research.summary.span_count + 7}`); + expect(screen.getByRole("banner")).toHaveTextContent(`${research.summary.span_count + 7} steps`); expect(screen.getByTestId("detail-pane")).toHaveAttribute("data-row-id", rootSpanId(research)); } finally { vi.useRealTimers(); @@ -476,7 +507,7 @@ describe("RunView", () => { await act(async () => { await vi.advanceTimersByTimeAsync(60_100); }); - expect(screen.getByRole("banner")).toHaveTextContent(`Steps ${research.summary.span_count}`); + expect(screen.getByRole("banner")).toHaveTextContent(`${research.summary.span_count} steps`); expect(vi.mocked(agentTraceCall)).toHaveBeenCalledTimes(2); } finally { vi.useRealTimers(); @@ -517,7 +548,7 @@ describe("RunView", () => { expect(screen.queryByRole("button", { name: "Load more steps" })).not.toBeInTheDocument(); expect(vi.mocked(agentTraceCall)).toHaveBeenCalledTimes(2); if (action === "Refresh run") { - expect(screen.getByRole("banner")).toHaveTextContent(`Steps ${updated.summary.span_count}`); + expect(screen.getByRole("banner")).toHaveTextContent(`${updated.summary.span_count} steps`); } } finally { vi.useRealTimers(); @@ -642,7 +673,7 @@ describe("RunView", () => { renderRun({ ...research, summary: { ...research.summary, status: "ok", error_count: 2 } }); const header = await screen.findByRole("banner"); expect(header).toHaveTextContent("Recorded"); - expect(header).toHaveTextContent("Step errors 2"); + expect(header).toHaveTextContent("Recorded · 2 step errors"); expect(header).not.toHaveTextContent("Failed"); }); @@ -684,7 +715,8 @@ describe("RunView", () => { const user = userEvent.setup(); renderRun(research); - await user.click(await screen.findByRole("button", { name: /copy link/i })); + fireEvent.click(await screen.findByRole("button", { name: "More run actions" })); + await user.click(await screen.findByRole("menuitem", { name: /copy link/i })); const url = new URL(vi.mocked(copyToClipboard).mock.calls[0][0] as string); expect(url.pathname).toBe(window.location.pathname); expect(url.searchParams.get("trace")).toBe(research.summary.trace_id); diff --git a/ui/litellm-dashboard/src/components/lens/traces/ui/RunSource.tsx b/ui/litellm-dashboard/src/components/lens/traces/ui/RunSource.tsx index fa4ad9e6f8a..1dc7ec99b92 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/ui/RunSource.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/ui/RunSource.tsx @@ -1,6 +1,6 @@ "use client"; -import { MessagesSquare } from "lucide-react"; +import { ArrowUpRight, MessagesSquare, UserRound } from "lucide-react"; import githubLogo from "../../../../../public/assets/logos/github.svg"; import jiraLogo from "../../../../../public/assets/logos/jira.svg"; @@ -52,25 +52,36 @@ function AppMark({ app, className }: { app: SourceApp; className: string }) { ); } -/** Links a run back to the conversation that started it, e.g. a Slack thread. */ +const chip = + "inline-flex h-6 min-w-0 shrink items-center gap-1.5 rounded-full border border-border px-2 text-xs text-foreground"; + +/** Who started the run, e.g. the person who asked in Slack. */ +export function RunUser({ user }: { user: string }) { + return ( + + + {user} + + ); +} + +/** Links a run back to the conversation that started it, e.g. a Slack thread; hover previews its title. */ export function RunSourceLink({ source }: { source: Source }) { const app = sourceApp(source); if (!app) return null; return ( - - Source - - - {app.label} - - + + + {app.label} thread + + @@ -78,7 +89,7 @@ export function RunSourceLink({ source }: { source: Source }) { - {app.label} + {source.user ? `${app.label} · ${source.user}` : app.label} diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 5060fc3575e..bf711498307 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -44979,6 +44979,8 @@ export interface components { type: "slack" | "teams" | "discord" | "linear" | "github" | "jira" | "custom"; /** Url */ url: string; + /** User */ + user?: string; }; /** SCIMEnterpriseUser */ SCIMEnterpriseUser: { From c912236fa55fa2a4372a085669c056dc9c257232 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:03:44 -0700 Subject: [PATCH 09/65] fix(proxy): declare the Moyai settings write's service target and allowlist its routes (#45220) * fix(proxy): declare the config_params service target for the Moyai UI settings write Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): exercise the Moyai settings write against a real DualCache Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): allowlist the Moyai quick-connect routes on the backend component Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): observe the Moyai settings write target through a composed DualCache Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- backend/routes/allowlist.py | 1 + litellm/proxy/moyai_endpoints.py | 3 +++ tests/unit/proxy/test_moyai_endpoints.py | 34 ++++++++++++++++++++++++ 3 files changed, 38 insertions(+) diff --git a/backend/routes/allowlist.py b/backend/routes/allowlist.py index ea56a1fd6df..44541276be0 100644 --- a/backend/routes/allowlist.py +++ b/backend/routes/allowlist.py @@ -23,6 +23,7 @@ BACKEND_PATH_PREFIXES: tuple[str, ...] = ( "/end_user/", "/sso/", "/liteadmin/slack/connect/", + "/moyai/connect/", "/login", "/v2/login", "/v3/login", diff --git a/litellm/proxy/moyai_endpoints.py b/litellm/proxy/moyai_endpoints.py index 655fc0560f4..a2335c0a49b 100644 --- a/litellm/proxy/moyai_endpoints.py +++ b/litellm/proxy/moyai_endpoints.py @@ -20,6 +20,7 @@ from urllib.parse import urlencode, urlparse from fastapi import APIRouter, Depends, HTTPException, Request, status from pydantic import BaseModel +from litellm._internal_context import with_service_target from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( @@ -28,6 +29,7 @@ from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import ( _ui_settings_db, normalize_moyai_url, ) +from litellm.proxy.utils import CONFIG_PARAMS_TARGET from litellm.repositories.config_repository import ConfigRepository from litellm.repositories.table_repositories import UISettingsRepository @@ -189,6 +191,7 @@ async def _moyai_key_alias(prisma_client, moyai_url: str) -> str: return alias +@with_service_target(CONFIG_PARAMS_TARGET) async def _persist_moyai_url(prisma_client, moyai_url: str) -> None: from litellm.proxy.proxy_server import user_api_key_cache diff --git a/tests/unit/proxy/test_moyai_endpoints.py b/tests/unit/proxy/test_moyai_endpoints.py index a0b0d56b637..92a1261d952 100644 --- a/tests/unit/proxy/test_moyai_endpoints.py +++ b/tests/unit/proxy/test_moyai_endpoints.py @@ -1,10 +1,13 @@ import json import time from types import SimpleNamespace +from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from litellm._internal_context import current_service_target +from litellm.caching.dual_cache import DualCache from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth @@ -272,6 +275,37 @@ async def test_exchange_concurrent_replay_claims_nonce_once(monkeypatch: pytest. assert len(mint_calls) == 1 +def _target_recording_cache(cache: DualCache) -> SimpleNamespace: + async def _set(key: str, value: object, **kwargs: object) -> None: + await cache.async_set_cache(key=f"{key}:service_target", value=current_service_target()) + await cache.async_set_cache(key=key, value=value, **kwargs) + + return SimpleNamespace(async_set_cache=_set) + + +@pytest.mark.asyncio +async def test_persist_moyai_url_writes_ui_settings_cache_under_config_params_target( + monkeypatch: pytest.MonkeyPatch, +) -> None: + from litellm.proxy import proxy_server + from litellm.proxy.moyai_endpoints import _persist_moyai_url + from litellm.proxy.ui_crud_endpoints.proxy_setting_endpoints import UI_SETTINGS_CACHE_KEY + from litellm.proxy.utils import CONFIG_PARAMS_TARGET + + cache: Final = DualCache() + monkeypatch.setattr(proxy_server, "user_api_key_cache", _target_recording_cache(cache)) + + prisma: Final = MagicMock() + prisma.db.litellm_uisettings.find_unique = AsyncMock(return_value=None) + prisma.db.litellm_uisettings.upsert = AsyncMock() + + await _persist_moyai_url(prisma, "https://moyai.example.com") + + assert await cache.async_get_cache(key=f"{UI_SETTINGS_CACHE_KEY}:service_target") == CONFIG_PARAMS_TARGET + assert await cache.async_get_cache(key=UI_SETTINGS_CACHE_KEY) == {"moyai_url": "https://moyai.example.com"} + assert current_service_target() is None + + @pytest.mark.asyncio async def test_exchange_replay_survives_fresh_worker_cache(monkeypatch: pytest.MonkeyPatch) -> None: from fastapi import HTTPException From 03ba79d261af22800fc3c603d3e0b610f700656b Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:14:30 -0700 Subject: [PATCH 10/65] feat(lens): show end-user feedback on traces, stored in ClickHouse (#45171) * feat(lens): add lens_feedback ClickHouse table for human trace scores ReplacingMergeTree(UpdatedAt, IsDeleted) keyed like agent_traces_by_key so feedback joins traces in sort order, with a bloom filter on TraceId, a score CHECK, and the same retention as traces Co-Authored-By: Claude Opus 5.5 * feat(lens): add scoped ClickHouse reads for trace feedback feedback_target resolves a visible trace's team, key and trace_ref before a write; feedback lists the latest live entry per author; feedback_summary returns count, average and lowest score for a batch of traces Co-Authored-By: Claude Opus 5.5 * test(lens): pin the Claude session to trace id hash shared with feedback Co-Authored-By: Claude Opus 5.5 * feat(lens): expose feedback queries to the Python trace bridge Co-Authored-By: Claude Opus 5.5 * feat(lens): add typed feedback models and ClickHouse feedback store Co-Authored-By: Claude Opus 5.5 * feat(lens): add /lens/feedback API for 0-10 human scores on traces PUT saves or replaces the caller's score and comment, GET lists every entry, DELETE removes the caller's, POST /summary feeds the trace list. Callers can target a trace_id or a Claude Code session_id Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): add typed feedback calls to the traces API Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): flag rated traces and filter the list by feedback A Feedback column shows each run's average score and rating count, marked red when any score is 4 or below, and a filter narrows to rated or low-score runs through the URL Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): show and edit human feedback inside a trace Co-Authored-By: Claude Opus 5.5 * feat(lens): let app keys post end-user feedback on their own traces Any key can now PUT/DELETE feedback on traces its team or key sent, naming the end user in an optional user field. Reading feedback stays with Lens admins Co-Authored-By: Claude Opus 5.5 * feat(lens-ui): show end-user feedback first in a run and flag low-scored rows Replace the edit popover and feedback filter with a read-only view: the list shows each run's score in a fixed-width badge and tints the row red when a user scored it 4 or below, and opening a run shows what users said with their score right under the header Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- litellm-rust/crates/lens/src/lib.rs | 20 +- litellm-rust/crates/lens/tests/receiver.rs | 70 ++- .../migrations/0017_lens_feedback.sql | 18 + .../traces-clickhouse/query/lens_feedback.sql | 12 + .../query/lens_feedback_summary.sql | 9 + .../query/lens_feedback_target.sql | 9 + .../crates/traces-clickhouse/src/insert.rs | 3 + .../traces-clickhouse/src/query/lens.rs | 110 ++++- .../crates/traces-clickhouse/src/schema.rs | 3 +- .../crates/traces-clickhouse/src/sql.rs | 7 + .../traces-clickhouse/src/wire_schema.rs | 12 + .../traces-clickhouse/tests/lens_feedback.rs | 408 ++++++++++++++++++ .../traces-clickhouse/tests/migrations.rs | 24 +- litellm-rust/crates/traces/src/query.rs | 3 + litellm-rust/crates/traces/tests/otlp.rs | 3 + litellm-rust/crates/traces/tests/query.rs | 3 + litellm/constants.py | 2 + litellm/proxy/_types.py | 3 + litellm/proxy/lens/feedback_endpoints.py | 124 ++++++ litellm/proxy/lens/feedback_models.py | 37 ++ litellm/proxy/lens/feedback_repository.py | 186 ++++++++ litellm/proxy/proxy_server.py | 2 + litellm/rust_bridge/trace/generated/models.py | 179 ++++++++ litellm/rust_bridge/trace/generated/types.py | 12 +- litellm/rust_bridge/trace/queries.py | 15 + litellm/tracing/remote.py | 8 +- .../traces-clickhouse/FeedbackRow.json | 53 +++ .../traces-clickhouse/FeedbackSummaryRow.json | 62 +++ .../traces-clickhouse/FeedbackTargetRow.json | 21 + .../traces-clickhouse/LensFeedbackParams.json | 34 ++ .../LensFeedbackSummaryParams.json | 33 ++ .../LensFeedbackTargetParams.json | 34 ++ .../traces-clickhouse/ReadQueryName.json | 5 +- .../proxy/lens/test_feedback_endpoints.py | 323 ++++++++++++++ tests/unit/tracing/test_remote.py | 17 +- .../lens/LensSetup.integration.test.tsx | 1 + .../lens/LensWorkspace.integration.test.tsx | 2 + .../lens/data/demo/createLensDemo.ts | 34 ++ .../src/components/lens/traces/api.ts | 16 + .../FeedbackPanel.integration.test.tsx | 88 ++++ .../traces/detail/feedback/FeedbackPanel.tsx | 87 ++++ .../traces/detail/feedback/feedback.test.ts | 31 ++ .../lens/traces/detail/feedback/feedback.ts | 18 + .../lens/traces/detail/run/RunView.tsx | 2 + .../AgentTracesSection.integration.test.tsx | 71 ++- .../lens/traces/list/AgentTracesSection.tsx | 3 + .../traces/list/AgentTracesTable.test.tsx | 44 ++ .../lens/traces/list/AgentTracesTable.tsx | 169 +++++--- .../useTraceFeedback.integration.test.tsx | 35 ++ .../lens/traces/list/useTraceFeedback.test.ts | 21 + .../lens/traces/list/useTraceFeedback.ts | 54 +++ .../src/components/lens/traces/types.ts | 5 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 250 +++++++++++ 53 files changed, 2712 insertions(+), 83 deletions(-) create mode 100644 litellm-rust/crates/traces-clickhouse/migrations/0017_lens_feedback.sql create mode 100644 litellm-rust/crates/traces-clickhouse/query/lens_feedback.sql create mode 100644 litellm-rust/crates/traces-clickhouse/query/lens_feedback_summary.sql create mode 100644 litellm-rust/crates/traces-clickhouse/query/lens_feedback_target.sql create mode 100644 litellm-rust/crates/traces-clickhouse/tests/lens_feedback.rs create mode 100644 litellm/proxy/lens/feedback_endpoints.py create mode 100644 litellm/proxy/lens/feedback_models.py create mode 100644 litellm/proxy/lens/feedback_repository.py create mode 100644 scripts/trace_codegen/schemas/traces-clickhouse/FeedbackRow.json create mode 100644 scripts/trace_codegen/schemas/traces-clickhouse/FeedbackSummaryRow.json create mode 100644 scripts/trace_codegen/schemas/traces-clickhouse/FeedbackTargetRow.json create mode 100644 scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackParams.json create mode 100644 scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackSummaryParams.json create mode 100644 scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackTargetParams.json create mode 100644 tests/unit/proxy/lens/test_feedback_endpoints.py create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.ts create mode 100644 ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.integration.test.tsx create mode 100644 ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.test.ts create mode 100644 ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.ts diff --git a/litellm-rust/crates/lens/src/lib.rs b/litellm-rust/crates/lens/src/lib.rs index 887f7f571fb..a3038a0f975 100644 --- a/litellm-rust/crates/lens/src/lib.rs +++ b/litellm-rust/crates/lens/src/lib.rs @@ -116,6 +116,7 @@ pub fn router(state: Arc) -> Router { Router::new() .route("/internal/read", post(read)) .route("/internal/spend", post(spend)) + .route("/internal/feedback", post(feedback)) .route("/internal/credentials", post(credentials)) .route("/internal/status", get(status)), ) @@ -233,6 +234,23 @@ async fn spend( AppState(state): AppState>, headers: HeaderMap, body: Body, +) -> Result { + insert(state, headers, body, InsertTable::SpendLogs).await +} + +async fn feedback( + AppState(state): AppState>, + headers: HeaderMap, + body: Body, +) -> Result { + insert(state, headers, body, InsertTable::LensFeedback).await +} + +async fn insert( + state: Arc, + headers: HeaderMap, + body: Body, + table: InsertTable, ) -> Result { auth::authorize_service(&headers, &state.service_token)?; state.require_storage()?; @@ -256,7 +274,7 @@ async fn spend( &state.storage.client, state.storage.config.storage().writer(), state.storage.config.storage().database(), - InsertTable::SpendLogs, + table, rows, ) .await?; diff --git a/litellm-rust/crates/lens/tests/receiver.rs b/litellm-rust/crates/lens/tests/receiver.rs index 1679fc2540d..bda1cecbbb3 100644 --- a/litellm-rust/crates/lens/tests/receiver.rs +++ b/litellm-rust/crates/lens/tests/receiver.rs @@ -116,6 +116,74 @@ async fn agent_picker_query_preserves_scope_through_the_internal_read_route() { assert_eq!(response.json::().await.unwrap(), result); } +#[rstest] +#[tokio::test] +async fn feedback_summary_query_preserves_scope_through_the_internal_read_route() { + let store = MockServer::start().await; + let result = json!({"data": [{ + "trace_id": "1234567890abcdef1234567890abcdef", "trace_ref": "REF", + "count": "2", "average": 5.5, "lowest": "2" + }]}); + Mock::given(method("POST")) + .and(body_string_contains("FROM lens_feedback FINAL")) + .and(query_param("param_all_teams", "0")) + .and(query_param("param_team", "feedback-team")) + .and(query_param( + "param_trace_ids", + "['1234567890abcdef1234567890abcdef']", + )) + .respond_with(ResponseTemplate::new(200).set_body_json(&result)) + .expect(1) + .mount(&store) + .await; + let server = serve(&store.uri(), true).await; + let response = http_client() + .unwrap() + .post(format!("{}/internal/read", server.url)) + .bearer_auth(SERVICE_TOKEN) + .json(&json!({ + "operation": "query", "name": "feedback_summary", "parameters": { + "all_teams": 0, "team": "feedback-team", "key_hash": "", + "trace_ids": ["1234567890abcdef1234567890abcdef"] + } + })) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200); + assert_eq!(response.json::().await.unwrap(), result); +} + +#[rstest] +#[tokio::test] +async fn feedback_rows_are_written_to_the_feedback_table() { + let store = MockServer::start().await; + Mock::given(method("POST")) + .and(query_param( + "query", + "INSERT INTO `litellm`.lens_feedback FORMAT JSONEachRow", + )) + .respond_with(ResponseTemplate::new(200)) + .expect(1) + .mount(&store) + .await; + let server = serve(&store.uri(), true).await; + let response = http_client() + .unwrap() + .post(format!("{}/internal/feedback", server.url)) + .bearer_auth(SERVICE_TOKEN) + .json(&json!([{ + "TeamId": "team", "ApiKeyHash": "", "TraceId": "1234567890abcdef1234567890abcdef", + "Author": "customer-1042", "Score": 2, "Comment": "wrong command", + "CreatedAt": "2026-10-07T21:57:01.414Z", "UpdatedAt": "2026-10-07T21:57:01.414Z", + "IsDeleted": 0 + }])) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 204); +} + #[rstest] #[tokio::test] async fn ingestion_confirms_storage_and_overwrites_exporter_tenant() { @@ -297,7 +365,7 @@ async fn ingestion_key_cannot_read_or_export_gateway_records() { let store = MockServer::start().await; let server = serve(&store.uri(), true).await; let client = http_client().unwrap(); - for path in ["/internal/read", "/internal/spend"] { + for path in ["/internal/read", "/internal/spend", "/internal/feedback"] { let response = client .post(format!("{}{path}", server.url)) .bearer_auth(KEY) diff --git a/litellm-rust/crates/traces-clickhouse/migrations/0017_lens_feedback.sql b/litellm-rust/crates/traces-clickhouse/migrations/0017_lens_feedback.sql new file mode 100644 index 00000000000..9cef629e223 --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/migrations/0017_lens_feedback.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS {database}.lens_feedback +( + TeamId LowCardinality(String), + ApiKeyHash String, + TraceId String CODEC(ZSTD(1)), + Author String, + Score UInt8, + Comment String CODEC(ZSTD(3)), + CreatedAt DateTime64(3), + UpdatedAt DateTime64(3), + IsDeleted UInt8, + EngineReceivedMs UInt64 DEFAULT 0, + INDEX idx_trace_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, + CONSTRAINT score_range CHECK Score <= 10 +) +ENGINE = ReplacingMergeTree(UpdatedAt, IsDeleted) +ORDER BY (TeamId, ApiKeyHash, TraceId, Author) +SETTINGS materialize_ttl_recalculate_only = 1, non_replicated_deduplication_window = 1000 diff --git a/litellm-rust/crates/traces-clickhouse/query/lens_feedback.sql b/litellm-rust/crates/traces-clickhouse/query/lens_feedback.sql new file mode 100644 index 00000000000..11540af6ddf --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/query/lens_feedback.sql @@ -0,0 +1,12 @@ +SELECT TraceId AS trace_id, + hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) AS trace_ref, + Author AS author, Score AS score, Comment AS comment, + formatDateTime(CreatedAt, '%FT%T.%fZ', 'UTC') AS created_at, + formatDateTime(UpdatedAt, '%FT%T.%fZ', 'UTC') AS updated_at +FROM lens_feedback FINAL +WHERE TraceId = {trace_id:String} + AND hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) = {trace_ref:String} + AND ({all_teams:UInt8}=1 OR TeamId={team:String}) + AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) + AND IsDeleted = 0 +ORDER BY UpdatedAt DESC, Author diff --git a/litellm-rust/crates/traces-clickhouse/query/lens_feedback_summary.sql b/litellm-rust/crates/traces-clickhouse/query/lens_feedback_summary.sql new file mode 100644 index 00000000000..8786ecdfa42 --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/query/lens_feedback_summary.sql @@ -0,0 +1,9 @@ +SELECT TraceId AS trace_id, + hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) AS trace_ref, + count() AS count, avg(Score) AS average, min(Score) AS lowest +FROM lens_feedback FINAL +WHERE TraceId IN {trace_ids:Array(String)} + AND ({all_teams:UInt8}=1 OR TeamId={team:String}) + AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) + AND IsDeleted = 0 +GROUP BY TeamId, ApiKeyHash, TraceId diff --git a/litellm-rust/crates/traces-clickhouse/query/lens_feedback_target.sql b/litellm-rust/crates/traces-clickhouse/query/lens_feedback_target.sql new file mode 100644 index 00000000000..e2b7f96d90b --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/query/lens_feedback_target.sql @@ -0,0 +1,9 @@ +SELECT TeamId AS team_id, ApiKeyHash AS key_hash, + hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId))) AS trace_ref +FROM agent_traces_by_key +WHERE TraceId = {trace_id:String} + AND ({all_teams:UInt8}=1 OR TeamId={team:String}) + AND ({key_hash:String}='' OR ApiKeyHash={key_hash:String}) + AND ({trace_ref:String}='' OR hex(SHA256(concat(TeamId, char(0), ApiKeyHash, char(0), TraceId)))={trace_ref:String}) +GROUP BY TeamId, ApiKeyHash, TraceId +LIMIT 2 diff --git a/litellm-rust/crates/traces-clickhouse/src/insert.rs b/litellm-rust/crates/traces-clickhouse/src/insert.rs index d45cc8d53b8..ed8db188c14 100644 --- a/litellm-rust/crates/traces-clickhouse/src/insert.rs +++ b/litellm-rust/crates/traces-clickhouse/src/insert.rs @@ -33,6 +33,7 @@ pub type InsertRow = BTreeMap>; pub enum InsertTable { OtelTraces, SpendLogs, + LensFeedback, } impl InsertTable { @@ -40,6 +41,7 @@ impl InsertTable { match value { "otel_traces" => Ok(Self::OtelTraces), "spend_logs" => Ok(Self::SpendLogs), + "lens_feedback" => Ok(Self::LensFeedback), _ => Err(Error::InvalidTable), } } @@ -48,6 +50,7 @@ impl InsertTable { match self { Self::OtelTraces => "otel_traces", Self::SpendLogs => "spend_logs", + Self::LensFeedback => "lens_feedback", } } } diff --git a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs index fcca4fe6f96..97b74de4c20 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs @@ -6,13 +6,16 @@ const SAMPLE_READ_LIMITS: ReadLimits = ReadLimits { ..litellm_storage_clickhouse::READ_LIMITS }; -pub const LENS_QUERIES: [litellm_traces::ReadQuery; 6] = [ +pub const LENS_QUERIES: [litellm_traces::ReadQuery; 9] = [ litellm_traces::ReadQuery::TraceAgents, litellm_traces::ReadQuery::Availability, litellm_traces::ReadQuery::Agents, litellm_traces::ReadQuery::Sample, litellm_traces::ReadQuery::Content, litellm_traces::ReadQuery::Evidence, + litellm_traces::ReadQuery::FeedbackTarget, + litellm_traces::ReadQuery::Feedback, + litellm_traces::ReadQuery::FeedbackSummary, ]; #[macro_rules_attribute::apply(wire_type)] @@ -339,3 +342,108 @@ impl Query for LensEvidence { const SQL: &'static str = include_str!("../../query/lens_evidence.sql"); } + +pub struct LensFeedbackTarget; + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Debug)] +#[serde(deny_unknown_fields)] +pub struct LensFeedbackTargetParams { + #[serde(flatten)] + pub access: LensAccessParams, + pub trace_id: String, + pub trace_ref: String, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Debug)] +#[cfg_attr(feature = "schema", schemars(rename = "FeedbackTargetRow"))] +pub struct LensFeedbackTargetRow { + pub team_id: String, + pub key_hash: String, + pub trace_ref: String, +} + +impl Query for LensFeedbackTarget { + type Params = LensFeedbackTargetParams; + type Row = LensFeedbackTargetRow; + + const SQL: &'static str = include_str!("../../query/lens_feedback_target.sql"); +} + +pub struct LensFeedback; + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Debug)] +#[serde(deny_unknown_fields)] +pub struct LensFeedbackParams { + #[serde(flatten)] + pub access: LensAccessParams, + pub trace_id: String, + pub trace_ref: String, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Debug)] +#[cfg_attr(feature = "schema", schemars(rename = "FeedbackRow"))] +pub struct LensFeedbackRow { + pub trace_id: String, + pub trace_ref: String, + pub author: String, + #[serde(deserialize_with = "super::number::deserialize")] + #[cfg_attr( + feature = "schema", + schemars(schema_with = "crate::wire_schema::u64_number") + )] + pub score: u64, + pub comment: String, + pub created_at: String, + pub updated_at: String, +} + +impl Query for LensFeedback { + type Params = LensFeedbackParams; + type Row = LensFeedbackRow; + + const SQL: &'static str = include_str!("../../query/lens_feedback.sql"); +} + +pub struct LensFeedbackSummary; + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Debug)] +#[serde(deny_unknown_fields)] +pub struct LensFeedbackSummaryParams { + #[serde(flatten)] + pub access: LensAccessParams, + pub trace_ids: Vec, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Debug)] +#[cfg_attr(feature = "schema", schemars(rename = "FeedbackSummaryRow"))] +pub struct LensFeedbackSummaryRow { + pub trace_id: String, + pub trace_ref: String, + #[serde(deserialize_with = "super::number::deserialize")] + #[cfg_attr( + feature = "schema", + schemars(schema_with = "crate::wire_schema::u64_number") + )] + pub count: u64, + #[serde(deserialize_with = "super::number::deserialize")] + pub average: f64, + #[serde(deserialize_with = "super::number::deserialize")] + #[cfg_attr( + feature = "schema", + schemars(schema_with = "crate::wire_schema::u64_number") + )] + pub lowest: u64, +} + +impl Query for LensFeedbackSummary { + type Params = LensFeedbackSummaryParams; + type Row = LensFeedbackSummaryRow; + + const SQL: &'static str = include_str!("../../query/lens_feedback_summary.sql"); +} diff --git a/litellm-rust/crates/traces-clickhouse/src/schema.rs b/litellm-rust/crates/traces-clickhouse/src/schema.rs index 562dbb976c2..5eb8bf0b973 100644 --- a/litellm-rust/crates/traces-clickhouse/src/schema.rs +++ b/litellm-rust/crates/traces-clickhouse/src/schema.rs @@ -14,10 +14,11 @@ static MIGRATOR: Migrator = Migrator { ..sqlx::migrate!("./migrations") }; -const RETENTION: [(&str, &str); 3] = [ +const RETENTION: [(&str, &str); 4] = [ ("otel_traces", "toDateTime(Timestamp)"), ("agent_traces_by_key", "toDateTime(StartTs)"), ("spend_logs", "toDateTime(start_time)"), + ("lens_feedback", "toDateTime(CreatedAt)"), ]; fn validate_schema(database: &str, retention_days: u32) -> Result<(), Error> { diff --git a/litellm-rust/crates/traces-clickhouse/src/sql.rs b/litellm-rust/crates/traces-clickhouse/src/sql.rs index 1f41f0f6f43..613159cf06d 100644 --- a/litellm-rust/crates/traces-clickhouse/src/sql.rs +++ b/litellm-rust/crates/traces-clickhouse/src/sql.rs @@ -37,6 +37,13 @@ pub async fn execute_named_read( ReadQuery::Sample => named_json::(client, connection, parameters).await, ReadQuery::Content => named_json::(client, connection, parameters).await, ReadQuery::Evidence => named_json::(client, connection, parameters).await, + ReadQuery::FeedbackTarget => { + named_json::(client, connection, parameters).await + } + ReadQuery::Feedback => named_json::(client, connection, parameters).await, + ReadQuery::FeedbackSummary => { + named_json::(client, connection, parameters).await + } } } diff --git a/litellm-rust/crates/traces-clickhouse/src/wire_schema.rs b/litellm-rust/crates/traces-clickhouse/src/wire_schema.rs index f7a52a9edd0..408d224f67b 100644 --- a/litellm-rust/crates/traces-clickhouse/src/wire_schema.rs +++ b/litellm-rust/crates/traces-clickhouse/src/wire_schema.rs @@ -81,6 +81,18 @@ pub fn schemas() -> BTreeMap<&'static str, Schema> { ("LensSampleParams", received::()), ("LensContentParams", received::()), ("LensEvidenceParams", received::()), + ( + "LensFeedbackTargetParams", + received::(), + ), + ("LensFeedbackParams", received::()), + ( + "LensFeedbackSummaryParams", + received::(), + ), + ("FeedbackTargetRow", received::()), + ("FeedbackRow", received::()), + ("FeedbackSummaryRow", received::()), ( "ActivityAvailability", received::(), diff --git a/litellm-rust/crates/traces-clickhouse/tests/lens_feedback.rs b/litellm-rust/crates/traces-clickhouse/tests/lens_feedback.rs new file mode 100644 index 00000000000..b2efea71823 --- /dev/null +++ b/litellm-rust/crates/traces-clickhouse/tests/lens_feedback.rs @@ -0,0 +1,408 @@ +use std::collections::BTreeMap; + +use litellm_traces_clickhouse::{ + Connection, InsertTable, Parameter, ReadQuery, ensure_schema, execute_named_read, execute_read, + insert_rows, +}; +use rstest::rstest; +use serde_json::{Value, json}; +use time::{Duration, OffsetDateTime, format_description::well_known::Rfc3339}; + +mod support; + +use support::{ClickHouseDatabase, TestResult, database}; + +struct Feedback<'a> { + team: &'a str, + key: &'a str, + trace: &'a str, + author: &'a str, + score: u8, + comment: &'a str, + edited_after_seconds: i64, + deleted: bool, +} + +const SAVED: Feedback<'static> = Feedback { + team: "team-a", + key: "key-a", + trace: "trace-1", + author: "alice", + score: 3, + comment: "missed the file", + edited_after_seconds: 0, + deleted: false, +}; + +fn now() -> TestResult { + Ok(OffsetDateTime::now_utc().replace_millisecond(0)?) +} + +fn iso(at: OffsetDateTime) -> TestResult { + Ok(at.format(&Rfc3339)?) +} + +fn row(feedback: &Feedback, created: OffsetDateTime) -> TestResult> { + let updated = created + Duration::seconds(feedback.edited_after_seconds); + Ok(serde_json::from_value(json!({ + "TeamId": feedback.team, "ApiKeyHash": feedback.key, "TraceId": feedback.trace, + "Author": feedback.author, "Score": feedback.score, "Comment": feedback.comment, + "CreatedAt": iso(created)?, "UpdatedAt": iso(updated)?, + "IsDeleted": u8::from(feedback.deleted) + }))?) +} + +async fn ready(database: &ClickHouseDatabase) -> TestResult { + let writer = Connection::writer(&database.url)?; + ensure_schema(&database.client, &writer, "trace_test", 7).await?; + Ok(writer) +} + +async fn save( + database: &ClickHouseDatabase, + writer: &Connection, + created: OffsetDateTime, + feedback: &[Feedback<'_>], +) -> TestResult { + let rows = feedback + .iter() + .map(|entry| row(entry, created)) + .collect::>>()?; + insert_rows( + &database.client, + writer, + "trace_test", + InsertTable::LensFeedback, + rows, + ) + .await?; + Ok(()) +} + +fn access(team: &str) -> BTreeMap { + BTreeMap::from([ + ( + "all_teams".into(), + Parameter::Integer(i64::from(team.is_empty())), + ), + ("team".into(), Parameter::Text(team.into())), + ("key_hash".into(), Parameter::Text(String::new())), + ]) +} + +async fn read( + database: &ClickHouseDatabase, + query: ReadQuery, + parameters: BTreeMap, +) -> TestResult> { + let reader = Connection::configured(&database.url, "trace_test", "default", "")?; + let body: Value = serde_json::from_str( + &execute_named_read(&database.client, &reader, query, ¶meters).await?, + )?; + Ok(body["data"] + .as_array() + .cloned() + .ok_or_else(|| format!("no data in {body}"))?) +} + +async fn trace_ref( + database: &ClickHouseDatabase, + team: &str, + key: &str, + trace: &str, +) -> TestResult { + let reader = Connection::configured(&database.url, "trace_test", "default", "")?; + let sql = format!( + "SELECT hex(SHA256(concat('{team}', char(0), '{key}', char(0), '{trace}'))) AS trace_ref" + ); + let response: Value = serde_json::from_str( + &execute_read(&database.client, &reader, &sql, &BTreeMap::new()).await?, + )?; + Ok(response["data"][0]["trace_ref"] + .as_str() + .unwrap_or_default() + .to_owned()) +} + +async fn feedback( + database: &ClickHouseDatabase, + team: &str, + trace: &str, + trace_ref: &str, +) -> TestResult> { + let mut parameters = access(team); + parameters.insert("trace_id".into(), Parameter::Text(trace.into())); + parameters.insert("trace_ref".into(), Parameter::Text(trace_ref.into())); + read(database, ReadQuery::Feedback, parameters).await +} + +fn timestamp(row: &Value, field: &str) -> TestResult { + Ok(OffsetDateTime::parse( + row[field].as_str().unwrap_or_default(), + &Rfc3339, + )?) +} + +#[rstest] +#[tokio::test] +async fn a_later_save_replaces_the_authors_feedback_and_keeps_other_authors( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = ready(&database).await?; + let created = now()?; + save(&database, &writer, created, &[SAVED]).await?; + save( + &database, + &writer, + created, + &[ + Feedback { + score: 8, + comment: "fine after retry", + edited_after_seconds: 300, + ..SAVED + }, + Feedback { + author: "bob", + score: 10, + comment: "", + ..SAVED + }, + ], + ) + .await?; + let reference = trace_ref(&database, "team-a", "key-a", "trace-1").await?; + + let rows = feedback(&database, "", "trace-1", &reference).await?; + + let shown: Vec<(&str, u64, &str)> = rows + .iter() + .map(|row| { + ( + row["author"].as_str().unwrap_or_default(), + row["score"].as_u64().unwrap_or(99), + row["comment"].as_str().unwrap_or_default(), + ) + }) + .collect(); + assert_eq!(shown, [("alice", 8, "fine after retry"), ("bob", 10, "")]); + assert_eq!(timestamp(&rows[0], "created_at")?, created); + assert_eq!( + timestamp(&rows[0], "updated_at")?, + created + Duration::seconds(300) + ); + Ok(()) +} + +#[rstest] +#[tokio::test] +async fn a_deleted_version_hides_the_authors_feedback( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = ready(&database).await?; + let created = now()?; + save( + &database, + &writer, + created, + &[ + SAVED, + Feedback { + author: "bob", + score: 6, + ..SAVED + }, + ], + ) + .await?; + save( + &database, + &writer, + created, + &[Feedback { + deleted: true, + edited_after_seconds: 540, + ..SAVED + }], + ) + .await?; + let reference = trace_ref(&database, "team-a", "key-a", "trace-1").await?; + + let authors: Vec = feedback(&database, "", "trace-1", &reference) + .await? + .into_iter() + .map(|row| row["author"].clone()) + .collect(); + + assert_eq!(authors, [json!("bob")]); + Ok(()) +} + +#[rstest] +#[tokio::test] +async fn summaries_aggregate_live_feedback_per_trace_and_respect_team_scope( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = ready(&database).await?; + save( + &database, + &writer, + now()?, + &[ + Feedback { score: 2, ..SAVED }, + Feedback { + author: "bob", + score: 7, + ..SAVED + }, + Feedback { + author: "carol", + score: 0, + deleted: true, + ..SAVED + }, + Feedback { + team: "team-b", + key: "key-b", + trace: "trace-2", + score: 9, + ..SAVED + }, + ], + ) + .await?; + let summary = |team: &str| { + let mut parameters = access(team); + parameters.insert( + "trace_ids".into(), + Parameter::Strings(vec![ + "trace-1".into(), + "trace-2".into(), + "trace-unrated".into(), + ]), + ); + parameters + }; + + let everyone = read(&database, ReadQuery::FeedbackSummary, summary("")).await?; + let team_a = read(&database, ReadQuery::FeedbackSummary, summary("team-a")).await?; + + let by_trace: BTreeMap<&str, (u64, f64, u64)> = everyone + .iter() + .map(|row| { + ( + row["trace_id"].as_str().unwrap_or_default(), + ( + row["count"].as_u64().unwrap_or(0), + row["average"].as_f64().unwrap_or(-1.0), + row["lowest"].as_u64().unwrap_or(99), + ), + ) + }) + .collect(); + assert_eq!( + by_trace, + BTreeMap::from([("trace-1", (2, 4.5, 2)), ("trace-2", (1, 9.0, 9))]) + ); + assert_eq!( + team_a + .iter() + .map(|row| row["trace_id"].clone()) + .collect::>(), + [json!("trace-1")] + ); + Ok(()) +} + +#[rstest] +#[tokio::test] +async fn feedback_for_another_teams_trace_is_not_readable( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = ready(&database).await?; + save(&database, &writer, now()?, &[SAVED]).await?; + let reference = trace_ref(&database, "team-a", "key-a", "trace-1").await?; + + assert!( + feedback(&database, "team-b", "trace-1", &reference) + .await? + .is_empty() + ); + assert_eq!( + feedback(&database, "team-a", "trace-1", &reference) + .await? + .len(), + 1 + ); + Ok(()) +} + +#[rstest] +#[tokio::test] +async fn the_table_rejects_scores_above_ten( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = ready(&database).await?; + + let rejected = save( + &database, + &writer, + now()?, + &[Feedback { score: 11, ..SAVED }], + ) + .await; + + assert!(rejected.is_err()); + Ok(()) +} + +#[rstest] +#[tokio::test] +async fn target_resolves_team_and_key_for_a_visible_trace_only( + #[future(awt)] database: TestResult, +) -> TestResult { + let database = database?; + let writer = ready(&database).await?; + let span: BTreeMap = serde_json::from_value(json!({ + "Timestamp": now()?.unix_timestamp_nanos() as i64, "TraceId": "trace-1", "SpanId": "root", + "ParentSpanId": "", "ServiceName": "agent", "SpanName": "run", + "ResourceAttributes": {"litellm.team_id": "team-a", "litellm.api_key_hash": "key-a"} + }))?; + insert_rows( + &database.client, + &writer, + "trace_test", + InsertTable::OtelTraces, + vec![span], + ) + .await?; + let target = |team: &str| { + let mut parameters = access(team); + parameters.insert("trace_id".into(), Parameter::Text("trace-1".into())); + parameters.insert("trace_ref".into(), Parameter::Text(String::new())); + parameters + }; + + let visible = read(&database, ReadQuery::FeedbackTarget, target("team-a")).await?; + let hidden = read(&database, ReadQuery::FeedbackTarget, target("team-b")).await?; + + assert_eq!(visible.len(), 1); + assert_eq!( + ( + visible[0]["team_id"].as_str(), + visible[0]["key_hash"].as_str() + ), + (Some("team-a"), Some("key-a")) + ); + assert_eq!( + visible[0]["trace_ref"].as_str().map(str::to_owned), + Some(trace_ref(&database, "team-a", "key-a", "trace-1").await?) + ); + assert!(hidden.is_empty()); + Ok(()) +} diff --git a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs index 9a3cd4b3050..a00f34a0002 100644 --- a/litellm-rust/crates/traces-clickhouse/tests/migrations.rs +++ b/litellm-rust/crates/traces-clickhouse/tests/migrations.rs @@ -335,10 +335,10 @@ async fn concurrent_schema_setup_succeeds( &database, "SELECT count() AS tables FROM system.tables \ WHERE database = 'trace_test' AND name IN \ - ('otel_traces', 'agent_traces_by_key', 'spend_logs')", + ('otel_traces', 'agent_traces_by_key', 'spend_logs', 'lens_feedback')", ) .await?; - assert_eq!(tables["data"][0]["tables"].as_u64(), Some(3)); + assert_eq!(tables["data"][0]["tables"].as_u64(), Some(4)); assert_eq!( migration_ledger_versions(&database).await?, migration_versions() @@ -1040,6 +1040,7 @@ async fn retention_changes_materialize_existing_rows_and_remain_idempotent( tables["data"], serde_json::json!([ {"name": "agent_traces_by_key"}, + {"name": "lens_feedback"}, {"name": "otel_traces"}, {"name": "spend_logs"} ]) @@ -1057,7 +1058,13 @@ async fn retention_changes_materialize_existing_rows_and_remain_idempotent( "start_time": old_timestamp_ms, "end_time": old_timestamp_ms + 1000 }))?; insert_rows(&database, "otel_traces", vec![span]).await?; + let old_iso = old_time.format(&time::format_description::well_known::Rfc3339)?; + let feedback = serde_json::from_value(serde_json::json!({ + "TeamId": "team-1", "ApiKeyHash": "", "TraceId": "expired", "Author": "admin", + "Score": 4, "Comment": "", "CreatedAt": old_iso, "UpdatedAt": old_iso, "IsDeleted": 0 + }))?; insert_rows(&database, "spend_logs", vec![spend]).await?; + insert_rows(&database, "lens_feedback", vec![feedback]).await?; assert_eq!(table_rows(&database, "agent_traces_by_key").await?, 1); ensure_schema(&database.client, &writer, "trace_test", 14).await?; let deadline = tokio::time::Instant::now() + Duration::from_secs(60); @@ -1087,6 +1094,8 @@ async fn retention_changes_materialize_existing_rows_and_remain_idempotent( ) .await?; execute_write(&database, "OPTIMIZE TABLE trace_test.spend_logs FINAL").await?; + execute_write(&database, "OPTIMIZE TABLE trace_test.lens_feedback FINAL").await?; + assert_eq!(table_rows(&database, "lens_feedback").await?, 0); assert_eq!(table_rows(&database, "otel_traces").await?, 0); assert_eq!(table_rows(&database, "agent_traces_by_key").await?, 0); assert_eq!(table_rows(&database, "spend_logs").await?, 0); @@ -1109,7 +1118,7 @@ async fn retention_reconciliation_updates_each_table_ttl( &database, "SELECT name, create_table_query FROM system.tables \ WHERE database = 'trace_test' AND name IN \ - ('otel_traces', 'agent_traces_by_key', 'spend_logs') ORDER BY name", + ('otel_traces', 'agent_traces_by_key', 'spend_logs', 'lens_feedback') ORDER BY name", ) .await?; let ttl_queries = ttl_queries["data"].as_array().expect("retention tables"); @@ -1118,7 +1127,12 @@ async fn retention_reconciliation_updates_each_table_ttl( .iter() .map(|row| row["name"].as_str().expect("table name")) .collect::>(), - ["agent_traces_by_key", "otel_traces", "spend_logs"] + [ + "agent_traces_by_key", + "lens_feedback", + "otel_traces", + "spend_logs" + ] ); for row in ttl_queries { let query = row["create_table_query"] @@ -1135,7 +1149,7 @@ async fn retention_reconciliation_updates_each_table_ttl( &database, "SELECT name, create_table_query FROM system.tables \ WHERE database = 'trace_test' AND name IN \ - ('otel_traces', 'agent_traces_by_key', 'spend_logs') ORDER BY name", + ('otel_traces', 'agent_traces_by_key', 'spend_logs', 'lens_feedback') ORDER BY name", ) .await?; for row in ttl_queries["data"].as_array().expect("retention tables") { diff --git a/litellm-rust/crates/traces/src/query.rs b/litellm-rust/crates/traces/src/query.rs index e2653e15b61..26eaa33d5ac 100644 --- a/litellm-rust/crates/traces/src/query.rs +++ b/litellm-rust/crates/traces/src/query.rs @@ -17,6 +17,9 @@ pub enum ReadQuery { Sample, Content, Evidence, + FeedbackTarget, + Feedback, + FeedbackSummary, } impl ReadQuery { diff --git a/litellm-rust/crates/traces/tests/otlp.rs b/litellm-rust/crates/traces/tests/otlp.rs index 68a25fb028b..1b817a0bef5 100644 --- a/litellm-rust/crates/traces/tests/otlp.rs +++ b/litellm-rust/crates/traces/tests/otlp.rs @@ -1650,6 +1650,9 @@ fn session_capture_joins_native_logs_and_traces_across_turns_without_changing_sp let first = litellm_traces::decode_otlp_logs(&logs.encode_to_vec(), None).unwrap(); let second = decode_otlp(&request.encode_to_vec(), None).unwrap(); assert_eq!(first[0].trace_id, second[0].trace_id); + // Lens feedback resolves session ids the same way; keep in sync with + // litellm/proxy/lens/feedback_repository.py::session_trace_id. + assert_eq!(second[0].trace_id, "5fddf060372c8501dca4f331b9da882b"); assert_eq!( first[0].attributes["lens.original_trace_id"], "01".repeat(16) diff --git a/litellm-rust/crates/traces/tests/query.rs b/litellm-rust/crates/traces/tests/query.rs index 67f422d6997..7c125b01379 100644 --- a/litellm-rust/crates/traces/tests/query.rs +++ b/litellm-rust/crates/traces/tests/query.rs @@ -14,6 +14,9 @@ use rstest::rstest; #[case::sample("sample", ReadQuery::Sample)] #[case::content("content", ReadQuery::Content)] #[case::evidence("evidence", ReadQuery::Evidence)] +#[case::feedback_target("feedback_target", ReadQuery::FeedbackTarget)] +#[case::feedback("feedback", ReadQuery::Feedback)] +#[case::feedback_summary("feedback_summary", ReadQuery::FeedbackSummary)] fn names_select_the_public_query(#[case] name: &str, #[case] query: ReadQuery) { assert_eq!(ReadQuery::parse(name).unwrap(), query); assert_eq!(query.as_ref(), name); diff --git a/litellm/constants.py b/litellm/constants.py index b09da3d6e7d..c4adc0de22f 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -64,6 +64,8 @@ AGENT_TRACING_AGENT_LIST_LIMIT: Final = get_env_int("AGENT_TRACING_AGENT_LIST_LI LENS_DATASET_MAX_CASES: Final = get_env_int("LENS_DATASET_MAX_CASES", 200) LENS_DATASET_MAX_CASE_CHARS: Final = get_env_int("LENS_DATASET_MAX_CASE_CHARS", 20_000) LENS_DATASET_TRACE_PAGE_SIZE: Final = 500 +LENS_FEEDBACK_MAX_COMMENT_CHARS: Final = 10_000 +LENS_FEEDBACK_MAX_SCORE: Final = 10 DEFAULT_S3_FLUSH_INTERVAL_SECONDS: Final = int(os.getenv("DEFAULT_S3_FLUSH_INTERVAL_SECONDS", 10)) DEFAULT_S3_BATCH_SIZE: Final = int(os.getenv("DEFAULT_S3_BATCH_SIZE", 512)) DEFAULT_S3_MAX_CONCURRENT_UPLOADS: Final = int(os.getenv("DEFAULT_S3_MAX_CONCURRENT_UPLOADS", "16")) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index d4229c825b6..ff36f3f78db 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -549,6 +549,8 @@ class LiteLLMRoutes(enum.Enum): "/lens/{lens_id}/executions/{execution_id}", "/lens/{lens_id}/cancel", "/lens/{lens_id}/findings/{finding_id}", + "/lens/feedback", + "/lens/feedback/summary", "/lens/preview/sample", "/lens/workers/register", "/lens/workers/{worker_id}", @@ -1064,6 +1066,7 @@ class LiteLLMRoutes(enum.Enum): admin_viewer_routes = ( [ "/lens/traces/findings", + "/lens/feedback/summary", "/user/list", "/user/available_users", "/user/available_roles", diff --git a/litellm/proxy/lens/feedback_endpoints.py b/litellm/proxy/lens/feedback_endpoints.py new file mode 100644 index 00000000000..f9479873a84 --- /dev/null +++ b/litellm/proxy/lens/feedback_endpoints.py @@ -0,0 +1,124 @@ +from datetime import datetime, timezone +from typing import Annotated, Final, TypeAlias + +from fastapi import APIRouter, Depends, HTTPException, Query, Response +from pydantic import Field, model_validator + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.lens.endpoints import Auth, user_scope +from litellm.proxy.lens.feedback_models import ( + Feedback, + FeedbackInput, + TraceFeedback, + TraceFeedbackRequest, + TraceFeedbackSummary, +) +from litellm.proxy.lens.feedback_repository import ( + ClickHouseFeedbackStore, + FeedbackStore, + FeedbackWrite, + session_trace_id, +) +from litellm.proxy.lens.models import Record, Scope, TraceIdentity +from litellm.proxy.tracing_runtime import provide_storage +from litellm.rust_bridge.trace.storage import ClickHouseStorage + +router: Final = APIRouter(prefix="/lens/feedback", tags=["Lens"]) +TRACE_NOT_FOUND: Final = "Trace not found" + + +class FeedbackTarget(Record): + trace_id: str | None = Field(default=None, min_length=1, max_length=128) + session_id: str | None = Field(default=None, min_length=1, max_length=512) + trace_ref: str = Field(default="", max_length=512) + + @model_validator(mode="after") + def one_target(self) -> "FeedbackTarget": + if (self.trace_id is None) == (self.session_id is None): + raise ValueError("Pass exactly one of trace_id or session_id") + return self + + def trace(self) -> TraceIdentity: + trace_id: Final = self.trace_id if self.trace_id is not None else session_trace_id(self.session_id or "") + return TraceIdentity(trace_id=trace_id, trace_ref=self.trace_ref) + + +class FeedbackSubmission(FeedbackTarget, FeedbackInput): + pass + + +class FeedbackDeletion(FeedbackTarget): + user: str = Field(default="", max_length=256) + + +def write_scope(auth: UserAPIKeyAuth) -> Scope: + if auth.user_role == LitellmUserRoles.PROXY_ADMIN: + return Scope(all_teams=True) + if auth.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: + raise HTTPException(403, "Admin viewers cannot write feedback") + return Scope(team_id=auth.team_id or "", api_key_hash="" if auth.team_id else auth.token or "") + + +def feedback_store( + storage: Annotated[ClickHouseStorage | None, Depends(provide_storage)], +) -> FeedbackStore: + if storage is None: + raise HTTPException(501, "Lens feedback needs agent tracing. Configure the Lens service and LITELLM_LENS_URL.") + return ClickHouseFeedbackStore(storage) + + +def stored_now() -> datetime: + now: Final = datetime.now(timezone.utc) + return now.replace(microsecond=now.microsecond // 1000 * 1000) + + +def author(auth: UserAPIKeyAuth, user: str) -> str: + identity: Final = user or auth.user_id or auth.token + if not identity: + raise HTTPException(422, "Name the user who left this feedback") + return identity + + +Store: TypeAlias = Annotated[FeedbackStore, Depends(feedback_store)] +Now: TypeAlias = Annotated[datetime, Depends(stored_now)] +Target: TypeAlias = Annotated[FeedbackTarget, Query()] +Deletion: TypeAlias = Annotated[FeedbackDeletion, Query()] + + +@router.get("", response_model=TraceFeedback) +async def read_feedback(target: Target, auth: Auth, store: Store) -> TraceFeedback: + feedback: Final = await store.for_trace(user_scope(auth), target.trace()) + if feedback is None: + raise HTTPException(404, TRACE_NOT_FOUND) + return feedback + + +@router.put("", response_model=Feedback) +async def submit_feedback(body: FeedbackSubmission, auth: Auth, store: Store, now: Now) -> Feedback: + saved: Final = await store.upsert( + write_scope(auth), + FeedbackWrite( + trace=body.trace(), + author=author(auth, body.user), + feedback=FeedbackInput(score=body.score, comment=body.comment), + at=now, + ), + ) + if saved is None: + raise HTTPException(404, TRACE_NOT_FOUND) + return saved + + +@router.delete("", status_code=204) +async def delete_feedback(target: Deletion, auth: Auth, store: Store, now: Now) -> Response: + deleted: Final = await store.delete(write_scope(auth), target.trace(), author(auth, target.user), now) + if deleted is None: + raise HTTPException(404, TRACE_NOT_FOUND) + if not deleted: + raise HTTPException(404, "No feedback from this user on this trace") + return Response(status_code=204) + + +@router.post("/summary", response_model=tuple[TraceFeedbackSummary, ...]) +async def feedback_summary(body: TraceFeedbackRequest, auth: Auth, store: Store) -> tuple[TraceFeedbackSummary, ...]: + return await store.summaries(user_scope(auth), body.traces) diff --git a/litellm/proxy/lens/feedback_models.py b/litellm/proxy/lens/feedback_models.py new file mode 100644 index 00000000000..31429c9dadc --- /dev/null +++ b/litellm/proxy/lens/feedback_models.py @@ -0,0 +1,37 @@ +from datetime import datetime +from typing import Annotated, TypeAlias + +from pydantic import Field + +from litellm.constants import LENS_FEEDBACK_MAX_COMMENT_CHARS, LENS_FEEDBACK_MAX_SCORE +from litellm.proxy.lens.models import Record, TraceIdentity + +FeedbackScore: TypeAlias = Annotated[int, Field(ge=0, le=LENS_FEEDBACK_MAX_SCORE)] + + +class FeedbackInput(Record): + score: FeedbackScore + comment: str = Field(default="", max_length=LENS_FEEDBACK_MAX_COMMENT_CHARS) + user: str = Field(default="", max_length=256) + + +class Feedback(TraceIdentity): + score: FeedbackScore + comment: str + author: str + created_at: datetime + updated_at: datetime + + +class TraceFeedback(TraceIdentity): + feedback: tuple[Feedback, ...] + + +class TraceFeedbackSummary(TraceIdentity): + count: int = Field(ge=0) + average: float | None = Field(ge=0, le=LENS_FEEDBACK_MAX_SCORE) + lowest: FeedbackScore | None + + +class TraceFeedbackRequest(Record): + traces: tuple[TraceIdentity, ...] = Field(min_length=1, max_length=500) diff --git a/litellm/proxy/lens/feedback_repository.py b/litellm/proxy/lens/feedback_repository.py new file mode 100644 index 00000000000..9ee1af164e2 --- /dev/null +++ b/litellm/proxy/lens/feedback_repository.py @@ -0,0 +1,186 @@ +import hashlib +from collections.abc import Mapping +from datetime import datetime, timezone +from typing import Final, Protocol + +from litellm.proxy.lens.feedback_models import Feedback, FeedbackInput, TraceFeedback, TraceFeedbackSummary +from litellm.proxy.lens.models import Record, Scope, TraceIdentity +from litellm.proxy.lens.sources import access_parameters +from litellm.rust_bridge.trace.generated.models import ( + FeedbackRow, + FeedbackSummaryRow, + LensFeedbackParams, + LensFeedbackSummaryParams, + LensFeedbackTargetParams, +) +from litellm.rust_bridge.trace.queries import LENS_FEEDBACK, LENS_FEEDBACK_SUMMARY, LENS_FEEDBACK_TARGET +from litellm.rust_bridge.trace.storage import ClickHouseStorage + +FEEDBACK_TABLE: Final = "lens_feedback" + + +def session_trace_id(session_id: str) -> str: + return hashlib.sha256(f"litellm.claude.session.v1\0{session_id}".encode()).digest()[:16].hex() + + +class FeedbackWrite(Record): + trace: TraceIdentity + author: str + feedback: FeedbackInput + at: datetime + + +class FeedbackStore(Protocol): + async def for_trace(self, scope: Scope, trace: TraceIdentity) -> TraceFeedback | None: ... + async def upsert(self, scope: Scope, write: FeedbackWrite) -> Feedback | None: ... + async def delete(self, scope: Scope, trace: TraceIdentity, author: str, at: datetime) -> bool | None: ... + async def summaries(self, scope: Scope, traces: tuple[TraceIdentity, ...]) -> tuple[TraceFeedbackSummary, ...]: ... + + +class _Target(Record): + trace_id: str + trace_ref: str + team_id: str + key_hash: str + + +def _iso(at: datetime) -> str: + return at.astimezone(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") + + +def _feedback(row: FeedbackRow) -> Feedback: + return Feedback( + trace_id=row.trace_id, + trace_ref=row.trace_ref, + score=row.score, + comment=row.comment, + author=row.author, + created_at=datetime.fromisoformat(row.created_at), + updated_at=datetime.fromisoformat(row.updated_at), + ) + + +class ClickHouseFeedbackStore: + def __init__(self, storage: ClickHouseStorage) -> None: + self.storage: Final = storage + + async def _target(self, scope: Scope, trace: TraceIdentity) -> _Target | None: + rows: Final = await self.storage.query( + LENS_FEEDBACK_TARGET, + LensFeedbackTargetParams( + **access_parameters(scope).model_dump(), trace_id=trace.trace_id, trace_ref=trace.trace_ref + ), + ) + if len(rows) != 1: + return None + return _Target( + trace_id=trace.trace_id, trace_ref=rows[0].trace_ref, team_id=rows[0].team_id, key_hash=rows[0].key_hash + ) + + async def _rows(self, scope: Scope, trace: TraceIdentity) -> tuple[Feedback, ...]: + rows: Final = await self.storage.query( + LENS_FEEDBACK, + LensFeedbackParams( + **access_parameters(scope).model_dump(), trace_id=trace.trace_id, trace_ref=trace.trace_ref + ), + ) + return tuple(_feedback(row) for row in rows) + + async def for_trace(self, scope: Scope, trace: TraceIdentity) -> TraceFeedback | None: + target: Final = await self._target(scope, trace) + if target is None: + return None + identity: Final = TraceIdentity(trace_id=target.trace_id, trace_ref=target.trace_ref) + return TraceFeedback( + trace_id=identity.trace_id, trace_ref=identity.trace_ref, feedback=await self._rows(scope, identity) + ) + + async def _write(self, target: _Target, author: str, values: Mapping[str, object]) -> None: + await self.storage.insert_rows( + FEEDBACK_TABLE, + ( + { + "TeamId": target.team_id, + "ApiKeyHash": target.key_hash, + "TraceId": target.trace_id, + "Author": author, + **values, + }, + ), + ) + + async def upsert(self, scope: Scope, write: FeedbackWrite) -> Feedback | None: + target: Final = await self._target(scope, write.trace) + if target is None: + return None + identity: Final = TraceIdentity(trace_id=target.trace_id, trace_ref=target.trace_ref) + previous: Final = next((f for f in await self._rows(scope, identity) if f.author == write.author), None) + created: Final = previous.created_at if previous else write.at + await self._write( + target, + write.author, + { + "Score": write.feedback.score, + "Comment": write.feedback.comment, + "CreatedAt": _iso(created), + "UpdatedAt": _iso(write.at), + "IsDeleted": 0, + }, + ) + return Feedback( + trace_id=target.trace_id, + trace_ref=target.trace_ref, + score=write.feedback.score, + comment=write.feedback.comment, + author=write.author, + created_at=created, + updated_at=write.at, + ) + + async def delete(self, scope: Scope, trace: TraceIdentity, author: str, at: datetime) -> bool | None: + target: Final = await self._target(scope, trace) + if target is None: + return None + identity: Final = TraceIdentity(trace_id=target.trace_id, trace_ref=target.trace_ref) + previous: Final = next((f for f in await self._rows(scope, identity) if f.author == author), None) + if previous is None: + return False + await self._write( + target, + author, + { + "Score": previous.score, + "Comment": "", + "CreatedAt": _iso(previous.created_at), + "UpdatedAt": _iso(at), + "IsDeleted": 1, + }, + ) + return True + + async def summaries(self, scope: Scope, traces: tuple[TraceIdentity, ...]) -> tuple[TraceFeedbackSummary, ...]: + rows: Final = await self.storage.query( + LENS_FEEDBACK_SUMMARY, + LensFeedbackSummaryParams( + **access_parameters(scope).model_dump(), trace_ids=sorted({t.trace_id for t in traces}) + ), + ) + return tuple(summary for trace in traces for summary in _summaries(trace, rows)) + + +def _summaries(trace: TraceIdentity, rows: tuple[FeedbackSummaryRow, ...]) -> tuple[TraceFeedbackSummary, ...]: + matched: Final = tuple( + row for row in rows if row.trace_id == trace.trace_id and trace.trace_ref in ("", row.trace_ref) + ) + if not matched: + return ( + TraceFeedbackSummary( + trace_id=trace.trace_id, trace_ref=trace.trace_ref, count=0, average=None, lowest=None + ), + ) + return tuple( + TraceFeedbackSummary( + trace_id=row.trace_id, trace_ref=row.trace_ref, count=row.count, average=row.average, lowest=row.lowest + ) + for row in matched + ) diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 8afacda2e4c..aa086e25411 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -595,6 +595,7 @@ from litellm.proxy.hooks.proxy_track_cost_callback import ( # noqa: F401, RUF10 from litellm.proxy.image_endpoints.endpoints import router as image_router from litellm.proxy.lens.dataset_endpoints import router as lens_dataset_router from litellm.proxy.lens.endpoints import router as lens_router +from litellm.proxy.lens.feedback_endpoints import router as lens_feedback_router from litellm.proxy.lens.repository import WriterDatabase from litellm.proxy.lens.signal_repository import SignalRepository from litellm.proxy.lens.signals import ( @@ -20307,6 +20308,7 @@ app.include_router(tag_management_router) app.include_router(workflow_management_router) app.include_router(memory_router) app.include_router(lens_dataset_router) +app.include_router(lens_feedback_router) app.include_router(lens_router) app.include_router(plugin_router) app.include_router(cost_tracking_settings_router) diff --git a/litellm/rust_bridge/trace/generated/models.py b/litellm/rust_bridge/trace/generated/models.py index 7a748102991..c9600cc545a 100644 --- a/litellm/rust_bridge/trace/generated/models.py +++ b/litellm/rust_bridge/trace/generated/models.py @@ -192,6 +192,141 @@ class ExecutionRow(LiteLLMBaseModel): selection_key: str = "" +Score: TypeAlias = Annotated[ + int, + Field( + ..., + ge=0, + json_schema_extra={ + "x-python-normalized": { + "type": "int", + "minimum": 0, + "maximum": 18446744073709551615, + } + }, + le=18446744073709551615, + ), +] + + +Score1: TypeAlias = Annotated[ + str, + Field( + ..., + json_schema_extra={ + "x-python-normalized": { + "type": "int", + "minimum": 0, + "maximum": 18446744073709551615, + } + }, + pattern="^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$", + ), +] + + +class FeedbackRow(LiteLLMBaseModel): + model_config = ConfigDict( + frozen=True, + ) + + trace_id: str + trace_ref: str + author: str + score: int = Field(..., ge=0, le=18446744073709551615) + comment: str + created_at: str + updated_at: str + + +Count2: TypeAlias = Annotated[ + int, + Field( + ..., + ge=0, + json_schema_extra={ + "x-python-normalized": { + "type": "int", + "minimum": 0, + "maximum": 18446744073709551615, + } + }, + le=18446744073709551615, + ), +] + + +Count3: TypeAlias = Annotated[ + str, + Field( + ..., + json_schema_extra={ + "x-python-normalized": { + "type": "int", + "minimum": 0, + "maximum": 18446744073709551615, + } + }, + pattern="^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$", + ), +] + + +Lowest: TypeAlias = Annotated[ + int, + Field( + ..., + ge=0, + json_schema_extra={ + "x-python-normalized": { + "type": "int", + "minimum": 0, + "maximum": 18446744073709551615, + } + }, + le=18446744073709551615, + ), +] + + +Lowest1: TypeAlias = Annotated[ + str, + Field( + ..., + json_schema_extra={ + "x-python-normalized": { + "type": "int", + "minimum": 0, + "maximum": 18446744073709551615, + } + }, + pattern="^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$", + ), +] + + +class FeedbackSummaryRow(LiteLLMBaseModel): + model_config = ConfigDict( + frozen=True, + ) + + trace_id: str + trace_ref: str + count: int = Field(..., ge=0, le=18446744073709551615) + average: float + lowest: int = Field(..., ge=0, le=18446744073709551615) + + +class FeedbackTargetRow(LiteLLMBaseModel): + model_config = ConfigDict( + frozen=True, + ) + + team_id: str + key_hash: str + trace_ref: str + + class LensAccessParams(LiteLLMBaseModel): model_config = ConfigDict( extra="forbid", @@ -239,6 +374,44 @@ class LensEvidenceParams(LiteLLMBaseModel): quote: str +class LensFeedbackParams(LiteLLMBaseModel): + model_config = ConfigDict( + extra="forbid", + frozen=True, + ) + + all_teams: Literal[0, 1] + team: str + key_hash: str + trace_id: str + trace_ref: str + + +class LensFeedbackSummaryParams(LiteLLMBaseModel): + model_config = ConfigDict( + extra="forbid", + frozen=True, + ) + + all_teams: Literal[0, 1] + team: str + key_hash: str + trace_ids: tuple[str, ...] + + +class LensFeedbackTargetParams(LiteLLMBaseModel): + model_config = ConfigDict( + extra="forbid", + frozen=True, + ) + + all_teams: Literal[0, 1] + team: str + key_hash: str + trace_id: str + trace_ref: str + + ExecutionSource: TypeAlias = Literal["traces", "requests", "both"] @@ -555,9 +728,15 @@ TraceWireModels: TypeAlias = Annotated[ | AgentRow | CountRow | ExecutionRow + | FeedbackRow + | FeedbackSummaryRow + | FeedbackTargetRow | LensAccessParams | LensContentParams | LensEvidenceParams + | LensFeedbackParams + | LensFeedbackSummaryParams + | LensFeedbackTargetParams | LensSampleParams | PartRow | TraceAgentRow diff --git a/litellm/rust_bridge/trace/generated/types.py b/litellm/rust_bridge/trace/generated/types.py index 12f38da6e28..16f22bc0b95 100644 --- a/litellm/rust_bridge/trace/generated/types.py +++ b/litellm/rust_bridge/trace/generated/types.py @@ -90,7 +90,17 @@ class TraceScope(typing_extensions.TypedDict): team_ids: ReadOnly[tuple[str, ...]] -ReadQueryName: TypeAlias = Literal["trace_agents", "availability", "agents", "sample", "content", "evidence"] +ReadQueryName: TypeAlias = Literal[ + "trace_agents", + "availability", + "agents", + "sample", + "content", + "evidence", + "feedback_target", + "feedback", + "feedback_summary", +] class UIFields(typing_extensions.TypedDict): diff --git a/litellm/rust_bridge/trace/queries.py b/litellm/rust_bridge/trace/queries.py index 8d1ee978b2b..5bfc2303057 100644 --- a/litellm/rust_bridge/trace/queries.py +++ b/litellm/rust_bridge/trace/queries.py @@ -11,9 +11,15 @@ from .generated.models import ( AgentRow, CountRow, ExecutionRow, + FeedbackRow, + FeedbackSummaryRow, + FeedbackTargetRow, LensAccessParams, LensContentParams, LensEvidenceParams, + LensFeedbackParams, + LensFeedbackSummaryParams, + LensFeedbackTargetParams, LensSampleParams, PartRow, TraceAgentRow, @@ -74,3 +80,12 @@ LENS_CONTENT: Final[ReadQuery[LensContentParams, PartRow]] = ReadQuery( LENS_EVIDENCE: Final[ReadQuery[LensEvidenceParams, CountRow]] = ReadQuery( "evidence", LensEvidenceParams, TypeAdapter(QueryResponse[CountRow]) ) +LENS_FEEDBACK_TARGET: Final[ReadQuery[LensFeedbackTargetParams, FeedbackTargetRow]] = ReadQuery( + "feedback_target", LensFeedbackTargetParams, TypeAdapter(QueryResponse[FeedbackTargetRow]) +) +LENS_FEEDBACK: Final[ReadQuery[LensFeedbackParams, FeedbackRow]] = ReadQuery( + "feedback", LensFeedbackParams, TypeAdapter(QueryResponse[FeedbackRow]) +) +LENS_FEEDBACK_SUMMARY: Final[ReadQuery[LensFeedbackSummaryParams, FeedbackSummaryRow]] = ReadQuery( + "feedback_summary", LensFeedbackSummaryParams, TypeAdapter(QueryResponse[FeedbackSummaryRow]) +) diff --git a/litellm/tracing/remote.py b/litellm/tracing/remote.py index 5d71557b8b2..187300d6e23 100644 --- a/litellm/tracing/remote.py +++ b/litellm/tracing/remote.py @@ -16,6 +16,7 @@ from litellm.rust_bridge.trace.generated.types import QueryScope, ReadQueryName, MAX_RESPONSE_BYTES: Final = 64 * 1024 * 1024 _JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +_INSERT_PATHS: Final[Mapping[str, str]] = {"spend_logs": "/internal/spend", "lens_feedback": "/internal/feedback"} @dataclass(frozen=True, slots=True, repr=False) @@ -125,9 +126,10 @@ class RemoteTraceStore: return _ReadFailure.INVALID_RESPONSE async def insert_rows(self, table: str, rows: Sequence[Mapping[str, object]]) -> None: - if table != "spend_logs": - raise ValueError("Lens only accepts gateway request records on this endpoint") - response: Final = await self.client.post("/internal/spend", json=tuple(dict(row) for row in rows)) + path: Final = _INSERT_PATHS.get(table) + if path is None: + raise ValueError("Lens only accepts gateway request records and feedback on this endpoint") + response: Final = await self.client.post(path, json=tuple(dict(row) for row in rows)) response.raise_for_status() async def ingest( diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/FeedbackRow.json b/scripts/trace_codegen/schemas/traces-clickhouse/FeedbackRow.json new file mode 100644 index 00000000000..a107aea47e3 --- /dev/null +++ b/scripts/trace_codegen/schemas/traces-clickhouse/FeedbackRow.json @@ -0,0 +1,53 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "author": { + "type": "string" + }, + "comment": { + "type": "string" + }, + "created_at": { + "type": "string" + }, + "score": { + "anyOf": [ + { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + { + "pattern": "^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$", + "type": "string" + } + ], + "x-python-normalized": { + "maximum": 18446744073709551615, + "minimum": 0, + "type": "int" + } + }, + "trace_id": { + "type": "string" + }, + "trace_ref": { + "type": "string" + }, + "updated_at": { + "type": "string" + } + }, + "required": [ + "trace_id", + "trace_ref", + "author", + "score", + "comment", + "created_at", + "updated_at" + ], + "title": "FeedbackRow", + "type": "object" +} diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/FeedbackSummaryRow.json b/scripts/trace_codegen/schemas/traces-clickhouse/FeedbackSummaryRow.json new file mode 100644 index 00000000000..53f0e7e306a --- /dev/null +++ b/scripts/trace_codegen/schemas/traces-clickhouse/FeedbackSummaryRow.json @@ -0,0 +1,62 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "average": { + "format": "double", + "type": "number" + }, + "count": { + "anyOf": [ + { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + { + "pattern": "^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$", + "type": "string" + } + ], + "x-python-normalized": { + "maximum": 18446744073709551615, + "minimum": 0, + "type": "int" + } + }, + "lowest": { + "anyOf": [ + { + "format": "uint64", + "maximum": 18446744073709551615, + "minimum": 0, + "type": "integer" + }, + { + "pattern": "^(?:0|[1-9][0-9]{0,18}|1[0-7][0-9]{18}|18[0-3][0-9]{17}|184[0-3][0-9]{16}|1844[0-5][0-9]{15}|18446[0-6][0-9]{14}|184467[0-3][0-9]{13}|1844674[0-3][0-9]{12}|184467440[0-6][0-9]{10}|1844674407[0-2][0-9]{9}|18446744073[0-6][0-9]{8}|1844674407370[0-8][0-9]{6}|18446744073709[0-4][0-9]{5}|184467440737095[0-4][0-9]{4}|1844674407370955[0-0][0-9]{3}|18446744073709551[0-5][0-9]{2}|184467440737095516[0-0][0-9]{1}|1844674407370955161[0-4][0-9]{0}|18446744073709551615)$", + "type": "string" + } + ], + "x-python-normalized": { + "maximum": 18446744073709551615, + "minimum": 0, + "type": "int" + } + }, + "trace_id": { + "type": "string" + }, + "trace_ref": { + "type": "string" + } + }, + "required": [ + "trace_id", + "trace_ref", + "count", + "average", + "lowest" + ], + "title": "FeedbackSummaryRow", + "type": "object" +} diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/FeedbackTargetRow.json b/scripts/trace_codegen/schemas/traces-clickhouse/FeedbackTargetRow.json new file mode 100644 index 00000000000..17b9adb2d68 --- /dev/null +++ b/scripts/trace_codegen/schemas/traces-clickhouse/FeedbackTargetRow.json @@ -0,0 +1,21 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "properties": { + "key_hash": { + "type": "string" + }, + "team_id": { + "type": "string" + }, + "trace_ref": { + "type": "string" + } + }, + "required": [ + "team_id", + "key_hash", + "trace_ref" + ], + "title": "FeedbackTargetRow", + "type": "object" +} diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackParams.json b/scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackParams.json new file mode 100644 index 00000000000..63794197547 --- /dev/null +++ b/scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackParams.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "all_teams": { + "enum": [ + 0, + 1 + ], + "type": "integer" + }, + "key_hash": { + "type": "string" + }, + "team": { + "type": "string" + }, + "trace_id": { + "type": "string" + }, + "trace_ref": { + "type": "string" + } + }, + "required": [ + "all_teams", + "team", + "key_hash", + "trace_id", + "trace_ref" + ], + "title": "LensFeedbackParams", + "type": "object" +} diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackSummaryParams.json b/scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackSummaryParams.json new file mode 100644 index 00000000000..dde3eb83612 --- /dev/null +++ b/scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackSummaryParams.json @@ -0,0 +1,33 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "all_teams": { + "enum": [ + 0, + 1 + ], + "type": "integer" + }, + "key_hash": { + "type": "string" + }, + "team": { + "type": "string" + }, + "trace_ids": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "all_teams", + "team", + "key_hash", + "trace_ids" + ], + "title": "LensFeedbackSummaryParams", + "type": "object" +} diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackTargetParams.json b/scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackTargetParams.json new file mode 100644 index 00000000000..4e5e475ce27 --- /dev/null +++ b/scripts/trace_codegen/schemas/traces-clickhouse/LensFeedbackTargetParams.json @@ -0,0 +1,34 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "additionalProperties": false, + "properties": { + "all_teams": { + "enum": [ + 0, + 1 + ], + "type": "integer" + }, + "key_hash": { + "type": "string" + }, + "team": { + "type": "string" + }, + "trace_id": { + "type": "string" + }, + "trace_ref": { + "type": "string" + } + }, + "required": [ + "all_teams", + "team", + "key_hash", + "trace_id", + "trace_ref" + ], + "title": "LensFeedbackTargetParams", + "type": "object" +} diff --git a/scripts/trace_codegen/schemas/traces-clickhouse/ReadQueryName.json b/scripts/trace_codegen/schemas/traces-clickhouse/ReadQueryName.json index 1a3fb1151f0..33d66e64a35 100644 --- a/scripts/trace_codegen/schemas/traces-clickhouse/ReadQueryName.json +++ b/scripts/trace_codegen/schemas/traces-clickhouse/ReadQueryName.json @@ -6,7 +6,10 @@ "agents", "sample", "content", - "evidence" + "evidence", + "feedback_target", + "feedback", + "feedback_summary" ], "title": "ReadQueryName", "type": "string" diff --git a/tests/unit/proxy/lens/test_feedback_endpoints.py b/tests/unit/proxy/lens/test_feedback_endpoints.py new file mode 100644 index 00000000000..3ee7c79f169 --- /dev/null +++ b/tests/unit/proxy/lens/test_feedback_endpoints.py @@ -0,0 +1,323 @@ +import hashlib +from collections.abc import Mapping, Sequence +from datetime import datetime, timedelta, timezone +from typing import Final + +import pytest +from fastapi import HTTPException +from pydantic import BaseModel, ValidationError + +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.lens.feedback_endpoints import ( + FeedbackDeletion, + FeedbackSubmission, + FeedbackTarget, + delete_feedback, + feedback_summary, + read_feedback, + submit_feedback, +) +from litellm.proxy.lens.feedback_models import TraceFeedbackRequest +from litellm.proxy.lens.feedback_repository import FEEDBACK_TABLE, ClickHouseFeedbackStore, session_trace_id +from litellm.proxy.lens.models import TraceIdentity +from litellm.rust_bridge.trace.generated.models import ( + FeedbackRow, + FeedbackSummaryRow, + FeedbackTargetRow, + LensFeedbackParams, + LensFeedbackSummaryParams, + LensFeedbackTargetParams, +) +from litellm.rust_bridge.trace.queries import ReadQuery +from litellm.rust_bridge.trace.storage import ClickHouseStorage + +ADMIN: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="admin") +OTHER_ADMIN: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN, user_id="other") +VIEWER: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, user_id="viewer") +INTERNAL: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, user_id="dev") +TEAM_APP: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, team_id="team-a", token="app-key") +SOLO_APP: Final = UserAPIKeyAuth(user_role=LitellmUserRoles.INTERNAL_USER, token="key-solo") +T0: Final = datetime(2026, 3, 1, 12, 0, tzinfo=timezone.utc) + + +def ref(team: str, key: str, trace: str) -> str: + return hashlib.sha256(f"{team}\0{key}\0{trace}".encode()).hexdigest().upper() + + +class FakeClickHouse(ClickHouseStorage): + """Mirrors lens_feedback: ReplacingMergeTree(UpdatedAt, IsDeleted) keyed by team, key, trace, author.""" + + def __init__(self, traces: Mapping[str, tuple[tuple[str, str], ...]]) -> None: + self.traces: Final = traces + self.rows: Final[list[Mapping[str, object]]] = [] # mutable-ok: stands in for the table + + async def insert_rows(self, table: str, rows: Sequence[Mapping[str, object]]) -> None: + assert table == FEEDBACK_TABLE + self.rows.extend(rows) + + def _visible( + self, team: str, key: str, access: LensFeedbackTargetParams | LensFeedbackParams | LensFeedbackSummaryParams + ) -> bool: + if access.all_teams: + return True + return team == access.team and (access.key_hash == "" or key == access.key_hash) + + def _latest(self) -> tuple[Mapping[str, object], ...]: + newest: Final = {} # mutable-ok: emulates FINAL collapse + for row in sorted(self.rows, key=lambda r: str(r["UpdatedAt"])): + newest[(row["TeamId"], row["ApiKeyHash"], row["TraceId"], row["Author"])] = row + return tuple(r for r in newest.values() if r["IsDeleted"] == 0) + + async def query(self, query: ReadQuery[BaseModel, object], parameters: BaseModel) -> tuple[object, ...]: # pyright: ignore[reportIncompatibleMethodOverride] # fake dispatches on the concrete params type + match parameters: + case LensFeedbackTargetParams(): + return tuple( + FeedbackTargetRow(team_id=team, key_hash=key, trace_ref=ref(team, key, parameters.trace_id)) + for team, key in self.traces.get(parameters.trace_id, ()) + if self._visible(team, key, parameters) + and parameters.trace_ref in ("", ref(team, key, parameters.trace_id)) + ) + case LensFeedbackParams(): + return tuple( + FeedbackRow( + trace_id=str(r["TraceId"]), + trace_ref=ref(str(r["TeamId"]), str(r["ApiKeyHash"]), str(r["TraceId"])), + author=str(r["Author"]), + score=int(str(r["Score"])), + comment=str(r["Comment"]), + created_at=str(r["CreatedAt"]), + updated_at=str(r["UpdatedAt"]), + ) + for r in self._latest() + if r["TraceId"] == parameters.trace_id + and ref(str(r["TeamId"]), str(r["ApiKeyHash"]), str(r["TraceId"])) == parameters.trace_ref + and self._visible(str(r["TeamId"]), str(r["ApiKeyHash"]), parameters) + ) + case LensFeedbackSummaryParams(): + live = tuple( + r + for r in self._latest() + if r["TraceId"] in parameters.trace_ids + and self._visible(str(r["TeamId"]), str(r["ApiKeyHash"]), parameters) + ) + keys = sorted({(str(r["TeamId"]), str(r["ApiKeyHash"]), str(r["TraceId"])) for r in live}) + return tuple( + FeedbackSummaryRow( + trace_id=trace, + trace_ref=ref(team, key, trace), + count=len(scores), + average=sum(scores) / len(scores), + lowest=min(scores), + ) + for team, key, trace in keys + for scores in [ + [ + int(str(r["Score"])) + for r in live + if (r["TeamId"], r["ApiKeyHash"], r["TraceId"]) == (team, key, trace) + ] + ] + ) + case _: + raise AssertionError(f"unexpected query {query.name}") + + +def store(**traces: tuple[tuple[str, str], ...]) -> ClickHouseFeedbackStore: + return ClickHouseFeedbackStore(FakeClickHouse(traces or {"t1": (("team-a", "key-a"),)})) + + +def submission(score: int, comment: str = "", **fields: str) -> FeedbackSubmission: + target: Final = {} if "trace_id" in fields or "session_id" in fields else {"trace_id": "t1"} + return FeedbackSubmission.model_validate({"score": score, "comment": comment, **target, **fields}) + + +@pytest.mark.asyncio +async def test_resubmitting_replaces_the_authors_feedback_and_keeps_other_authors() -> None: + feedback: Final = store() + first: Final = await submit_feedback(submission(3, "wrong file"), ADMIN, feedback, T0) + await submit_feedback(submission(9, "great"), OTHER_ADMIN, feedback, T0) + second: Final = await submit_feedback( + submission(7, "fixed after retry"), ADMIN, feedback, T0 + timedelta(minutes=5) + ) + + listed: Final = await read_feedback(FeedbackTarget(trace_id="t1"), VIEWER, feedback) + + assert (first.created_at, second.created_at, second.updated_at) == (T0, T0, T0 + timedelta(minutes=5)) + assert {f.author: f.created_at for f in listed.feedback}["admin"] == T0 + assert listed.trace_ref == ref("team-a", "key-a", "t1") + assert {(f.author, f.score, f.comment) for f in listed.feedback} == { + ("admin", 7, "fixed after retry"), + ("other", 9, "great"), + } + + +@pytest.mark.asyncio +async def test_session_id_resolves_to_the_trace_lens_derives_at_ingest() -> None: + session_trace: Final = session_trace_id("session-one") + feedback: Final = store(**{session_trace: (("team-a", "key-a"),)}) + + saved: Final = await submit_feedback(submission(4, session_id="session-one"), ADMIN, feedback, T0) + listed: Final = await read_feedback(FeedbackTarget(session_id="session-one"), ADMIN, feedback) + + assert saved.trace_id == session_trace + assert listed.trace_ref == ref("team-a", "key-a", session_trace) + assert [f.score for f in listed.feedback] == [4] + + +def test_session_trace_id_matches_the_rust_ingest_hash() -> None: + # Pinned in litellm-rust/crates/traces/tests/otlp.rs (session_capture_joins_native_logs_...). + assert session_trace_id("session-one") == "5fddf060372c8501dca4f331b9da882b" + + +@pytest.mark.asyncio +async def test_unknown_traces_are_not_found_and_write_nothing() -> None: + feedback: Final = store() + + with pytest.raises(HTTPException) as write: + await submit_feedback(submission(5, trace_id="missing"), ADMIN, feedback, T0) + with pytest.raises(HTTPException) as read: + await read_feedback(FeedbackTarget(trace_id="missing"), ADMIN, feedback) + + assert (write.value.status_code, read.value.status_code) == (404, 404) + assert isinstance(feedback.storage, FakeClickHouse) and feedback.storage.rows == [] + + +@pytest.mark.parametrize("score", (-1, 11)) +def test_scores_outside_zero_to_ten_are_rejected(score: int) -> None: + with pytest.raises(ValidationError): + submission(score) + + +@pytest.mark.parametrize("target", ({}, {"trace_id": "t1", "session_id": "s1"})) +def test_target_needs_exactly_one_of_trace_or_session(target: dict[str, str]) -> None: + with pytest.raises(ValidationError): + FeedbackTarget.model_validate(target) + + +@pytest.mark.asyncio +async def test_viewers_can_read_but_not_write_and_non_admins_cannot_read_in_lens() -> None: + feedback: Final = store() + await read_feedback(FeedbackTarget(trace_id="t1"), VIEWER, feedback) + + with pytest.raises(HTTPException) as write: + await submit_feedback(submission(5), VIEWER, feedback, T0) + with pytest.raises(HTTPException) as read: + await read_feedback(FeedbackTarget(trace_id="t1"), INTERNAL, feedback) + + assert (write.value.status_code, read.value.status_code) == (403, 403) + + +@pytest.mark.asyncio +async def test_tenant_comes_from_the_trace_and_author_defaults_to_the_caller() -> None: + feedback: Final = store() + saved: Final = await submit_feedback(submission(5), OTHER_ADMIN, feedback, T0) + + assert saved.author == "other" + assert isinstance(feedback.storage, FakeClickHouse) + assert {(r["TeamId"], r["ApiKeyHash"], r["Author"]) for r in feedback.storage.rows} == { + ("team-a", "key-a", "other") + } + with pytest.raises(ValidationError): + FeedbackSubmission.model_validate({"trace_id": "t1", "score": 5, "team_id": "someone-else"}) + + +@pytest.mark.asyncio +async def test_delete_hides_only_the_callers_feedback() -> None: + feedback: Final = store() + await submit_feedback(submission(2), ADMIN, feedback, T0) + await submit_feedback(submission(8), OTHER_ADMIN, feedback, T0) + + await delete_feedback(FeedbackDeletion(trace_id="t1"), ADMIN, feedback, T0 + timedelta(minutes=9)) + with pytest.raises(HTTPException) as missing: + await delete_feedback(FeedbackDeletion(trace_id="t1"), ADMIN, feedback, T0 + timedelta(minutes=9)) + + remaining: Final = await read_feedback(FeedbackTarget(trace_id="t1"), ADMIN, feedback) + assert missing.value.status_code == 404 + assert [f.author for f in remaining.feedback] == ["other"] + + +@pytest.mark.asyncio +async def test_summary_flags_rated_traces_and_leaves_unrated_ones_empty() -> None: + feedback: Final = store(t1=(("team-a", "key-a"),), t2=(("team-a", "key-a"),)) + await submit_feedback(submission(2), ADMIN, feedback, T0) + await submit_feedback(submission(8), OTHER_ADMIN, feedback, T0) + rated: Final = TraceIdentity(trace_id="t1", trace_ref=ref("team-a", "key-a", "t1")) + unrated: Final = TraceIdentity(trace_id="t2", trace_ref=ref("team-a", "key-a", "t2")) + + summaries: Final = await feedback_summary(TraceFeedbackRequest(traces=(rated, unrated)), VIEWER, feedback) + + assert {s.trace_id: (s.count, s.average, s.lowest) for s in summaries} == { + "t1": (2, 5.0, 2), + "t2": (0, None, None), + } + + +@pytest.mark.asyncio +async def test_a_trace_id_shared_by_two_keys_needs_its_trace_ref() -> None: + feedback: Final = store(t1=(("team-a", "key-a"), ("team-a", "key-b"))) + + with pytest.raises(HTTPException) as ambiguous: + await submit_feedback(submission(5), ADMIN, feedback, T0) + saved: Final = await submit_feedback( + submission(5, trace_id="t1", trace_ref=ref("team-a", "key-b", "t1")), ADMIN, feedback, T0 + ) + + assert ambiguous.value.status_code == 404 + assert saved.trace_ref == ref("team-a", "key-b", "t1") + + +@pytest.mark.asyncio +async def test_summary_without_trace_ref_reports_the_rated_trace_with_its_resolved_ref() -> None: + feedback: Final = store() + await submit_feedback(submission(6), ADMIN, feedback, T0) + + summaries: Final = await feedback_summary( + TraceFeedbackRequest(traces=(TraceIdentity(trace_id="t1"),)), VIEWER, feedback + ) + + assert [(s.trace_ref, s.count, s.lowest) for s in summaries] == [(ref("team-a", "key-a", "t1"), 1, 6)] + + +@pytest.mark.asyncio +async def test_an_app_key_records_its_end_users_feedback_on_its_own_teams_trace() -> None: + feedback: Final = store() + await submit_feedback(submission(2, "It ignored my file", user="customer-1"), TEAM_APP, feedback, T0) + await submit_feedback(submission(9, "Perfect", user="customer-2"), TEAM_APP, feedback, T0) + await submit_feedback( + submission(4, "Better after retry", user="customer-1"), TEAM_APP, feedback, T0 + timedelta(minutes=1) + ) + + listed: Final = await read_feedback(FeedbackTarget(trace_id="t1"), ADMIN, feedback) + + assert {(f.author, f.score, f.comment) for f in listed.feedback} == { + ("customer-1", 4, "Better after retry"), + ("customer-2", 9, "Perfect"), + } + + +@pytest.mark.asyncio +async def test_an_app_key_cannot_write_feedback_on_another_tenants_trace() -> None: + feedback: Final = store(t1=(("team-b", "key-b"),), solo=(("", "key-solo"),), other=(("", "key-other"),)) + + with pytest.raises(HTTPException) as other_team: + await submit_feedback(submission(5, user="customer-1"), TEAM_APP, feedback, T0) + with pytest.raises(HTTPException) as other_key: + await submit_feedback(submission(5, trace_id="other", user="customer-1"), SOLO_APP, feedback, T0) + saved: Final = await submit_feedback(submission(5, trace_id="solo", user="customer-1"), SOLO_APP, feedback, T0) + + assert (other_team.value.status_code, other_key.value.status_code) == (404, 404) + assert saved.author == "customer-1" + + +@pytest.mark.asyncio +async def test_an_app_can_remove_one_end_users_feedback() -> None: + feedback: Final = store() + await submit_feedback(submission(2, user="customer-1"), TEAM_APP, feedback, T0) + await submit_feedback(submission(9, user="customer-2"), TEAM_APP, feedback, T0) + + await delete_feedback( + FeedbackDeletion(trace_id="t1", user="customer-1"), TEAM_APP, feedback, T0 + timedelta(minutes=1) + ) + + listed: Final = await read_feedback(FeedbackTarget(trace_id="t1"), ADMIN, feedback) + assert [f.author for f in listed.feedback] == ["customer-2"] diff --git a/tests/unit/tracing/test_remote.py b/tests/unit/tracing/test_remote.py index 2572c2a2044..92922dbd3ca 100644 --- a/tests/unit/tracing/test_remote.py +++ b/tests/unit/tracing/test_remote.py @@ -168,7 +168,7 @@ async def test_gateway_cannot_relay_otlp_or_write_arbitrary_tables() -> None: await store.ensure_schema() with pytest.raises(RuntimeError, match="directly"): await store.ingest(b"{}", "application/json", {}) - with pytest.raises(ValueError, match="request records"): + with pytest.raises(ValueError, match="request records and feedback"): await store.insert_rows("otel_traces", ()) @@ -185,3 +185,18 @@ async def test_request_records_use_the_internal_service_endpoint() -> None: request: Final = requests.get_nowait() assert request.url.path == "/internal/spend" assert json.loads(request.content) == [{"request_id": "r"}] + + +@pytest.mark.asyncio +async def test_feedback_rows_use_the_internal_feedback_endpoint() -> None: + requests: Final = asyncio.Queue[httpx.Request]() + + def accept(request: httpx.Request) -> httpx.Response: + requests.put_nowait(request) + return httpx.Response(204) + + async with httpx.AsyncClient(base_url="http://lens", transport=httpx.MockTransport(accept)) as client: + await RemoteTraceStore(client).insert_rows("lens_feedback", ({"TraceId": "t", "Score": 2},)) + request: Final = requests.get_nowait() + assert request.url.path == "/internal/feedback" + assert json.loads(request.content) == [{"TraceId": "t", "Score": 2}] diff --git a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx index 1a81235b361..e3b79d04496 100644 --- a/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensSetup.integration.test.tsx @@ -40,6 +40,7 @@ function serve({ enabled = false, traces = false, requests = false, connected = return Response.json({ agents: traces ? rollUpAgents([data.runs[0].trace.summary]) : [] }); if (path === "/lens/activity/available") return Response.json({ traces, requests }); if (path === "/lens/traces/findings") return Response.json([]); + if (path === "/lens/feedback/summary") return Response.json([]); if (path === "/lens" && method === "POST") { const saved = { ...data.lenses[0], settings: { ...data.lenses[0].settings, ...(body as object) } }; list.mockResolvedValue({ lenses: [saved], workers: [worker()], tracing_enabled: true }); diff --git a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx index f3891376882..41bf0fc6730 100644 --- a/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/LensWorkspace.integration.test.tsx @@ -178,6 +178,7 @@ describe("Lens interactive demo", () => { if (path.endsWith("/runs")) return Response.json(saved.jobs); if (path === "/v1/traces") return Response.json({ data: data.runs.map((run) => run.trace.summary) }); if (path === "/lens/traces/findings") return Response.json([]); + if (path === "/lens/feedback/summary") return Response.json([]); return Response.json({ data: [], traces: true, requests: false }); }); renderWithProviders(, { @@ -358,6 +359,7 @@ describe("Lens interactive demo", () => { if (path.startsWith("/lens/preview")) return Response.json({ eligible: 0, selected: 0, executions: [] }); if (path === "/v1/traces") return Response.json({ data: [createLensDemoData().runs[0].trace.summary] }); if (path === "/lens/traces/findings") return Response.json([]); + if (path === "/lens/feedback/summary") return Response.json([]); return Response.json({ data: [], traces: true, requests: false }); }); renderWithProviders(, { diff --git a/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts b/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts index 48c993eb6ed..1cc2d672f81 100644 --- a/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts +++ b/ui/litellm-dashboard/src/components/lens/data/demo/createLensDemo.ts @@ -1,6 +1,7 @@ import { ApiError } from "@/lib/http/client"; import type { TracesApi } from "@/components/lens/traces/api"; import { rollUpAgents } from "@/components/lens/agents/agentRollup"; +import type { Feedback, TraceSummary } from "@/components/lens/traces/types"; import type { LensServices } from "../LensServices"; import type { LensApi } from "../service"; import { demoDatasetsApi } from "./demoDatasets"; @@ -50,6 +51,22 @@ function demoLensApi(data: LensDemoData): LensApi { }; } +const DEMO_FEEDBACK = [ + { score: 3, comment: "It edited the wrong file and I had to ask twice.", author: "customer-1042" }, + { score: 9, comment: "Exactly what I asked for.", author: "customer-2210" }, +] as const; + +function demoFeedback(runs: LensDemoData["runs"]): Feedback[] { + return DEMO_FEEDBACK.flatMap((entry, index) => { + const summary: TraceSummary | undefined = runs[index]?.trace.summary; + if (!summary) return []; + const at = summary.start_time; + return [ + { ...entry, trace_id: summary.trace_id, trace_ref: summary.trace_ref ?? "", created_at: at, updated_at: at }, + ]; + }); +} + const summariesIn = (data: LensDemoData, startMs: number, endMs: number) => data.runs .map((item) => item.trace.summary) @@ -57,6 +74,8 @@ const summariesIn = (data: LensDemoData, startMs: number, endMs: number) => function demoTracesApi(data: LensDemoData): TracesApi { const run = (traceId: string) => data.runs.find(({ trace }) => trace.summary.trace_id === traceId); + const feedback = demoFeedback(data.runs); + const feedbackFor = (traceId: string) => feedback.filter((entry) => entry.trace_id === traceId); return { live: false, handoff: (traceId, spanId) => { @@ -96,6 +115,21 @@ function demoTracesApi(data: LensDemoData): TracesApi { }), signals: async (traces) => traces.map((trace) => ({ ...trace, status: "unclassified" as const, flags: [], model: "", classified_at: null })), + feedbackSummary: async (traces) => + traces.map((trace) => { + const scores = feedbackFor(trace.trace_id).map((entry) => entry.score); + return { + trace_id: trace.trace_id, + trace_ref: trace.trace_ref ?? "", + count: scores.length, + average: scores.length ? scores.reduce((sum, score) => sum + score, 0) / scores.length : null, + lowest: scores.length ? Math.min(...scores) : null, + }; + }), + feedback: async (traceId) => { + const summary = (await found(run(traceId))).trace.summary; + return { trace_id: traceId, trace_ref: summary.trace_ref ?? "", feedback: feedbackFor(traceId) }; + }, anyRecorded: async () => data.runs.length > 0, trace: (traceId) => found(run(traceId)?.trace), span: (traceId, spanId) => found(run(traceId)?.details.find((span) => span.span_id === spanId)), diff --git a/ui/litellm-dashboard/src/components/lens/traces/api.ts b/ui/litellm-dashboard/src/components/lens/traces/api.ts index a4846311028..422d2f15773 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/api.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/api.ts @@ -21,6 +21,10 @@ import type { TraceFindingCount, TraceFindingsRequest, TraceSignals, + FeedbackQuery, + TraceFeedback, + TraceFeedbackRequest, + TraceFeedbackSummary, TraceAgentList, TraceAgentsQuery, } from "./types"; @@ -44,6 +48,8 @@ export interface TracesApi { agents(window: TraceWindow): Promise; findings(traces: TraceFindingsRequest["traces"]): Promise; signals(traces: TraceFindingsRequest["traces"]): Promise; + feedbackSummary(traces: TraceFeedbackRequest["traces"]): Promise; + feedback(traceId: string, traceRef?: string): Promise; anyRecorded(): Promise; trace(traceId: string, traceRef?: string, cursor?: string | null): Promise; span(traceId: string, spanId: string, traceRef?: string): Promise; @@ -100,6 +106,16 @@ export function liveTracesApi(accessToken: string): TracesApi { accessToken, body: { traces } satisfies TraceFindingsRequest, }), + feedbackSummary: (traces) => + apiClient.post("/lens/feedback/summary", { + accessToken, + body: { traces } satisfies TraceFeedbackRequest, + }), + feedback: (traceId, traceRef) => + apiClient.get("/lens/feedback", { + accessToken, + query: { trace_id: traceId, trace_ref: traceRef ?? "" } satisfies FeedbackQuery, + }), anyRecorded: async () => { const page = await apiClient.get("/v1/traces", { accessToken, diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.integration.test.tsx new file mode 100644 index 00000000000..645fe30de7f --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.integration.test.tsx @@ -0,0 +1,88 @@ +import { screen, within } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { renderWithProviders, testQueryClient } from "../../../../../../tests/test-utils"; +import { type TracesApi, TracesApiContext } from "../../api"; +import type { Feedback } from "../../types"; +import { FeedbackPanel } from "./FeedbackPanel"; + +const summary = { trace_id: "trace-1", trace_ref: "REF1" }; + +const entry = (author: string, score: number, comment: string, updated_at = "2026-03-01T12:00:00Z"): Feedback => ({ + ...summary, + author, + score, + comment, + created_at: updated_at, + updated_at, +}); + +const renderPanel = (feedback: Feedback[]) => { + const api = { live: true, feedback: vi.fn(async () => ({ ...summary, feedback })) }; + renderWithProviders( + + + , + ); + return api; +}; + +describe("FeedbackPanel", () => { + beforeEach(() => testQueryClient.clear()); + + it("shows what the end user said and their score, flagged when it is low", async () => { + const api = renderPanel([entry("customer-1042", 2, "It edited the wrong file and I had to ask twice")]); + + const panel = await screen.findByRole("region", { name: "User feedback" }); + expect(panel).toHaveAttribute("data-low", "true"); + const row = within(panel).getByTestId("feedback-entry"); + expect(row).toHaveTextContent("2/10"); + expect(row).toHaveTextContent("“It edited the wrong file and I had to ask twice”"); + expect(row).toHaveTextContent("customer-1042"); + expect(api.feedback).toHaveBeenCalledWith("trace-1", "REF1"); + }); + + it("lists every user's feedback newest first with the average when several users rated the run", async () => { + renderPanel([ + entry("customer-1", 9, "Perfect", "2026-03-01T12:00:00Z"), + entry("customer-2", 3, "Too slow", "2026-03-01T12:05:00Z"), + ]); + + const panel = await screen.findByRole("region", { name: "User feedback" }); + expect( + within(panel) + .getAllByTestId("feedback-entry") + .map((row) => row.textContent), + ).toEqual([expect.stringContaining("customer-2"), expect.stringContaining("customer-1")]); + expect(panel).toHaveTextContent("6/10 avg from 2 users"); + expect(panel).toHaveAttribute("data-low", "true"); + }); + + it("does not flag a run every user scored well and offers no way to edit feedback", async () => { + renderPanel([entry("customer-1", 8, "")]); + + const panel = await screen.findByRole("region", { name: "User feedback" }); + expect(panel).not.toHaveAttribute("data-low"); + expect(within(panel).getByText("No comment")).toBeInTheDocument(); + expect(within(panel).queryByRole("button")).not.toBeInTheDocument(); + expect(within(panel).queryByRole("textbox")).not.toBeInTheDocument(); + }); + + it("stays out of the way when feedback cannot be loaded so the run still reads cleanly", async () => { + const api = { live: true, feedback: vi.fn(() => Promise.reject(new Error("ClickHouse down"))) }; + renderWithProviders( + + + , + ); + await vi.waitFor(() => expect(api.feedback).toHaveBeenCalled()); + expect(screen.queryByRole("alert")).not.toBeInTheDocument(); + expect(screen.queryByRole("region", { name: "User feedback" })).not.toBeInTheDocument(); + }); + + it("renders nothing when no end user rated the run", async () => { + const api = renderPanel([]); + await vi.waitFor(() => expect(api.feedback).toHaveBeenCalled()); + expect(screen.queryByRole("region", { name: "User feedback" })).not.toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.tsx new file mode 100644 index 00000000000..f06591acce8 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.tsx @@ -0,0 +1,87 @@ +"use client"; + +import { useQuery } from "@tanstack/react-query"; +import { MessageSquareQuote, Star } from "lucide-react"; + +import { cn } from "@/lib/cva.config"; +import { formatActivityTimestamp } from "@/utils/activityTimestamp"; + +import { useTracesApi } from "../../api"; +import { formatScore } from "../../list/AgentTracesTable"; +import { LOW_SCORE } from "../../list/useTraceFeedback"; +import type { Feedback, TraceSummary } from "../../types"; +import { feedbackView } from "./feedback"; + +interface FeedbackPanelProps { + summary: Pick; + accessToken: string; +} + +const isLow = (score: number) => score <= LOW_SCORE; + +function Entry({ entry }: { entry: Feedback }) { + const low = isLow(entry.score); + return ( +
  • + + {entry.score} + /10 + +
    + {entry.comment ? ( +

    “{entry.comment}”

    + ) : ( +

    No comment

    + )} + + {entry.author} · {formatActivityTimestamp(entry.updated_at)} + +
    +
    • + ); +} + +/** End-user feedback on this run, shown first so a developer reads what the user said before the steps. */ +export function FeedbackPanel({ summary, accessToken }: FeedbackPanelProps) { + const api = useTracesApi(accessToken); + const traceRef = summary.trace_ref ?? ""; + const feedback = useQuery({ + queryKey: ["traceFeedbackDetail", accessToken, summary.trace_id, traceRef], + queryFn: () => api.feedback(summary.trace_id, traceRef), + retry: false, + }); + const view = feedback.data ? feedbackView(feedback.data.feedback) : null; + if (!view) return null; + const low = isLow(view.lowest); + return ( +
    +
    + + User feedback + {view.entries.length > 1 && ( + + + {formatScore(view.average)}/10 avg from {view.entries.length} users + + )} +
    +
      + {view.entries.map((entry) => ( + + ))} +
    +
    + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.test.ts b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.test.ts new file mode 100644 index 00000000000..4af6cf5862d --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.test.ts @@ -0,0 +1,31 @@ +import { describe, expect, it } from "vitest"; + +import type { Feedback } from "../../types"; +import { feedbackView } from "./feedback"; + +const entry = (author: string, score: number, updated_at: string): Feedback => ({ + author, + score, + comment: "", + trace_id: "t1", + trace_ref: "", + created_at: updated_at, + updated_at, +}); + +describe("feedbackView", () => { + it("lists every end user's feedback newest first with the average and lowest score", () => { + const view = feedbackView([ + entry("alice", 2, "2026-03-01T12:00:00Z"), + entry("bob", 9, "2026-03-01T12:05:00Z"), + entry("carol", 7, "2026-03-01T12:03:00Z"), + ]); + expect(view?.entries.map((item) => item.author)).toEqual(["bob", "carol", "alice"]); + expect(view?.average).toBe(6); + expect(view?.lowest).toBe(2); + }); + + it("has nothing to show when nobody rated the run", () => { + expect(feedbackView([])).toBeNull(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.ts b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.ts new file mode 100644 index 00000000000..6a572997657 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.ts @@ -0,0 +1,18 @@ +import type { Feedback } from "../../types"; + +export interface FeedbackView { + readonly entries: readonly Feedback[]; + readonly average: number; + readonly lowest: number; +} + +/** End-user feedback on one run, newest first, or null when nobody has rated it. */ +export function feedbackView(feedback: readonly Feedback[]): FeedbackView | null { + if (feedback.length === 0) return null; + const scores = feedback.map((entry) => entry.score); + return { + entries: [...feedback].sort((a, b) => Date.parse(b.updated_at) - Date.parse(a.updated_at)), + average: scores.reduce((sum, score) => sum + score, 0) / scores.length, + lowest: Math.min(...scores), + }; +} diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.tsx index 43b3c00048e..b325ef2fbd9 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/run/RunView.tsx @@ -17,6 +17,7 @@ import type { Trace } from "../../types"; import { PagingBanner } from "./PagingBanner"; import { RunBody } from "./RunBody"; import { RunHeader } from "./RunHeader"; +import { FeedbackPanel } from "../feedback/FeedbackPanel"; import { useTraceSignalFlags } from "../../list/useTraceSignals"; interface RunViewProps { @@ -175,6 +176,7 @@ function LoadedRun({ onLiveChange={toggleLive} signals={signals} /> + {traceQuery.isRefetchError && (
    Could not refresh this run. Previously received steps are still shown. diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx index eee113f6f0c..f0ba76367b3 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx @@ -12,7 +12,7 @@ import { filterRuns } from "./runSearch/runQuery"; import type { RelativeRangeState } from "@/components/shared/timeRange/useRelativeRange"; import { AgentTracesSection } from "./AgentTracesSection"; -import type { TraceFindingCount, TracePage, TraceSummary } from "../types"; +import type { TraceFeedbackSummary, TraceFindingCount, TracePage, TraceSummary } from "../types"; vi.mock("../../../networking", () => ({ apiClient: { get: vi.fn(), post: vi.fn() }, @@ -71,6 +71,26 @@ const renderWindowed = (timeControls?: RelativeRangeState) => , ); +type TraceKey = { trace_id: string; trace_ref?: string }; +const unrated = (traces: TraceKey[]): TraceFeedbackSummary[] => + traces.map((trace) => ({ + trace_id: trace.trace_id, + trace_ref: trace.trace_ref ?? "", + count: 0, + average: null, + lowest: null, + })); + +/** Routes the shared POST mock: findings get `findings`, the feedback summary gets `feedback`. */ +const stubPost = ( + findings: (traces: TraceKey[]) => Promise, + feedback: (traces: TraceKey[]) => Promise = async (traces) => unrated(traces), +) => + vi.mocked(apiClient.post).mockImplementation(async (path, options) => { + const traces = (options?.body as { traces: TraceKey[] }).traces; + return path === "/lens/feedback/summary" ? feedback(traces) : findings(traces); + }); + const bucketRunCounts = () => screen.getAllByTestId("timeline-bucket").map((bucket) => Number(bucket.getAttribute("data-total"))); @@ -96,10 +116,7 @@ describe("AgentTracesSection", () => { testQueryClient.clear(); vi.mocked(agentTraceListCall).mockReset(); serveService(true); - vi.mocked(apiClient.post).mockImplementation(async (_path, options) => { - const body = options?.body as { traces: { trace_id: string; trace_ref?: string }[] }; - return body.traces.map((trace) => ({ ...trace, finding_count: null })); - }); + stubPost(async (traces) => traces.map((trace) => ({ ...trace, finding_count: null }))); }); it("loads the next page only once the list scrolls near its end, then stops at the last page", async () => { @@ -316,12 +333,13 @@ describe("AgentTracesSection", () => { expect(screen.getByRole("columnheader", { name: "Findings" })).toBeInTheDocument(); expect(screen.queryByRole("columnheader", { name: "Failed" })).not.toBeInTheDocument(); expect(screen.getByRole("columnheader", { name: "Cost" })).toBeInTheDocument(); - expect(within(failed).getByText("—")).toBeInTheDocument(); + const costColumn = screen.getAllByRole("columnheader").findIndex((header) => header.textContent === "Cost"); + expect(within(failed).getAllByRole("cell")[costColumn]).toHaveTextContent("—"); }); it("distinguishes uninvestigated traces, completed clean investigations, and findings", async () => { vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs, next_cursor: null }); - vi.mocked(apiClient.post).mockResolvedValue( + stubPost(async () => runs.map((run, index) => ({ trace_id: run.trace_id, trace_ref: run.trace_ref ?? "", @@ -347,7 +365,7 @@ describe("AgentTracesSection", () => { it("does not present a failed findings lookup as an uninvestigated or clean trace", async () => { vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs.slice(0, 1), next_cursor: null }); - vi.mocked(apiClient.post).mockRejectedValue(new ApiError("Unavailable", 503, {})); + stubPost(() => Promise.reject(new ApiError("Unavailable", 503, {}))); renderSection(); expect(await screen.findByTitle("Could not load findings")).toHaveTextContent("Unavailable"); expect(screen.queryByTitle("No conclusive investigation for this trace")).not.toBeInTheDocument(); @@ -357,7 +375,7 @@ describe("AgentTracesSection", () => { const user = userEvent.setup(); const pending = Promise.withResolvers(); vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs.slice(0, 1), next_cursor: null }); - vi.mocked(apiClient.post).mockReturnValue(pending.promise); + stubPost(() => pending.promise); renderSection(); expect(await screen.findByTestId("agent-trace-row")).toBeVisible(); await user.click(screen.getByRole("button", { name: "Columns" })); @@ -375,7 +393,7 @@ describe("AgentTracesSection", () => { renderSection(false); expect(await screen.findAllByTestId("agent-trace-row")).toHaveLength(runs.length); expect(screen.queryByRole("columnheader", { name: "Findings" })).not.toBeInTheDocument(); - expect(apiClient.post).not.toHaveBeenCalled(); + expect(apiClient.post).not.toHaveBeenCalledWith("/lens/traces/findings", expect.anything()); }); it("shows the spend returned for a run", async () => { @@ -521,6 +539,39 @@ describe("AgentTracesSection", () => { expect(lastUrl(onUrlUpdate).has("fullscreen")).toBe(false); }); + it("shows each run's end-user score and flags the whole row when a user scored it low", async () => { + vi.mocked(agentTraceListCall).mockResolvedValue({ data: runs, next_cursor: null }); + const [low, good] = runs; + const score = (trace: TraceKey, average: number, lowest: number) => ({ + trace_id: trace.trace_id, + trace_ref: trace.trace_ref ?? "", + count: 1, + average, + lowest, + }); + stubPost( + async (traces) => traces.map((trace) => ({ ...trace, finding_count: null })), + async (traces) => + traces.map((trace) => { + if (trace.trace_id === low.trace_id) return score(trace, 2, 2); + if (trace.trace_id === good.trace_id) return score(trace, 9, 9); + return unrated([trace])[0]; + }), + ); + renderSection(); + + await waitFor(() => expect(screen.getAllByTestId("feedback-score")).toHaveLength(2)); + const rows = screen.getAllByTestId("agent-trace-row"); + expect(rows).toHaveLength(runs.length); + const lowRow = rows.find((row) => within(row).queryByText("2/10")); + const goodRow = rows.find((row) => within(row).queryByText("9/10")); + expect(lowRow).toHaveAttribute("data-low-feedback", "true"); + expect(lowRow).toHaveAttribute("data-flagged", "true"); + expect(goodRow).not.toHaveAttribute("data-flagged"); + expect(rows.filter((row) => row.hasAttribute("data-low-feedback"))).toHaveLength(1); + expect(screen.queryByRole("combobox", { name: "Filter traces by feedback" })).not.toBeInTheDocument(); + }); + it("narrows the list to the zoom window named in the URL and clears it on request", async () => { vi.mocked(agentTraceListCall).mockResolvedValue(traceList as TracePage); const startMs = Date.parse(runs[0].start_time); diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.tsx index 97eae300106..1a83ebb9f04 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.tsx @@ -13,6 +13,7 @@ import { Button } from "@/components/ui/button"; import { AgentTracesTable } from "./AgentTracesTable"; import { useTraceFindings } from "./useTraceFindings"; +import { useTraceFeedback } from "./useTraceFeedback"; import { useTraceSignals } from "./useTraceSignals"; import { useOptionalLensApi } from "../../data/LensServices"; import { lensKeys } from "../../data/queries"; @@ -125,6 +126,7 @@ export function AgentTracesSection({ [traces.traces, query, agent, status], ); const runs = useMemo(() => (zoom ? filterByWindow(filtered, zoom) : filtered), [filtered, zoom]); + const feedback = useTraceFeedback(accessToken, runs, isActive); const runRefs = useMemo(() => runs.map(traceRefOf), [runs]); const findings = useTraceFindings(accessToken, runs, isActive, canViewFindings); const signalSetup = useSignalSetup(isActive && canViewFindings !== false); @@ -231,6 +233,7 @@ export function AgentTracesSection({ ( { expect(screen.getAllByTestId("agent-trace-row").some((row) => row.hasAttribute("data-flagged"))).toBe(false); }); }); + +describe("AgentTracesTable feedback column", () => { + const template = (traceList as TracePage).data[0] as TraceSummary; + const renderFeedback = (state: TraceFeedbackState) => + renderWithProviders( + inList( + , + ), + ); + + it("shows the score and flags both the score and the row when a user scored the run low", () => { + renderFeedback({ status: "ready", summary: { count: 2, average: 5.5, lowest: LOW_SCORE } }); + const score = screen.getByTestId("feedback-score"); + expect(score).toHaveTextContent(/^5\.5\/10$/); + expect(score).toHaveAttribute("data-low", "true"); + expect(score).toHaveAttribute("title", `User feedback: 2 ratings, lowest ${LOW_SCORE}/10`); + const row = screen.getByTestId("agent-trace-row"); + expect(row).toHaveAttribute("data-flagged", "true"); + expect(row).toHaveAttribute("data-low-feedback", "true"); + }); + + it("shows a single user's whole-number score plainly and leaves a well scored row unflagged", () => { + renderFeedback({ status: "ready", summary: { count: 1, average: 9, lowest: LOW_SCORE + 1 } }); + expect(screen.getByTestId("feedback-score")).toHaveTextContent(/^9\/10$/); + expect(screen.getByTestId("feedback-score")).not.toHaveAttribute("data-low"); + expect(screen.getByTestId("agent-trace-row")).not.toHaveAttribute("data-flagged"); + }); + + it("shows a dash for a run nobody has rated", () => { + renderFeedback({ status: "ready", summary: { count: 0, average: null, lowest: null } }); + expect(screen.queryByTestId("feedback-score")).not.toBeInTheDocument(); + expect(screen.getByTitle("No feedback yet")).toHaveTextContent("—"); + }); +}); diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.tsx index cb39eea63e0..dcc6161a31d 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesTable.tsx @@ -1,7 +1,7 @@ "use client"; import { getCoreRowModel, useReactTable, type ColumnDef, type TableOptions } from "@tanstack/react-table"; -import { ArrowDown, ChevronRight, Plus } from "lucide-react"; +import { ArrowDown, ChevronRight, Plus, Star } from "lucide-react"; import { createContext, useContext, useEffect } from "react"; import { useInView } from "react-intersection-observer"; @@ -16,6 +16,7 @@ import { formatActivityTimestamp, formatRunTimestamp, localTimeZoneAbbreviation import { SpanIcon } from "../ui/SpanIcon"; import type { TraceFindingState } from "./useTraceFindings"; import { flaggedSignals, isFlagged, type TraceSignalState } from "./useTraceSignals"; +import { LOW_SCORE, isLowFeedback, type TraceFeedbackState } from "./useTraceFeedback"; import { SignalPills } from "../ui/SignalPills"; import { FrameworkLogo, traceFramework } from "../ui/TraceFramework"; import type { TraceSummary } from "../types"; @@ -25,6 +26,7 @@ import { fmtMs, previewText, traceAgentNames } from "../utils"; interface AgentTracesTableProps { traces: TraceSummary[]; findings: ReadonlyMap; + feedback?: ReadonlyMap; canViewFindings?: boolean; signals?: ReadonlyMap; showSignals?: boolean; @@ -89,6 +91,9 @@ const SignalsContext = createContext>(new const NO_SIGNALS: ReadonlyMap = new Map(); const SignalSetupContext = createContext<{ configured: boolean; onSetUp?: () => void }>({ configured: false }); const FLAGGED_ROW = "bg-destructive/[0.04] shadow-[inset_2px_0_0_var(--color-destructive)] hover:bg-destructive/[0.07]"; +const FeedbackContext = createContext>(new Map()); +const NO_FEEDBACK: ReadonlyMap = new Map(); +const feedbackOrEmpty = (feedback?: ReadonlyMap) => feedback ?? NO_FEEDBACK; function AgentCell({ run }: { run: TraceSummary }) { const framework = traceFramework(run); @@ -181,6 +186,32 @@ function SignalsCell({ run }: { run: TraceSummary }) { return ; } +export const formatScore = (score: number): string => (Number.isInteger(score) ? String(score) : score.toFixed(1)); + +export function FeedbackScore({ run }: { run: TraceSummary }) { + const state = useContext(FeedbackContext).get(runKey(run)); + if (!state || state.status === "pending") + return ; + if (state.status === "error") return Unavailable; + const { count, average, lowest } = state.summary; + if (count === 0 || average == null) return —; + const low = lowest != null && lowest <= LOW_SCORE; + return ( + + + {formatScore(average)}/10 + + ); +} + const RUN_COLUMNS: ColumnDef[] = [ { id: "time", @@ -263,6 +294,13 @@ const RUN_COLUMNS: ColumnDef[] = [ cell: ({ row }) => , meta: { numeric: true, className: NUM }, }, + { + id: "feedback", + size: 96, + header: "Feedback", + cell: ({ row }) => , + meta: { numeric: true, className: NUM }, + }, { id: "open", size: 32, @@ -309,12 +347,24 @@ function EmptyRuns({ rangeEmpty, onSetUpTracing }: { rangeEmpty: boolean; onSetU ); } +function rowFlags( + run: TraceSummary, + feedback: ReadonlyMap | undefined, + signals: ReadonlyMap, + showSignals: boolean, +) { + const lowFeedback = isLowFeedback(feedback?.get(runKey(run))); + const signalled = showSignals && isFlagged(signals.get(runKey(run))); + return { lowFeedback, flagged: lowFeedback || signalled }; +} + const bodyClassName = (blurred?: boolean) => cn("transition-[filter]", blurred && "blur-[1.5px]"); /** Devtool-dense runs list: one row per agent run, newest first. */ export function AgentTracesTable({ traces, findings, + feedback, canViewFindings = true, signals = NO_SIGNALS, showSignals = false, @@ -350,64 +400,67 @@ export function AgentTracesTable({ const table = useReactTable(tableOptions); return ( - - - - - - - className={bodyClassName(isPlaceholder)} - rowHeight={() => ROW_HEIGHT} - after={ - <> - {isLoading && SKELETON_ROWS.map((row) => )} - {autoContinue && } - - } - > - {(row) => { - const flagged = showSignals && isFlagged(signals.get(runKey(row.original))); - return ( - - ); - }} - - - {isLoading && ( -

    - Loading runs… -

    - )} - {error && ( -
    - - {traces.length ? "Could not load more runs" : "Could not load runs"}: {error.message} - - {onRetry && ( - + )} +
    + )} + {canContinue && traces.length === 0 && ( +
    + No loaded runs match these filters. + - )} -
    - )} - {canContinue && traces.length === 0 && ( -
    - No loaded runs match these filters. - -
    - )} - {isEmpty && } -     -     -     +
    + )} + {isEmpty && } + + + + ); } diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.integration.test.tsx new file mode 100644 index 00000000000..13710ecb606 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.integration.test.tsx @@ -0,0 +1,35 @@ +import { screen } from "@testing-library/react"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +import { renderWithProviders, testQueryClient } from "../../../../../tests/test-utils"; +import { type TracesApi, TracesApiContext } from "../api"; +import type { TraceSummary } from "../types"; +import { useTraceFeedback } from "./useTraceFeedback"; + +const run = { trace_id: "trace-1", trace_ref: "REF1" } as TraceSummary; + +function Probe() { + const state = useTraceFeedback("sk-test", [run], true).get("REF1"); + return {state?.status ?? "missing"}; +} + +const renderWith = (feedbackSummary: () => Promise) => + renderWithProviders( + + + , + ); + +describe("useTraceFeedback", () => { + beforeEach(() => testQueryClient.clear()); + + it("reads a run's summary from the batch response", async () => { + renderWith(async () => [{ ...run, count: 1, average: 2, lowest: 2 }]); + expect(await screen.findByText("ready")).toBeInTheDocument(); + }); + + it("marks feedback unavailable instead of crashing when the proxy answers with something other than a list", async () => { + renderWith(vi.fn(async () => ({ detail: "Not Found" }))); + expect(await screen.findByText("error")).toBeInTheDocument(); + }); +}); diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.test.ts b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.test.ts new file mode 100644 index 00000000000..f4f943290a6 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; + +import { LOW_SCORE, isLowFeedback, type TraceFeedbackState } from "./useTraceFeedback"; + +const ready = (count: number, lowest: number | null): TraceFeedbackState => ({ + status: "ready", + summary: { count, average: lowest, lowest }, +}); + +describe("isLowFeedback", () => { + it.each([ + { name: "a run an end user scored at the threshold", state: ready(1, LOW_SCORE), low: true }, + { name: "a run whose lowest score is just above the threshold", state: ready(3, LOW_SCORE + 1), low: false }, + { name: "a run nobody scored", state: ready(0, null), low: false }, + { name: "a run whose feedback is still loading", state: { status: "pending" } as const, low: false }, + { name: "a run whose feedback failed to load", state: { status: "error" } as const, low: false }, + { name: "a run with no feedback state", state: undefined, low: false }, + ])("flags $name only when it is low", ({ state, low }) => { + expect(isLowFeedback(state)).toBe(low); + }); +}); diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.ts b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.ts new file mode 100644 index 00000000000..b35542a0943 --- /dev/null +++ b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.ts @@ -0,0 +1,54 @@ +import { useQueries } from "@tanstack/react-query"; +import { chunk } from "es-toolkit"; + +import { useTracesApi } from "../api"; +import type { TraceFeedbackSummary, TraceSummary } from "../types"; + +export const LOW_SCORE = 4; + +export type TraceFeedbackState = + | { status: "ready"; summary: Pick } + | { status: "pending" } + | { status: "error" }; + +export const traceFeedbackKey = (accessToken: string) => ["traceFeedback", accessToken] as const; + +/** A run some end user scored at or below the low-score threshold. */ +export const isLowFeedback = (state: TraceFeedbackState | undefined): boolean => + state?.status === "ready" && state.summary.count > 0 && (state.summary.lowest ?? Infinity) <= LOW_SCORE; + +export function useTraceFeedback(accessToken: string, runs: TraceSummary[], isActive: boolean) { + const api = useTracesApi(accessToken); + const batches = chunk( + runs.map(({ trace_id, trace_ref }) => ({ trace_id, trace_ref: trace_ref ?? "" })), + 500, + ); + const queries = useQueries({ + queries: batches.map((traces) => ({ + queryKey: [...traceFeedbackKey(accessToken), traces], + queryFn: async () => { + const summaries: unknown = await api.feedbackSummary(traces); + if (!Array.isArray(summaries)) throw new Error("Unexpected feedback summary response"); + return summaries as TraceFeedbackSummary[]; + }, + enabled: isActive, + staleTime: 15000, + refetchInterval: isActive && api.live ? 15000 : false, + retry: false, + })), + }); + return new Map( + batches.flatMap((traces, index) => { + const query = queries[index]; + const summaries = new Map(query.data?.map((summary) => [summary.trace_ref || summary.trace_id, summary])); + return traces.map((trace): [string, TraceFeedbackState] => { + const key = trace.trace_ref || trace.trace_id; + if (query.isError) return [key, { status: "error" }]; + if (query.isPending) return [key, { status: "pending" }]; + const summary = summaries.get(key); + if (!summary) return [key, { status: "error" }]; + return [key, { status: "ready", summary }]; + }); + }), + ); +} diff --git a/ui/litellm-dashboard/src/components/lens/traces/types.ts b/ui/litellm-dashboard/src/components/lens/traces/types.ts index ecd9586e42a..04525fbb34c 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/types.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/types.ts @@ -18,6 +18,11 @@ export type TraceFindingsRequest = components["schemas"]["TraceFindingsRequest"] export type TraceFindingCount = components["schemas"]["TraceFindingCount"]; export type TraceSignals = components["schemas"]["TraceSignals"]; export type SignalFlag = NonNullable[number]; +export type TraceFeedback = components["schemas"]["TraceFeedback"]; +export type Feedback = components["schemas"]["Feedback"]; +export type TraceFeedbackRequest = components["schemas"]["TraceFeedbackRequest"]; +export type TraceFeedbackSummary = components["schemas"]["TraceFeedbackSummary"]; +export type FeedbackQuery = NonNullable; type ApiSpanDetail = paths["/v1/traces/{trace_id}/spans/{span_id}"]["get"]["responses"][200]["content"]["application/json"]; export type Span = Trace["spans"][number]; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bf711498307..54de1d30d3e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -9088,6 +9088,42 @@ export interface paths { patch?: never; trace?: never; }; + "/lens/feedback": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + /** Read Feedback */ + get: operations["read_feedback_lens_feedback_get"]; + /** Submit Feedback */ + put: operations["submit_feedback_lens_feedback_put"]; + post?: never; + /** Delete Feedback */ + delete: operations["delete_feedback_lens_feedback_delete"]; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; + "/lens/feedback/summary": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Feedback Summary */ + post: operations["feedback_summary_lens_feedback_summary_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/lens/internal/ingestion-credentials": { parameters: { query?: never; @@ -32530,6 +32566,56 @@ export interface components { */ model: string; }; + /** Feedback */ + Feedback: { + /** Author */ + author: string; + /** Comment */ + comment: string; + /** + * Created At + * Format: date-time + */ + created_at: string; + /** Score */ + score: number; + /** Trace Id */ + trace_id: string; + /** + * Trace Ref + * @default + */ + trace_ref: string; + /** + * Updated At + * Format: date-time + */ + updated_at: string; + }; + /** FeedbackSubmission */ + FeedbackSubmission: { + /** + * Comment + * @default + */ + comment: string; + /** Score */ + score: number; + /** Session Id */ + session_id?: string | null; + /** Trace Id */ + trace_id?: string | null; + /** + * Trace Ref + * @default + */ + trace_ref: string; + /** + * User + * @default + */ + user: string; + }; /** FieldDetail */ FieldDetail: { /** Field Default Value */ @@ -47997,6 +48083,39 @@ export interface components { /** Agents */ agents: components["schemas"]["TraceAgent"][]; }; + /** TraceFeedback */ + TraceFeedback: { + /** Feedback */ + feedback: components["schemas"]["Feedback"][]; + /** Trace Id */ + trace_id: string; + /** + * Trace Ref + * @default + */ + trace_ref: string; + }; + /** TraceFeedbackRequest */ + TraceFeedbackRequest: { + /** Traces */ + traces: components["schemas"]["TraceIdentity"][]; + }; + /** TraceFeedbackSummary */ + TraceFeedbackSummary: { + /** Average */ + average: number | null; + /** Count */ + count: number; + /** Lowest */ + lowest: number | null; + /** Trace Id */ + trace_id: string; + /** + * Trace Ref + * @default + */ + trace_ref: string; + }; /** TraceFindingCount */ TraceFindingCount: { /** Finding Count */ @@ -63613,6 +63732,137 @@ export interface operations { }; }; }; + read_feedback_lens_feedback_get: { + parameters: { + query?: { + trace_id?: string | null; + session_id?: string | null; + trace_ref?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TraceFeedback"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + submit_feedback_lens_feedback_put: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["FeedbackSubmission"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["Feedback"]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + delete_feedback_lens_feedback_delete: { + parameters: { + query?: { + trace_id?: string | null; + session_id?: string | null; + trace_ref?: string; + user?: string; + }; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 204: { + headers: { + [name: string]: unknown; + }; + content?: never; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; + feedback_summary_lens_feedback_summary_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["TraceFeedbackRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["TraceFeedbackSummary"][]; + }; + }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; + }; + }; ingestion_credentials_lens_internal_ingestion_credentials_get: { parameters: { query?: never; From 9506f9e58b56931ef905064a3a3c77e9a9497a81 Mon Sep 17 00:00:00 2001 From: Mayuri <163738104+mayuriphad@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:45:57 +0530 Subject: [PATCH 11/65] fix(proxy): admit litellm_proxy/hosted_vllm in provider-endpoint discovery (#38617) * fix(proxy): admit litellm_proxy/hosted_vllm in provider-endpoint discovery get_provider_models gated all endpoint discovery on membership in the static litellm.models_by_provider dict, which litellm_proxy and hosted_vllm are intentionally absent from since their model list only exists behind the provider's own endpoint. GET /v1/models returned the literal wildcard string instead of the expanded model list for these providers. Now falls through to ProviderConfigManager, which already knows about them, before giving up. * fix(tests): stop patching an SDK internal in the discovery regression test TQ008 flagged patching litellm.proxy.auth.model_checks.get_valid_models. Assert the gate's own return value instead of mocking past it. * chore: drop redundant comment per repo convention * test(proxy): cover litellm_proxy wildcard model discovery end to end --------- Signed-off-by: mayuriphad <163738104+mayuriphad@users.noreply.github.com> --- litellm/proxy/auth/model_checks.py | 11 +++- tests/unit/proxy/auth/test_model_checks.py | 75 ++++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/litellm/proxy/auth/model_checks.py b/litellm/proxy/auth/model_checks.py index f48f5dd7162..133e95eadba 100644 --- a/litellm/proxy/auth/model_checks.py +++ b/litellm/proxy/auth/model_checks.py @@ -13,7 +13,7 @@ from litellm.router import Router from litellm.router_utils.fallback_event_handlers import get_fallback_model_group from litellm.types.router import CredentialLiteLLMParams, LiteLLM_Params from litellm.types.utils import LlmProviders -from litellm.utils import get_valid_models +from litellm.utils import ProviderConfigManager, get_valid_models _CREDENTIAL_LITELLM_PARAM_FIELDS = set(CredentialLiteLLMParams.model_fields) @@ -45,7 +45,14 @@ def get_provider_models(provider: str, litellm_params: LiteLLM_Params | None = N if provider in litellm.models_by_provider: provider_models: Final = get_valid_models(custom_llm_provider=provider, litellm_params=litellm_params) return provider_models - return None + + try: + llm_provider: Final = LlmProviders(provider) + except ValueError: + return None + if ProviderConfigManager.get_provider_model_info(model=None, provider=llm_provider) is None: + return None + return get_valid_models(custom_llm_provider=provider, litellm_params=litellm_params) def _get_models_from_access_groups( diff --git a/tests/unit/proxy/auth/test_model_checks.py b/tests/unit/proxy/auth/test_model_checks.py index 13171a42cda..88c2d9c684a 100644 --- a/tests/unit/proxy/auth/test_model_checks.py +++ b/tests/unit/proxy/auth/test_model_checks.py @@ -967,6 +967,81 @@ def test_get_complete_model_list_sentinel_only_grants_nothing(): assert result == [] +def test_get_provider_models_admits_providers_without_a_static_catalog(): + """Providers without a static model list are no longer rejected up front. + + With endpoint discovery off, get_valid_models falls back to the (empty) + static list, so the result is [] rather than None. Before the fix this + returned None and the wildcard was never expanded. + """ + import litellm + from litellm.proxy.auth.model_checks import get_provider_models + from litellm.types.router import LiteLLM_Params + + assert "litellm_proxy" not in litellm.models_by_provider + assert "hosted_vllm" not in litellm.models_by_provider + + result = get_provider_models( + "litellm_proxy", + litellm_params=LiteLLM_Params( + model="litellm_proxy/*", + api_base="http://upstream:4000", + api_key="sk-upstream", + ), + ) + + assert result == [] + + +def test_get_complete_model_list_discovers_litellm_proxy_wildcard_models(monkeypatch): + """A litellm_proxy/* deployment lists the upstream proxy's models when endpoint discovery is on.""" + import litellm + from litellm import Router + from litellm.llms.openai.chat.gpt_transformation import OpenAIGPTConfig + from litellm.proxy.auth.model_checks import get_complete_model_list + + monkeypatch.setattr(litellm, "check_provider_endpoint", True) + captured = {} + + def fake_get_models(self, api_key=None, api_base=None): + captured["api_key"] = api_key + captured["api_base"] = api_base + return ["gpt-4o", "claude-sonnet"] + + monkeypatch.setattr(OpenAIGPTConfig, "get_models", fake_get_models) + + router = Router( + model_list=[ + { + "model_name": "litellm_proxy/*", + "litellm_params": { + "model": "litellm_proxy/*", + "api_base": "http://upstream:4000", + "api_key": "sk-upstream", + }, + } + ] + ) + result = get_complete_model_list( + key_models=[], + team_models=[], + proxy_model_list=["litellm_proxy/*"], + user_model=None, + infer_model_from_keys=False, + llm_router=router, + ) + + assert captured == {"api_key": "sk-upstream", "api_base": "http://upstream:4000"} + assert "litellm_proxy/gpt-4o" in result + assert "litellm_proxy/claude-sonnet" in result + + +def test_get_provider_models_returns_none_for_an_unknown_provider(): + from litellm.proxy.auth.model_checks import get_provider_models + + assert get_provider_models("not-a-real-provider") is None + + def test_transcribe_is_a_known_provider_for_wildcard_expansion(): import litellm from litellm.proxy.auth.model_checks import ( From 8fc31b1e4e340bef600cdbde0fedac1702224d35 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:41:03 -0700 Subject: [PATCH 12/65] feat(decisions): serve System One format at /v1/systemone and OpenAI format at /v1/decisions (#45184) * feat(decisions): serve System One format at /v1/systemone and OpenAI format at /v1/decisions The System One request format moves to /v1/systemone and /systemone. /v1/decisions and /decisions now accept the OpenAI Decisions API format, translate it into a System One request, route it through the same pipeline, and translate the answers back. * fix(decisions): import assert_never from typing_extensions for Python 3.10 * fix(decisions): cap OpenAI-format questions at the System One limit /v1/decisions accepted up to 200 questions, but the System One request it translates to takes at most 128, so 129 to 200 questions failed with an internal validation error. Both now share MAX_DECISION_QUESTIONS. * fix(decisions): return 400 for bodies that are not JSON * test(decisions): send System One bodies to /v1/systemone in integration tests * fix(decisions): return guardrail_information on OpenAI-format decisions when requested * refactor(decisions): validate proxy request data before reading guardrail settings --- gateway/routes/allowlist.py | 2 + litellm/decisions/openai_transformation.py | 127 +++++++++++ litellm/proxy/_lazy_features.py | 2 +- litellm/proxy/_lazy_openapi_snapshot.json | 48 ++++ litellm/proxy/_types.py | 2 + .../auth/managed_authorization.py | 2 + .../proxy/decisions_endpoints/endpoints.py | 145 ++++++++++--- litellm/types/decisions.py | 171 ++++++++++++++- litellm/types/utils.py | 2 + .../cost_calculation/cost_tracking_case.py | 2 +- .../cost_calculation/cost_tracking_cases.json | 2 +- .../cost_calculation/test_cost_tracking.py | 2 +- .../providers/test_decisions_chaos.py | 4 +- .../providers/test_decisions_wire.py | 14 +- .../decisions/test_openai_transformation.py | 32 +++ .../decisions_endpoints/test_endpoints.py | 205 +++++++++++++++++- .../SystemOneUI.integration.test.tsx | 16 +- .../components/systemOneUI/SystemOneUI.tsx | 16 +- .../systemOneUI/lib/decisions.test.ts | 2 +- .../components/systemOneUI/lib/schemas.ts | 2 +- .../systemOneUI/lib/validatePayload.ts | 2 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 74 +++++++ 22 files changed, 803 insertions(+), 71 deletions(-) create mode 100644 litellm/decisions/openai_transformation.py create mode 100644 tests/unit/decisions/test_openai_transformation.py diff --git a/gateway/routes/allowlist.py b/gateway/routes/allowlist.py index 31c05f21b1b..3cfe9569b34 100644 --- a/gateway/routes/allowlist.py +++ b/gateway/routes/allowlist.py @@ -62,6 +62,8 @@ GATEWAY_PATH_PREFIXES: tuple[str, ...] = ( "/rerank", "/v1/decisions", "/decisions", + "/v1/systemone", + "/systemone", "/v1/ocr", "/ocr", "/v1/rag/", diff --git a/litellm/decisions/openai_transformation.py b/litellm/decisions/openai_transformation.py new file mode 100644 index 00000000000..af2c838b9ab --- /dev/null +++ b/litellm/decisions/openai_transformation.py @@ -0,0 +1,127 @@ +from collections.abc import Mapping, Sequence +from typing import Final + +from typing_extensions import assert_never + +from litellm.types.decisions import ( + ChoiceAnswer, + DecisionAnswer, + DecisionsResponse, + DecisionsUsage, + NoulAnswer, + OpenAIChoiceAnswer, + OpenAIChoiceProbability, + OpenAIChoiceQuestion, + OpenAIDecisionAnswer, + OpenAIDecisionInputMessage, + OpenAIDecisionQuestion, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, + OpenAIDecisionUsage, + OpenAIPredicateAnswer, + OpenAIPredicateQuestion, + OpenAIRefusalAnswer, + OpenAIScoreAnswer, + OpenAIScoreLevel, + OpenAIScoreProbability, + OpenAIScoreQuestion, + ScoreAnswer, + systemone_choice_key, +) + +_OPENAI_ONLY_FIELDS: Final = frozenset({"input", "questions", "safety_identifier"}) + + +def _message_text(message: OpenAIDecisionInputMessage) -> str: + if isinstance(message.content, str): + return message.content + return "\n\n".join(part.text for part in message.content) + + +def _state(decision_input: str | Sequence[OpenAIDecisionInputMessage]) -> str: + if isinstance(decision_input, str): + return decision_input + return "\n\n".join(_message_text(message) for message in decision_input) + + +def _level_criterion(level: OpenAIScoreLevel) -> str: + return level.label if level.description is None else f"{level.label}: {level.description}" + + +def _systemone_question(question: OpenAIDecisionQuestion) -> Mapping[str, object]: + match question: + case OpenAIPredicateQuestion(): + return {"type": "noul", "instructions": question.instructions} + case OpenAIChoiceQuestion(): + return { + "type": "choice", + "instructions": question.instructions, + "criteria": {systemone_choice_key(option.value): option.description for option in question.choices}, + } + case OpenAIScoreQuestion(): + return { + "type": "score", + "instructions": question.instructions, + "criteria": [_level_criterion(level) for level in question.levels], + } + case _: + assert_never(question) + + +def to_systemone_request(request_data: Mapping[str, object], body: OpenAIDecisionRequestBody) -> Mapping[str, object]: + return { + **{key: value for key, value in request_data.items() if key not in _OPENAI_ONLY_FIELDS}, + "state": _state(body.input), + "questions": {str(index): _systemone_question(question) for index, question in enumerate(body.questions)}, + } + + +def _openai_answer(question: OpenAIDecisionQuestion, answer: DecisionAnswer | None) -> OpenAIDecisionAnswer: + match question, answer: + case OpenAIPredicateQuestion(), NoulAnswer(): + return OpenAIPredicateAnswer(name=question.name, probability=answer.noul) + case OpenAIChoiceQuestion(), ChoiceAnswer(): + typed_values: Final = {systemone_choice_key(option.value): option.value for option in question.choices} + return OpenAIChoiceAnswer( + name=question.name, + choice=typed_values.get(answer.choice, answer.choice), + probabilities=tuple( + OpenAIChoiceProbability( + value=option.value, + probability=answer.probabilities.get(systemone_choice_key(option.value), 0.0), + ) + for option in question.choices + ), + confidence=answer.confidence, + ) + case OpenAIScoreQuestion(), ScoreAnswer(): + return OpenAIScoreAnswer( + name=question.name, + score=answer.score, + probabilities=tuple( + OpenAIScoreProbability( + value=index, label=level.label, probability=answer.probabilities.get(str(index), 0.0) + ) + for index, level in enumerate(question.levels) + ), + confidence=answer.confidence, + ) + case _: + return OpenAIRefusalAnswer(name=question.name) + + +def to_openai_response( + response: DecisionsResponse, questions: Sequence[OpenAIDecisionQuestion], requested_model: str +) -> OpenAIDecisionResponse: + usage: Final = response.usage or DecisionsUsage() + return OpenAIDecisionResponse( + model=response.model or requested_model, + answers=tuple( + _openai_answer(question, response.answers.get(str(index))) for index, question in enumerate(questions) + ), + usage=OpenAIDecisionUsage( + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + ), + ) diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index f018029058f..517c0916e13 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -267,7 +267,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( LazyFeature( name="decisions", module_path="litellm.proxy.decisions_endpoints.endpoints", - path_prefixes=("/v1/decisions", "/decisions"), + path_prefixes=("/v1/decisions", "/decisions", "/v1/systemone", "/systemone"), ), LazyFeature( name="claude_code_marketplace", diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 06f1a666a70..9322fb77615 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9521,6 +9521,30 @@ ] } }, + "/systemone": { + "post": { + "operationId": "systemone_systemone_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Systemone", + "tags": [ + "decisions" + ] + } + }, "/v1/decisions": { "post": { "operationId": "decisions_v1_decisions_post", @@ -9544,6 +9568,30 @@ "decisions" ] } + }, + "/v1/systemone": { + "post": { + "operationId": "systemone_v1_systemone_post", + "responses": { + "200": { + "content": { + "application/json": { + "schema": {} + } + }, + "description": "Successful Response" + } + }, + "security": [ + { + "APIKeyHeader": [] + } + ], + "summary": "Systemone", + "tags": [ + "decisions" + ] + } } } }, diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index ff36f3f78db..bc27b2352a9 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -488,6 +488,8 @@ class LiteLLMRoutes(enum.Enum): "/v1/search/{search_tool_name}", "/decisions", "/v1/decisions", + "/systemone", + "/v1/systemone", # OCR "/ocr", "/v1/ocr", diff --git a/litellm/proxy/agent_endpoints/auth/managed_authorization.py b/litellm/proxy/agent_endpoints/auth/managed_authorization.py index 5d8287227a5..64e50cd74e7 100644 --- a/litellm/proxy/agent_endpoints/auth/managed_authorization.py +++ b/litellm/proxy/agent_endpoints/auth/managed_authorization.py @@ -30,6 +30,7 @@ _MANAGED_MODEL_ROUTES: Final = frozenset( "moderations", "rerank", "decisions", + "systemone", "ocr", ), ) @@ -74,6 +75,7 @@ _MODEL_ROUTE_KINDS: Final[ "/audio/speech": "speech", "/rerank": "body", "/decisions": "body", + "/systemone": "body", "/messages/count_tokens": "body", ":countTokens": "path", } diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index e05d37963cb..265dbbb2957 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -1,39 +1,64 @@ +from collections.abc import Mapping from typing import Annotated, Final from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract from pydantic import TypeAdapter, ValidationError +from litellm.decisions.openai_transformation import to_openai_response, to_systemone_request from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth -from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.decisions import DecisionsRequestBody +from litellm.proxy.common_request_processing import ( + ProxyBaseLLMRequestProcessing, + attach_guardrail_information, + include_guardrail_response_requested, +) +from litellm.types.decisions import DecisionsRequestBody, DecisionsResponse, OpenAIDecisionRequestBody router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter( + OpenAIDecisionRequestBody +) +_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) -@router.post( - "/v1/decisions", - dependencies=[Depends(user_api_key_auth)], - response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract - tags=["decisions"], -) -@router.post( - "/decisions", - dependencies=[Depends(user_api_key_auth)], - response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract - tags=["decisions"], -) -async def decisions( +async def _invalid_request( + raw_data: Mapping[str, object], error: ValidationError, user_api_key_dict: UserAPIKeyAuth +) -> Exception: + from litellm.proxy.proxy_server import proxy_logging_obj, version + + return await ProxyBaseLLMRequestProcessing(data=dict(raw_data)).handle_llm_api_exception( + e=BadRequestError( + message=f"Invalid Decisions request: {error}", + model=str(raw_data.get("model", "")), + llm_provider="", + ), + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> dict[str, object]: + body: Final = await request.body() + try: + return _REQUEST_DATA_ADAPTER.validate_json(body) + except ValidationError as error: + raise await _invalid_request(raw_data={}, error=error, user_api_key_dict=user_api_key_dict) + + +async def _process_systemone( request: Request, fastapi_response: Response, - user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], -): + user_api_key_dict: UserAPIKeyAuth, + raw_data: Mapping[str, object], + openai_body: OpenAIDecisionRequestBody | None, +) -> object: from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, ) @@ -55,7 +80,7 @@ async def decisions( user_temperature as proxy_user_temperature, ) - data: Final = _REQUEST_DATA_ADAPTER.validate_json(await request.body()) + data: Final = dict(raw_data if openai_body is None else to_systemone_request(raw_data, openai_body)) general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base) user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model) @@ -63,7 +88,7 @@ async def decisions( processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: _DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data) - return await processor.base_process_llm_request( + result: Final[object] = await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -81,18 +106,21 @@ async def decisions( user_api_base=user_api_base, version=version, ) + if openai_body is None or isinstance(result, Response): + return result + openai_response: Final = to_openai_response( + _DECISIONS_RESPONSE_ADAPTER.validate_python(result), + openai_body.questions, + str(data.get("model", "")), + ) + request_data: Final = _REQUEST_DATA_ADAPTER.validate_python( + processor.data # pyright: ignore[reportUnknownMemberType] # ProxyBaseLLMRequestProcessing.data is a bare dict + ) + if include_guardrail_response_requested(request_data): + return attach_guardrail_information(response=openai_response, request_data=request_data) + return openai_response except ValidationError as error: - bad_request_error: Final = BadRequestError( - message=f"Invalid Decisions request: {error}", - model=str(data.get("model", "")), - llm_provider="", - ) - raise await processor.handle_llm_api_exception( - e=bad_request_error, - user_api_key_dict=user_api_key_dict, - proxy_logging_obj=proxy_logging_obj, - version=version, - ) + raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) except Exception as error: raise await processor.handle_llm_api_exception( e=error, @@ -100,3 +128,60 @@ async def decisions( proxy_logging_obj=proxy_logging_obj, version=version, ) + + +@router.post( + "/v1/systemone", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +@router.post( + "/systemone", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +async def systemone( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + return await _process_systemone( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + raw_data=await _request_data(request, user_api_key_dict), + openai_body=None, + ) + + +@router.post( + "/v1/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +@router.post( + "/decisions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract + tags=["decisions"], +) +async def decisions( + request: Request, + fastapi_response: Response, + user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], +): + raw_data: Final = await _request_data(request, user_api_key_dict) + try: + openai_body: Final = _OPENAI_DECISION_REQUEST_BODY_ADAPTER.validate_python(raw_data) + except ValidationError as error: + raise await _invalid_request(raw_data=raw_data, error=error, user_api_key_dict=user_api_key_dict) + return await _process_systemone( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + raw_data=raw_data, + openai_body=openai_body, + ) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index 8bc535ffebe..b543b3b32df 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -1,5 +1,5 @@ from collections.abc import Mapping, Sequence -from typing import Annotated, Literal, TypeAlias +from typing import Annotated, Final, Literal, TypeAlias from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config from typing_extensions import ReadOnly, Required, TypedDict @@ -8,6 +8,7 @@ from litellm.types.llms.base import LiteLLMPydanticObjectBase DecisionsJSON: TypeAlias = str | Mapping[str, object] | Sequence[object] NoulCriteria: TypeAlias = Mapping[Literal["true", "false"], DecisionsJSON | None] +MAX_DECISION_QUESTIONS: Final = 128 class NoulQuestion(LiteLLMPydanticObjectBase): @@ -47,7 +48,7 @@ DecisionQuestion: TypeAlias = Annotated[ DecisionQuestionMap: TypeAlias = Annotated[ Mapping[Annotated[str, Field(min_length=1)], DecisionQuestion], - Field(min_length=1, max_length=128), + Field(min_length=1, max_length=MAX_DECISION_QUESTIONS), ] @@ -128,3 +129,169 @@ class DecisionsResponse(LiteLLMPydanticObjectBase): def set_hidden_params(self, params: Mapping[str, object]) -> None: self._hidden_params.update(params) + + +class OpenAIDecisionInputText(LiteLLMPydanticObjectBase): + type: Literal["input_text"] + text: str + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class OpenAIDecisionInputMessage(LiteLLMPydanticObjectBase): + role: Literal["user"] = "user" + type: Literal["message"] = "message" + content: str | Sequence[OpenAIDecisionInputText] + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase): + type: Literal["predicate"] + name: str | None = None + instructions: str + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class OpenAIChoiceOption(LiteLLMPydanticObjectBase): + value: str | bool + description: str | None = None + + model_config = ConfigDict(extra="forbid", frozen=True) + + +def systemone_choice_key(value: str | bool) -> str: + if isinstance(value, bool): + return "true" if value else "false" + return value + + +class OpenAIChoiceQuestion(LiteLLMPydanticObjectBase): + type: Literal["choice"] + name: str | None = None + instructions: str + choices: Annotated[Sequence[OpenAIChoiceOption], Field(min_length=2, max_length=255)] + + model_config = ConfigDict(extra="forbid", frozen=True) + + @model_validator(mode="after") + def require_unique_systemone_keys(self) -> "OpenAIChoiceQuestion": + keys: Final = frozenset(systemone_choice_key(option.value) for option in self.choices) + if len(keys) != len(self.choices): + raise ValueError("Choice values must be unique, and a boolean cannot share its text with a string choice") + return self + + +class OpenAIScoreLevel(LiteLLMPydanticObjectBase): + label: str + description: str | None = None + + model_config = ConfigDict(extra="forbid", frozen=True) + + +class OpenAIScoreQuestion(LiteLLMPydanticObjectBase): + type: Literal["score"] + name: str | None = None + instructions: str + levels: Annotated[Sequence[OpenAIScoreLevel], Field(min_length=2, max_length=10)] + + model_config = ConfigDict(extra="forbid", frozen=True) + + +OpenAIDecisionQuestion: TypeAlias = Annotated[ + OpenAIPredicateQuestion | OpenAIChoiceQuestion | OpenAIScoreQuestion, + Field(discriminator="type"), +] + + +class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): + input: str | Sequence[OpenAIDecisionInputMessage] + questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] + safety_identifier: str | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + +class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase): + type: Literal["predicate"] = "predicate" + name: str | None + probability: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIChoiceProbability(LiteLLMPydanticObjectBase): + value: str | bool + probability: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIChoiceAnswer(LiteLLMPydanticObjectBase): + type: Literal["choice"] = "choice" + name: str | None + choice: str | bool + probabilities: tuple[OpenAIChoiceProbability, ...] + confidence: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIScoreProbability(LiteLLMPydanticObjectBase): + value: int + label: str + probability: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIScoreAnswer(LiteLLMPydanticObjectBase): + type: Literal["score"] = "score" + name: str | None + score: float + probabilities: tuple[OpenAIScoreProbability, ...] + confidence: float + + model_config = ConfigDict(frozen=True) + + +class OpenAIRefusalAnswer(LiteLLMPydanticObjectBase): + type: Literal["refusal"] = "refusal" + name: str | None + + model_config = ConfigDict(frozen=True) + + +OpenAIDecisionAnswer: TypeAlias = OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer + + +class OpenAIDecisionInputTokensDetails(LiteLLMPydanticObjectBase): + cached_tokens: int = 0 + cache_write_tokens: int = 0 + + model_config = ConfigDict(frozen=True) + + +class OpenAIDecisionOutputTokensDetails(LiteLLMPydanticObjectBase): + reasoning_tokens: int = 0 + + model_config = ConfigDict(frozen=True) + + +class OpenAIDecisionUsage(LiteLLMPydanticObjectBase): + input_tokens: int + input_tokens_details: OpenAIDecisionInputTokensDetails = OpenAIDecisionInputTokensDetails() + output_tokens: int + output_tokens_details: OpenAIDecisionOutputTokensDetails = OpenAIDecisionOutputTokensDetails() + total_tokens: int + + model_config = ConfigDict(frozen=True) + + +class OpenAIDecisionResponse(LiteLLMPydanticObjectBase): + model: str + answers: tuple[OpenAIDecisionAnswer, ...] + usage: OpenAIDecisionUsage + + model_config = ConfigDict(extra="allow", frozen=True) diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 507bfa6c5e1..a99c22e9479 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -788,6 +788,8 @@ API_ROUTE_TO_CALL_TYPES: Final[Mapping[str, Sequence[CallTypes]]] = { "/v1/search": [CallTypes.asearch, CallTypes.search], "/decisions": [CallTypes.adecisions, CallTypes.decisions], "/v1/decisions": [CallTypes.adecisions, CallTypes.decisions], + "/systemone": [CallTypes.adecisions, CallTypes.decisions], + "/v1/systemone": [CallTypes.adecisions, CallTypes.decisions], # Batches "/batches": [CallTypes.acreate_batch, CallTypes.create_batch], "/v1/batches": [CallTypes.acreate_batch, CallTypes.create_batch], diff --git a/tests/integration/cost_calculation/cost_tracking_case.py b/tests/integration/cost_calculation/cost_tracking_case.py index 466fdc46555..8c9a53262b6 100644 --- a/tests/integration/cost_calculation/cost_tracking_case.py +++ b/tests/integration/cost_calculation/cost_tracking_case.py @@ -250,7 +250,7 @@ class CostTrackingTestCase(BaseModel): "/v1/audio/speech", "/v1/images/generations", "/v1/images/edits", - "/v1/decisions", + "/v1/systemone", ] | Annotated[str, Field(pattern=r"^/(gemini|anthropic|bedrock)/")] ) = "/v1/chat/completions" diff --git a/tests/integration/cost_calculation/cost_tracking_cases.json b/tests/integration/cost_calculation/cost_tracking_cases.json index 94571f2a63c..81beff68c7c 100644 --- a/tests/integration/cost_calculation/cost_tracking_cases.json +++ b/tests/integration/cost_calculation/cost_tracking_cases.json @@ -31207,7 +31207,7 @@ "name": "perplexity/pplx-decider-v1-27b-decisions", "covers": "quota_management.spend_tracking.decisions_costs", "model": "perplexity/pplx-decider-v1-27b", - "endpoint": "/v1/decisions", + "endpoint": "/v1/systemone", "request": { "model": "$MODEL", "state": { diff --git a/tests/integration/cost_calculation/test_cost_tracking.py b/tests/integration/cost_calculation/test_cost_tracking.py index 379b2c13f5b..97dd7e494f1 100644 --- a/tests/integration/cost_calculation/test_cost_tracking.py +++ b/tests/integration/cost_calculation/test_cost_tracking.py @@ -268,7 +268,7 @@ def test_case_bills_expected_cost(gateway: Gateway, case: CostTrackingTestCase) assert row.spend == 0, f"{case.name}: failure spend was {row.spend}" return assert response.is_success, f"{case.name}: proxy returned {response.status_code}: {response.text[:400]}" - if case.endpoint == "/v1/decisions": + if case.endpoint == "/v1/systemone": observed: Final = JSON_OBJECT.validate_json( httpx.get(f"{gateway.upstream_url}/__observations", timeout=5, trust_env=False).content ) diff --git a/tests/integration/providers/test_decisions_chaos.py b/tests/integration/providers/test_decisions_chaos.py index 2e88cdbbc7f..63b4a482e92 100644 --- a/tests/integration/providers/test_decisions_chaos.py +++ b/tests/integration/providers/test_decisions_chaos.py @@ -27,7 +27,7 @@ _API_KEY: Final = "synthetic-decisions-key" _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) _STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") _QUESTIONS: Final[dict[str, JsonValue]] = {"fine": {"type": "noul", "instructions": "Is the state fine?"}} -_ROUTES: Final = ("/v1/decisions", "/decisions") +_ROUTES: Final = ("/v1/systemone", "/systemone") @dataclass(frozen=True, slots=True) @@ -221,7 +221,7 @@ async def test_worker_sigkill_mid_burst_leaves_the_sibling_serving_the_default_m release.set() served: Final = await burst assert len(served) == held_by[survivor_pid], (held_by, len(served)) - follow_up: Final = _Call(route="/decisions", marker=f"ok-{uuid.uuid4().hex}", fail=False) + follow_up: Final = _Call(route="/systemone", marker=f"ok-{uuid.uuid4().hex}", fail=False) (answered,) = await _burst(base_url, candidate.key, None, (follow_up,)) await asyncio.to_thread( eventually, diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 548a0d27f89..84b31088d60 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -158,7 +158,7 @@ def _deployment(scenario: Scenario, handle: ScenarioHandle, provider: _Provider) def _decide(gateway: Gateway, model: str, *, key: str | None = None, **extra: JsonValue) -> httpx.Response: return gateway.request( - "POST", "/v1/decisions", {"model": model, "state": _STATE, "questions": _QUESTIONS, **extra}, key=key + "POST", "/v1/systemone", {"model": model, "state": _STATE, "questions": _QUESTIONS, **extra}, key=key ) @@ -310,7 +310,7 @@ def test_invalid_bodies_are_refused_at_the_gateway_without_an_upstream_call(gate handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) model: Final = _deployment(scenario, handle, _PERPLEXITY) for label, body in _INVALID_BODIES: - response: Final = gateway.request("POST", "/v1/decisions", {"model": model, **body}) + response: Final = gateway.request("POST", "/v1/systemone", {"model": model, **body}) assert response.status_code == 400, (label, response.text) assert "Invalid Decisions request" in response.text, (label, response.text) assert _upstream_calls(gateway, handle) == [] @@ -329,7 +329,7 @@ def test_key_checks_match_chat(gateway: Gateway) -> None: handle: Final = _register(scenario, _answer_body(_PERPLEXITY)) model: Final = _deployment(scenario, handle, _PERPLEXITY) anonymous: Final = gateway.client.post( - "/v1/decisions", json={"model": model, "state": _STATE, "questions": _QUESTIONS} + "/v1/systemone", json={"model": model, "state": _STATE, "questions": _QUESTIONS} ) assert anonymous.status_code == 401, anonymous.text restricted: Final = scenario.key(models=[f"other-{uuid.uuid4().hex}"]) @@ -395,7 +395,7 @@ def test_a_deployment_opted_into_client_api_base_sends_decisions_and_chat_to_the assert _calls_to(observed, configured) == [] -def test_a_config_pass_through_at_v1_decisions_keeps_answering_and_the_native_api_serves_decisions( +def test_a_config_pass_through_at_v1_decisions_keeps_answering_and_the_native_api_serves_system_one( gateway: Gateway, tmp_path: Path ) -> None: with gateway.scenario() as scenario: @@ -407,9 +407,11 @@ def test_a_config_pass_through_at_v1_decisions_keeps_answering_and_the_native_ap tmp_path, f"{pass_through_target.api_base()}/v1/decisions", native_target.api_base() ) with owned_proxy_process(gateway, tmp_path, {}, config=config) as owned: - through: Final = _decide(owned.gateway, _PASS_THROUGH_MODEL) + through: Final = owned.gateway.request( + "POST", "/v1/decisions", {"model": _PASS_THROUGH_MODEL, "state": _STATE, "questions": _QUESTIONS} + ) native: Final = owned.gateway.request( - "POST", "/decisions", {"model": _PASS_THROUGH_NEIGHBOUR, "state": _STATE, "questions": _QUESTIONS} + "POST", "/systemone", {"model": _PASS_THROUGH_NEIGHBOUR, "state": _STATE, "questions": _QUESTIONS} ) assert through.status_code == 200, through.text assert through.json() == {"model": _PASS_THROUGH_MODEL, "answers": _ANSWERS, "usage": _USAGE} diff --git a/tests/unit/decisions/test_openai_transformation.py b/tests/unit/decisions/test_openai_transformation.py new file mode 100644 index 00000000000..a3cc1583d40 --- /dev/null +++ b/tests/unit/decisions/test_openai_transformation.py @@ -0,0 +1,32 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +from pydantic import TypeAdapter, ValidationError + +from litellm.decisions.openai_transformation import to_systemone_request +from litellm.types.decisions import MAX_DECISION_QUESTIONS, DecisionsRequestBody, OpenAIDecisionRequestBody + +_OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) +_SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) + + +def _openai_request(question_count: int) -> Mapping[str, object]: + return { + "model": "decider", + "input": "The package arrived with a broken screen.", + "questions": [{"type": "predicate", "instructions": f"Question {index}?"} for index in range(question_count)], + } + + +def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_request() -> None: + raw: Final = _openai_request(MAX_DECISION_QUESTIONS) + + translated: Final = _SYSTEMONE_BODY.validate_python(to_systemone_request(raw, _OPENAI_BODY.validate_python(raw))) + + assert len(translated.questions) == MAX_DECISION_QUESTIONS + + +def test_an_openai_request_with_more_questions_than_systemone_takes_is_rejected_before_translation() -> None: + with pytest.raises(ValidationError, match="questions"): + _OPENAI_BODY.validate_python(_openai_request(MAX_DECISION_QUESTIONS + 1)) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index 6b1ac9e3404..b00f2be092c 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -16,7 +16,7 @@ from starlette.routing import Match import litellm from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features -from litellm.proxy.decisions_endpoints.endpoints import decisions +from litellm.proxy.decisions_endpoints.endpoints import decisions, systemone from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder from litellm.proxy.proxy_server import ( app, @@ -77,7 +77,7 @@ def client(monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClient]: litellm.in_memory_llm_clients_cache.flush_cache() -@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions")) +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/systemone")) def test_proxy_decisions_route_returns_answers_and_cost( client: TestClient, respx_mock: respx.MockRouter, @@ -126,7 +126,7 @@ def test_proxy_decisions_dispatches_typesafe_deployment( upstream: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) response: Final = client.post( - "/v1/decisions", + "/v1/systemone", json={ "model": "jev", "state": {"source": "proxy-test"}, @@ -165,7 +165,7 @@ def test_proxy_decisions_sends_the_env_key_to_the_deployment_api_base( monkeypatch.setattr(litellm.proxy.proxy_server, "llm_router", router) upstream: Final = respx_mock.post("https://egress.example/perplexity/v1/decisions").respond(json=_RESPONSE) - response: Final = client.post("/v1/decisions", json=_REQUEST) + response: Final = client.post("/v1/systemone", json=_REQUEST) assert response.status_code == 200, response.text assert upstream.call_count == 1 @@ -177,7 +177,7 @@ def test_proxy_decisions_unknown_model_is_a_client_error( respx_mock: respx.MockRouter, ) -> None: response: Final = client.post( - "/v1/decisions", + "/v1/systemone", json={ "model": "missing-model", "state": "review", @@ -210,7 +210,7 @@ def test_proxy_decisions_missing_required_field_is_a_client_error( ) -> None: upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) - response: Final = client.post("/v1/decisions", json=request_body) + response: Final = client.post("/v1/systemone", json=request_body) assert response.status_code == 400, response.text assert not upstream.called @@ -238,7 +238,7 @@ def test_proxy_decisions_dispatches_strands_decider( upstream: Final = respx_mock.post("https://strands.example/v1/systemone").respond(json=_STRANDS_RESPONSE) response: Final = client.post( - "/v1/decisions", + "/v1/systemone", json={ "model": "strands", "state": {"source": "proxy-test"}, @@ -266,7 +266,7 @@ def test_proxy_decisions_without_model_uses_the_proxy_default_model( upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) response: Final = client.post( - "/v1/decisions", json={key: value for key, value in _REQUEST.items() if key != "model"} + "/v1/systemone", json={key: value for key, value in _REQUEST.items() if key != "model"} ) assert response.status_code == 200, response.text @@ -275,6 +275,193 @@ def test_proxy_decisions_without_model_uses_the_proxy_default_model( assert json.loads(upstream.calls[0].request.content)["model"] == "pplx-decider-v1-27b" +_OPENAI_FORMAT_REQUEST: Final[Mapping[str, object]] = { + "model": "decider", + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "The package arrived with a broken screen."}, + {"type": "input_text", "text": "I want a refund."}, + ], + }, + {"role": "user", "content": "Order 1234."}, + ], + "questions": [ + {"type": "predicate", "name": "damaged", "instructions": "Does the customer report a damaged item?"}, + { + "type": "choice", + "instructions": "Should we refund?", + "choices": [{"value": True, "description": "Refund now"}, {"value": "escalate"}], + }, + { + "type": "score", + "name": "severity", + "instructions": "How severe is the issue?", + "levels": [{"label": "minor"}, {"label": "major", "description": "Product unusable"}], + }, + {"type": "predicate", "name": "fraud", "instructions": "Is this fraud?"}, + ], + "safety_identifier": "end-user-1", +} +_SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST: Final[Mapping[str, object]] = { + "model": "pplx-decider-v1-27b", + "answers": { + "0": {"type": "noul", "noul": 0.95}, + "1": {"type": "choice", "choice": "true", "confidence": 0.8, "probabilities": {"true": 0.9, "escalate": 0.1}}, + "2": { + "type": "score", + "score": 0.7, + "confidence": 0.6, + "legend": {"0": "minor", "1": "major: Product unusable"}, + "probabilities": {"0": 0.3, "1": 0.7}, + }, + }, + "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, +} + + +@pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions")) +def test_openai_format_decisions_translate_through_systemone( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond( + json=_SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST + ) + + response: Final = client.post(endpoint, json=_OPENAI_FORMAT_REQUEST) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "pplx-decider-v1-27b", + "state": "The package arrived with a broken screen.\n\nI want a refund.\n\nOrder 1234.", + "questions": { + "0": {"type": "noul", "instructions": "Does the customer report a damaged item?"}, + "1": { + "type": "choice", + "instructions": "Should we refund?", + "criteria": {"true": "Refund now", "escalate": None}, + }, + "2": { + "type": "score", + "instructions": "How severe is the issue?", + "criteria": ["minor", "major: Product unusable"], + }, + "3": {"type": "noul", "instructions": "Is this fraud?"}, + }, + } + body: Final = response.json() + assert body["model"] == _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST["model"] + assert body["answers"] == [ + {"type": "predicate", "name": "damaged", "probability": 0.95}, + { + "type": "choice", + "name": None, + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": "major", "probability": 0.7}, + ], + "confidence": 0.6, + }, + {"type": "refusal", "name": "fraud"}, + ] + assert body["usage"] == { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + } + assert float(response.headers["x-litellm-response-cost"]) > 0 + + +@pytest.mark.parametrize( + ("endpoint", "request_body", "upstream_response"), + ( + ("/v1/systemone", _REQUEST, _RESPONSE), + ("/v1/decisions", _OPENAI_FORMAT_REQUEST, _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST), + ), + ids=("systemone", "openai_format"), +) +def test_decisions_return_guardrail_information_when_requested( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, + request_body: Mapping[str, object], + upstream_response: Mapping[str, object], +) -> None: + respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=upstream_response) + + response: Final = client.post(endpoint, json={**request_body, "include_guardrail_response": True}) + + assert response.status_code == 200, response.text + assert response.json()["guardrail_information"] == [] + + +@pytest.mark.parametrize( + "request_body", + ( + _REQUEST, + { + "model": "decider", + "input": "review", + "questions": [ + { + "type": "choice", + "instructions": "Pick one", + "choices": [{"value": True}, {"value": "true"}], + } + ], + }, + { + "model": "decider", + "input": [ + {"role": "user", "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}]} + ], + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + }, + ), + ids=("systemone_body", "colliding_choice_values", "image_input"), +) +def test_openai_format_decisions_rejects_bodies_it_cannot_translate( + client: TestClient, + respx_mock: respx.MockRouter, + request_body: Mapping[str, object], +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post("/v1/decisions", json=request_body) + + assert response.status_code == 400, response.text + assert not upstream.called + + +@pytest.mark.parametrize("endpoint", ("/v1/systemone", "/v1/decisions")) +@pytest.mark.parametrize("raw_body", (b"", b"{not json"), ids=("empty", "malformed")) +def test_a_body_that_is_not_json_is_a_client_error( + client: TestClient, + respx_mock: respx.MockRouter, + endpoint: str, + raw_body: bytes, +) -> None: + upstream: Final = respx_mock.post("https://api.perplexity.ai/v1/decisions").respond(json=_RESPONSE) + + response: Final = client.post(endpoint, content=raw_body, headers={"Content-Type": "application/json"}) + + assert response.status_code == 400, response.text + assert not upstream.called + + def _decisions_feature() -> LazyFeature: return next(feature for feature in LAZY_FEATURES if feature.name == "decisions") @@ -301,6 +488,8 @@ def test_a_config_pass_through_at_v1_decisions_keeps_its_route_and_the_native_ap assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} assert _serving_endpoint(bare, "/v1/decisions") is pass_through assert _serving_endpoint(bare, "/decisions") is decisions + assert _serving_endpoint(bare, "/v1/systemone") is systemone + assert _serving_endpoint(bare, "/systemone") is systemone def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_wins( diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx index 2cc7e704499..0812e40d3e4 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx @@ -75,7 +75,7 @@ describe("SystemOneUI integration", () => { render(); screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); await user.keyboard("{ArrowDown}"); - await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); + await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" })); expect(screen.getByRole("note", { name: "Decision endpoint notice" })).toHaveTextContent( "omit model to use the proxy's configured default.", @@ -162,7 +162,7 @@ describe("SystemOneUI integration", () => { render(); screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); await user.keyboard("{ArrowDown}"); - await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); + await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" })); const editor = screen.getByRole("textbox", { name: "System One JSON payload" }); const draft = JSON.stringify({ model: "my-decider", @@ -185,12 +185,12 @@ describe("SystemOneUI integration", () => { screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); await user.keyboard("{ArrowDown}"); - await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); + await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" })); expect(editor).toHaveValue(draft); expect(screen.queryByText("Selected choice")).not.toBeInTheDocument(); await user.click(screen.getByRole("button", { name: "Send" })); expect(await screen.findByText("Selected choice")).toBeInTheDocument(); - expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/decisions$/); + expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/systemone$/); expect(JSON.parse(mockFetch.mock.calls[1]?.[1]?.body as string)).toEqual(JSON.parse(draft)); }); @@ -199,7 +199,7 @@ describe("SystemOneUI integration", () => { render(); screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); await user.keyboard("{ArrowDown}"); - await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); + await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" })); const payload = { state: "An outage", questions: { urgent: { type: "noul", instructions: "Is this urgent?" } } }; fireEvent.change(screen.getByRole("textbox", { name: "System One JSON payload" }), { target: { value: JSON.stringify(payload) }, @@ -207,7 +207,7 @@ describe("SystemOneUI integration", () => { expect(screen.getByRole("button", { name: "Send" })).toBeEnabled(); await user.click(screen.getByRole("button", { name: "Send" })); expect(await screen.findByText("jev-1.13.0")).toBeInTheDocument(); - expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/decisions$/); + expect(mockFetch.mock.calls[0]?.[0]).toMatch(/\/v1\/systemone$/); const body = JSON.parse(mockFetch.mock.calls[0]?.[1]?.body as string); expect(body).toEqual(payload); expect(body).not.toHaveProperty("model"); @@ -227,7 +227,7 @@ describe("SystemOneUI integration", () => { await screen.findByRole("button", { name: "Cancel request" }); screen.getByRole("combobox", { name: "Decision endpoint" }).focus(); await user.keyboard("{ArrowDown}"); - await user.click(await screen.findByRole("option", { name: "Decisions · /v1/decisions" })); + await user.click(await screen.findByRole("option", { name: "System One · /v1/systemone" })); expect(mockFetch.mock.calls[0]?.[1]?.signal?.aborted).toBe(true); expect(screen.getByRole("button", { name: "Send" })).toBeEnabled(); @@ -249,7 +249,7 @@ describe("SystemOneUI integration", () => { await user.click(screen.getByRole("button", { name: "Send" })); expect(await screen.findByText("jev-1.13.0")).toBeInTheDocument(); expect(screen.getByText("Selected choice")).toBeInTheDocument(); - expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/decisions$/); + expect(mockFetch.mock.calls[1]?.[0]).toMatch(/\/v1\/systemone$/); expect(mockFetch).toHaveBeenCalledTimes(2); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx index 7b312eab081..e092efce161 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx @@ -42,11 +42,11 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = const [customApiKey, setCustomApiKey] = useState(""); const [endpoint, setEndpoint] = useState("/typesafe/v1/systemone"); const [payloads, setPayloads] = useState>({ - "/v1/decisions": DECISIONS_EXAMPLE_PAYLOAD, + "/v1/systemone": DECISIONS_EXAMPLE_PAYLOAD, "/typesafe/v1/systemone": EXAMPLE_PAYLOAD, }); const rawPayload = payloads[endpoint]; - const examplePayload = endpoint === "/v1/decisions" ? DECISIONS_EXAMPLE_PAYLOAD : EXAMPLE_PAYLOAD; + const examplePayload = endpoint === "/v1/systemone" ? DECISIONS_EXAMPLE_PAYLOAD : EXAMPLE_PAYLOAD; const activeController = useRef(null); const validation = useMemo(() => validateSystemOnePayload(rawPayload, endpoint), [rawPayload, endpoint]); const effectiveApiKey = apiKeySource === "session" ? accessToken || "" : customApiKey.trim(); @@ -105,7 +105,7 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = @@ -182,11 +182,11 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = - {endpoint === "/v1/decisions" ? "Decision models · Jev format" : "TypeSafe Jev · System One"} + {endpoint === "/v1/systemone" ? "Decision models · System One" : "TypeSafe Jev · System One"} - {endpoint === "/v1/decisions" - ? "Sends choice, noul, and score questions through /v1/decisions. Replace the example model with a decision model configured on your proxy, or omit model to use the proxy's configured default." + {endpoint === "/v1/systemone" + ? "Sends choice, noul, and score questions through /v1/systemone. Replace the example model with a decision model configured on your proxy, or omit model to use the proxy's configured default." : "Sends requests through /typesafe/v1/systemone and requires TYPESAFE_API_KEY on the proxy."}{" "} Give us feedback on what you want for decision models diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/decisions.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/decisions.test.ts index 1ae9b2a130f..90f1b63b09f 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/decisions.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/decisions.test.ts @@ -12,7 +12,7 @@ const request = { }, provider_option: { enabled: true }, }; -const validate = (value: unknown) => validateSystemOnePayload(JSON.stringify(value), "/v1/decisions"); +const validate = (value: unknown) => validateSystemOnePayload(JSON.stringify(value), "/v1/systemone"); describe("native decisions validation", () => { it("accepts structured Jev criteria, optional instructions, and provider extensions without dropping fields", () => { diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts index 585a2a36921..9da040256bc 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/schemas.ts @@ -1,6 +1,6 @@ import { z } from "zod"; -export type DecisionEndpoint = "/v1/decisions" | "/typesafe/v1/systemone"; +export type DecisionEndpoint = "/v1/systemone" | "/typesafe/v1/systemone"; const decisionsJson = z.union([z.string(), z.record(z.string(), z.unknown()), z.array(z.unknown())]); const decisionInstructions = decisionsJson.nullish(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/validatePayload.ts b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/validatePayload.ts index eac96e67045..ac53f871333 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/validatePayload.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/validatePayload.ts @@ -54,7 +54,7 @@ export function validateSystemOnePayload( return invalid("syntax", `Invalid JSON syntax: ${json.message}`); } - const schema = endpoint === "/v1/decisions" ? decisionsRequestSchema : systemOneRequestSchema; + const schema = endpoint === "/v1/systemone" ? decisionsRequestSchema : systemOneRequestSchema; const result = schema.safeParse(json.value); if (!result.success) { return { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 54de1d30d3e..1108d4a8743 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -16321,6 +16321,23 @@ export interface paths { patch?: never; trace?: never; }; + "/systemone": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Systemone */ + post: operations["systemone_systemone_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/tag/daily/activity": { parameters: { query?: never; @@ -22434,6 +22451,23 @@ export interface paths { patch?: never; trace?: never; }; + "/v1/systemone": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** Systemone */ + post: operations["systemone_v1_systemone_post"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/v1/threads": { parameters: { query?: never; @@ -73321,6 +73355,26 @@ export interface operations { }; }; }; + systemone_systemone_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; get_tag_daily_activity_tag_daily_activity_get: { parameters: { query?: { @@ -81595,6 +81649,26 @@ export interface operations { }; }; }; + systemone_v1_systemone_post: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody?: never; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": unknown; + }; + }; + }; + }; create_threads_v1_threads_post: { parameters: { query?: never; From 33d908e0ae2c0a257eeb5d546df08527d348a670 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:57:41 -0700 Subject: [PATCH 13/65] feat(decisions): add OpenAI as a Decisions provider behind a shared decisions format (#45214) * feat(decisions): serve System One format at /v1/systemone and OpenAI format at /v1/decisions The System One request format moves to /v1/systemone and /systemone. /v1/decisions and /decisions now accept the OpenAI Decisions API format, translate it into a System One request, route it through the same pipeline, and translate the answers back. * fix(decisions): import assert_never from typing_extensions for Python 3.10 * fix(decisions): cap OpenAI-format questions at the System One limit /v1/decisions accepted up to 200 questions, but the System One request it translates to takes at most 128, so 129 to 200 questions failed with an internal validation error. Both now share MAX_DECISION_QUESTIONS. * fix(decisions): return 400 for bodies that are not JSON * test(decisions): send System One bodies to /v1/systemone in integration tests * feat(decisions): add OpenAI as a Decisions provider System One requests to openai/ models are translated to OpenAI's Decisions shape on the way out and OpenAI's answers are translated back, so both /v1/systemone and /v1/decisions can route to gpt-6-luna. * refactor(decisions): move the OpenAI wire translation into llms/openai * feat(decisions): route both request formats through a shared decisions IR System One and OpenAI-format bodies now convert to one internal representation, and each provider translates it to its own wire format. OpenAI deployments receive the caller's messages, images, names, typed choice values, level descriptions and safety_identifier unchanged. System One providers return a 400 for image input. Cached and cache-write tokens are priced on both response shapes, and provider response extras survive the round trip. * fix(decisions): keep provider fields inside System One answers * fix(decisions): load on Python 3.10 and 3.11 and bill cached tokens once under custom pricing Decisions IR dataclasses used a mappingproxy default, which is only hashable from Python 3.12, so import litellm failed on 3.10 and 3.11. Decisions usage now reaches cost_per_token as a Usage with prompt_tokens_details, so custom pricing no longer adds cached and cache-write tokens on top of input_tokens that already include them * fix(decisions): honor litellm OpenAI key and base settings for the openai provider OpenAI decisions now resolve the key from litellm.api_key, litellm.openai_key, then OPENAI_API_KEY, and the base from litellm.api_base, OPENAI_BASE_URL, then OPENAI_API_BASE, matching other OpenAI calls. Providers own their configured key and base lookup through the endpoint config. * test(decisions): clear every OpenAI base setting in the proxy OpenAI deployment test * fix(decisions): send OpenAI instructions for System One questions without them and reject single-option questions * fix(decisions): return guardrail_information on OpenAI-format decisions when requested * refactor(decisions): validate proxy request data before reading guardrail settings --- litellm/__init__.py | 3 + litellm/_lazy_imports_registry.py | 2 + litellm/cost_calculator.py | 41 ++- litellm/decisions/main.py | 144 ++++++-- litellm/decisions/openai_transformation.py | 127 ------- litellm/litellm_core_utils/litellm_logging.py | 3 +- .../llms/base_llm/decisions/transformation.py | 326 ++++++++++++++++-- litellm/llms/custom_httpx/llm_http_handler.py | 25 +- .../llms/openai/decisions/transformation.py | 320 +++++++++++++++++ ...odel_prices_and_context_window_backup.json | 3 +- .../proxy/decisions_endpoints/endpoints.py | 54 +-- litellm/types/decisions.py | 175 +++++++++- litellm/utils.py | 6 +- model_prices_and_context_window.json | 3 +- tests/unit/decisions/test_main.py | 251 +++++++++++++- .../decisions/test_openai_transformation.py | 32 -- .../test_base_decisions_transformation.py | 180 ++++++++++ tests/unit/llms/openai/decisions/__init__.py | 0 .../test_openai_decisions_transformation.py | 256 ++++++++++++++ .../decisions_endpoints/test_endpoints.py | 110 ++++-- tests/unit/test_utils.py | 1 + 21 files changed, 1753 insertions(+), 309 deletions(-) delete mode 100644 litellm/decisions/openai_transformation.py create mode 100644 litellm/llms/openai/decisions/transformation.py delete mode 100644 tests/unit/decisions/test_openai_transformation.py create mode 100644 tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py create mode 100644 tests/unit/llms/openai/decisions/__init__.py create mode 100644 tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py diff --git a/litellm/__init__.py b/litellm/__init__.py index 4cc1399ee9e..3f93166ecc8 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -1682,6 +1682,9 @@ if TYPE_CHECKING: from .llms.strands_decider.decisions.transformation import ( StrandsDeciderDecisionsConfig as StrandsDeciderDecisionsConfig, ) + from .llms.openai.decisions.transformation import ( + OpenAIDecisionsConfig as OpenAIDecisionsConfig, + ) from .llms.nvidia_nim.rerank.transformation import ( NvidiaNimRerankConfig as NvidiaNimRerankConfig, ) diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index f95679684b8..abefab95e9f 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -161,6 +161,7 @@ LLM_CONFIG_NAMES: Final = ( "OpenRouterDecisionsConfig", "CloudflareDecisionsConfig", "StrandsDeciderDecisionsConfig", + "OpenAIDecisionsConfig", "NvidiaNimRerankConfig", "NvidiaNimRankingConfig", "VertexAIRerankConfig", @@ -720,6 +721,7 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.strands_decider.decisions.transformation", "StrandsDeciderDecisionsConfig", ), + "OpenAIDecisionsConfig": (".llms.openai.decisions.transformation", "OpenAIDecisionsConfig"), "NvidiaNimRerankConfig": ( ".llms.nvidia_nim.rerank.transformation", "NvidiaNimRerankConfig", diff --git a/litellm/cost_calculator.py b/litellm/cost_calculator.py index 4d5f882ea10..da420ab0578 100644 --- a/litellm/cost_calculator.py +++ b/litellm/cost_calculator.py @@ -9,7 +9,7 @@ from typing import TYPE_CHECKING, Any, Final, Literal, cast from httpx import Response from pydantic import BaseModel -from typing_extensions import ReadOnly, TypedDict +from typing_extensions import ReadOnly, TypedDict, assert_never import litellm import litellm._logging @@ -100,7 +100,7 @@ from litellm.llms.vertex_ai.cost_calculator import cost_router as google_cost_ro from litellm.llms.xai.cost_calculator import cost_per_token as xai_cost_per_token from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse -from litellm.types.decisions import DecisionsResponse, DecisionsUsage +from litellm.types.decisions import DecisionsResponse, DecisionsUsage, OpenAIDecisionResponse, OpenAIDecisionUsage from litellm.types.llms.base import CachedTokensDetails, LiteLLMBaseModel from litellm.types.llms.openai import ( HttpxBinaryResponseContent, @@ -1034,6 +1034,8 @@ def get_usage_object( ), ) + if isinstance(completion_response, (DecisionsResponse, OpenAIDecisionResponse)): + return None if completion_response.usage is None else _decisions_usage(completion_response.usage) if usage_obj is None: return None if isinstance(usage_obj, Usage): @@ -1064,12 +1066,37 @@ def get_usage_object( return None +def _decisions_prompt_tokens_details(usage: DecisionsUsage | OpenAIDecisionUsage) -> PromptTokensDetailsWrapper: + match usage: + case DecisionsUsage(): + return PromptTokensDetailsWrapper( + cached_tokens=usage.cached_tokens, cache_write_tokens=usage.cache_write_tokens + ) + case OpenAIDecisionUsage(): + return PromptTokensDetailsWrapper( + cached_tokens=usage.input_tokens_details.cached_tokens, + cache_write_tokens=usage.input_tokens_details.cache_write_tokens, + ) + case _: + assert_never(usage) + + +def _decisions_usage(usage: DecisionsUsage | OpenAIDecisionUsage) -> Usage: + return Usage( + prompt_tokens=usage.input_tokens, + completion_tokens=usage.output_tokens, + total_tokens=usage.input_tokens + usage.output_tokens, + prompt_tokens_details=_decisions_prompt_tokens_details(usage), + ) + + def _is_known_usage_objects(usage_obj): """Returns True if the usage obj is a known Usage type""" return ( isinstance(usage_obj, litellm.Usage) or isinstance(usage_obj, ResponseAPIUsage) or isinstance(usage_obj, DecisionsUsage) + or isinstance(usage_obj, OpenAIDecisionUsage) or TranscriptionUsageObjectTransformation.is_transcription_usage_object(usage_obj) ) @@ -1481,11 +1508,8 @@ def completion_cost( "usage", litellm.Usage(**_usage_for_dump.model_dump()), ) - if isinstance(usage_obj, DecisionsUsage): - _usage = { - "prompt_tokens": usage_obj.input_tokens, - "completion_tokens": usage_obj.output_tokens, - } + if isinstance(usage_obj, (DecisionsUsage, OpenAIDecisionUsage)): + _usage = _decisions_usage(usage_obj).model_dump() elif usage_obj is None: _usage = {} elif isinstance(usage_obj, BaseModel): @@ -1977,7 +2001,8 @@ def response_cost_calculator( | OpenAIModerationResponse | Response | SearchResponse - | DecisionsResponse, + | DecisionsResponse + | OpenAIDecisionResponse, model: str, custom_llm_provider: str | None, call_type: Literal[ diff --git a/litellm/decisions/main.py b/litellm/decisions/main.py index ecb2669db9c..25ec501fd48 100644 --- a/litellm/decisions/main.py +++ b/litellm/decisions/main.py @@ -1,34 +1,56 @@ -from collections.abc import Mapping +from collections.abc import Mapping, Sequence from dataclasses import dataclass, field -from typing import Final +from typing import Final, TypeAlias import httpx from pydantic import TypeAdapter, ValidationError +from typing_extensions import assert_never import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException -from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig +from litellm.llms.base_llm.decisions.transformation import ( + BaseDecisionsConfig, + ir_to_systemone_response, + systemone_request_to_ir, +) from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler +from litellm.llms.openai.decisions.transformation import ir_to_openai_response, openai_request_to_ir from litellm.types.decisions import ( DecisionQuestion, + DecisionsIRRequest, + DecisionsIRResponse, DecisionsJSON, - DecisionsRequest, + DecisionsRequestBody, DecisionsResponse, + OpenAIDecisionInput, + OpenAIDecisionQuestion, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, + UnsupportedDecisionsRequest, ) from litellm.types.utils import LlmProviders from litellm.utils import ProviderConfigManager, client -_DECISIONS_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequest]] = TypeAdapter(DecisionsRequest) +DecisionsQuestions: TypeAlias = ( + Mapping[str, DecisionQuestion | Mapping[str, object]] | Sequence[OpenAIDecisionQuestion | Mapping[str, object]] +) +DecisionsRequestFormat: TypeAlias = DecisionsRequestBody | OpenAIDecisionRequestBody + +_SYSTEMONE_REQUEST_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_REQUEST_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) _HANDLER: Final = BaseLLMHTTPHandler() @dataclass(frozen=True, slots=True, repr=False) class _DecisionsCall: model: str + requested_model: str custom_llm_provider: str provider_config: BaseDecisionsConfig - request: DecisionsRequest + request: DecisionsRequestFormat + ir_request: DecisionsIRRequest + body: Mapping[str, object] = field(repr=False) api_base: str api_key: str | None = field(repr=False) logging_obj: LiteLLMLoggingObj | None @@ -59,11 +81,37 @@ def _provider_config(model: str, custom_llm_provider: str) -> BaseDecisionsConfi return provider_config +def _validate_request( + *, + state: DecisionsJSON | None, + questions: DecisionsQuestions | None, + decision_input: OpenAIDecisionInput | None, + safety_identifier: str | None, +) -> DecisionsRequestFormat: + if decision_input is None: + return _SYSTEMONE_REQUEST_ADAPTER.validate_python({"state": state, "questions": questions}) + return _OPENAI_REQUEST_ADAPTER.validate_python( + {"input": decision_input, "questions": questions, "safety_identifier": safety_identifier} + ) + + +def _ir_request(request: DecisionsRequestFormat) -> DecisionsIRRequest: + match request: + case DecisionsRequestBody(): + return systemone_request_to_ir(request) + case OpenAIDecisionRequestBody(): + return openai_request_to_ir(request) + case _: + assert_never(request) + + def _prepare_call( *, model: str, - state: DecisionsJSON, - questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + state: DecisionsJSON | None, + questions: DecisionsQuestions | None, + decision_input: OpenAIDecisionInput | None, + safety_identifier: str | None, api_key: str | None, api_base: str | None, timeout: float | httpx.Timeout | None, @@ -85,9 +133,15 @@ def _prepare_call( model=model, llm_provider=provider, ) + if state is not None and decision_input is not None: + raise litellm.BadRequestError( + message="Pass either state (System One format) or input (OpenAI format) to the Decisions API, not both", + model=model, + llm_provider=provider, + ) try: - request: Final = _DECISIONS_REQUEST_ADAPTER.validate_python( - {"model": canonical_model, "state": state, "questions": questions} + request: Final = _validate_request( + state=state, questions=questions, decision_input=decision_input, safety_identifier=safety_identifier ) except ValidationError as error: raise litellm.BadRequestError( @@ -111,6 +165,17 @@ def _prepare_call( llm_provider=provider, ) + ir_request: Final = _ir_request(request) + body: Final = provider_config.transform_decisions_request( + model=canonical_model, request=ir_request, custom_llm_provider=provider + ) + if isinstance(body, UnsupportedDecisionsRequest): + raise litellm.BadRequestError( + message=f"Decisions provider '{provider}' cannot serve this request: {body.reason}", + model=model, + llm_provider=provider, + ) + logging_obj: Final = kwargs.get("litellm_logging_obj") if isinstance(logging_obj, LiteLLMLoggingObj): logging_obj.update_from_kwargs( @@ -124,9 +189,12 @@ def _prepare_call( ) return _DecisionsCall( model=canonical_model, + requested_model=model, custom_llm_provider=provider, provider_config=provider_config, request=request, + ir_request=ir_request, + body=body, api_base=resolved_api_base, api_key=resolved_api_key, logging_obj=logging_obj if isinstance(logging_obj, LiteLLMLoggingObj) else None, @@ -135,6 +203,30 @@ def _prepare_call( ) +def _format_response(response: DecisionsIRResponse, call: _DecisionsCall) -> DecisionsResponse | OpenAIDecisionResponse: + formatted: Final = _formatted_response(response, call) + formatted.set_hidden_params( + { + "model": f"{call.custom_llm_provider}/{call.model}", + "custom_llm_provider": call.custom_llm_provider, + "provider_response_model": f"{call.custom_llm_provider}/{call.model}", + } + ) + return formatted + + +def _formatted_response( + response: DecisionsIRResponse, call: _DecisionsCall +) -> DecisionsResponse | OpenAIDecisionResponse: + match call.request: + case DecisionsRequestBody(): + return ir_to_systemone_response(response, call.ir_request) + case OpenAIDecisionRequestBody(): + return ir_to_openai_response(response, call.ir_request, call.requested_model) + case _: + assert_never(call.request) + + def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception: if isinstance(error, BaseLLMException) and error.status_code_is_synthesized: provider_label: Final = f"{call.custom_llm_provider[0].upper()}{call.custom_llm_provider[1:]}Exception" @@ -153,19 +245,23 @@ def _map_upstream_exception(error: Exception, call: _DecisionsCall) -> Exception @client async def adecisions( model: str, - state: DecisionsJSON, - questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + state: DecisionsJSON | None = None, + questions: DecisionsQuestions | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, extra_headers: Mapping[str, str] | None = None, + input: OpenAIDecisionInput | None = None, + safety_identifier: str | None = None, **kwargs: object, -) -> DecisionsResponse: +) -> DecisionsResponse | OpenAIDecisionResponse: call: Final = _prepare_call( model=model, state=state, questions=questions, + decision_input=input, + safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, timeout=timeout, @@ -174,12 +270,13 @@ async def adecisions( kwargs=kwargs, ) try: - return await _HANDLER.adecisions( + response: Final = await _HANDLER.adecisions( model=call.model, custom_llm_provider=call.custom_llm_provider, logging_obj=call.logging_obj, provider_config=call.provider_config, - request=call.request, + request=call.ir_request, + body=call.body, api_base=call.api_base, api_key=call.api_key, headers=call.headers, @@ -187,24 +284,29 @@ async def adecisions( ) except Exception as error: raise _map_upstream_exception(error, call) from error + return _format_response(response, call) @client def decisions( model: str, - state: DecisionsJSON, - questions: Mapping[str, DecisionQuestion | Mapping[str, object]], + state: DecisionsJSON | None = None, + questions: DecisionsQuestions | None = None, api_key: str | None = None, api_base: str | None = None, timeout: float | httpx.Timeout | None = None, custom_llm_provider: str | None = None, extra_headers: Mapping[str, str] | None = None, + input: OpenAIDecisionInput | None = None, + safety_identifier: str | None = None, **kwargs: object, -) -> DecisionsResponse: +) -> DecisionsResponse | OpenAIDecisionResponse: call: Final = _prepare_call( model=model, state=state, questions=questions, + decision_input=input, + safety_identifier=safety_identifier, api_key=api_key, api_base=api_base, timeout=timeout, @@ -213,12 +315,13 @@ def decisions( kwargs=kwargs, ) try: - return _HANDLER.decisions( + response: Final = _HANDLER.decisions( model=call.model, custom_llm_provider=call.custom_llm_provider, logging_obj=call.logging_obj, provider_config=call.provider_config, - request=call.request, + request=call.ir_request, + body=call.body, api_base=call.api_base, api_key=call.api_key, headers=call.headers, @@ -226,6 +329,7 @@ def decisions( ) except Exception as error: raise _map_upstream_exception(error, call) from error + return _format_response(response, call) __all__ = ["adecisions", "decisions"] diff --git a/litellm/decisions/openai_transformation.py b/litellm/decisions/openai_transformation.py deleted file mode 100644 index af2c838b9ab..00000000000 --- a/litellm/decisions/openai_transformation.py +++ /dev/null @@ -1,127 +0,0 @@ -from collections.abc import Mapping, Sequence -from typing import Final - -from typing_extensions import assert_never - -from litellm.types.decisions import ( - ChoiceAnswer, - DecisionAnswer, - DecisionsResponse, - DecisionsUsage, - NoulAnswer, - OpenAIChoiceAnswer, - OpenAIChoiceProbability, - OpenAIChoiceQuestion, - OpenAIDecisionAnswer, - OpenAIDecisionInputMessage, - OpenAIDecisionQuestion, - OpenAIDecisionRequestBody, - OpenAIDecisionResponse, - OpenAIDecisionUsage, - OpenAIPredicateAnswer, - OpenAIPredicateQuestion, - OpenAIRefusalAnswer, - OpenAIScoreAnswer, - OpenAIScoreLevel, - OpenAIScoreProbability, - OpenAIScoreQuestion, - ScoreAnswer, - systemone_choice_key, -) - -_OPENAI_ONLY_FIELDS: Final = frozenset({"input", "questions", "safety_identifier"}) - - -def _message_text(message: OpenAIDecisionInputMessage) -> str: - if isinstance(message.content, str): - return message.content - return "\n\n".join(part.text for part in message.content) - - -def _state(decision_input: str | Sequence[OpenAIDecisionInputMessage]) -> str: - if isinstance(decision_input, str): - return decision_input - return "\n\n".join(_message_text(message) for message in decision_input) - - -def _level_criterion(level: OpenAIScoreLevel) -> str: - return level.label if level.description is None else f"{level.label}: {level.description}" - - -def _systemone_question(question: OpenAIDecisionQuestion) -> Mapping[str, object]: - match question: - case OpenAIPredicateQuestion(): - return {"type": "noul", "instructions": question.instructions} - case OpenAIChoiceQuestion(): - return { - "type": "choice", - "instructions": question.instructions, - "criteria": {systemone_choice_key(option.value): option.description for option in question.choices}, - } - case OpenAIScoreQuestion(): - return { - "type": "score", - "instructions": question.instructions, - "criteria": [_level_criterion(level) for level in question.levels], - } - case _: - assert_never(question) - - -def to_systemone_request(request_data: Mapping[str, object], body: OpenAIDecisionRequestBody) -> Mapping[str, object]: - return { - **{key: value for key, value in request_data.items() if key not in _OPENAI_ONLY_FIELDS}, - "state": _state(body.input), - "questions": {str(index): _systemone_question(question) for index, question in enumerate(body.questions)}, - } - - -def _openai_answer(question: OpenAIDecisionQuestion, answer: DecisionAnswer | None) -> OpenAIDecisionAnswer: - match question, answer: - case OpenAIPredicateQuestion(), NoulAnswer(): - return OpenAIPredicateAnswer(name=question.name, probability=answer.noul) - case OpenAIChoiceQuestion(), ChoiceAnswer(): - typed_values: Final = {systemone_choice_key(option.value): option.value for option in question.choices} - return OpenAIChoiceAnswer( - name=question.name, - choice=typed_values.get(answer.choice, answer.choice), - probabilities=tuple( - OpenAIChoiceProbability( - value=option.value, - probability=answer.probabilities.get(systemone_choice_key(option.value), 0.0), - ) - for option in question.choices - ), - confidence=answer.confidence, - ) - case OpenAIScoreQuestion(), ScoreAnswer(): - return OpenAIScoreAnswer( - name=question.name, - score=answer.score, - probabilities=tuple( - OpenAIScoreProbability( - value=index, label=level.label, probability=answer.probabilities.get(str(index), 0.0) - ) - for index, level in enumerate(question.levels) - ), - confidence=answer.confidence, - ) - case _: - return OpenAIRefusalAnswer(name=question.name) - - -def to_openai_response( - response: DecisionsResponse, questions: Sequence[OpenAIDecisionQuestion], requested_model: str -) -> OpenAIDecisionResponse: - usage: Final = response.usage or DecisionsUsage() - return OpenAIDecisionResponse( - model=response.model or requested_model, - answers=tuple( - _openai_answer(question, response.answers.get(str(index))) for index, question in enumerate(questions) - ), - usage=OpenAIDecisionUsage( - input_tokens=usage.input_tokens, - output_tokens=usage.output_tokens, - total_tokens=usage.input_tokens + usage.output_tokens, - ), - ) diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index eb11bd9c11c..715e492b159 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -120,7 +120,7 @@ from litellm.llms.base_llm.search.transformation import SearchResponse from litellm.responses.utils import ResponseAPILoggingUtils from litellm.types.agents import LiteLLMSendMessageResponse from litellm.types.containers.main import ContainerObject -from litellm.types.decisions import DecisionsResponse +from litellm.types.decisions import DecisionsResponse, OpenAIDecisionResponse from litellm.types.integrations.s3_v2 import S3PartitionGranularity from litellm.types.interactions import ( InteractionsAPIResponse, @@ -2650,6 +2650,7 @@ class Logging(LiteLLMLoggingBaseClass): or isinstance(logging_result, OCRResponse) # OCR or isinstance(logging_result, SearchResponse) # Search API or isinstance(logging_result, DecisionsResponse) + or isinstance(logging_result, OpenAIDecisionResponse) or ( isinstance(logging_result, InteractionsAPIResponse) and logging_result.usage is not None diff --git a/litellm/llms/base_llm/decisions/transformation.py b/litellm/llms/base_llm/decisions/transformation.py index 36739555cea..a8d14ca59a3 100644 --- a/litellm/llms/base_llm/decisions/transformation.py +++ b/litellm/llms/base_llm/decisions/transformation.py @@ -1,17 +1,299 @@ +import json from abc import ABC -from collections.abc import Mapping +from collections.abc import Mapping, Sequence +from types import MappingProxyType from typing import Final import httpx from pydantic import TypeAdapter, ValidationError +from typing_extensions import assert_never from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.secret_managers.main import get_secret_str -from litellm.types.decisions import DecisionsRequest, DecisionsResponse +from litellm.types.decisions import ( + ChoiceAnswer, + ChoiceQuestion, + DecisionAnswer, + DecisionQuestion, + DecisionsIRAnswer, + DecisionsIRChoiceAnswer, + DecisionsIRChoiceOption, + DecisionsIRChoiceProbability, + DecisionsIRChoiceQuestion, + DecisionsIRMessages, + DecisionsIRPredicateAnswer, + DecisionsIRPredicateQuestion, + DecisionsIRQuestion, + DecisionsIRRefusal, + DecisionsIRRequest, + DecisionsIRResponse, + DecisionsIRScoreAnswer, + DecisionsIRScoreLevel, + DecisionsIRScoreProbability, + DecisionsIRScoreQuestion, + DecisionsIRState, + DecisionsIRUsage, + DecisionsJSON, + DecisionsRequestBody, + DecisionsResponse, + DecisionsUsage, + NoulAnswer, + NoulQuestion, + OpenAIDecisionInputImage, + OpenAIDecisionInputMessage, + OpenAIDecisionInputText, + ScoreAnswer, + ScoreQuestion, + UnsupportedDecisionsRequest, + systemone_choice_key, +) -PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) -_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) +_PAYLOAD_ADAPTER: Final[TypeAdapter[object]] = TypeAdapter(object) +_SYSTEMONE_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) _RESERVED_HEADERS: Final[frozenset[str]] = frozenset({"authorization", "content-type"}) +_TEXT_ONLY: Final = UnsupportedDecisionsRequest( + reason="input_image content parts are not supported because System One providers accept text input only" +) + + +def decisions_text(value: DecisionsJSON) -> str: + return value if isinstance(value, str) else json.dumps(value) + + +def systemone_keys(questions: Sequence[DecisionsIRQuestion]) -> tuple[str, ...]: + names: Final = tuple(question.name for question in questions if question.name is not None) + if len(frozenset(names)) == len(questions): + return names + return tuple(str(index) for index in range(len(questions))) + + +def _ir_question(name: str, question: DecisionQuestion) -> DecisionsIRQuestion: + extra: Final = MappingProxyType(question.model_extra or {}) + match question: + case NoulQuestion(): + return DecisionsIRPredicateQuestion( + name=name, instructions=question.instructions, criteria=question.criteria, extra=extra + ) + case ChoiceQuestion(): + return DecisionsIRChoiceQuestion( + name=name, + instructions=question.instructions, + choices=tuple( + DecisionsIRChoiceOption(value=value, description=description) + for value, description in question.criteria.items() + ), + extra=extra, + ) + case ScoreQuestion(): + return DecisionsIRScoreQuestion( + name=name, + instructions=question.instructions, + levels=tuple( + DecisionsIRScoreLevel(label=criterion, description=None) for criterion in question.criteria + ), + extra=extra, + ) + case _: + assert_never(question) + + +def systemone_request_to_ir(request: DecisionsRequestBody) -> DecisionsIRRequest: + return DecisionsIRRequest( + input=DecisionsIRState(state=request.state), + questions=tuple(_ir_question(name, question) for name, question in request.questions.items()), + ) + + +def _has_image(message: OpenAIDecisionInputMessage) -> bool: + return not isinstance(message.content, str) and any( + isinstance(part, OpenAIDecisionInputImage) for part in message.content + ) + + +def _message_text(message: OpenAIDecisionInputMessage) -> str: + if isinstance(message.content, str): + return message.content + return "\n\n".join(part.text for part in message.content if isinstance(part, OpenAIDecisionInputText)) + + +def _systemone_state( + decision_input: DecisionsIRState | DecisionsIRMessages, +) -> DecisionsJSON | UnsupportedDecisionsRequest: + match decision_input: + case DecisionsIRState(): + return decision_input.state + case DecisionsIRMessages(): + if any(_has_image(message) for message in decision_input.messages): + return _TEXT_ONLY + return "\n\n".join(_message_text(message) for message in decision_input.messages) + case _: + assert_never(decision_input) + + +def _optional(key: str, value: object) -> Mapping[str, object]: + return {} if value is None else {key: value} + + +def _level_criterion(level: DecisionsIRScoreLevel) -> DecisionsJSON: + return level.label if level.description is None else f"{decisions_text(level.label)}: {level.description}" + + +def _systemone_question(question: DecisionsIRQuestion) -> Mapping[str, object]: + match question: + case DecisionsIRPredicateQuestion(): + return { + "type": "noul", + **_optional("instructions", question.instructions), + **_optional("criteria", question.criteria), + **question.extra, + } + case DecisionsIRChoiceQuestion(): + return { + "type": "choice", + **_optional("instructions", question.instructions), + "criteria": {systemone_choice_key(option.value): option.description for option in question.choices}, + **question.extra, + } + case DecisionsIRScoreQuestion(): + return { + "type": "score", + **_optional("instructions", question.instructions), + "criteria": [_level_criterion(level) for level in question.levels], + **question.extra, + } + case _: + assert_never(question) + + +def ir_to_systemone_request( + model: str, request: DecisionsIRRequest +) -> Mapping[str, object] | UnsupportedDecisionsRequest: + state: Final = _systemone_state(request.input) + if isinstance(state, UnsupportedDecisionsRequest): + return state + keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) + return { + "model": model, + "state": state, + "questions": {key: _systemone_question(question) for key, question in keyed_questions}, + } + + +def _ir_choice_answer(question: DecisionsIRChoiceQuestion, answer: ChoiceAnswer) -> DecisionsIRChoiceAnswer: + typed_values: Final = {systemone_choice_key(option.value): option.value for option in question.choices} + return DecisionsIRChoiceAnswer( + choice=typed_values.get(answer.choice, answer.choice), + confidence=answer.confidence, + probabilities=tuple( + DecisionsIRChoiceProbability(value=typed_values.get(key, key), probability=probability) + for key, probability in answer.probabilities.items() + ), + extra=MappingProxyType(answer.model_extra or {}), + ) + + +def _ir_score_answer(question: DecisionsIRScoreQuestion, answer: ScoreAnswer) -> DecisionsIRScoreAnswer: + return DecisionsIRScoreAnswer( + score=answer.score, + confidence=answer.confidence, + probabilities=tuple( + DecisionsIRScoreProbability( + value=index, + label=answer.legend.get(str(index), level.label), + probability=answer.probabilities.get(str(index), 0.0), + ) + for index, level in enumerate(question.levels) + ), + extra=MappingProxyType(answer.model_extra or {}), + ) + + +def _ir_answer(question: DecisionsIRQuestion, answer: DecisionAnswer | None) -> DecisionsIRAnswer: + match question, answer: + case DecisionsIRPredicateQuestion(), NoulAnswer(): + return DecisionsIRPredicateAnswer(probability=answer.noul, extra=MappingProxyType(answer.model_extra or {})) + case DecisionsIRChoiceQuestion(), ChoiceAnswer(): + return _ir_choice_answer(question, answer) + case DecisionsIRScoreQuestion(), ScoreAnswer(): + return _ir_score_answer(question, answer) + case _: + return DecisionsIRRefusal() + + +def systemone_response_to_ir(response: DecisionsResponse, request: DecisionsIRRequest) -> DecisionsIRResponse: + usage: Final = response.usage or DecisionsUsage() + keyed_questions: Final = zip(systemone_keys(request.questions), request.questions, strict=True) + return DecisionsIRResponse( + model=response.model, + answers=tuple(_ir_answer(question, response.answers.get(key)) for key, question in keyed_questions), + usage=DecisionsIRUsage( + input_tokens=usage.input_tokens, + output_tokens=usage.output_tokens, + cached_tokens=usage.cached_tokens, + cache_write_tokens=usage.cache_write_tokens, + extra=MappingProxyType(usage.model_extra or {}), + ), + extra=MappingProxyType(response.model_extra or {}), + ) + + +def parse_systemone_response(payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + return systemone_response_to_ir(_SYSTEMONE_RESPONSE_ADAPTER.validate_python(payload), request) + + +def _systemone_answer(answer: DecisionsIRAnswer) -> DecisionAnswer | None: + match answer: + case DecisionsIRPredicateAnswer(): + return NoulAnswer.model_validate({**answer.extra, "type": "noul", "noul": answer.probability}) + case DecisionsIRChoiceAnswer(): + return ChoiceAnswer.model_validate( + { + **answer.extra, + "type": "choice", + "choice": systemone_choice_key(answer.choice), + "confidence": answer.confidence, + "probabilities": { + systemone_choice_key(item.value): item.probability for item in answer.probabilities + }, + } + ) + case DecisionsIRScoreAnswer(): + return ScoreAnswer.model_validate( + { + **answer.extra, + "type": "score", + "score": answer.score, + "confidence": answer.confidence, + "legend": {str(item.value): item.label for item in answer.probabilities}, + "probabilities": {str(item.value): item.probability for item in answer.probabilities}, + } + ) + case DecisionsIRRefusal(): + return None + case _: + assert_never(answer) + + +def ir_to_systemone_response(response: DecisionsIRResponse, request: DecisionsIRRequest) -> DecisionsResponse: + keyed_answers: Final = zip( + systemone_keys(request.questions), (_systemone_answer(answer) for answer in response.answers), strict=True + ) + return DecisionsResponse.model_validate( + { + **response.extra, + "model": response.model, + "answers": {key: answer for key, answer in keyed_answers if answer is not None}, + "usage": DecisionsUsage.model_validate( + { + **response.usage.extra, + "input_tokens": response.usage.input_tokens, + "output_tokens": response.usage.output_tokens, + "cached_tokens": response.usage.cached_tokens, + "cache_write_tokens": response.usage.cache_write_tokens, + } + ), + } + ) class BaseDecisionsConfig(ABC): @@ -55,48 +337,32 @@ class BaseDecisionsConfig(ABC): def transform_decisions_request( self, model: str, - request: DecisionsRequest, + request: DecisionsIRRequest, custom_llm_provider: str, - ) -> dict[str, object]: - return { - "model": self.request_model(model), - "state": request.state, - "questions": { - name: question.model_dump(mode="json", exclude_none=True) - for name, question in request.questions.items() - }, - } + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + return ir_to_systemone_request(self.request_model(model), request) def unwrap_response(self, payload: object) -> object: return payload + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + return parse_systemone_response(self.unwrap_response(payload), request) + def transform_decisions_response( self, model: str, custom_llm_provider: str, raw_response: httpx.Response, - request: DecisionsRequest, - ) -> DecisionsResponse: - payload: Final[object] = PAYLOAD_ADAPTER.validate_json(raw_response.content) + request: DecisionsIRRequest, + ) -> DecisionsIRResponse: + payload: Final[object] = _PAYLOAD_ADAPTER.validate_json(raw_response.content) try: - response: Final = _RESPONSE_ADAPTER.validate_python(self.unwrap_response(payload)) + return self.parse_response(payload, request) except ValidationError as error: raise BaseLLMException( status_code=500, message=f"Decisions provider '{custom_llm_provider}' returned an unexpected response: {error}", ) from error - self.set_hidden_params(response, model, custom_llm_provider) - return response - - @staticmethod - def set_hidden_params(response: DecisionsResponse, model: str, custom_llm_provider: str) -> None: - response.set_hidden_params( - { - "model": f"{custom_llm_provider}/{model}", - "custom_llm_provider": custom_llm_provider, - "provider_response_model": f"{custom_llm_provider}/{model}", - } - ) def get_error_class( self, diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index de271cb09ba..124f45e9bf8 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -123,7 +123,7 @@ from litellm.types.containers.main import ( ContainerObject, DeleteContainerResult, ) -from litellm.types.decisions import DecisionsRequest, DecisionsResponse +from litellm.types.decisions import DecisionsIRRequest, DecisionsIRResponse from litellm.types.files import StreamingMediaUploadConfig, TwoStepFileUploadConfig from litellm.types.integrations.custom_logger import ( NON_CODE_INTERPRETER_INTERCEPTION_INTERNAL_PREFIXES, @@ -1486,19 +1486,16 @@ class BaseLLMHTTPHandler: def _prepare_decisions_request( self, model: str, - custom_llm_provider: str, logging_obj: LiteLLMLoggingObj | None, provider_config: BaseDecisionsConfig, - request: DecisionsRequest, + body: Mapping[str, object], api_base: str, api_key: str | None, headers: Mapping[str, str], ) -> tuple[str, dict[str, str], dict[str, object]]: outbound_headers: Final = provider_config.validate_environment(headers=headers, model=model, api_key=api_key) url: Final = provider_config.get_complete_url(api_base=api_base, model=model) - data: Final = provider_config.transform_decisions_request( - model=model, request=request, custom_llm_provider=custom_llm_provider - ) + data: Final = dict(body) if logging_obj is not None: logging_obj.pre_call( input=data, @@ -1514,19 +1511,19 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, logging_obj: LiteLLMLoggingObj | None, provider_config: BaseDecisionsConfig, - request: DecisionsRequest, + request: DecisionsIRRequest, + body: Mapping[str, object], api_base: str, api_key: str | None, headers: Mapping[str, str], timeout: float | httpx.Timeout | None, client: HTTPHandler | None = None, - ) -> DecisionsResponse: + ) -> DecisionsIRResponse: url, outbound_headers, data = self._prepare_decisions_request( model=model, - custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, provider_config=provider_config, - request=request, + body=body, api_base=api_base, api_key=api_key, headers=headers, @@ -1548,19 +1545,19 @@ class BaseLLMHTTPHandler: custom_llm_provider: str, logging_obj: LiteLLMLoggingObj | None, provider_config: BaseDecisionsConfig, - request: DecisionsRequest, + request: DecisionsIRRequest, + body: Mapping[str, object], api_base: str, api_key: str | None, headers: Mapping[str, str], timeout: float | httpx.Timeout | None, client: AsyncHTTPHandler | None = None, - ) -> DecisionsResponse: + ) -> DecisionsIRResponse: url, outbound_headers, data = self._prepare_decisions_request( model=model, - custom_llm_provider=custom_llm_provider, logging_obj=logging_obj, provider_config=provider_config, - request=request, + body=body, api_base=api_base, api_key=api_key, headers=headers, diff --git a/litellm/llms/openai/decisions/transformation.py b/litellm/llms/openai/decisions/transformation.py new file mode 100644 index 00000000000..0afbe1a434e --- /dev/null +++ b/litellm/llms/openai/decisions/transformation.py @@ -0,0 +1,320 @@ +from collections.abc import Mapping, Sequence +from typing import Final + +from pydantic import TypeAdapter +from typing_extensions import assert_never + +import litellm +from litellm.llms.base_llm.decisions.transformation import BaseDecisionsConfig, decisions_text +from litellm.secret_managers.main import get_secret_str +from litellm.types.decisions import ( + DecisionsIRAnswer, + DecisionsIRChoiceAnswer, + DecisionsIRChoiceOption, + DecisionsIRChoiceProbability, + DecisionsIRChoiceQuestion, + DecisionsIRMessages, + DecisionsIRPredicateAnswer, + DecisionsIRPredicateQuestion, + DecisionsIRQuestion, + DecisionsIRRefusal, + DecisionsIRRequest, + DecisionsIRResponse, + DecisionsIRScoreAnswer, + DecisionsIRScoreLevel, + DecisionsIRScoreProbability, + DecisionsIRScoreQuestion, + DecisionsIRState, + DecisionsIRUsage, + DecisionsJSON, + OpenAIChoiceAnswer, + OpenAIChoiceProbability, + OpenAIChoiceQuestion, + OpenAIDecisionAnswer, + OpenAIDecisionInput, + OpenAIDecisionInputTokensDetails, + OpenAIDecisionOutputTokensDetails, + OpenAIDecisionQuestion, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, + OpenAIDecisionUsage, + OpenAIPredicateAnswer, + OpenAIPredicateQuestion, + OpenAIRefusalAnswer, + OpenAIScoreAnswer, + OpenAIScoreProbability, + OpenAIScoreQuestion, + UnsupportedDecisionsRequest, + systemone_choice_key, +) + +_OPENAI_RESPONSE_ADAPTER: Final[TypeAdapter[OpenAIDecisionResponse]] = TypeAdapter(OpenAIDecisionResponse) +_SINGLE_OPTION: Final = UnsupportedDecisionsRequest( + reason="OpenAI needs at least 2 choices or levels on every choice or score question" +) + + +def _ir_input(decision_input: OpenAIDecisionInput) -> DecisionsIRState | DecisionsIRMessages: + if isinstance(decision_input, str): + return DecisionsIRState(state=decision_input) + return DecisionsIRMessages(messages=tuple(decision_input)) + + +def _ir_question(question: OpenAIDecisionQuestion) -> DecisionsIRQuestion: + match question: + case OpenAIPredicateQuestion(): + return DecisionsIRPredicateQuestion(name=question.name, instructions=question.instructions) + case OpenAIChoiceQuestion(): + return DecisionsIRChoiceQuestion( + name=question.name, + instructions=question.instructions, + choices=tuple( + DecisionsIRChoiceOption(value=option.value, description=option.description) + for option in question.choices + ), + ) + case OpenAIScoreQuestion(): + return DecisionsIRScoreQuestion( + name=question.name, + instructions=question.instructions, + levels=tuple( + DecisionsIRScoreLevel(label=level.label, description=level.description) for level in question.levels + ), + ) + case _: + assert_never(question) + + +def openai_request_to_ir(request: OpenAIDecisionRequestBody) -> DecisionsIRRequest: + return DecisionsIRRequest( + input=_ir_input(request.input), + questions=tuple(_ir_question(question) for question in request.questions), + safety_identifier=request.safety_identifier, + ) + + +def _text_field(key: str, value: DecisionsJSON | None) -> Mapping[str, str]: + return {} if value is None else {key: decisions_text(value)} + + +def _instructions(value: DecisionsJSON | None, default: str) -> str: + return default if value is None else decisions_text(value) + + +def _predicate_instructions(question: DecisionsIRPredicateQuestion) -> str: + instructions: Final = () if question.instructions is None else (decisions_text(question.instructions),) + criteria: Final = tuple( + f"Answer {answer} when: {decisions_text(rule)}" + for answer, rule in (question.criteria or {}).items() + if rule is not None + ) + return "\n\n".join((*instructions, *criteria)) + + +def _openai_question(question: DecisionsIRQuestion) -> Mapping[str, object]: + match question: + case DecisionsIRPredicateQuestion(): + return { + "type": "predicate", + **_text_field("name", question.name), + "instructions": _predicate_instructions(question) or "Is this true of the input?", + } + case DecisionsIRChoiceQuestion(): + return { + "type": "choice", + **_text_field("name", question.name), + "instructions": _instructions(question.instructions, "Which choice best fits the input?"), + "choices": [ + {"value": option.value, **_text_field("description", option.description)} + for option in question.choices + ], + } + case DecisionsIRScoreQuestion(): + return { + "type": "score", + **_text_field("name", question.name), + "instructions": _instructions(question.instructions, "Which level best fits the input?"), + "levels": [ + {"label": decisions_text(level.label), **_text_field("description", level.description)} + for level in question.levels + ], + } + case _: + assert_never(question) + + +def _openai_input(decision_input: DecisionsIRState | DecisionsIRMessages) -> str | Sequence[Mapping[str, object]]: + match decision_input: + case DecisionsIRState(): + return decisions_text(decision_input.state) + case DecisionsIRMessages(): + return [message.model_dump(mode="json", exclude_none=True) for message in decision_input.messages] + case _: + assert_never(decision_input) + + +def _has_one_option(question: DecisionsIRQuestion) -> bool: + match question: + case DecisionsIRChoiceQuestion(): + return len(question.choices) < 2 + case DecisionsIRScoreQuestion(): + return len(question.levels) < 2 + case _: + return False + + +def ir_to_openai_request(model: str, request: DecisionsIRRequest) -> Mapping[str, object] | UnsupportedDecisionsRequest: + if any(_has_one_option(question) for question in request.questions): + return _SINGLE_OPTION + return { + "model": model, + "input": _openai_input(request.input), + "questions": [_openai_question(question) for question in request.questions], + **_text_field("safety_identifier", request.safety_identifier), + } + + +def _level_label(question: DecisionsIRScoreQuestion, probability: OpenAIScoreProbability) -> DecisionsJSON: + if 0 <= probability.value < len(question.levels): + return question.levels[probability.value].label + return probability.label + + +def _ir_answer(question: DecisionsIRQuestion, answer: OpenAIDecisionAnswer | None) -> DecisionsIRAnswer: + match question, answer: + case DecisionsIRPredicateQuestion(), OpenAIPredicateAnswer(): + return DecisionsIRPredicateAnswer(probability=answer.probability) + case DecisionsIRChoiceQuestion(), OpenAIChoiceAnswer(): + return DecisionsIRChoiceAnswer( + choice=answer.choice, + confidence=answer.confidence, + probabilities=tuple( + DecisionsIRChoiceProbability(value=item.value, probability=item.probability) + for item in answer.probabilities + ), + ) + case DecisionsIRScoreQuestion(), OpenAIScoreAnswer(): + return DecisionsIRScoreAnswer( + score=answer.score, + confidence=answer.confidence, + probabilities=tuple( + DecisionsIRScoreProbability( + value=item.value, label=_level_label(question, item), probability=item.probability + ) + for item in answer.probabilities + ), + ) + case _: + return DecisionsIRRefusal() + + +def openai_response_to_ir(response: OpenAIDecisionResponse, request: DecisionsIRRequest) -> DecisionsIRResponse: + answers: Final = response.answers + return DecisionsIRResponse( + model=response.model, + answers=tuple( + _ir_answer(question, answers[index] if index < len(answers) else None) + for index, question in enumerate(request.questions) + ), + usage=DecisionsIRUsage( + input_tokens=response.usage.input_tokens, + output_tokens=response.usage.output_tokens, + cached_tokens=response.usage.input_tokens_details.cached_tokens, + cache_write_tokens=response.usage.input_tokens_details.cache_write_tokens, + reasoning_tokens=response.usage.output_tokens_details.reasoning_tokens, + ), + ) + + +def _openai_choice_answer(question: DecisionsIRChoiceQuestion, answer: DecisionsIRChoiceAnswer) -> OpenAIChoiceAnswer: + probabilities: Final = {systemone_choice_key(item.value): item.probability for item in answer.probabilities} + return OpenAIChoiceAnswer( + name=question.name, + choice=answer.choice, + probabilities=tuple( + OpenAIChoiceProbability( + value=option.value, probability=probabilities.get(systemone_choice_key(option.value), 0.0) + ) + for option in question.choices + ), + confidence=answer.confidence, + ) + + +def _openai_score_answer(question: DecisionsIRScoreQuestion, answer: DecisionsIRScoreAnswer) -> OpenAIScoreAnswer: + probabilities: Final = {item.value: item.probability for item in answer.probabilities} + return OpenAIScoreAnswer( + name=question.name, + score=answer.score, + probabilities=tuple( + OpenAIScoreProbability( + value=index, label=decisions_text(level.label), probability=probabilities.get(index, 0.0) + ) + for index, level in enumerate(question.levels) + ), + confidence=answer.confidence, + ) + + +def _openai_answer(question: DecisionsIRQuestion, answer: DecisionsIRAnswer) -> OpenAIDecisionAnswer: + match question, answer: + case DecisionsIRPredicateQuestion(), DecisionsIRPredicateAnswer(): + return OpenAIPredicateAnswer(name=question.name, probability=answer.probability) + case DecisionsIRChoiceQuestion(), DecisionsIRChoiceAnswer(): + return _openai_choice_answer(question, answer) + case DecisionsIRScoreQuestion(), DecisionsIRScoreAnswer(): + return _openai_score_answer(question, answer) + case _: + return OpenAIRefusalAnswer(name=question.name) + + +def ir_to_openai_response( + response: DecisionsIRResponse, request: DecisionsIRRequest, requested_model: str +) -> OpenAIDecisionResponse: + return OpenAIDecisionResponse( + model=response.model or requested_model, + answers=tuple( + _openai_answer(question, answer) + for question, answer in zip(request.questions, response.answers, strict=True) + ), + usage=OpenAIDecisionUsage( + input_tokens=response.usage.input_tokens, + input_tokens_details=OpenAIDecisionInputTokensDetails( + cached_tokens=response.usage.cached_tokens, + cache_write_tokens=response.usage.cache_write_tokens, + ), + output_tokens=response.usage.output_tokens, + output_tokens_details=OpenAIDecisionOutputTokensDetails(reasoning_tokens=response.usage.reasoning_tokens), + total_tokens=response.usage.input_tokens + response.usage.output_tokens, + ), + ) + + +class OpenAIDecisionsConfig(BaseDecisionsConfig): + path = "/v1/decisions" + + def get_default_api_base(self) -> str | None: + return "https://api.openai.com" + + def resolve_api_base(self, api_base: str | None) -> str | None: + return ( + api_base + or litellm.api_base + or get_secret_str("OPENAI_BASE_URL") + or get_secret_str("OPENAI_API_BASE") + or self.get_default_api_base() + ) + + def resolve_api_key(self, api_key: str | None) -> str | None: + return api_key or litellm.api_key or litellm.openai_key or get_secret_str("OPENAI_API_KEY") + + def transform_decisions_request( + self, + model: str, + request: DecisionsIRRequest, + custom_llm_provider: str, + ) -> Mapping[str, object] | UnsupportedDecisionsRequest: + return ir_to_openai_request(self.request_model(model), request) + + def parse_response(self, payload: object, request: DecisionsIRRequest) -> DecisionsIRResponse: + return openai_response_to_ir(_OPENAI_RESPONSE_ADAPTER.validate_python(payload), request) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5453985dadb..f34606ce127 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -34527,7 +34527,8 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", - "/v1/responses" + "/v1/responses", + "/v1/decisions" ], "supported_modalities": [ "text", diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 265dbbb2957..f2891c184bf 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -5,15 +5,10 @@ from fastapi import APIRouter, Depends, Request, Response from fastapi.responses import ORJSONResponse # pyright: ignore[reportDeprecated] # required endpoint contract from pydantic import TypeAdapter, ValidationError -from litellm.decisions.openai_transformation import to_openai_response, to_systemone_request from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth -from litellm.proxy.common_request_processing import ( - ProxyBaseLLMRequestProcessing, - attach_guardrail_information, - include_guardrail_response_requested, -) -from litellm.types.decisions import DecisionsRequestBody, DecisionsResponse, OpenAIDecisionRequestBody +from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing +from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) @@ -21,7 +16,6 @@ _DECISIONS_REQUEST_BODY_ADAPTER: Final[TypeAdapter[DecisionsRequestBody]] = Type _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter( OpenAIDecisionRequestBody ) -_DECISIONS_RESPONSE_ADAPTER: Final[TypeAdapter[DecisionsResponse]] = TypeAdapter(DecisionsResponse) _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) @@ -52,12 +46,11 @@ async def _request_data(request: Request, user_api_key_dict: UserAPIKeyAuth) -> raise await _invalid_request(raw_data={}, error=error, user_api_key_dict=user_api_key_dict) -async def _process_systemone( +async def _process_decisions( request: Request, fastapi_response: Response, user_api_key_dict: UserAPIKeyAuth, - raw_data: Mapping[str, object], - openai_body: OpenAIDecisionRequestBody | None, + body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], ) -> object: from litellm.proxy.proxy_server import ( general_settings as proxy_general_settings, @@ -80,15 +73,18 @@ async def _process_systemone( user_temperature as proxy_user_temperature, ) - data: Final = dict(raw_data if openai_body is None else to_systemone_request(raw_data, openai_body)) + data: Final = await _request_data(request, user_api_key_dict) + try: + body_adapter.validate_python(data) + except ValidationError as error: + raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) general_settings: Final = _GENERAL_SETTINGS_ADAPTER.validate_python(proxy_general_settings) user_api_base: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_api_base) user_model: Final = _OPTIONAL_STRING_ADAPTER.validate_python(proxy_user_model) user_temperature: Final = _OPTIONAL_FLOAT_ADAPTER.validate_python(proxy_user_temperature) processor: Final = ProxyBaseLLMRequestProcessing(data=data) try: - _DECISIONS_REQUEST_BODY_ADAPTER.validate_python(data) - result: Final[object] = await processor.base_process_llm_request( + return await processor.base_process_llm_request( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, @@ -106,21 +102,6 @@ async def _process_systemone( user_api_base=user_api_base, version=version, ) - if openai_body is None or isinstance(result, Response): - return result - openai_response: Final = to_openai_response( - _DECISIONS_RESPONSE_ADAPTER.validate_python(result), - openai_body.questions, - str(data.get("model", "")), - ) - request_data: Final = _REQUEST_DATA_ADAPTER.validate_python( - processor.data # pyright: ignore[reportUnknownMemberType] # ProxyBaseLLMRequestProcessing.data is a bare dict - ) - if include_guardrail_response_requested(request_data): - return attach_guardrail_information(response=openai_response, request_data=request_data) - return openai_response - except ValidationError as error: - raise await _invalid_request(raw_data=data, error=error, user_api_key_dict=user_api_key_dict) except Exception as error: raise await processor.handle_llm_api_exception( e=error, @@ -147,12 +128,11 @@ async def systemone( fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): - return await _process_systemone( + return await _process_decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, - raw_data=await _request_data(request, user_api_key_dict), - openai_body=None, + body_adapter=_DECISIONS_REQUEST_BODY_ADAPTER, ) @@ -173,15 +153,9 @@ async def decisions( fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): - raw_data: Final = await _request_data(request, user_api_key_dict) - try: - openai_body: Final = _OPENAI_DECISION_REQUEST_BODY_ADAPTER.validate_python(raw_data) - except ValidationError as error: - raise await _invalid_request(raw_data=raw_data, error=error, user_api_key_dict=user_api_key_dict) - return await _process_systemone( + return await _process_decisions( request=request, fastapi_response=fastapi_response, user_api_key_dict=user_api_key_dict, - raw_data=raw_data, - openai_body=openai_body, + body_adapter=_OPENAI_DECISION_REQUEST_BODY_ADAPTER, ) diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index b543b3b32df..b24690dbd9d 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -1,4 +1,6 @@ from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from types import MappingProxyType from typing import Annotated, Final, Literal, TypeAlias from pydantic import ConfigDict, Field, PrivateAttr, model_validator, with_config @@ -110,17 +112,13 @@ DecisionAnswer: TypeAlias = Annotated[ class DecisionsUsage(LiteLLMPydanticObjectBase): input_tokens: int = 0 output_tokens: int = 0 + cached_tokens: Annotated[int, Field(exclude=True)] = 0 + cache_write_tokens: Annotated[int, Field(exclude=True)] = 0 model_config = ConfigDict(extra="allow", frozen=True) -class DecisionsResponse(LiteLLMPydanticObjectBase): - model: str | None = None - answers: Mapping[str, DecisionAnswer] - usage: DecisionsUsage | None = None - - model_config = ConfigDict(extra="allow", frozen=True) - +class _HiddenParamsResponse(LiteLLMPydanticObjectBase): _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) @property @@ -131,6 +129,14 @@ class DecisionsResponse(LiteLLMPydanticObjectBase): self._hidden_params.update(params) +class DecisionsResponse(_HiddenParamsResponse): + model: str | None = None + answers: Mapping[str, DecisionAnswer] + usage: DecisionsUsage | None = None + + model_config = ConfigDict(extra="allow", frozen=True) + + class OpenAIDecisionInputText(LiteLLMPydanticObjectBase): type: Literal["input_text"] text: str @@ -138,14 +144,31 @@ class OpenAIDecisionInputText(LiteLLMPydanticObjectBase): model_config = ConfigDict(extra="forbid", frozen=True) +class OpenAIDecisionInputImage(LiteLLMPydanticObjectBase): + type: Literal["input_image"] + image_url: str + detail: str | None = None + + model_config = ConfigDict(extra="forbid", frozen=True) + + +OpenAIDecisionContentPart: TypeAlias = Annotated[ + OpenAIDecisionInputText | OpenAIDecisionInputImage, + Field(discriminator="type"), +] + + class OpenAIDecisionInputMessage(LiteLLMPydanticObjectBase): role: Literal["user"] = "user" type: Literal["message"] = "message" - content: str | Sequence[OpenAIDecisionInputText] + content: str | Sequence[OpenAIDecisionContentPart] model_config = ConfigDict(extra="forbid", frozen=True) +OpenAIDecisionInput: TypeAlias = str | Sequence[OpenAIDecisionInputMessage] + + class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase): type: Literal["predicate"] name: str | None = None @@ -206,7 +229,7 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[ class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): - input: str | Sequence[OpenAIDecisionInputMessage] + input: OpenAIDecisionInput questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] safety_identifier: str | None = None @@ -263,7 +286,10 @@ class OpenAIRefusalAnswer(LiteLLMPydanticObjectBase): model_config = ConfigDict(frozen=True) -OpenAIDecisionAnswer: TypeAlias = OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer +OpenAIDecisionAnswer: TypeAlias = Annotated[ + OpenAIPredicateAnswer | OpenAIChoiceAnswer | OpenAIScoreAnswer | OpenAIRefusalAnswer, + Field(discriminator="type"), +] class OpenAIDecisionInputTokensDetails(LiteLLMPydanticObjectBase): @@ -289,9 +315,136 @@ class OpenAIDecisionUsage(LiteLLMPydanticObjectBase): model_config = ConfigDict(frozen=True) -class OpenAIDecisionResponse(LiteLLMPydanticObjectBase): +class OpenAIDecisionResponse(_HiddenParamsResponse): model: str answers: tuple[OpenAIDecisionAnswer, ...] usage: OpenAIDecisionUsage model_config = ConfigDict(extra="allow", frozen=True) + + +_NO_EXTRA: Final[Mapping[str, object]] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRState: + state: DecisionsJSON + + +@dataclass(frozen=True, slots=True) +class DecisionsIRMessages: + messages: tuple[OpenAIDecisionInputMessage, ...] + + +@dataclass(frozen=True, slots=True) +class DecisionsIRPredicateQuestion: + name: str | None + instructions: DecisionsJSON | None + criteria: NoulCriteria | None = None + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRChoiceOption: + value: str | bool + description: DecisionsJSON | None + + +@dataclass(frozen=True, slots=True) +class DecisionsIRChoiceQuestion: + name: str | None + instructions: DecisionsJSON | None + choices: tuple[DecisionsIRChoiceOption, ...] + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRScoreLevel: + label: DecisionsJSON + description: str | None + + +@dataclass(frozen=True, slots=True) +class DecisionsIRScoreQuestion: + name: str | None + instructions: DecisionsJSON | None + levels: tuple[DecisionsIRScoreLevel, ...] + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + +DecisionsIRQuestion: TypeAlias = DecisionsIRPredicateQuestion | DecisionsIRChoiceQuestion | DecisionsIRScoreQuestion + + +@dataclass(frozen=True, slots=True) +class DecisionsIRRequest: + input: DecisionsIRState | DecisionsIRMessages + questions: tuple[DecisionsIRQuestion, ...] + safety_identifier: str | None = None + + +@dataclass(frozen=True, slots=True) +class DecisionsIRPredicateAnswer: + probability: float + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRChoiceProbability: + value: str | bool + probability: float + + +@dataclass(frozen=True, slots=True) +class DecisionsIRChoiceAnswer: + choice: str | bool + confidence: float + probabilities: tuple[DecisionsIRChoiceProbability, ...] + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRScoreProbability: + value: int + label: DecisionsJSON + probability: float + + +@dataclass(frozen=True, slots=True) +class DecisionsIRScoreAnswer: + score: float + confidence: float + probabilities: tuple[DecisionsIRScoreProbability, ...] + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRRefusal: + pass + + +DecisionsIRAnswer: TypeAlias = ( + DecisionsIRPredicateAnswer | DecisionsIRChoiceAnswer | DecisionsIRScoreAnswer | DecisionsIRRefusal +) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRUsage: + input_tokens: int = 0 + output_tokens: int = 0 + cached_tokens: int = 0 + cache_write_tokens: int = 0 + reasoning_tokens: int = 0 + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class DecisionsIRResponse: + model: str | None + answers: tuple[DecisionsIRAnswer, ...] + usage: DecisionsIRUsage + extra: Mapping[str, object] = field(default_factory=lambda: _NO_EXTRA) + + +@dataclass(frozen=True, slots=True) +class UnsupportedDecisionsRequest: + reason: str diff --git a/litellm/utils.py b/litellm/utils.py index 7207c6bacaa..89544cf31e2 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1219,8 +1219,8 @@ def function_setup( else search_query ) elif call_type in (CallTypes.decisions.value, CallTypes.adecisions.value): - decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state", "") - messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state) + decisions_state: Final = args[1] if len(args) > 1 else kwargs.get("state") or kwargs.get("input") or "" + messages = decisions_state if isinstance(decisions_state, str) else json.dumps(decisions_state, default=str) elif call_type in (CallTypes.image_edit.value, CallTypes.aimage_edit.value): messages = args[1] if len(args) > 1 else kwargs.get("prompt") elif call_type in (CallTypes.ocr.value, CallTypes.aocr.value): @@ -8974,6 +8974,8 @@ class ProviderConfigManager: return litellm.CloudflareDecisionsConfig() if provider == LlmProviders.STRANDS_DECIDER: return litellm.StrandsDeciderDecisionsConfig() + if provider == LlmProviders.OPENAI: + return litellm.OpenAIDecisionsConfig() return None @staticmethod diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5453985dadb..f34606ce127 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -34527,7 +34527,8 @@ "supported_endpoints": [ "/v1/chat/completions", "/v1/batch", - "/v1/responses" + "/v1/responses", + "/v1/decisions" ], "supported_modalities": [ "text", diff --git a/tests/unit/decisions/test_main.py b/tests/unit/decisions/test_main.py index e5964f972b9..820451ed943 100644 --- a/tests/unit/decisions/test_main.py +++ b/tests/unit/decisions/test_main.py @@ -18,6 +18,7 @@ from litellm.types.decisions import ( DecisionsResponse, DecisionsUsage, NoulAnswer, + OpenAIDecisionResponse, ScoreAnswer, ) @@ -30,6 +31,8 @@ _QUESTIONS: Final[Mapping[str, object]] = MappingProxyType( ) _INPUT_TOKENS: Final[int] = 367 _OUTPUT_TOKENS: Final[int] = 3 +_CACHED_TOKENS: Final[int] = 256 +_CACHE_WRITE_TOKENS: Final[int] = 64 _RESPONSE: Final[Mapping[str, object]] = { "model": "jev-1.13", "answers": { @@ -80,6 +83,21 @@ _PROVIDERS: Final[tuple[tuple[str, str, str, str], ...]] = ( ), ) +_OPENAI_RESPONSE: Final[Mapping[str, object]] = { + "model": "gpt-6-luna", + "answers": [ + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + {"type": "refusal", "name": "sentiment"}, + ], + "usage": { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": _CACHED_TOKENS, "cache_write_tokens": _CACHE_WRITE_TOKENS}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + }, +} + class _RecordingLogger(CustomLogger): def __init__(self) -> None: @@ -265,6 +283,46 @@ def test_decisions_cost_uses_litellm_token_pricing() -> None: assert cost == pytest.approx(expected_cost) +@pytest.mark.parametrize( + "response", + ( + DecisionsResponse( + model="jev-latest", + answers={}, + usage=DecisionsUsage( + input_tokens=_INPUT_TOKENS, + output_tokens=_OUTPUT_TOKENS, + cached_tokens=_CACHED_TOKENS, + cache_write_tokens=_CACHE_WRITE_TOKENS, + ), + ), + OpenAIDecisionResponse.model_validate(_OPENAI_RESPONSE), + ), + ids=("systemone", "openai"), +) +def test_custom_token_pricing_bills_cached_decisions_input_tokens_once( + response: DecisionsResponse | OpenAIDecisionResponse, +) -> None: + cost: Final = litellm.completion_cost( + completion_response=response, + model="gpt-6-luna", + custom_llm_provider="openai", + custom_cost_per_token={ + "input_cost_per_token": 1.0, + "output_cost_per_token": 2.0, + "cache_read_input_token_cost": 0.1, + "cache_creation_input_token_cost": 1.25, + }, + ) + + assert cost == pytest.approx( + (_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * 1.0 + + _CACHED_TOKENS * 0.1 + + _CACHE_WRITE_TOKENS * 1.25 + + _OUTPUT_TOKENS * 2.0 + ) + + def test_decisions_response_hidden_params_getter_preserves_mutable_identity() -> None: response: Final = DecisionsResponse(model="decider", answers={}, usage=None) @@ -487,7 +545,7 @@ async def test_cloudflare_clef_resolves_model_and_response_envelope( "state": "review", "questions": {"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, } - assert response.answers == DecisionsResponse.model_validate(_RESPONSE).answers + assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} assert response._hidden_params["model"] == "cloudflare/@cf/cloudflare/clef" @@ -673,3 +731,194 @@ async def test_strands_decider_provider_resolution_and_router_dispatch( assert provider_resolution[:2] == ("strands-decider-2B-hobson-v19", "strands_decider") assert route.called assert response.model == _STRANDS_RESPONSE["model"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("api_base", "url"), + ( + (None, "https://api.openai.com/v1/decisions"), + ("https://gateway.example/v1", "https://gateway.example/v1/decisions"), + ), +) +async def test_openai_decisions_translate_systemone_to_the_openai_wire_contract_and_back( + api_base: str | None, + url: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) + + response: Final = await litellm.adecisions( + model="openai/gpt-6-luna", + state="The package arrived broken.", + questions={ + "is_defect": {"type": "noul", "instructions": "Is this a defect?"}, + "sentiment": { + "type": "choice", + "instructions": "How does the customer feel?", + "criteria": {"positive": None, "negative": "unhappy"}, + }, + }, + api_key="caller-key", + api_base=api_base, + ) + + assert route.called + request: Final = respx_mock.calls[0].request + assert request.headers["authorization"] == "Bearer caller-key" + assert json.loads(request.content) == { + "model": "gpt-6-luna", + "input": "The package arrived broken.", + "questions": [ + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "sentiment", + "instructions": "How does the customer feel?", + "choices": [{"value": "positive"}, {"value": "negative", "description": "unhappy"}], + }, + ], + } + assert response.answers == {"is_defect": NoulAnswer(type="noul", noul=0.9)} + assert response.hidden_params["custom_llm_provider"] == "openai" + luna_cost: Final = litellm.model_cost["gpt-6-luna"] + expected_cost: Final = ( + (_INPUT_TOKENS - _CACHED_TOKENS - _CACHE_WRITE_TOKENS) * float(luna_cost["input_cost_per_token"]) + + _CACHED_TOKENS * float(luna_cost["cache_read_input_token_cost"]) + + _CACHE_WRITE_TOKENS * float(luna_cost["cache_creation_input_token_cost"]) + + _OUTPUT_TOKENS * float(luna_cost["output_cost_per_token"]) + ) + assert expected_cost > 0 + assert litellm.completion_cost(completion_response=response) == pytest.approx(expected_cost) + + +@pytest.mark.parametrize( + ("settings", "env", "url", "authorization"), + ( + ({"openai_key": "sdk-key"}, {}, "https://api.openai.com/v1/decisions", "Bearer sdk-key"), + ( + {"api_key": "global-key", "openai_key": "sdk-key"}, + {"OPENAI_API_KEY": "env-key"}, + "https://api.openai.com/v1/decisions", + "Bearer global-key", + ), + ( + {}, + {"OPENAI_API_KEY": "env-key", "OPENAI_API_BASE": "https://legacy.example/v1"}, + "https://legacy.example/v1/decisions", + "Bearer env-key", + ), + ( + {"api_base": "https://sdk.example"}, + {"OPENAI_API_KEY": "env-key", "OPENAI_BASE_URL": "https://env.example"}, + "https://sdk.example/v1/decisions", + "Bearer env-key", + ), + ), + ids=("openai_key", "api_key_before_env", "openai_api_base_env", "api_base_before_env"), +) +def test_openai_decisions_use_the_same_settings_as_other_openai_calls( + settings: Mapping[str, str], + env: Mapping[str, str], + url: str, + authorization: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + for name in ("api_key", "openai_key", "api_base"): + monkeypatch.setattr(litellm, name, settings.get(name)) + for name in ("OPENAI_API_KEY", "OPENAI_BASE_URL", "OPENAI_API_BASE"): + monkeypatch.delenv(name, raising=False) + for name, value in env.items(): + monkeypatch.setenv(name, value) + route: Final = respx_mock.post(url).respond(json=_OPENAI_RESPONSE) + + litellm.decisions( + model="openai/gpt-6-luna", + state="review", + questions={"is_defect": {"type": "noul", "instructions": "Is this a defect?"}}, + ) + + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == authorization + + +@pytest.mark.asyncio +async def test_openai_format_calls_to_a_systemone_provider_get_openai_format_answers( + respx_mock: respx.MockRouter, +) -> None: + route: Final = respx_mock.post("https://api.typesafe.ai/v1/systemone").respond(json=_RESPONSE) + + response: Final = await litellm.adecisions( + model="typesafe/jev-1.13", + input="review", + questions=[ + {"type": "predicate", "name": "is_defect", "instructions": "Is this a defect?"}, + { + "type": "choice", + "name": "sentiment", + "instructions": "Tone?", + "choices": [{"value": "positive"}, {"value": "negative"}], + }, + ], + api_key="caller-key", + ) + + assert route.called + assert tuple(json.loads(respx_mock.calls[0].request.content)["questions"]) == ("is_defect", "sentiment") + assert isinstance(response, OpenAIDecisionResponse) + assert [answer.model_dump(mode="json") for answer in response.answers] == [ + {"type": "predicate", "name": "is_defect", "probability": 0.9}, + { + "type": "choice", + "name": "sentiment", + "choice": "positive", + "probabilities": [{"value": "positive", "probability": 0.8}, {"value": "negative", "probability": 0.2}], + "confidence": 0.8, + }, + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "request_kwargs", "message"), + ( + ( + "typesafe/jev-1.13", + { + "input": [ + { + "role": "user", + "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA=="}], + } + ], + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + }, + "cannot serve this request", + ), + ( + "openai/gpt-6-luna", + { + "state": "review", + "input": "review", + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + }, + "not both", + ), + ), + ids=("image_to_systemone_provider", "state_and_input"), +) +async def test_requests_a_provider_cannot_serve_are_rejected_before_http( + respx_mock: respx.MockRouter, + model: str, + request_kwargs: Mapping[str, object], + message: str, +) -> None: + with pytest.raises(litellm.BadRequestError, match=message): + await litellm.adecisions(model=model, api_key="caller-key", **request_kwargs) + + assert len(respx_mock.calls) == 0 diff --git a/tests/unit/decisions/test_openai_transformation.py b/tests/unit/decisions/test_openai_transformation.py deleted file mode 100644 index a3cc1583d40..00000000000 --- a/tests/unit/decisions/test_openai_transformation.py +++ /dev/null @@ -1,32 +0,0 @@ -from collections.abc import Mapping -from typing import Final - -import pytest -from pydantic import TypeAdapter, ValidationError - -from litellm.decisions.openai_transformation import to_systemone_request -from litellm.types.decisions import MAX_DECISION_QUESTIONS, DecisionsRequestBody, OpenAIDecisionRequestBody - -_OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) -_SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) - - -def _openai_request(question_count: int) -> Mapping[str, object]: - return { - "model": "decider", - "input": "The package arrived with a broken screen.", - "questions": [{"type": "predicate", "instructions": f"Question {index}?"} for index in range(question_count)], - } - - -def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_request() -> None: - raw: Final = _openai_request(MAX_DECISION_QUESTIONS) - - translated: Final = _SYSTEMONE_BODY.validate_python(to_systemone_request(raw, _OPENAI_BODY.validate_python(raw))) - - assert len(translated.questions) == MAX_DECISION_QUESTIONS - - -def test_an_openai_request_with_more_questions_than_systemone_takes_is_rejected_before_translation() -> None: - with pytest.raises(ValidationError, match="questions"): - _OPENAI_BODY.validate_python(_openai_request(MAX_DECISION_QUESTIONS + 1)) diff --git a/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py new file mode 100644 index 00000000000..a452a621d2e --- /dev/null +++ b/tests/unit/llms/base_llm/decisions/test_base_decisions_transformation.py @@ -0,0 +1,180 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm.llms.base_llm.decisions.transformation import ( + ir_to_systemone_request, + ir_to_systemone_response, + parse_systemone_response, + systemone_request_to_ir, +) +from litellm.llms.openai.decisions.transformation import openai_request_to_ir +from litellm.types.decisions import ( + MAX_DECISION_QUESTIONS, + DecisionsIRRefusal, + DecisionsIRRequest, + DecisionsRequestBody, + OpenAIDecisionRequestBody, + UnsupportedDecisionsRequest, +) + +_SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) + +_SYSTEMONE_REQUEST: Final[Mapping[str, object]] = { + "state": {"ticket": 1234, "text": "Screen cracked"}, + "questions": { + "damaged": { + "type": "noul", + "instructions": "Is the item damaged?", + "criteria": {"true": "Visible damage", "false": None}, + "provider_field": "dropped", + }, + "rubric_only": {"type": "noul", "criteria": {"true": {"signal": "refund"}}}, + "action": { + "type": "choice", + "instructions": {"policy": "refund-v2"}, + "criteria": {"refund": "Within 30 days", "escalate": None}, + }, + "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, + }, +} + + +def _openai_ir(raw: Mapping[str, object]) -> DecisionsIRRequest: + return openai_request_to_ir(_OPENAI_BODY.validate_python(raw)) + + +def test_a_systemone_request_reaches_a_systemone_provider_unchanged() -> None: + ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python(_SYSTEMONE_REQUEST)) + + assert ir_to_systemone_request("jev-latest", ir) == {"model": "jev-latest", **_SYSTEMONE_REQUEST} + + +@pytest.mark.parametrize( + ("names", "keys"), + ( + (("damaged", "action"), ("damaged", "action")), + (("damaged", None), ("0", "1")), + (("damaged", "damaged"), ("0", "1")), + ), + ids=("all_named", "one_unnamed", "duplicate_names"), +) +def test_openai_questions_are_keyed_by_name_only_when_every_name_is_unique( + names: tuple[str | None, str | None], keys: tuple[str, str] +) -> None: + questions: Final = [ + {"type": "predicate", "instructions": f"Question {index}?", **({} if name is None else {"name": name})} + for index, name in enumerate(names) + ] + + body: Final = ir_to_systemone_request("jev-latest", _openai_ir({"input": "review", "questions": questions})) + + assert tuple(_SYSTEMONE_BODY.validate_python(body).questions) == keys + + +def test_openai_messages_become_systemone_state_text() -> None: + ir: Final = _openai_ir( + { + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "The package arrived broken."}, + {"type": "input_text", "text": "I want a refund."}, + ], + }, + {"role": "user", "content": "Order 1234."}, + ], + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + } + ) + + body: Final = ir_to_systemone_request("jev-latest", ir) + + assert isinstance(body, Mapping) + assert body["state"] == "The package arrived broken.\n\nI want a refund.\n\nOrder 1234." + + +def test_image_input_is_unsupported_by_systemone_providers() -> None: + ir: Final = _openai_ir( + { + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": "Is the screen cracked?"}, + {"type": "input_image", "image_url": "data:image/png;base64,AA=="}, + ], + } + ], + "questions": [{"type": "predicate", "instructions": "Is this a defect?"}], + } + ) + + assert isinstance(ir_to_systemone_request("jev-latest", ir), UnsupportedDecisionsRequest) + + +def test_the_largest_openai_request_accepted_translates_to_a_valid_systemone_request() -> None: + ir: Final = _openai_ir( + { + "input": "review", + "questions": [ + {"type": "predicate", "instructions": f"Question {index}?"} for index in range(MAX_DECISION_QUESTIONS) + ], + } + ) + + translated: Final = _SYSTEMONE_BODY.validate_python(ir_to_systemone_request("jev-latest", ir)) + + assert len(translated.questions) == MAX_DECISION_QUESTIONS + + +def test_a_systemone_response_reaches_the_caller_unchanged_with_provider_extras() -> None: + payload: Final = { + "model": "jev-latest", + "answers": { + "damaged": {"type": "noul", "noul": 0.95, "rationale": "crack visible"}, + "action": { + "type": "choice", + "choice": "refund", + "confidence": 0.8, + "probabilities": {"refund": 0.9, "escalate": 0.1}, + "calibrated": True, + }, + "severity": { + "type": "score", + "score": 0.4, + "confidence": 0.6, + "legend": {"0": "minor", "1": {"label": "major"}}, + "probabilities": {"0": 0.6, "1": 0.4}, + "raw_logits": [0.1, 0.2], + }, + }, + "usage": {"input_tokens": 383, "output_tokens": 2, "cost": 0.25}, + "latency_ms": 3722.17, + } + ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python(_SYSTEMONE_REQUEST)) + + response: Final = ir_to_systemone_response(parse_systemone_response(payload, ir), ir) + + assert response.model_dump(mode="json") == payload + + +def test_missing_or_mismatched_systemone_answers_are_refusals_left_out_of_the_response() -> None: + ir: Final = systemone_request_to_ir(_SYSTEMONE_BODY.validate_python(_SYSTEMONE_REQUEST)) + payload: Final = { + "model": "jev-latest", + "answers": { + "damaged": {"type": "noul", "noul": 0.95}, + "action": {"type": "noul", "noul": 0.5}, + }, + "usage": {"input_tokens": 10, "output_tokens": 1}, + } + + parsed: Final = parse_systemone_response(payload, ir) + + assert parsed.answers[1:] == (DecisionsIRRefusal(), DecisionsIRRefusal(), DecisionsIRRefusal()) + assert tuple(ir_to_systemone_response(parsed, ir).answers) == ("damaged",) diff --git a/tests/unit/llms/openai/decisions/__init__.py b/tests/unit/llms/openai/decisions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py new file mode 100644 index 00000000000..e73c5b92ab6 --- /dev/null +++ b/tests/unit/llms/openai/decisions/test_openai_decisions_transformation.py @@ -0,0 +1,256 @@ +from collections.abc import Mapping +from typing import Final + +import pytest +from pydantic import TypeAdapter + +from litellm.llms.base_llm.decisions.transformation import ir_to_systemone_response, systemone_request_to_ir +from litellm.llms.openai.decisions.transformation import ( + OpenAIDecisionsConfig, + ir_to_openai_request, + ir_to_openai_response, + openai_request_to_ir, +) +from litellm.types.decisions import ( + DecisionsIRRequest, + DecisionsRequestBody, + OpenAIDecisionRequestBody, + UnsupportedDecisionsRequest, +) + +_SYSTEMONE_BODY: Final[TypeAdapter[DecisionsRequestBody]] = TypeAdapter(DecisionsRequestBody) +_OPENAI_BODY: Final[TypeAdapter[OpenAIDecisionRequestBody]] = TypeAdapter(OpenAIDecisionRequestBody) + +_OPENAI_REQUEST: Final[Mapping[str, object]] = { + "input": [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "The screen is cracked."}, + {"type": "input_image", "image_url": "data:image/png;base64,AA==", "detail": "high"}, + ], + }, + {"type": "message", "role": "user", "content": "Order 1234."}, + ], + "questions": [ + {"type": "predicate", "name": "damaged", "instructions": "Is the item damaged?"}, + { + "type": "choice", + "instructions": "Should we refund?", + "choices": [{"value": True, "description": "Refund now"}, {"value": "escalate"}], + }, + { + "type": "score", + "name": "severity", + "instructions": "How severe is it?", + "levels": [{"label": "minor"}, {"label": "major", "description": "Product unusable"}], + }, + {"type": "predicate", "name": "fraud", "instructions": "Is this fraud?"}, + ], + "safety_identifier": "end-user-1", +} +_PREDICATE_ANSWER: Final[Mapping[str, object]] = {"type": "predicate", "name": "damaged", "probability": 0.95} +_CHOICE_ANSWER: Final[Mapping[str, object]] = { + "type": "choice", + "name": None, + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, +} +_REFUSAL_ANSWER: Final[Mapping[str, object]] = {"type": "refusal", "name": "fraud"} +_USAGE: Final[Mapping[str, object]] = { + "input_tokens": 383, + "input_tokens_details": {"cached_tokens": 256, "cache_write_tokens": 64}, + "output_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 1}, + "total_tokens": 385, +} +_OPENAI_RESPONSE: Final[Mapping[str, object]] = { + "model": "gpt-6-luna", + "answers": [ + _PREDICATE_ANSWER, + _CHOICE_ANSWER, + { + "type": "score", + "name": "severity", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": "major", "probability": 0.7}, + ], + "confidence": 0.6, + }, + _REFUSAL_ANSWER, + ], + "usage": _USAGE, +} + + +def _openai_ir(raw: Mapping[str, object]) -> DecisionsIRRequest: + return openai_request_to_ir(_OPENAI_BODY.validate_python(raw)) + + +def test_an_openai_request_reaches_openai_unchanged() -> None: + assert ir_to_openai_request("gpt-6-luna", _openai_ir(_OPENAI_REQUEST)) == { + "model": "gpt-6-luna", + **_OPENAI_REQUEST, + } + + +def test_an_openai_response_reaches_the_caller_unchanged() -> None: + ir: Final = _openai_ir(_OPENAI_REQUEST) + + parsed: Final = OpenAIDecisionsConfig().parse_response(_OPENAI_RESPONSE, ir) + + assert ir_to_openai_response(parsed, ir, "requested").model_dump(mode="json") == _OPENAI_RESPONSE + + +def test_answers_openai_did_not_return_are_refusals() -> None: + ir: Final = _openai_ir(_OPENAI_REQUEST) + payload: Final = {**_OPENAI_RESPONSE, "answers": [_PREDICATE_ANSWER]} + + response: Final = ir_to_openai_response(OpenAIDecisionsConfig().parse_response(payload, ir), ir, "requested") + + assert [answer.type for answer in response.answers] == ["predicate", "refusal", "refusal", "refusal"] + assert [answer.name for answer in response.answers] == ["damaged", None, "severity", "fraud"] + + +def test_a_systemone_request_becomes_an_openai_request_with_questions_named_by_their_keys() -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": {"ticket": 1234, "text": "Screen cracked"}, + "questions": { + "damaged": { + "type": "noul", + "instructions": "Is the item damaged?", + "criteria": {"true": "Visible damage", "false": None}, + "provider_field": "dropped", + }, + "rubric_only": {"type": "noul", "criteria": {"true": {"signal": "refund"}}}, + "action": { + "type": "choice", + "instructions": {"policy": "refund-v2"}, + "criteria": {"refund": "Within 30 days", "escalate": None}, + }, + "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, + }, + } + ) + + assert ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)) == { + "model": "gpt-6-luna", + "input": '{"ticket": 1234, "text": "Screen cracked"}', + "questions": [ + { + "type": "predicate", + "name": "damaged", + "instructions": "Is the item damaged?\n\nAnswer true when: Visible damage", + }, + {"type": "predicate", "name": "rubric_only", "instructions": 'Answer true when: {"signal": "refund"}'}, + { + "type": "choice", + "name": "action", + "instructions": '{"policy": "refund-v2"}', + "choices": [{"value": "refund", "description": "Within 30 days"}, {"value": "escalate"}], + }, + { + "type": "score", + "name": "severity", + "instructions": "Which level best fits the input?", + "levels": [{"label": "minor"}, {"label": '{"label": "major"}'}], + }, + ], + } + + +def test_systemone_questions_without_instructions_become_valid_openai_questions() -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": "Screen cracked", + "questions": { + "damaged": {"type": "noul", "criteria": {"false": None}}, + "action": {"type": "choice", "criteria": {"refund": None, "escalate": None}}, + "severity": {"type": "score", "criteria": ["minor", "major"]}, + }, + } + ) + + body: Final = ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)) + + assert all(question.instructions for question in _OPENAI_BODY.validate_python(body).questions) + + +@pytest.mark.parametrize( + "question", + ({"type": "choice", "criteria": {"refund": None}}, {"type": "score", "criteria": ["minor"]}), + ids=("choice", "score"), +) +def test_a_systemone_question_with_one_option_is_unsupported_by_openai(question: Mapping[str, object]) -> None: + request: Final = _SYSTEMONE_BODY.validate_python( + { + "state": "Screen cracked", + "questions": {"damaged": {"type": "noul", "instructions": "Damaged?"}, "q": question}, + } + ) + + assert isinstance(ir_to_openai_request("gpt-6-luna", systemone_request_to_ir(request)), UnsupportedDecisionsRequest) + + +def test_an_openai_response_becomes_systemone_answers_with_the_callers_score_labels() -> None: + ir: Final = systemone_request_to_ir( + _SYSTEMONE_BODY.validate_python( + { + "state": "Screen cracked", + "questions": { + "damaged": {"type": "noul", "instructions": "Damaged?"}, + "action": {"type": "choice", "criteria": {"true": None, "escalate": None}}, + "severity": {"type": "score", "criteria": ["minor", {"label": "major"}]}, + "fraud": {"type": "noul", "instructions": "Fraud?"}, + }, + } + ) + ) + payload: Final = { + **_OPENAI_RESPONSE, + "answers": [ + _PREDICATE_ANSWER, + _CHOICE_ANSWER, + { + "type": "score", + "name": "severity", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": '{"label": "major"}', "probability": 0.7}, + ], + "confidence": 0.6, + }, + _REFUSAL_ANSWER, + ], + } + + response: Final = ir_to_systemone_response(OpenAIDecisionsConfig().parse_response(payload, ir), ir) + + assert response.model_dump(mode="json") == { + "model": "gpt-6-luna", + "answers": { + "damaged": {"type": "noul", "noul": 0.95}, + "action": { + "type": "choice", + "choice": "true", + "confidence": 0.8, + "probabilities": {"true": 0.9, "escalate": 0.1}, + }, + "severity": { + "type": "score", + "score": 0.7, + "confidence": 0.6, + "legend": {"0": "minor", "1": {"label": "major"}}, + "probabilities": {"0": 0.3, "1": 0.7}, + }, + }, + "usage": {"input_tokens": 383, "output_tokens": 2}, + } + assert response.usage is not None + assert (response.usage.cached_tokens, response.usage.cache_write_tokens) == (256, 64) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index b00f2be092c..ccfa12e10d6 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -320,6 +320,28 @@ _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST: Final[Mapping[str, object]] = { "usage": {"input_tokens": _INPUT_TOKENS, "output_tokens": _OUTPUT_TOKENS}, } +_OPENAI_FORMAT_ANSWERS: Final = [ + {"type": "predicate", "name": "damaged", "probability": 0.95}, + { + "type": "choice", + "name": None, + "choice": True, + "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], + "confidence": 0.8, + }, + { + "type": "score", + "name": "severity", + "score": 0.7, + "probabilities": [ + {"value": 0, "label": "minor", "probability": 0.3}, + {"value": 1, "label": "major", "probability": 0.7}, + ], + "confidence": 0.6, + }, + {"type": "refusal", "name": "fraud"}, +] + @pytest.mark.parametrize("endpoint", ("/v1/decisions", "/decisions")) def test_openai_format_decisions_translate_through_systemone( @@ -354,27 +376,7 @@ def test_openai_format_decisions_translate_through_systemone( } body: Final = response.json() assert body["model"] == _SYSTEMONE_ANSWERS_FOR_OPENAI_REQUEST["model"] - assert body["answers"] == [ - {"type": "predicate", "name": "damaged", "probability": 0.95}, - { - "type": "choice", - "name": None, - "choice": True, - "probabilities": [{"value": True, "probability": 0.9}, {"value": "escalate", "probability": 0.1}], - "confidence": 0.8, - }, - { - "type": "score", - "name": "severity", - "score": 0.7, - "probabilities": [ - {"value": 0, "label": "minor", "probability": 0.3}, - {"value": 1, "label": "major", "probability": 0.7}, - ], - "confidence": 0.6, - }, - {"type": "refusal", "name": "fraud"}, - ] + assert body["answers"] == _OPENAI_FORMAT_ANSWERS assert body["usage"] == { "input_tokens": _INPUT_TOKENS, "input_tokens_details": {"cached_tokens": 0, "cache_write_tokens": 0}, @@ -462,6 +464,72 @@ def test_a_body_that_is_not_json_is_a_client_error( assert not upstream.called +def test_openai_format_decisions_reach_an_openai_deployment_unchanged_including_images( + client: TestClient, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + monkeypatch.setattr(litellm, "api_base", None) + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + monkeypatch.setattr( + litellm.proxy.proxy_server, + "llm_router", + litellm.Router( + model_list=[{"model_name": "decider", "litellm_params": {"model": "openai/gpt-6-luna", "api_key": "k"}}] + ), + ) + image_message: Final = { + "type": "message", + "role": "user", + "content": [{"type": "input_image", "image_url": "data:image/png;base64,AA==", "detail": "low"}], + } + request_body: Final = {**_OPENAI_FORMAT_REQUEST, "input": [*_OPENAI_FORMAT_REQUEST["input"], image_message]} + cached_tokens: Final = 128 + upstream_usage: Final = { + "input_tokens": _INPUT_TOKENS, + "input_tokens_details": {"cached_tokens": cached_tokens, "cache_write_tokens": 0}, + "output_tokens": _OUTPUT_TOKENS, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": _INPUT_TOKENS + _OUTPUT_TOKENS, + } + upstream: Final = respx_mock.post("https://api.openai.com/v1/decisions").respond( + json={"model": "gpt-6-luna", "answers": _OPENAI_FORMAT_ANSWERS, "usage": upstream_usage} + ) + + response: Final = client.post("/v1/decisions", json=request_body) + + assert response.status_code == 200, response.text + assert json.loads(upstream.calls[0].request.content) == { + "model": "gpt-6-luna", + "input": [ + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "The package arrived with a broken screen."}, + {"type": "input_text", "text": "I want a refund."}, + ], + }, + {"type": "message", "role": "user", "content": "Order 1234."}, + image_message, + ], + "questions": _OPENAI_FORMAT_REQUEST["questions"], + "safety_identifier": "end-user-1", + } + body: Final = response.json() + assert body["answers"] == _OPENAI_FORMAT_ANSWERS + assert body["usage"] == upstream_usage + luna_cost: Final = litellm.model_cost["gpt-6-luna"] + expected_cost: Final = ( + (_INPUT_TOKENS - cached_tokens) * float(luna_cost["input_cost_per_token"]) + + cached_tokens * float(luna_cost["cache_read_input_token_cost"]) + + _OUTPUT_TOKENS * float(luna_cost["output_cost_per_token"]) + ) + assert expected_cost > 0 + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(expected_cost) + + def _decisions_feature() -> LazyFeature: return next(feature for feature in LAZY_FEATURES if feature.name == "decisions") diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ac5dc6e7c45..7551a310977 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -1064,6 +1064,7 @@ def test_aaamodel_prices_and_context_window_json_is_valid(): "/vertex_ai/live", "/v1/listen", "/v1/systemone", + "/v1/decisions", "/v1beta/interactions", ], }, From a3b280f69d068e8ed2350dcd4c5314073f69ce0f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:18:00 -0700 Subject: [PATCH 14/65] feat(lint): add LIT015 requiring pydantic models to be frozen (#42348) * feat(lint): add LIT013 requiring pydantic models to be frozen Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lint): let an explicit frozen=False override earlier or inherited frozen=True in LIT013 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(lint): annotate LIT013 helper locals with Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(lint): document the frozen pydantic model rule as LIT015 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(lint): honor the frozen class keyword in LIT015 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(lint): annotate main and scan_paths locals as Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lint): cover LiteLLMBaseModel, BaseSettings, and SettingsConfigDict in LIT015 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lint): resolve LIT015 model bases per class node Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lint): treat the OpenAIObject alias as a LIT015 model base Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lint): resolve shared LIT015 config constants and list frozen-ok in the gate hint Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lint): resolve LIT015 shared config at class definition Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: mateo Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- AGENTS.md | 1 + scripts/check_type_discipline.py | 185 ++++++++++++++- scripts/type_discipline_gate.py | 5 +- tests/unit/test_check_type_discipline.py | 287 ++++++++++++++++++++++- type-discipline-budget.json | 3 + 5 files changed, 458 insertions(+), 23 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a7d7256eeb4..ce3bfa94a0d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -97,6 +97,7 @@ Follow these coding conventions for new/updated code (a three-line fix in a lega - Annotate every variable with `: Final` (LIT010). Unpacking and walrus targets cannot carry the annotation, so they are implicitly final. Don't rebind them. Never rebind or mutate function parameters (LIT011); `self`/`cls` attribute stores are the exception. If rebinding or in-place mutation is truly unavoidable, suppress with `# rebind-ok: ` - Qualify every TypedDict field with `ReadOnly[...]` (LIT012), which nests freely with `Required` / `NotRequired` / `Annotated` in any order. If making the key writable is truly unavoidable, suppress with `# writable-ok: ` - Comprehensions take at most one `for` clause and one `if` clause (LIT014); split stacked clauses into a helper generator, a named intermediate, or a plain loop. Suppress with `# comprehension-ok: ` only when unavoidable + - Every pydantic model must be frozen (LIT015), set via `model_config = ConfigDict(frozen=True)`, a dict-literal `model_config`, an inner `class Config`, or the class keywords. Subclasses inherit it unless they override it. Replace in-place field writes with `model_copy(update=...)`. If making the model mutable is truly unavoidable, suppress with `# frozen-ok: ` on the `class` line - Use dependency injection - Fully typed; no `Any` or coarse types like `dict[str, Any]` or just `dict`. Every function parameter must be strongly typed - Use tagged unions + match diff --git a/scripts/check_type_discipline.py b/scripts/check_type_discipline.py index 8adc3ac27b7..cafec8d249b 100644 --- a/scripts/check_type_discipline.py +++ b/scripts/check_type_discipline.py @@ -18,8 +18,8 @@ LIT003 noqa suppression without rule codes or without a reason. LIT004 pyright/mypy ignore without bracketed codes or without a reason. Required shape: `# pyright: ignore[reportArgumentType] # ` LIT005 A `# mutable-ok` / `# cast-ok` / `# guard-ok` / `# kwargs-ok` / - `# rebind-ok` / `# writable-ok` / `# comprehension-ok` suppression - without a reason. + `# rebind-ok` / `# writable-ok` / `# comprehension-ok` / `# frozen-ok` + suppression without a reason. LIT006 `cast(...)` call. typing.cast is an unchecked assertion (the moral equivalent of TypeScript's `as`); it lies to the type checker with zero runtime guarantee. Validate into a concrete frozen type at the boundary instead. @@ -91,6 +91,16 @@ LIT014 Comprehension with more than one `for` clause or more than one `if` clau `# comprehension-ok: ` on any line the comprehension spans. The marker belongs to the innermost violating comprehension spanning that line, and also to any single-line violating comprehension on that line. +LIT015 Pydantic model class that is not frozen. Set `frozen=True` in + `model_config = ConfigDict(...)`, `SettingsConfigDict(...)`, a dict-literal + `model_config`, an inner `class Config`, or the class keywords. Classes + inherit the frozen setting from in-module model bases, unless their own + configuration overrides it. Detection is name-based: `BaseModel`, + `pydantic.BaseModel`, `LiteLLMBaseModel`, `BaseLiteLLMOpenAIResponseObject`, + `LiteLLMPydanticObjectBase`, `OpenAIObject`, `RootModel`, and `BaseSettings` identify + models, while `TypedDict` classes are exempt. Replace in-place field writes + with `model_copy(update=...)`. Suppress with `# frozen-ok: ` on the + `class` line. LIT000 Setup failure: a target file could not be read, or contains a syntax error. Reported as a violation rather than crashing the run. @@ -110,12 +120,13 @@ import os import re import sys import tokenize +from collections.abc import Iterable, Iterator, Mapping, Sequence from dataclasses import dataclass +from itertools import groupby from multiprocessing import Pool from pathlib import Path -from collections.abc import Iterable, Iterator, Mapping, Sequence from types import MappingProxyType -from typing import NamedTuple +from typing import Final, NamedTuple # Mutable collection types, banned in *every* annotation. Name-based, so `dict`, # `typing.Dict`, `collections.deque`, and `collections.abc.MutableMapping` all match @@ -149,6 +160,19 @@ READONLY_QUALIFIER = "ReadOnly" # first argument is type syntax, the rest is metadata and never qualifies the field. FIELD_QUALIFIER_WRAPPERS = frozenset(("Required", "NotRequired", "Annotated")) TYPEDDICT_BASE = "TypedDict" +# Base names that mark a class as a pydantic model (LIT015). +PYDANTIC_BASES: Final = frozenset( + ( + "BaseModel", + "LiteLLMBaseModel", + "BaseLiteLLMOpenAIResponseObject", + "LiteLLMPydanticObjectBase", + "OpenAIObject", + "RootModel", + "BaseSettings", + ) +) +PYDANTIC_CONFIG_FACTORIES: Final = frozenset(("ConfigDict", "SettingsConfigDict")) MIN_REASON_LEN = 3 NOQA_RE = re.compile( @@ -166,6 +190,8 @@ KWARGS_OK_RE = re.compile(r"#\s*kwargs-ok(?::\s*(?P.*))?") REBIND_OK_RE = re.compile(r"#\s*rebind-ok(?::\s*(?P.*))?") WRITABLE_OK_RE = re.compile(r"#\s*writable-ok(?::\s*(?P.*))?") COMPREHENSION_OK_RE = re.compile(r"#\s*comprehension-ok(?::\s*(?P.*))?") +FROZEN_OK_RE: Final = re.compile(r"#\s*frozen-ok(?::\s*(?P.*))?") + @dataclass(frozen=True, slots=True) class _OkToken: @@ -185,6 +211,7 @@ OK_SUPPRESSIONS: Final[tuple[_OkToken, ...]] = ( _OkToken("rebind-ok", REBIND_OK_RE, frozenset(("LIT010", "LIT011"))), _OkToken("writable-ok", WRITABLE_OK_RE, frozenset(("LIT012",))), _OkToken("comprehension-ok", COMPREHENSION_OK_RE, frozenset(("LIT014",))), + _OkToken("frozen-ok", FROZEN_OK_RE, frozenset(("LIT015",))), ) @@ -833,6 +860,147 @@ def iter_typeddict_violations(path: Path, tree: ast.AST) -> Iterator[Violation]: ) +# --------------------------------------------------------------------------- # +# Unfrozen pydantic models (LIT015) +# --------------------------------------------------------------------------- # + + +def _pydantic_classes(tree: ast.AST) -> tuple[ast.ClassDef, ...]: + classes: Final = tuple(node for node in ast.walk(tree) if isinstance(node, ast.ClassDef)) + + def expand(known: frozenset[str]) -> frozenset[str]: + grown: Final = known | frozenset(cls.name for cls in classes if _base_names(cls) & known) + return grown if grown == known else expand(grown) + + model_names: Final = expand(PYDANTIC_BASES) + typeddict_names: Final = expand(frozenset((TYPEDDICT_BASE,))) + return tuple(cls for cls in classes if _base_names(cls) & model_names and not _base_names(cls) & typeddict_names) + + +def _bool_constant(value: ast.expr) -> bool | None: + return value.value if isinstance(value, ast.Constant) and isinstance(value.value, bool) else None + + +def _module_assignment_items(tree: ast.AST) -> Iterator[tuple[str, tuple[int, ast.expr]]]: + if not isinstance(tree, ast.Module): + return + for stmt in tree.body: + if isinstance(stmt, ast.Assign): + yield from ( + (target.id, (stmt.lineno, stmt.value)) for target in stmt.targets if isinstance(target, ast.Name) + ) + elif isinstance(stmt, ast.AnnAssign) and isinstance(stmt.target, ast.Name) and stmt.value is not None: + yield stmt.target.id, (stmt.lineno, stmt.value) + + +def _model_config_frozen( + value: ast.expr, + module_assignments: Mapping[str, tuple[tuple[int, ast.expr], ...]], + class_lineno: int, +) -> bool | None: + config: Final = ( + next( + ( + expression + for lineno, expression in reversed(module_assignments.get(value.id, ())) + if lineno < class_lineno + ), + None, + ) + if isinstance(value, ast.Name) + else value + ) + if isinstance(config, ast.Call) and _head_name(config.func) in PYDANTIC_CONFIG_FACTORIES: + flags: Final = tuple(_bool_constant(kw.value) for kw in config.keywords if kw.arg == "frozen") + return flags[-1] if flags else None + if isinstance(config, ast.Dict): + flags: Final = tuple( + _bool_constant(item) + for key, item in zip(config.keys, config.values) + if isinstance(key, ast.Constant) and key.value == "frozen" + ) + return flags[-1] if flags else None + return None + + +def _assigns_name(stmt: ast.stmt, name: str) -> ast.expr | None: + value: Final = stmt.value if isinstance(stmt, (ast.Assign, ast.AnnAssign)) else None + targets: Final = ( + stmt.targets if isinstance(stmt, ast.Assign) else (stmt.target,) if isinstance(stmt, ast.AnnAssign) else () + ) + if value is not None and any(isinstance(target, ast.Name) and target.id == name for target in targets): + return value + return None + + +def _config_class_frozen(node: ast.ClassDef) -> bool | None: + values: Final = tuple(_assigns_name(stmt, "frozen") for stmt in node.body) + flags: Final = tuple(_bool_constant(value) for value in values if value is not None) + return flags[-1] if flags else None + + +def _stmt_frozen_flag( + stmt: ast.stmt, + module_assignments: Mapping[str, tuple[tuple[int, ast.expr], ...]], + class_lineno: int, +) -> bool | None: + config_value: Final = _assigns_name(stmt, "model_config") + if config_value is not None: + return _model_config_frozen(config_value, module_assignments, class_lineno) + if isinstance(stmt, ast.ClassDef) and stmt.name == "Config": + return _config_class_frozen(stmt) + return None + + +def _class_frozen_override( + cls: ast.ClassDef, + module_assignments: Mapping[str, tuple[tuple[int, ast.expr], ...]], + class_lineno: int, +) -> bool | None: + keyword_flags: Final = tuple(_bool_constant(keyword.value) for keyword in cls.keywords if keyword.arg == "frozen") + if keyword_flags: + return keyword_flags[-1] + body_flags: Final = tuple(_stmt_frozen_flag(stmt, module_assignments, class_lineno) for stmt in cls.body) + return next((flag for flag in reversed(body_flags) if flag is not None), None) + + +def iter_pydantic_violations(path: Path, tree: ast.AST) -> Iterator[Violation]: + module_assignments: Final = MappingProxyType( + { + name: tuple(binding for _, binding in assignments) + for name, assignments in groupby( + sorted(_module_assignment_items(tree), key=lambda item: item[0]), + key=lambda item: item[0], + ) + } + ) + models: Final = _pydantic_classes(tree) + bases_of: Final = {cls: _base_names(cls) for cls in models} + override_of: Final = {cls: _class_frozen_override(cls, module_assignments, cls.lineno) for cls in models} + + def frozen(known: frozenset[str]) -> frozenset[str]: + grown: Final = known | frozenset( + cls.name + for cls in models + if override_of[cls] is True or (override_of[cls] is None and bases_of[cls] & known) + ) + return grown if grown == known else frozen(grown) + + frozen_names: Final = frozen(frozenset()) + for cls in models: + if override_of[cls] is True or (override_of[cls] is None and bases_of[cls] & frozen_names): + continue + yield Violation( + path, + cls.lineno, + "LIT015", + f"pydantic model `{cls.name}` is not frozen: set `frozen=True` in " + f"`model_config`, an inner `class Config`, or the class keywords, and " + f"replace in-place field writes with `model_copy(update=...)` " + f"(suppress: `# frozen-ok: `)", + ) + + # --------------------------------------------------------------------------- # # Stacked comprehension clauses (LIT014) # --------------------------------------------------------------------------- # @@ -978,6 +1146,7 @@ def check_file(path: Path) -> tuple[Violation, ...]: *iter_final_violations(path, tree), *iter_param_violations(path, tree), *iter_typeddict_violations(path, tree), + *iter_pydantic_violations(path, tree), *(v for v, owned in comprehension_violations if owned), ), suppressions, @@ -1009,7 +1178,7 @@ def _worker_count(path_count: int) -> int: def scan_paths(paths: Sequence[Path]) -> tuple[Violation, ...]: """check_file over every path. Pure per-file work, so it fans out across processes; callers sort, which is what keeps output order stable.""" - workers = _worker_count(len(paths)) + workers: Final = _worker_count(len(paths)) if workers == 1: return tuple(v for path in paths for v in check_file(path)) with Pool(workers) as pool: @@ -1017,13 +1186,13 @@ def scan_paths(paths: Sequence[Path]) -> tuple[Violation, ...]: def main(argv: Sequence[str]) -> int: - paths = tuple(a for a in argv if not a.startswith("-")) + paths: Final = tuple(a for a in argv if not a.startswith("-")) if not paths: print("usage: check_type_discipline.py ...", file=sys.stderr) return 2 - targets = tuple(collect_paths(paths)) - violations = sorted(scan_paths(targets)) + targets: Final = tuple(collect_paths(paths)) + violations: Final = sorted(scan_paths(targets)) for v in violations: print(v.render()) diff --git a/scripts/type_discipline_gate.py b/scripts/type_discipline_gate.py index 3293f32d565..3a0564c3928 100644 --- a/scripts/type_discipline_gate.py +++ b/scripts/type_discipline_gate.py @@ -17,7 +17,8 @@ LIT012 (TypedDict field without a `ReadOnly[...]` qualifier; suppress with `# writable-ok: `), and LIT014 (comprehension with more than one `for` or `if` clause; suppress with `# comprehension-ok: ` on a spanned line, which belongs to the innermost violating comprehension spanning it and -to any single-line violating comprehension on that line) carry limits at +to any single-line violating comprehension on that line), and LIT015 +(pydantic model not frozen; suppress with `# frozen-ok: `) carry limits at or above their current count to ratchet down; LIT005 (`*-ok` suppression without a reason) is frozen at limit 0 so any net-new reasonless suppression trips the gate; LIT013 (`*-ok` suppression @@ -202,7 +203,7 @@ def cmd_check(base: str) -> None: "`# pyright: ignore[rule] # `, `# mutable-ok: `, " "`# cast-ok: `, `# guard-ok: `, `# kwargs-ok: `, " "`# rebind-ok: `, `# writable-ok: `, " - "`# comprehension-ok: `), or remove an equal " + "`# comprehension-ok: `, `# frozen-ok: `), or remove an equal " "number elsewhere; the ceiling " "is the limit in type-discipline-budget.json." ) diff --git a/tests/unit/test_check_type_discipline.py b/tests/unit/test_check_type_discipline.py index 5b8ea7b56ac..abb7ea1408c 100644 --- a/tests/unit/test_check_type_discipline.py +++ b/tests/unit/test_check_type_discipline.py @@ -13,6 +13,7 @@ import re import subprocess import sys from pathlib import Path +from typing import Final import pytest @@ -579,6 +580,277 @@ def test_writable_ok_without_reason_is_lit005_and_does_not_suppress(tmp_path): assert "LIT012" in codes +# --------------------------------------------------------------------------- # +# Unfrozen pydantic models (LIT015) +# --------------------------------------------------------------------------- # + + +def test_unfrozen_basemodel_is_flagged(tmp_path): + src = "from pydantic import BaseModel\nclass P(BaseModel):\n a: int\n" + assert "LIT015" in _codes(tmp_path, src) + + +def test_configdict_frozen_true_is_clean(tmp_path): + src = ( + "from pydantic import BaseModel, ConfigDict\n" + "class P(BaseModel):\n" + " model_config = ConfigDict(extra='allow', frozen=True)\n" + ) + assert "LIT015" not in _codes(tmp_path, src) + + +def test_shared_frozen_configdict_constant_is_clean(tmp_path): + src: Final = ( + "from typing import Final\n" + "from pydantic import BaseModel, ConfigDict\n" + "_RESPONSE_CONFIG: Final = ConfigDict(frozen=True)\n" + "class Foo(BaseModel):\n" + " model_config = _RESPONSE_CONFIG\n" + " x: int\n" + ) + assert "LIT015" not in _codes(tmp_path, src) + + +def test_mutable_config_before_class_is_not_hidden_by_frozen_reassignment(tmp_path): + src: Final = ( + "from pydantic import BaseModel, ConfigDict\n" + "_RESPONSE_CONFIG = ConfigDict(frozen=False)\n" + "class Foo(BaseModel):\n" + " model_config = _RESPONSE_CONFIG\n" + " x: int\n" + "_RESPONSE_CONFIG = ConfigDict(frozen=True)\n" + ) + assert "LIT015" in _codes(tmp_path, src) + + +def test_frozen_config_before_class_survives_mutable_reassignment(tmp_path): + src: Final = ( + "from pydantic import BaseModel, ConfigDict\n" + "_RESPONSE_CONFIG = ConfigDict(frozen=True)\n" + "class Foo(BaseModel):\n" + " model_config = _RESPONSE_CONFIG\n" + " x: int\n" + "_RESPONSE_CONFIG = ConfigDict(frozen=False)\n" + ) + assert "LIT015" not in _codes(tmp_path, src) + + +def test_config_assigned_only_after_class_is_unresolved(tmp_path): + src: Final = ( + "from pydantic import BaseModel, ConfigDict\n" + "class Foo(BaseModel):\n" + " model_config = _RESPONSE_CONFIG\n" + " x: int\n" + "_RESPONSE_CONFIG = ConfigDict(frozen=True)\n" + ) + assert "LIT015" in _codes(tmp_path, src) + + +def test_unknown_model_config_name_is_flagged(tmp_path): + src: Final = ( + "from pydantic import BaseModel\nclass Foo(BaseModel):\n model_config = UNKNOWN_CONFIG\n x: int\n" + ) + assert "LIT015" in _codes(tmp_path, src) + + +def test_dict_literal_model_config_frozen_true_is_clean(tmp_path): + src = "from pydantic import BaseModel\nclass P(BaseModel):\n model_config = {'frozen': True, 'extra': 'allow'}\n" + assert "LIT015" not in _codes(tmp_path, src) + + +def test_subclass_of_in_file_frozen_model_is_clean(tmp_path): + src = ( + "from pydantic import BaseModel, ConfigDict\n" + "class Base(BaseModel):\n" + " model_config = ConfigDict(frozen=True)\n" + "class Child(Base):\n" + " a: int\n" + ) + assert "LIT015" not in _codes(tmp_path, src) + + +def test_subclass_of_in_file_unfrozen_model_flags_both(tmp_path): + src = "from pydantic import BaseModel\nclass Base(BaseModel):\n pass\nclass Child(Base):\n a: int\n" + assert _codes(tmp_path, src).count("LIT015") == 2 + + +def test_litellm_pydantic_object_base_without_frozen_is_flagged(tmp_path): + src = "class P(LiteLLMPydanticObjectBase):\n a: int\n" + assert "LIT015" in _codes(tmp_path, src) + + +def test_inner_config_class_frozen_true_is_clean(tmp_path): + src = "from pydantic import BaseModel\nclass P(BaseModel):\n class Config:\n frozen = True\n" + assert "LIT015" not in _codes(tmp_path, src) + + +def test_frozen_false_is_flagged(tmp_path): + src = ( + "from pydantic import BaseModel, ConfigDict\nclass P(BaseModel):\n model_config = ConfigDict(frozen=False)\n" + ) + assert "LIT015" in _codes(tmp_path, src) + + +def test_later_model_config_frozen_false_overrides_earlier_frozen_true(tmp_path): + src = ( + "from pydantic import BaseModel, ConfigDict\n" + "class P(BaseModel):\n" + " model_config = ConfigDict(frozen=True)\n" + " model_config = ConfigDict(frozen=False)\n" + ) + assert "LIT015" in _codes(tmp_path, src) + + +def test_subclass_frozen_false_overrides_frozen_parent(tmp_path): + src = ( + "from pydantic import BaseModel, ConfigDict\n" + "class Base(BaseModel):\n" + " model_config = ConfigDict(frozen=True)\n" + "class Writable(Base):\n" + " model_config = ConfigDict(frozen=False)\n" + "class StillFrozen(Base):\n" + " model_config = ConfigDict(extra='allow')\n" + ) + assert _codes(tmp_path, src).count("LIT015") == 1 + + +def test_root_model_without_frozen_is_flagged(tmp_path): + src = "from pydantic import RootModel\nclass P(RootModel):\n root: int\n" + assert "LIT015" in _codes(tmp_path, src) + + +def test_qualified_pydantic_basemodel_is_flagged(tmp_path): + src = "import pydantic\nclass P(pydantic.BaseModel):\n a: int\n" + assert "LIT015" in _codes(tmp_path, src) + + +def test_keyword_frozen_model_is_clean(tmp_path): + src = "from pydantic import BaseModel\nclass M(BaseModel, frozen=True):\n a: int\n" + assert "LIT015" not in _codes(tmp_path, src) + + +def test_subclass_of_keyword_frozen_model_is_clean(tmp_path): + src = ( + "from pydantic import BaseModel\n" + "class Parent(BaseModel, frozen=True):\n" + " pass\n" + "class Child(Parent):\n" + " a: int\n" + ) + assert "LIT015" not in _codes(tmp_path, src) + + +def test_frozen_false_keyword_overrides_body_frozen_config(tmp_path): + src = ( + "from pydantic import BaseModel, ConfigDict\n" + "class M(BaseModel, frozen=False):\n" + " model_config = ConfigDict(frozen=True)\n" + ) + assert "LIT015" in _codes(tmp_path, src) + + +def test_frozen_ok_with_reason_suppresses_and_is_not_lit013(tmp_path): + src = ( + "from pydantic import BaseModel\n" + "class P(BaseModel): # frozen-ok: mutated during build before handoff\n" + " a: int\n" + ) + codes = _codes(tmp_path, src) + assert "LIT015" not in codes + assert "LIT013" not in codes + + +def test_frozen_ok_on_frozen_model_is_lit013(tmp_path): + src = ( + "from pydantic import BaseModel\n" + "class P(BaseModel, frozen=True): # frozen-ok: mutable during construction\n" + " a: int\n" + ) + assert _codes(tmp_path, src) == ["LIT013"] + + +def test_frozen_ok_without_reason_is_lit005_and_does_not_suppress(tmp_path): + src = "from pydantic import BaseModel\nclass P(BaseModel): # frozen-ok\n a: int\n" + codes = _codes(tmp_path, src) + assert "LIT005" in codes + assert "LIT015" in codes + + +def test_typeddict_and_plain_classes_are_not_models(tmp_path): + src = "from typing import TypedDict\nclass T(TypedDict):\n a: int\nclass C:\n a: int\n" + assert "LIT015" not in _codes(tmp_path, src) + + +def test_extra_allow_does_not_exempt(tmp_path): + src = ( + "from pydantic import BaseModel, ConfigDict\n" + "class P(BaseModel):\n" + " model_config = ConfigDict(extra='allow')\n" + ) + assert "LIT015" in _codes(tmp_path, src) + + +def test_litellm_base_model_is_flagged(tmp_path): + src = "class Foo(LiteLLMBaseModel):\n x: int\n" + assert "LIT015" in _codes(tmp_path, src) + + +def test_litellm_openai_response_base_is_flagged(tmp_path): + src = "class R(BaseLiteLLMOpenAIResponseObject):\n x: int\n" + assert "LIT015" in _codes(tmp_path, src) + + +def test_base_settings_is_flagged(tmp_path): + src = "class S(BaseSettings):\n x: int\n" + assert "LIT015" in _codes(tmp_path, src) + + +def test_settings_config_dict_frozen_true_is_clean(tmp_path): + src = ( + "from pydantic_settings import BaseSettings, SettingsConfigDict\n" + "class S(BaseSettings):\n" + " model_config = SettingsConfigDict(frozen=True)\n" + ) + assert "LIT015" not in _codes(tmp_path, src) + + +def test_openai_object_base_is_flagged_and_can_be_frozen(tmp_path): + assert "LIT015" in _codes(tmp_path, "class Foo(OpenAIObject):\n x: int\n") + assert "LIT015" not in _codes( + tmp_path, + "from pydantic import ConfigDict\n" + "class Foo(OpenAIObject):\n" + " model_config = ConfigDict(frozen=True)\n" + " x: int\n", + ) + + +def test_same_named_models_use_each_classes_own_override(tmp_path): + src = ( + "from pydantic import BaseModel\n" + "class Dup(BaseModel, frozen=True):\n" + " x: int\n" + "class Dup(BaseModel):\n" + " x: int\n" + ) + assert _codes(tmp_path, src) == ["LIT015"] + + +def test_pydantic_model_discovery_uses_each_class_nodes_bases(tmp_path): + src: Final = ( + "from pydantic import BaseModel\n" + "class Config(BaseModel):\n" + " x: int\n" + "class Consumer(BaseModel, frozen=True):\n" + " class Config:\n" + " arbitrary_types_allowed = True\n" + ) + path: Final = tmp_path / "snippet.py" + path.write_text(src, encoding="utf-8") + violations: Final = checker.check_file(path) + assert [(violation.line, violation.code) for violation in violations] == [(2, "LIT015")] + + # --------------------------------------------------------------------------- # # Stacked comprehension clauses (LIT014) # --------------------------------------------------------------------------- # @@ -619,23 +891,12 @@ def test_comprehension_ok_with_reason_suppresses_lit014(tmp_path: Path): def test_comprehension_ok_on_any_spanned_line_suppresses_lit014(tmp_path: Path): - src = ( - "y = [\n" - " x for a in xs\n" - " for x in a\n" - "] # comprehension-ok: cartesian product is the clearest form\n" - ) + src = "y = [\n x for a in xs\n for x in a\n] # comprehension-ok: cartesian product is the clearest form\n" assert "LIT014" not in _codes(tmp_path, src) def test_comprehension_ok_after_the_closing_line_does_not_suppress(tmp_path: Path): - src = ( - "y = [\n" - " x for a in xs\n" - " for x in a\n" - "]\n" - "# comprehension-ok: cartesian product is the clearest form\n" - ) + src = "y = [\n x for a in xs\n for x in a\n]\n# comprehension-ok: cartesian product is the clearest form\n" f = tmp_path / "snippet.py" f.write_text(src, encoding="utf-8") violations = checker.check_file(f) diff --git a/type-discipline-budget.json b/type-discipline-budget.json index 11bc9722d81..aeb77212e8e 100644 --- a/type-discipline-budget.json +++ b/type-discipline-budget.json @@ -37,5 +37,8 @@ }, "LIT014": { "limit": 369 + }, + "LIT015": { + "limit": 1448 } } From a7ee03859228cfbdc14263bae3f38f9b77b01d7d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:25:12 -0700 Subject: [PATCH 15/65] fix(router): keep include_fallback_errors off the provider call on the sync Router path (#45218) * fix(router): keep include_fallback_errors off the provider call on the sync Router path Router.completion spread include_fallback_errors into litellm.completion, so OpenAI answered 400 Unknown parameter and a fallback walk lost the backup's reply. Only Router._acompletion popped the router-only flags. Both paths now build the provider call through one shared helper that drops them, and a wire-level respx test covers each path. * test(router): pin include_fallback_errors off the wire across the sync, async, and proxy paths --- litellm/router.py | 35 +- litellm/router_utils/common_utils.py | 10 +- .../test_include_fallback_errors_wire.py | 666 ++++++++++++++++++ ...est_router_include_fallback_errors_wire.py | 364 ++++++++++ tests/unit/test_router/test_router.py | 85 +++ 5 files changed, 1143 insertions(+), 17 deletions(-) create mode 100644 tests/integration/routing/test_include_fallback_errors_wire.py create mode 100644 tests/integration/sdk/test_router_include_fallback_errors_wire.py diff --git a/litellm/router.py b/litellm/router.py index d6730db8748..88c4a7011ef 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -199,6 +199,7 @@ from litellm.router_utils.common_utils import ( resolve_model_group_alias, truncate_fallback_error_detail, warn_on_provider_credential_mismatch, + without_router_only_kwargs, ) from litellm.router_utils.cooldown_cache import CooldownCache from litellm.router_utils.cooldown_handlers import ( @@ -2728,13 +2729,15 @@ class Router: if model in self.model_names or not self.has_model_id(model): self.routing_strategy_pre_call_checks(deployment=deployment) - input_kwargs: Final = { - **litellm_params, - "messages": messages, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } + input_kwargs: Final = without_router_only_kwargs( + { + **litellm_params, + "messages": messages, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } + ) response: Final = litellm.completion(**input_kwargs) verbose_router_logger.info("litellm.completion(model=%s)\x1b[32m 200 OK\x1b[0m", model_name) @@ -3880,15 +3883,15 @@ class Router: ) self.total_calls[model_name] += 1 - input_kwargs: Final = { - **litellm_params, - "messages": messages, - "caching": self.cache_responses, - "client": model_client, - **kwargs, - } - input_kwargs.pop("silent_model", None) - input_kwargs.pop("include_fallback_errors", None) + input_kwargs: Final = without_router_only_kwargs( + { + **litellm_params, + "messages": messages, + "caching": self.cache_responses, + "client": model_client, + **kwargs, + } + ) logging_obj: Final[LiteLLMLogging | None] = kwargs.get("litellm_logging_obj", None) diff --git a/litellm/router_utils/common_utils.py b/litellm/router_utils/common_utils.py index 6e400738b8e..4d6af710bb9 100644 --- a/litellm/router_utils/common_utils.py +++ b/litellm/router_utils/common_utils.py @@ -2,7 +2,7 @@ import hashlib import json from collections.abc import Mapping, Sequence from types import MappingProxyType -from typing import TYPE_CHECKING, Final +from typing import TYPE_CHECKING, Final, TypeVar if TYPE_CHECKING: from litellm.types.llms.openai import OpenAIFileObject @@ -16,6 +16,14 @@ from litellm.litellm_core_utils.sensitive_data_masker import mask_sensitive_stru from litellm.types.router import CredentialLiteLLMParams from litellm.types.utils import LlmProviders +_V = TypeVar("_V") + +ROUTER_ONLY_CALL_KWARGS: Final = frozenset({"silent_model", "include_fallback_errors"}) + + +def without_router_only_kwargs(kwargs: Mapping[str, _V]) -> dict[str, _V]: + return {key: value for key, value in kwargs.items() if key not in ROUTER_ONLY_CALL_KWARGS} + def is_proxy_admin_request(request_kwargs: Mapping[str, object] | None) -> bool: if request_kwargs is None: diff --git a/tests/integration/routing/test_include_fallback_errors_wire.py b/tests/integration/routing/test_include_fallback_errors_wire.py new file mode 100644 index 00000000000..9075f5e41d3 --- /dev/null +++ b/tests/integration/routing/test_include_fallback_errors_wire.py @@ -0,0 +1,666 @@ +from __future__ import annotations + +import json +import os +import re +import uuid +from collections.abc import Callable, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from pathlib import Path +from types import MappingProxyType +from typing import Final, Literal +from urllib.parse import unquote + +import httpx +import pytest +import yaml +from integration._support.anthropic_sse import error_body, message_json, message_stream, parse_sse, stream_reply +from integration._support.client import Gateway, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.openai_wire import chat_reply, responses_reply +from integration._support.process import graceful_stop_seconds, owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +from litellm.constants import PROXY_CONFIG_RELOAD_INTERVAL_SECONDS + +Endpoint = Literal["chat", "messages", "responses"] + +_OPENAI_MODEL: Final = "gpt-5.4" +_ANTHROPIC_MODEL: Final = "claude-haiku-4-5" +_API_KEY: Final = "synthetic-fallback-errors-key" +_ROUTER_ONLY_KEYS: Final = ("include_fallback_errors", "silent_model") +_ANSWER: Final = "answered by" +_UNAUTHORIZED_MESSAGE: Final = "scripted 401: the primary key was revoked" +_NONCE: Final = re.compile(r"nonce=([0-9a-f]{32})") +_JSON: Final = TypeAdapter(dict[str, JsonValue]) +_ERRORS: Final = TypeAdapter(list[dict[str, JsonValue]]) +_ITEMS: Final = TypeAdapter(list[JsonValue]) +_UPSTREAM_URL_PLACEHOLDER: Final = "upstream-url" +_NOT_YET_ON_EVERY_WORKER: Final = ( + "Invalid model name passed in model=", + "There are no healthy deployments for this model", +) +_PROXY_WORKERS: Final = int(os.environ.get("INTEGRATION_PROXY_WORKERS", "1")) +_WORKER_SYNC_SECONDS: Final = 0.0 if _PROXY_WORKERS == 1 else PROXY_CONFIG_RELOAD_INTERVAL_SECONDS + 5.0 +_FRESH_CONNECTION: Final = MappingProxyType({"Connection": "close"}) +_SPEND_ROW_SECONDS: Final = 70 +_BURST: Final = 24 +_ENDPOINTS: Final[tuple[Endpoint, ...]] = ("chat", "messages", "responses") +_PATHS: Final = MappingProxyType( + {"chat": "/v1/chat/completions", "messages": "/v1/messages", "responses": "/v1/responses"} +) +_PROVIDERS: Final = MappingProxyType( + { + "chat": f"openai/{_OPENAI_MODEL}", + "messages": f"anthropic/{_ANTHROPIC_MODEL}", + "responses": f"openai/{_OPENAI_MODEL}", + } +) +_STREAMING: Final = (pytest.param(False, id="non-stream"), pytest.param(True, id="stream")) +_NON_BOOLEAN_FLAGS: Final = ( + pytest.param(1, True, id="int"), + pytest.param("", False, id="empty-string"), + pytest.param([], False, id="list"), + pytest.param("x" * 5120, True, id="five-kilobytes"), +) +_OPENAI_UNAUTHORIZED: Final = Reply( + status=401, + body=json.dumps( + { + "error": { + "message": _UNAUTHORIZED_MESSAGE, + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key", + } + } + ).encode(), +) +_ANTHROPIC_UNAUTHORIZED: Final = Reply(status=401, body=error_body(401, _UNAUTHORIZED_MESSAGE)) + +_EXPOSED_OPENAI_PRIMARY: Final = "exposed-openai-primary" +_EXPOSED_OPENAI_BACKUP: Final = "exposed-openai-backup" +_EXPOSED_OPENAI_SERVING: Final = "exposed-openai-serving" +_EXPOSED_OPENAI_FLAKY: Final = "exposed-openai-flaky" +_EXPOSED_ANTHROPIC_PRIMARY: Final = "exposed-anthropic-primary" +_EXPOSED_ANTHROPIC_BACKUP: Final = "exposed-anthropic-backup" + +_CHAT_BRIDGE_INHERITED_LEAKS: Final = frozenset( + f"{deployment}:include_fallback_errors" for deployment in (_EXPOSED_ANTHROPIC_PRIMARY, _EXPOSED_ANTHROPIC_BACKUP) +) + +pytestmark = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + + +def _prompt(nonce: str, *, outage: bool = False) -> str: + return f"which deployment answers this? nonce={nonce} outage={int(outage)}" + + +def _nonce_of(text: str) -> str: + found: Final = _NONCE.search(text) + assert found is not None, text + return found.group(1) + + +def _outage(deployment: str, raw: str) -> bool: + return deployment.endswith("primary") or (deployment.endswith("flaky") and "outage=1" in raw) + + +def _identity(endpoint: Endpoint, deployment: str, nonce: str) -> str: + match endpoint: + case "chat": + return f"chatcmpl-{deployment}-{nonce}" + case "messages": + return f"msg_{deployment}_{nonce}" + case "responses": + return f"resp_{deployment}_{nonce}" + + +def _peer(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + deployment, _, route = unquote(request.target).lstrip("/").partition("/") + raw: Final = request.body.decode() + body: Final = _JSON.validate_json(request.body) + nonce: Final = _nonce_of(raw) + stream: Final = body.get("stream") is True + text: Final = f"{_ANSWER} {deployment}" + if route == "v1/messages": + if _outage(deployment, raw): + return _ANTHROPIC_UNAUTHORIZED + identity: Final = _identity("messages", deployment, nonce) + if stream: + return stream_reply(message_stream(identity, _ANTHROPIC_MODEL, text)) + return Reply(body=message_json(identity, _ANTHROPIC_MODEL, text)) + if route.endswith("responses"): + if _outage(deployment, raw): + return _OPENAI_UNAUTHORIZED + return responses_reply(_identity("responses", deployment, nonce), _OPENAI_MODEL, text, stream=stream) + assert route == "chat/completions", request.target + if _outage(deployment, raw): + return _OPENAI_UNAUTHORIZED + return chat_reply(_identity("chat", deployment, nonce), _OPENAI_MODEL, text, stream=stream) + + +@dataclass(frozen=True, slots=True) +class _Posted: + deployment: str + route: str + raw: str + + def leaked(self) -> tuple[str, ...]: + return tuple(key for key in _ROUTER_ONLY_KEYS if key in self.raw) + + +def _posted_of(request: Request) -> _Posted: + deployment, _, route = unquote(request.target).lstrip("/").partition("/") + return _Posted(deployment, route, request.body.decode()) + + +def _all_posted(wire: Wire) -> tuple[_Posted, ...]: + return tuple(_posted_of(request) for request in wire.drain() if request.method == "POST") + + +def _posted(wire: Wire, nonce: str) -> tuple[_Posted, ...]: + return tuple(item for item in _all_posted(wire) if nonce in item.raw) + + +def _header(response: httpx.Response, name: str) -> str | None: + return response.headers[name] if name in response.headers else None + + +def _leaks(posted: tuple[_Posted, ...]) -> tuple[str, ...]: + return tuple(f"{item.deployment}:{','.join(item.leaked())}" for item in posted if item.leaked()) + + +def _hit(posted: tuple[_Posted, ...]) -> tuple[str, ...]: + return tuple(item.deployment for item in posted) + + +def _body(endpoint: Endpoint, model: str, nonce: str, *, stream: bool, outage: bool = False) -> dict[str, JsonValue]: + prompt: Final = _prompt(nonce, outage=outage) + match endpoint: + case "chat": + return {"model": model, "stream": stream, "messages": [{"role": "user", "content": prompt}]} + case "messages": + return { + "model": model, + "stream": stream, + "max_tokens": 32, + "messages": [{"role": "user", "content": prompt}], + } + case "responses": + return {"model": model, "stream": stream, "input": prompt} + + +def _output_item_identity(item: JsonValue) -> str: + return str(_JSON.validate_python(item)["id"]).removeprefix("msg_") + + +def _served_id(endpoint: Endpoint, response: httpx.Response, *, stream: bool) -> str: + if not stream: + body: Final = _JSON.validate_json(response.content) + if endpoint == "responses": + return _output_item_identity(_ITEMS.validate_python(body["output"])[0]) + return str(body["id"]) + events: Final = parse_sse(response.text) + match endpoint: + case "chat": + return str(events[0].data["id"]) + case "messages": + start: Final = next(event for event in events if event.event == "message_start") + return str(_JSON.validate_python(start.data["message"])["id"]) + case "responses": + done: Final = next(event for event in events if event.data.get("type") == "response.output_item.done") + return _output_item_identity(done.data["item"]) + + +def _settled(text: str) -> bool: + return not any(phrase in text for phrase in _NOT_YET_ON_EVERY_WORKER) + + +@dataclass(frozen=True, slots=True) +class _Sent: + nonce: str + response: httpx.Response + + +def _send( + gateway: Gateway, path: str, body: Callable[[str], Mapping[str, JsonValue]], *, key: str | None = None +) -> _Sent: + def attempt() -> _Sent: + nonce: Final = uuid.uuid4().hex + return _Sent(nonce, gateway.request("POST", path, body(nonce), key=key, headers=_FRESH_CONNECTION)) + + return eventually(attempt, lambda sent: _settled(sent.response.text), seconds=_WORKER_SYNC_SECONDS + 10) + + +@dataclass(frozen=True, slots=True) +class _Observed: + status: int + served_id: str + spend_id: str + text: str + attempted: str | None + errors_header: str | None + hit: tuple[str, ...] + leaks: tuple[str, ...] + + +def _observe(endpoint: Endpoint, wire: Wire, sent: _Sent, *, stream: bool) -> _Observed: + posted: Final = _posted(wire, sent.nonce) + response: Final = sent.response + return _Observed( + status=response.status_code, + served_id=_served_id(endpoint, response, stream=stream) if response.status_code == 200 else "", + spend_id=_spend_id(response) if response.status_code == 200 and not stream else "", + text=response.text, + attempted=_header(response, "x-litellm-attempted-fallbacks"), + errors_header=_header(response, "x-litellm-fallback-errors"), + hit=_hit(posted), + leaks=_leaks(posted), + ) + + +def _spend_id(response: httpx.Response) -> str: + return str(_JSON.validate_json(response.content)["id"]) + + +def _spend_row_lands(spend_id: str) -> None: + eventually( + lambda: read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (spend_id,)), + lambda rows: len(rows) == 1, + seconds=_SPEND_ROW_SECONDS, + ) + + +@dataclass(frozen=True, slots=True) +class _Registered: + gateway: Gateway + wire: Wire + primary: Mapping[Endpoint, str] + backup: Mapping[Endpoint, str] + + +@pytest.fixture(scope="module") +def registered() -> Iterator[_Registered]: + with gateway_from_environment() as gateway, wire_server(_peer) as wire, gateway.scenario() as scenario: + primary: Final[dict[Endpoint, str]] = { + endpoint: scenario.model(model=_PROVIDERS[endpoint], api_key=_API_KEY, api_base=f"{wire.url}/primary") + for endpoint in _ENDPOINTS + } + backup: Final[dict[Endpoint, str]] = { + endpoint: scenario.model(model=_PROVIDERS[endpoint], api_key=_API_KEY, api_base=f"{wire.url}/backup") + for endpoint in _ENDPOINTS + } + yield _Registered(gateway, wire, MappingProxyType(primary), MappingProxyType(backup)) + + +@pytest.mark.parametrize("endpoint", _ENDPOINTS) +@pytest.mark.parametrize("stream", _STREAMING) +def test_default_gateway_keeps_the_flag_off_the_wire_on_a_fallback( + registered: _Registered, endpoint: Endpoint, stream: bool +) -> None: + sent: Final = _send( + registered.gateway, + _PATHS[endpoint], + lambda nonce: { + **_body(endpoint, registered.primary[endpoint], nonce, stream=stream), + "fallbacks": [registered.backup[endpoint]], + "include_fallback_errors": True, + }, + ) + observed: Final = _observe(endpoint, registered.wire, sent, stream=stream) + assert observed.status == 200, observed + assert f"{_ANSWER} backup" in observed.text, observed + assert observed.served_id == _identity(endpoint, "backup", sent.nonce), observed + assert observed.leaks == (), observed + assert observed.hit == ("primary", "backup"), observed + assert observed.errors_header is None, observed + if not stream: + _spend_row_lands(observed.spend_id) + if endpoint == "chat" and not stream: + assert observed.attempted == "1", observed + + +@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS) +def test_default_gateway_keeps_a_non_boolean_flag_off_the_wire( + registered: _Registered, value: JsonValue, errors_reported: bool +) -> None: + sent: Final = _send( + registered.gateway, + _PATHS["chat"], + lambda nonce: { + **_body("chat", registered.primary["chat"], nonce, stream=False), + "fallbacks": [registered.backup["chat"]], + "include_fallback_errors": value, + }, + ) + observed: Final = _observe("chat", registered.wire, sent, stream=False) + assert observed.status == 200, observed + assert f"{_ANSWER} backup" in observed.text, observed + assert observed.leaks == (), observed + assert observed.hit == ("primary", "backup"), observed + assert observed.errors_header is None, observed + + +def test_default_gateway_keeps_a_duplicated_raw_flag_off_the_wire(registered: _Registered) -> None: + def raw_body(nonce: str) -> str: + body: Final = { + **_body("chat", registered.primary["chat"], nonce, stream=False), + "fallbacks": [registered.backup["chat"]], + } + return json.dumps(body)[:-1] + ', "include_fallback_errors": true, "include_fallback_errors": true}' + + def attempt() -> _Sent: + nonce: Final = uuid.uuid4().hex + response: Final = registered.gateway.client.post( + _PATHS["chat"], + content=raw_body(nonce), + headers={ + "Authorization": f"Bearer {registered.gateway.key}", + "Content-Type": "application/json", + **_FRESH_CONNECTION, + }, + ) + return _Sent(nonce, response) + + sent: Final = eventually(attempt, lambda s: _settled(s.response.text), seconds=_WORKER_SYNC_SECONDS + 10) + observed: Final = _observe("chat", registered.wire, sent, stream=False) + assert observed.status == 200, observed + assert observed.served_id == _identity("chat", "backup", sent.nonce), observed + assert observed.leaks == (), observed + assert observed.hit == ("primary", "backup"), observed + assert observed.errors_header is None, observed + + +def test_default_gateway_rejects_an_unauthenticated_flagged_request_before_the_wire(registered: _Registered) -> None: + nonce: Final = uuid.uuid4().hex + response: Final = registered.gateway.request( + "POST", + _PATHS["chat"], + { + **_body("chat", registered.primary["chat"], nonce, stream=False), + "fallbacks": [registered.backup["chat"]], + "include_fallback_errors": True, + }, + key="sk-not-a-key", + headers=_FRESH_CONNECTION, + ) + assert response.status_code == 401, response.text + assert _posted(registered.wire, nonce) == () + + +def _exposed_deployment(name: str, provider: str) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": { + "model": provider, + "api_base": f"{_UPSTREAM_URL_PLACEHOLDER}/{name}", + "api_key": _API_KEY, + }, + } + + +def _exposed_config(wire: Wire, directory: Path) -> Path: + config: Final = _JSON.validate_python(yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config["model_list"] = [ + _exposed_deployment(_EXPOSED_OPENAI_PRIMARY, _PROVIDERS["chat"]), + _exposed_deployment(_EXPOSED_OPENAI_BACKUP, _PROVIDERS["chat"]), + _exposed_deployment(_EXPOSED_OPENAI_SERVING, _PROVIDERS["chat"]), + _exposed_deployment(_EXPOSED_OPENAI_FLAKY, _PROVIDERS["chat"]), + _exposed_deployment(_EXPOSED_ANTHROPIC_PRIMARY, _PROVIDERS["messages"]), + _exposed_deployment(_EXPOSED_ANTHROPIC_BACKUP, _PROVIDERS["messages"]), + ] + config["general_settings"] = { + **_JSON.validate_python(config["general_settings"]), + "expose_fallback_errors_to_caller": True, + } + config["router_settings"] = { + "num_retries": 0, + "disable_cooldowns": True, + "fallbacks": [ + {_EXPOSED_OPENAI_PRIMARY: [_EXPOSED_OPENAI_BACKUP]}, + {_EXPOSED_OPENAI_FLAKY: [_EXPOSED_OPENAI_BACKUP]}, + {_EXPOSED_ANTHROPIC_PRIMARY: [_EXPOSED_ANTHROPIC_BACKUP]}, + ], + } + path: Final = directory / "include-fallback-errors-exposed.yaml" + path.write_text(yaml.safe_dump(config).replace(_UPSTREAM_URL_PLACEHOLDER, wire.url)) + return path + + +@dataclass(frozen=True, slots=True) +class _Exposed: + proxy: Gateway + wire: Wire + + +@pytest.fixture(scope="module") +def exposed(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Exposed]: + directory: Final = tmp_path_factory.mktemp("include-fallback-errors-exposed") + with gateway_from_environment() as gateway, wire_server(_peer) as wire: + with owned_proxy(gateway, directory, {}, config=_exposed_config(wire, directory), workers=2) as proxy: + yield _Exposed(proxy, wire) + + +def _errors_mention_the_outage(errors_header: str | None) -> bool: + if errors_header is None: + return False + errors: Final = _ERRORS.validate_json(errors_header) + return len(errors) == 1 and _UNAUTHORIZED_MESSAGE in str(errors[0]["message"]) + + +@pytest.mark.parametrize("stream", _STREAMING) +def test_exposed_chat_fallback_reports_the_errors_without_putting_the_flag_on_the_wire( + exposed: _Exposed, stream: bool +) -> None: + sent: Final = _send( + exposed.proxy, + _PATHS["chat"], + lambda nonce: { + **_body("chat", _EXPOSED_OPENAI_PRIMARY, nonce, stream=stream), + "include_fallback_errors": True, + }, + ) + observed: Final = _observe("chat", exposed.wire, sent, stream=stream) + assert observed.status == 200, observed + assert f"{_ANSWER} {_EXPOSED_OPENAI_BACKUP}" in observed.text, observed + assert observed.served_id == _identity("chat", _EXPOSED_OPENAI_BACKUP, sent.nonce), observed + assert observed.leaks == (), observed + assert observed.hit == (_EXPOSED_OPENAI_PRIMARY, _EXPOSED_OPENAI_BACKUP), observed + if not stream: + assert observed.attempted == "1", observed + assert _errors_mention_the_outage(observed.errors_header), observed + + +def test_exposed_chat_without_a_fallback_keeps_the_flag_off_the_wire_and_the_errors_header_off( + exposed: _Exposed, +) -> None: + sent: Final = _send( + exposed.proxy, + _PATHS["chat"], + lambda nonce: { + **_body("chat", _EXPOSED_OPENAI_SERVING, nonce, stream=False), + "include_fallback_errors": True, + }, + ) + observed: Final = _observe("chat", exposed.wire, sent, stream=False) + assert observed.status == 200, observed + assert observed.served_id == _identity("chat", _EXPOSED_OPENAI_SERVING, sent.nonce), observed + assert observed.leaks == (), observed + assert observed.hit == (_EXPOSED_OPENAI_SERVING,), observed + assert observed.attempted == "0", observed + assert observed.errors_header is None, observed + + +@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS) +def test_exposed_chat_keeps_a_non_boolean_flag_off_the_wire_and_the_errors_header_follows_its_truthiness( + exposed: _Exposed, value: JsonValue, errors_reported: bool +) -> None: + sent: Final = _send( + exposed.proxy, + _PATHS["chat"], + lambda nonce: { + **_body("chat", _EXPOSED_OPENAI_PRIMARY, nonce, stream=False), + "include_fallback_errors": value, + }, + ) + observed: Final = _observe("chat", exposed.wire, sent, stream=False) + assert observed.status == 200, observed + assert observed.served_id == _identity("chat", _EXPOSED_OPENAI_BACKUP, sent.nonce), observed + assert observed.leaks == (), observed + assert observed.hit == (_EXPOSED_OPENAI_PRIMARY, _EXPOSED_OPENAI_BACKUP), observed + assert observed.attempted == "1", observed + assert _errors_mention_the_outage(observed.errors_header) is errors_reported, observed + + +def test_exposed_proxy_rejects_an_unauthenticated_flagged_request_before_the_wire(exposed: _Exposed) -> None: + nonce: Final = uuid.uuid4().hex + response: Final = exposed.proxy.request( + "POST", + _PATHS["chat"], + {**_body("chat", _EXPOSED_OPENAI_PRIMARY, nonce, stream=False), "include_fallback_errors": True}, + key="sk-not-a-key", + headers=_FRESH_CONNECTION, + ) + assert response.status_code == 401, response.text + assert _posted(exposed.wire, nonce) == () + + +@pytest.mark.parametrize("stream", _STREAMING) +def test_exposed_messages_fallback_keeps_the_flag_off_the_wire(exposed: _Exposed, stream: bool) -> None: + sent: Final = _send( + exposed.proxy, + _PATHS["messages"], + lambda nonce: { + **_body("messages", _EXPOSED_ANTHROPIC_PRIMARY, nonce, stream=stream), + "include_fallback_errors": True, + }, + ) + observed: Final = _observe("messages", exposed.wire, sent, stream=stream) + assert observed.status == 200, observed + assert observed.served_id == _identity("messages", _EXPOSED_ANTHROPIC_BACKUP, sent.nonce), observed + assert observed.hit == (_EXPOSED_ANTHROPIC_PRIMARY, _EXPOSED_ANTHROPIC_BACKUP), observed + assert observed.leaks == (), observed + + +@pytest.mark.parametrize("stream", _STREAMING) +def test_exposed_responses_fallback_keeps_the_flag_off_the_wire(exposed: _Exposed, stream: bool) -> None: + sent: Final = _send( + exposed.proxy, + _PATHS["responses"], + lambda nonce: { + **_body("responses", _EXPOSED_OPENAI_PRIMARY, nonce, stream=stream), + "include_fallback_errors": True, + }, + ) + observed: Final = _observe("responses", exposed.wire, sent, stream=stream) + assert observed.status == 200, observed + assert observed.served_id == _identity("responses", _EXPOSED_OPENAI_BACKUP, sent.nonce), observed + assert observed.hit == (_EXPOSED_OPENAI_PRIMARY, _EXPOSED_OPENAI_BACKUP), observed + assert observed.leaks == (), observed + + +@pytest.mark.parametrize("stream", _STREAMING) +def test_exposed_messages_on_the_openai_deployment_bridges_the_fallback_with_a_clean_wire( + exposed: _Exposed, stream: bool +) -> None: + sent: Final = _send( + exposed.proxy, + _PATHS["messages"], + lambda nonce: { + **_body("messages", _EXPOSED_OPENAI_PRIMARY, nonce, stream=stream), + "include_fallback_errors": True, + }, + ) + observed: Final = _observe("messages", exposed.wire, sent, stream=stream) + assert observed.status == 200, observed + assert f"{_ANSWER} {_EXPOSED_OPENAI_BACKUP}" in observed.text, observed + assert observed.hit == (_EXPOSED_OPENAI_PRIMARY, _EXPOSED_OPENAI_BACKUP), observed + assert observed.leaks == (), observed + + +@pytest.mark.parametrize("stream", _STREAMING) +def test_exposed_responses_on_the_anthropic_deployment_falls_back_through_the_chat_bridge_with_no_new_key_on_the_wire( + exposed: _Exposed, stream: bool +) -> None: + sent: Final = _send( + exposed.proxy, + _PATHS["responses"], + lambda nonce: { + **_body("responses", _EXPOSED_ANTHROPIC_PRIMARY, nonce, stream=stream), + "include_fallback_errors": True, + }, + ) + observed: Final = _observe("responses", exposed.wire, sent, stream=stream) + assert observed.status == 200, observed + assert f"{_ANSWER} {_EXPOSED_ANTHROPIC_BACKUP}" in observed.text, observed + assert observed.hit == (_EXPOSED_ANTHROPIC_PRIMARY, _EXPOSED_ANTHROPIC_BACKUP), observed + assert set(observed.leaks) <= _CHAT_BRIDGE_INHERITED_LEAKS, observed + + +@dataclass(frozen=True, slots=True) +class _BurstRequest: + endpoint: Endpoint + stream: bool + outage: bool + nonce: str + + +def _burst_plan() -> tuple[_BurstRequest, ...]: + shapes: Final[tuple[tuple[Endpoint, bool], ...]] = (("chat", False), ("chat", True), ("responses", False)) + return tuple( + _BurstRequest(endpoint, stream, index % 2 == 1, uuid.uuid4().hex) + for index, (endpoint, stream) in enumerate(shapes * (_BURST // len(shapes))) + ) + + +def _fire(proxy: Gateway, request: _BurstRequest) -> httpx.Response: + return proxy.request( + "POST", + _PATHS[request.endpoint], + { + **_body( + request.endpoint, _EXPOSED_OPENAI_FLAKY, request.nonce, stream=request.stream, outage=request.outage + ), + "include_fallback_errors": True, + }, + headers=_FRESH_CONNECTION, + ) + + +def _expected_hit(request: _BurstRequest) -> tuple[str, ...]: + if request.outage: + return (_EXPOSED_OPENAI_FLAKY, _EXPOSED_OPENAI_BACKUP) + return (_EXPOSED_OPENAI_FLAKY,) + + +def _expected_served_id(request: _BurstRequest) -> str: + deployment: Final = _EXPOSED_OPENAI_BACKUP if request.outage else _EXPOSED_OPENAI_FLAKY + return _identity(request.endpoint, deployment, request.nonce) + + +def _check_burst_row(request: _BurstRequest, response: httpx.Response, posted: tuple[_Posted, ...]) -> None: + own: Final = tuple(item for item in posted if request.nonce in item.raw) + assert response.status_code == 200, (request, response.text) + assert _served_id(request.endpoint, response, stream=request.stream) == _expected_served_id(request), request + assert _hit(own) == _expected_hit(request), (request, own) + assert _leaks(own) == (), (request, own) + if request.endpoint == "chat" and not request.stream: + assert _header(response, "x-litellm-attempted-fallbacks") == ("1" if request.outage else "0"), request + assert _errors_mention_the_outage(_header(response, "x-litellm-fallback-errors")) is request.outage, request + + +def test_exposed_burst_with_scripted_outages_lands_every_prompt_once_per_hop_with_a_clean_wire( + exposed: _Exposed, +) -> None: + plan: Final = _burst_plan() + with ThreadPoolExecutor(max_workers=_BURST) as pool: + responses: Final = tuple(pool.map(partial(_fire, exposed.proxy), plan)) + posted: Final = _all_posted(exposed.wire) + for request, response in zip(plan, responses, strict=True): + _check_burst_row(request, response, posted) diff --git a/tests/integration/sdk/test_router_include_fallback_errors_wire.py b/tests/integration/sdk/test_router_include_fallback_errors_wire.py new file mode 100644 index 00000000000..fb17e05b02a --- /dev/null +++ b/tests/integration/sdk/test_router_include_fallback_errors_wire.py @@ -0,0 +1,364 @@ +from __future__ import annotations + +import asyncio +import json +import re +import threading +import uuid +from collections.abc import Mapping +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from typing import Final + +import pytest +from integration._support.client import eventually +from integration._support.openai_wire import chat_reply +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +import litellm +from litellm import CustomStreamWrapper, Router +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.llms.openai import AllMessageValues, ChatCompletionUserMessage +from litellm.types.utils import Choices, ModelResponse, ModelResponseStream + +_MODEL: Final = "gpt-5.4" +_API_KEY: Final = "synthetic-fallback-errors-key" +_ROUTER_ONLY_KEYS: Final = ("include_fallback_errors", "silent_model") +_ANSWER: Final = "answered by" +_UNAUTHORIZED_MESSAGE: Final = "scripted 401: the primary key was revoked" +_NONCE: Final = re.compile(r"nonce=([0-9a-f]{32})") +_JSON: Final = TypeAdapter(dict[str, JsonValue]) +_OBJECT: Final = TypeAdapter(dict[str, object]) +_ERRORS: Final = TypeAdapter(list[dict[str, JsonValue]]) +_BURST: Final = 12 +_CALLBACK_WINDOW: Final = 15.0 +_STREAMING: Final = (pytest.param(False, id="non-stream"), pytest.param(True, id="stream")) +_NON_BOOLEAN_FLAGS: Final = ( + pytest.param(1, True, id="int"), + pytest.param("", False, id="empty-string"), + pytest.param([], False, id="list"), + pytest.param("x" * 5120, True, id="five-kilobytes"), + pytest.param(False, False, id="false"), +) +_UNAUTHORIZED: Final = Reply( + status=401, + body=json.dumps( + { + "error": { + "message": _UNAUTHORIZED_MESSAGE, + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key", + } + } + ).encode(), +) + + +def _prompt(nonce: str) -> str: + return f"which deployment answers this? nonce={nonce}" + + +def _nonce_of(body: Mapping[str, JsonValue]) -> str: + found: Final = _NONCE.search(json.dumps(body)) + assert found is not None, body + return found.group(1) + + +def _peer(request: Request) -> Reply: + if request.method == "GET": + return Reply(body=json.dumps({"object": "list", "data": []}).encode()) + deployment, _, route = request.target.lstrip("/").partition("/") + assert route == "chat/completions", request.target + if deployment == "primary": + return _UNAUTHORIZED + body: Final = _JSON.validate_json(request.body) + identity: Final = f"chatcmpl-{deployment}-{_nonce_of(body)}" + return chat_reply(identity, _MODEL, f"{_ANSWER} {deployment}", stream=body.get("stream") is True) + + +def _deployment(name: str, wire: Wire, **extra: JsonValue) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": { + "model": f"openai/{_MODEL}", + "api_base": f"{wire.url}/{name}", + "api_key": _API_KEY, + **extra, + }, + } + + +def _router(wire: Wire, *, cache_responses: bool = False) -> Router: + return Router( + model_list=[ + _deployment("primary", wire), + _deployment("backup", wire), + _deployment("serving", wire), + _deployment("mirrored", wire, silent_model="shadow"), + _deployment("shadow", wire), + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + disable_cooldowns=True, + cache_responses=cache_responses, + ) + + +@dataclass(frozen=True, slots=True) +class _Posted: + deployment: str + raw: str + + def leaked(self) -> tuple[str, ...]: + return tuple(key for key in _ROUTER_ONLY_KEYS if key in self.raw) + + def nonce(self) -> str: + found: Final = _NONCE.search(self.raw) + assert found is not None, self.raw + return found.group(1) + + +def _posted(wire: Wire) -> tuple[_Posted, ...]: + return tuple( + _Posted(request.target.lstrip("/").partition("/")[0], request.body.decode()) + for request in wire.drain() + if request.method == "POST" + ) + + +def _leaks(posted: tuple[_Posted, ...]) -> tuple[str, ...]: + return tuple(f"{item.deployment}:{','.join(item.leaked())}" for item in posted if item.leaked()) + + +def _hit(posted: tuple[_Posted, ...]) -> tuple[str, ...]: + return tuple(item.deployment for item in posted) + + +def _nonces_at(posted: tuple[_Posted, ...], deployment: str) -> tuple[str, ...]: + return tuple(sorted(item.nonce() for item in posted if item.deployment == deployment)) + + +def _hidden(response: object) -> Mapping[str, object]: + return _OBJECT.validate_python(getattr(response, "_hidden_params", None) or {}) + + +def _headers(response: object) -> Mapping[str, object]: + return _OBJECT.validate_python(_hidden(response).get("additional_headers") or {}) + + +def _cache_hit(response: object) -> bool: + return _hidden(response).get("cache_hit") is True + + +def _error_messages(headers: Mapping[str, object]) -> tuple[str, ...]: + raw: Final = headers.get("x-litellm-fallback-errors") + if raw is None: + return () + return tuple(str(error["message"]) for error in _ERRORS.validate_json(str(raw))) + + +def _delta(chunk: object) -> str: + assert isinstance(chunk, ModelResponseStream), chunk + return "".join(str(choice.delta.content or "") for choice in chunk.choices) + + +def _messages(nonce: str) -> list[dict[str, str]]: + return [{"role": "user", "content": _prompt(nonce)}] + + +def _typed_messages(nonce: str) -> list[AllMessageValues]: + return [ChatCompletionUserMessage(role="user", content=_prompt(nonce))] + + +def _content(response: object) -> str: + assert isinstance(response, ModelResponse), response + choice: Final = response.choices[0] + assert isinstance(choice, Choices), choice + return str(choice.message.content) + + +@dataclass(frozen=True, slots=True) +class _Served: + text: str + headers: Mapping[str, object] + cache_hit: bool + + +@dataclass(frozen=True, slots=True) +class _Outcome: + text: str + attempted: object + errors: tuple[str, ...] + hit: tuple[str, ...] + leaks: tuple[str, ...] + + +def _outcome(wire: Wire, served: _Served) -> _Outcome: + posted: Final = _posted(wire) + return _Outcome( + text=served.text, + attempted=served.headers.get("x-litellm-attempted-fallbacks"), + errors=_error_messages(served.headers), + hit=_hit(posted), + leaks=_leaks(posted), + ) + + +def _complete(router: Router, model: str, *, stream: bool, nonce: str | None = None, **request: object) -> _Served: + response: Final = router.completion( + model=model, messages=_messages(nonce or uuid.uuid4().hex), stream=stream, **request + ) + if isinstance(response, CustomStreamWrapper): + return _Served("".join(_delta(chunk) for chunk in response), _headers(response), _cache_hit(response)) + return _Served(_content(response), _headers(response), _cache_hit(response)) + + +async def _acomplete(router: Router, model: str, *, stream: bool, **request: object) -> _Served: + response: Final = await router.acompletion( + model=model, messages=_typed_messages(uuid.uuid4().hex), stream=stream, **request + ) + if isinstance(response, CustomStreamWrapper): + parts: Final = [_delta(chunk) async for chunk in response] + return _Served("".join(parts), _headers(response), _cache_hit(response)) + return _Served(_content(response), _headers(response), _cache_hit(response)) + + +@pytest.mark.parametrize("stream", _STREAMING) +def test_sync_fallback_keeps_the_flag_off_the_wire_and_reports_the_errors(stream: bool) -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + outcome: Final = _outcome(wire, _complete(router, "primary", stream=stream, include_fallback_errors=True)) + assert outcome.leaks == (), outcome + assert outcome.hit == ("primary", "backup"), outcome + assert outcome.text == f"{_ANSWER} backup", outcome + assert outcome.attempted == 1, outcome + if not stream: + assert len(outcome.errors) == 1 and _UNAUTHORIZED_MESSAGE in outcome.errors[0], outcome + + +@pytest.mark.parametrize("stream", _STREAMING) +def test_sync_matches_the_async_twin(stream: bool) -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + twin: Final = _outcome( + wire, asyncio.run(_acomplete(router, "primary", stream=stream, include_fallback_errors=True)) + ) + observed: Final = _outcome(wire, _complete(router, "primary", stream=stream, include_fallback_errors=True)) + assert observed == twin, (observed, twin) + assert twin.leaks == (), twin + assert twin.hit == ("primary", "backup"), twin + + +def test_without_a_fallback_the_flag_still_stays_off_the_wire() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + outcome: Final = _outcome(wire, _complete(router, "serving", stream=False, include_fallback_errors=True)) + assert outcome.leaks == (), outcome + assert outcome.hit == ("serving",), outcome + assert outcome.text == f"{_ANSWER} serving", outcome + assert outcome.attempted == 0, outcome + assert outcome.errors == (), outcome + + +@pytest.mark.parametrize(("value", "errors_reported"), _NON_BOOLEAN_FLAGS) +def test_a_non_boolean_flag_stays_off_the_wire_and_the_errors_header_follows_its_truthiness( + value: object, errors_reported: bool +) -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + outcome: Final = _outcome(wire, _complete(router, "primary", stream=False, include_fallback_errors=value)) + assert outcome.leaks == (), outcome + assert outcome.hit == ("primary", "backup"), outcome + assert outcome.text == f"{_ANSWER} backup", outcome + assert outcome.attempted == 1, outcome + assert (len(outcome.errors) == 1) is errors_reported, outcome + + +class _Recorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.mentions_the_flag: bool | None = None + self.fired = threading.Event() + + def _record(self, kwargs: Mapping[str, object]) -> None: + self.mentions_the_flag = "include_fallback_errors" in json.dumps(kwargs, default=str) + self.fired.set() + + def log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs) + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object + ) -> None: + self._record(kwargs) + + +async def _acomplete_and_await_the_logger(router: Router, recorder: _Recorder) -> _Served: + served: Final = await _acomplete(router, "serving", stream=False, include_fallback_errors=True) + assert await asyncio.get_running_loop().run_in_executor(None, recorder.fired.wait, _CALLBACK_WINDOW) + return served + + +def test_sync_logger_kwargs_carry_the_flag_exactly_as_the_async_ones_do(monkeypatch: pytest.MonkeyPatch) -> None: + sync_recorder: Final = _Recorder() + async_recorder: Final = _Recorder() + with wire_server(_peer) as wire: + router: Final = _router(wire) + monkeypatch.setattr(litellm, "callbacks", [sync_recorder]) + _complete(router, "serving", stream=False, include_fallback_errors=True) + assert sync_recorder.fired.wait(_CALLBACK_WINDOW) + monkeypatch.setattr(litellm, "callbacks", [async_recorder]) + asyncio.run(_acomplete_and_await_the_logger(router, async_recorder)) + posted: Final = _posted(wire) + assert _leaks(posted) == (), posted + assert (sync_recorder.mentions_the_flag, async_recorder.mentions_the_flag) == (False, False) + + +def test_silent_model_shadow_traffic_carries_neither_router_only_key() -> None: + with wire_server(_peer) as wire: + router: Final = _router(wire) + served: Final = _complete(router, "mirrored", stream=False, include_fallback_errors=True) + eventually(wire.received.qsize, lambda count: count >= 2, seconds=20) + posted: Final = _posted(wire) + assert served.text == f"{_ANSWER} mirrored", served + assert tuple(sorted(_hit(posted))) == ("mirrored", "shadow"), posted + assert _leaks(posted) == (), posted + + +def test_a_cache_hit_repeats_the_answer_without_a_wire_request(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "cache", None) + with wire_server(_peer) as wire: + router: Final = _router(wire, cache_responses=True) + nonce: Final = uuid.uuid4().hex + first: Final = _complete(router, "serving", stream=False, nonce=nonce, include_fallback_errors=True) + posted: Final = _posted(wire) + second: Final = _complete(router, "serving", stream=False, nonce=nonce, include_fallback_errors=True) + again: Final = _posted(wire) + assert _hit(posted) == ("serving",), posted + assert _leaks(posted) == (), posted + assert again == (), again + assert second.text == first.text == f"{_ANSWER} serving" + assert (first.cache_hit, second.cache_hit) == (False, True), (first, second) + + +def _flagged(router: Router, nonce: str) -> _Served: + return _complete(router, "primary", stream=False, nonce=nonce, include_fallback_errors=True) + + +def test_a_sync_burst_lands_every_prompt_once_on_each_side_of_the_fallback() -> None: + nonces: Final = tuple(uuid.uuid4().hex for _ in range(_BURST)) + with wire_server(_peer) as wire: + router: Final = _router(wire) + with ThreadPoolExecutor(max_workers=_BURST) as pool: + served: Final = tuple(pool.map(partial(_flagged, router), nonces)) + posted: Final = _posted(wire) + assert tuple(item.text for item in served) == (f"{_ANSWER} backup",) * _BURST, served + assert tuple(item.headers.get("x-litellm-attempted-fallbacks") for item in served) == (1,) * _BURST, served + assert _leaks(posted) == (), posted + assert _nonces_at(posted, "primary") == tuple(sorted(nonces)), posted + assert _nonces_at(posted, "backup") == tuple(sorted(nonces)), posted diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index ec9e4861fdd..e3e6326574b 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -24593,3 +24593,88 @@ class CompletionCustomHandler( except Exception: print(f"Assertion Error: {traceback.format_exc()}") self.errors.append(traceback.format_exc()) + + +_FALLBACK_WIRE_PRIMARY: Final = "http://primary.wire.test/v1" +_FALLBACK_WIRE_BACKUP: Final = "http://backup.wire.test/v1" +_FALLBACK_WIRE_BACKUP_REPLY: Final = { + "id": "chatcmpl-backup", + "object": "chat.completion", + "created": 1, + "model": "gpt-5.6", + "choices": [{"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "pong"}}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, +} + + +def _fallback_wire_router() -> Router: + return Router( + model_list=[ + { + "model_name": "primary", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-primary", + "api_base": _FALLBACK_WIRE_PRIMARY, + "max_retries": 0, + }, + }, + { + "model_name": "backup", + "litellm_params": { + "model": "openai/gpt-5.6", + "api_key": "sk-backup", + "api_base": _FALLBACK_WIRE_BACKUP, + "max_retries": 0, + }, + }, + ], + fallbacks=[{"primary": ["backup"]}], + num_retries=0, + ) + + +def _mock_fallback_wire(respx_mock: respx.MockRouter) -> tuple[respx.Route, respx.Route]: + primary = respx_mock.post(f"{_FALLBACK_WIRE_PRIMARY}/chat/completions").mock( + return_value=httpx.Response(500, json={"error": {"message": "primary overloaded", "type": "server_error"}}) + ) + backup = respx_mock.post(f"{_FALLBACK_WIRE_BACKUP}/chat/completions").mock( + return_value=httpx.Response(200, json=_FALLBACK_WIRE_BACKUP_REPLY) + ) + return primary, backup + + +def _assert_fallback_errors_reached_the_caller_and_not_the_wire( + response: object, primary: respx.Route, backup: respx.Route +) -> None: + for route in (primary, backup): + assert route.called + for call in route.calls: + assert "include_fallback_errors" not in json.loads(call.request.content) + assert isinstance(response, litellm.ModelResponse) + assert response.choices[0].message.content == "pong" + headers = response._hidden_params["additional_headers"] + assert headers["x-litellm-attempted-fallbacks"] == 1 + errors = json.loads(headers["x-litellm-fallback-errors"]) + assert len(errors) == 1 + assert "primary overloaded" in errors[0]["message"] + + +def test_sync_completion_keeps_include_fallback_errors_off_the_wire_and_returns_the_errors(): + with respx.mock(assert_all_called=True) as respx_mock: + primary, backup = _mock_fallback_wire(respx_mock) + response = _fallback_wire_router().completion( + model="primary", messages=[{"role": "user", "content": "hi"}], include_fallback_errors=True + ) + _assert_fallback_errors_reached_the_caller_and_not_the_wire(response, primary, backup) + + +@pytest.mark.asyncio +async def test_acompletion_keeps_include_fallback_errors_off_the_wire_and_returns_the_errors(monkeypatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + with respx.mock(assert_all_called=True) as respx_mock: + primary, backup = _mock_fallback_wire(respx_mock) + response = await _fallback_wire_router().acompletion( + model="primary", messages=[{"role": "user", "content": "hi"}], include_fallback_errors=True + ) + _assert_fallback_errors_reached_the_caller_and_not_the_wire(response, primary, backup) From 328f5a720c5f491c3fafe992d3d1891453b57505 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:25:22 -0700 Subject: [PATCH 16/65] fix(router): honor a per-request fallbacks list on a mid-stream fallback (#45227) * fix(router): honor a per-request fallbacks list on a mid-stream fallback The completion entrypoints record the per-request fallbacks, context_window_fallbacks, and content_policy_fallbacks in the mid-stream controls carrier the Responses and Messages paths already use, and each completion attempt restores them into the kwargs its stream re-enters the fallback chain with, so a streaming request that dies before its first chunk fails over to the list the request named instead of the router-level list only. The sync vs async parity cell now asserts the backup's text on both twins. * test(router): annotate the new fallback test locals as Final and flatten the leak check * fix(proxy): skip null key and team router settings when merging per-request overrides * test(proxy): annotate the null router settings test locals and drop a redundant comment --- litellm/proxy/route_llm_request.py | 3 +- litellm/router.py | 19 ++- .../router_utils/fallback_event_handlers.py | 30 +++- .../test_router_sync_stream_fallback_wire.py | 35 ++--- tests/unit/proxy/test_route_llm_request.py | 34 +++++ .../test_fallback_event_handlers.py | 25 ++++ tests/unit/test_router/test_router.py | 135 +++++++++++++++++- 7 files changed, 246 insertions(+), 35 deletions(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 22b04062534..2a9312dc0e3 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -678,9 +678,8 @@ async def _route_request_single_attempt( # noqa: ANN202 # returns unawaited pr "enable_tag_filtering", ] - # Merge override settings into data (only if not already set in request) for key in per_request_settings: - if key in override_settings and key not in data: + if override_settings.get(key) is not None and key not in data: data[key] = override_settings[key] # Use main router with overridden kwargs diff --git a/litellm/router.py b/litellm/router.py index 88c4a7011ef..7bedb714338 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -228,6 +228,7 @@ from litellm.router_utils.fallback_event_handlers import ( get_pre_routing_selection, has_unattempted_fallback_target, mid_stream_fallback_hop_kwargs, + mid_stream_fallback_snapshot_kwargs, mid_stream_retry_kwargs, per_request_fallback_controls, record_disable_fallbacks, @@ -2661,6 +2662,8 @@ class Router: kwargs["model"] = model kwargs["messages"] = messages kwargs["original_function"] = self._completion + controls: Final = per_request_fallback_controls(kwargs) + kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop self._update_kwargs_before_fallbacks(model=model, kwargs=kwargs) response: Final = self.function_with_fallbacks(**kwargs) @@ -2672,10 +2675,10 @@ class Router: model_name = None deployment = None try: - # Capture kwargs before deployment selection so the streaming - # fallback iterator can re-dispatch with the original model group. - input_kwargs_for_streaming_fallback: Final = kwargs.copy() - input_kwargs_for_streaming_fallback["model"] = model + controls: Final = kwargs.pop(MID_STREAM_FALLBACK_CONTROLS_KEY, None) + input_kwargs_for_streaming_fallback: Final = mid_stream_fallback_snapshot_kwargs( + model=model, controls=controls, kwargs=kwargs + ) # pick the one that is available (lowest TPM/RPM) deployment = self.get_available_deployment( @@ -2923,6 +2926,8 @@ class Router: messages=messages, kwargs=kwargs, ) + controls: Final = per_request_fallback_controls(kwargs) + kwargs[MID_STREAM_FALLBACK_CONTROLS_KEY] = controls # rebind-ok: forwarded to every hop if request_priority is not None and isinstance(request_priority, int): response = await self.schedule_acompletion(**kwargs) else: @@ -3818,8 +3823,10 @@ class Router: deployment = None _timeout_debug_deployment_dict = {} # this is a temporary dict to debug timeout issues try: - input_kwargs_for_streaming_fallback: Final = kwargs.copy() - input_kwargs_for_streaming_fallback["model"] = model + controls: Final = kwargs.pop(MID_STREAM_FALLBACK_CONTROLS_KEY, None) + input_kwargs_for_streaming_fallback: Final = mid_stream_fallback_snapshot_kwargs( + model=model, controls=controls, kwargs=kwargs + ) parent_otel_span: Final = get_parent_otel_span_from_kwargs(kwargs) start_time: Final = time.time() diff --git a/litellm/router_utils/fallback_event_handlers.py b/litellm/router_utils/fallback_event_handlers.py index e33897e3b63..239ee400503 100644 --- a/litellm/router_utils/fallback_event_handlers.py +++ b/litellm/router_utils/fallback_event_handlers.py @@ -335,6 +335,28 @@ def per_request_fallback_controls(kwargs: Mapping[str, object]) -> MidStreamFall ) +def mid_stream_fallback_snapshot_kwargs( + model: str, + controls: object, + kwargs: Mapping[str, object], +) -> dict[str, object]: # mutable-ok: the streaming iterators rewrite it in place when they re-enter the chain + """ + The kwargs a completion attempt's stream re-enters the fallback chain with if it fails. + + async_function_with_retries popped the per-request fallback lists before the attempt ran, so + the carrier restores them here and rides along into every hop this re-entry opens. A shallow + copy keeps the metadata buckets shared with the live kwargs, the way the attempt's own + in-place bucket writes expect. + """ + hop_controls: Final = controls if isinstance(controls, MidStreamFallbackControls) else _NO_FALLBACK_CONTROLS + return { + **kwargs, + **hop_controls.overrides, + MID_STREAM_FALLBACK_CONTROLS_KEY: hop_controls, + "model": model, + } + + def mid_stream_fallback_hop_kwargs( model: str, original_generic_function: Callable[..., object], @@ -342,22 +364,18 @@ def mid_stream_fallback_hop_kwargs( kwargs: Mapping[str, object], ) -> dict[str, object]: # mutable-ok: the streaming iterators rewrite it in place when they re-enter the chain """ - The kwargs one streaming attempt re-enters the fallback chain with if its stream fails. + The kwargs one generic-endpoint streaming attempt re-enters the fallback chain with if its stream fails. A shallow copy keeps ``attempted_targets`` shared with the outer chain, so entries this request already tried are never retried; the metadata buckets are copied key by key because the attempt writes deployment-specific fields into them in place. """ - hop_controls: Final = controls if isinstance(controls, MidStreamFallbackControls) else _NO_FALLBACK_CONTROLS copied_buckets: Final = MappingProxyType( {name: safe_deep_copy(kwargs[name]) for name in _ROUTER_METADATA_BUCKETS if isinstance(kwargs.get(name), dict)} ) return { - **kwargs, + **mid_stream_fallback_snapshot_kwargs(model=model, controls=controls, kwargs=kwargs), **copied_buckets, - **hop_controls.overrides, - MID_STREAM_FALLBACK_CONTROLS_KEY: hop_controls, - "model": model, "original_generic_function": original_generic_function, } diff --git a/tests/integration/sdk/test_router_sync_stream_fallback_wire.py b/tests/integration/sdk/test_router_sync_stream_fallback_wire.py index f5095c5a778..fc5fb028123 100644 --- a/tests/integration/sdk/test_router_sync_stream_fallback_wire.py +++ b/tests/integration/sdk/test_router_sync_stream_fallback_wire.py @@ -199,28 +199,23 @@ def test_router_retries_configured(client: str) -> None: assert _deployments_hit(wire) == ("primary", "backup") -@dataclass(frozen=True, slots=True) -class _Outcome: - text: str | None - error: str | None - hit: tuple[str, ...] - - -def _outcome(client: str, wire: Wire, router: Router, **request: object) -> _Outcome: - try: - streamed: Final = _stream(client, router, **request) - except litellm.APIConnectionError as error: - return _Outcome(text=None, error=type(error).__name__, hit=_deployments_hit(wire)) - return _Outcome(text=streamed.text, error=None, hit=_deployments_hit(wire)) - - -def test_per_request_fallback_list_behaves_like_the_async_twin() -> None: +@pytest.mark.parametrize("client", _CLIENTS) +def test_per_request_fallback_list(client: str) -> None: with wire_server(_peer(_PRIMARY_DIES)) as wire: router: Final = _router(wire, ("primary", "backup")) - twin: Final = _outcome("async", wire, router, fallbacks=_PRIMARY_TO_BACKUP) - observed: Final = _outcome("sync", wire, router, fallbacks=_PRIMARY_TO_BACKUP) - assert observed == twin, (observed, twin) - assert observed.hit[:1] == ("primary",), observed + streamed: Final = _stream(client, router, fallbacks=_PRIMARY_TO_BACKUP) + assert streamed.text == "answered by the backup", streamed + assert streamed.attempted_fallbacks == 1, streamed + assert _deployments_hit(wire) == ("primary", "backup") + + +@pytest.mark.parametrize("client", _CLIENTS) +def test_per_request_fallbacks_none_turns_the_router_list_off(client: str) -> None: + with wire_server(_peer(_PRIMARY_DIES)) as wire: + router: Final = _router(wire, ("primary", "backup"), fallbacks=_PRIMARY_TO_BACKUP) + with pytest.raises(litellm.APIConnectionError, match="overloaded"): + _stream(client, router, fallbacks=None) + assert _deployments_hit(wire) == ("primary",) @pytest.mark.parametrize("client", _CLIENTS) diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index a891e1079d8..d5546da024b 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -1728,3 +1728,37 @@ async def test_route_request_without_model_on_model_routed_endpoint_is_a_400(): assert exc_info.value.code == "400" assert exc_info.value.param == "model" + + +@pytest.mark.asyncio +async def test_route_request_router_settings_override_skips_null_fields(): + """ + A key or team saved from the dashboard stores every unset router setting as null. Those nulls + must not reach the router as explicit per-request values, or they switch the router-level + fallbacks and retries off for that key. + """ + data: Final = { + "model": "gpt-3.5-turbo", + "messages": [{"role": "user", "content": "Hello"}], + "stream": True, + "router_settings_override": { + "fallbacks": None, + "context_window_fallbacks": None, + "num_retries": None, + "model_group_retry_policy": None, + "timeout": 600, + }, + } + + llm_router: Final = MagicMock() + llm_router.acompletion.return_value = "success" + + response: Final = await route_request(data, llm_router, None, "acompletion") + + assert response == "success" + call_kwargs: Final = llm_router.acompletion.call_args[1] + assert call_kwargs["timeout"] == 600 + assert "fallbacks" not in call_kwargs + assert "context_window_fallbacks" not in call_kwargs + assert "num_retries" not in call_kwargs + assert "model_group_retry_policy" not in call_kwargs diff --git a/tests/unit/router_utils/test_fallback_event_handlers.py b/tests/unit/router_utils/test_fallback_event_handlers.py index adfce2c34be..38c13decaa7 100644 --- a/tests/unit/router_utils/test_fallback_event_handlers.py +++ b/tests/unit/router_utils/test_fallback_event_handlers.py @@ -30,6 +30,7 @@ from litellm.router_utils.fallback_event_handlers import ( get_pre_routing_selection, log_failure_fallback_event, log_success_fallback_event, + mid_stream_fallback_snapshot_kwargs, mid_stream_retry_kwargs, record_pre_routing_selection, record_retry_attempt, @@ -1471,6 +1472,30 @@ def test_get_fallback_model_group_never_resolves_a_provider_without_a_prefixed_k resolver.assert_not_called() +def test_mid_stream_fallback_snapshot_kwargs_restores_the_popped_lists_and_shares_the_buckets(): + controls: Final = MidStreamFallbackControls( + MappingProxyType({"fallbacks": [{"primary": ["backup"]}], "context_window_fallbacks": None}) + ) + metadata: Final = {"model_group": "primary"} + kwargs: Final = {"messages": [{"role": "user", "content": "hi"}], "stream": True, "metadata": metadata} + + snapshot: Final = mid_stream_fallback_snapshot_kwargs(model="primary", controls=controls, kwargs=kwargs) + + assert snapshot == { + **kwargs, + "fallbacks": [{"primary": ["backup"]}], + "context_window_fallbacks": None, + MID_STREAM_FALLBACK_CONTROLS_KEY: controls, + "model": "primary", + } + assert snapshot["metadata"] is metadata + assert "fallbacks" not in kwargs + + bare: Final = mid_stream_fallback_snapshot_kwargs(model="primary", controls=None, kwargs=kwargs) + assert "fallbacks" not in bare + assert bare[MID_STREAM_FALLBACK_CONTROLS_KEY] == MidStreamFallbackControls(MappingProxyType({})) + + def test_mid_stream_retry_kwargs_strips_what_the_retry_wrapper_pops_and_keeps_the_controls_carrier(): def generic_function(**kwargs) -> None: return None diff --git a/tests/unit/test_router/test_router.py b/tests/unit/test_router/test_router.py index e3e6326574b..cd490e127cf 100644 --- a/tests/unit/test_router/test_router.py +++ b/tests/unit/test_router/test_router.py @@ -63,7 +63,10 @@ from litellm.router_utils.cooldown_handlers import ( async_get_cooldown_deployments, get_cooldown_deployments, ) -from litellm.router_utils.fallback_event_handlers import DISABLE_FALLBACKS_METADATA_KEY +from litellm.router_utils.fallback_event_handlers import ( + DISABLE_FALLBACKS_METADATA_KEY, + MID_STREAM_FALLBACK_CONTROLS_KEY, +) from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute from litellm.scheduler import FlowItem from litellm.types.llms.openai import ChatCompletionRequest @@ -3868,6 +3871,136 @@ async def test_acompletion_mid_stream_fallback_walks_every_entry_of_the_configur ] +class _DiesBeforeFirstChunk(CustomStreamWrapper): + def __init__(self, model: str): + super().__init__(completion_stream=object(), model=model, custom_llm_provider="openai", logging_obj=MagicMock()) + + def _mid_stream_error(self) -> MidStreamFallbackError: + return MidStreamFallbackError( + message=f"provider 500 from {self.model}", + model=self.model, + llm_provider="openai", + generated_content="", + is_pre_first_chunk=True, + original_exception=litellm.InternalServerError( + message=f"provider 500 from {self.model}", model=self.model, llm_provider="openai" + ), + ) + + def __iter__(self): + return self + + def __next__(self): + raise self._mid_stream_error() + + def __aiter__(self): + return self + + async def __anext__(self): + raise self._mid_stream_error() + + +class _Answers(_DiesBeforeFirstChunk): + def __init__(self, model: str): + super().__init__(model) + self._chunks = iter( + [litellm.ModelResponseStream(choices=[{"index": 0, "delta": {"content": f"ok-from-{model}"}}])] + ) + + def __next__(self): + return next(self._chunks) + + async def __anext__(self): + try: + return next(self._chunks) + except StopIteration: + raise StopAsyncIteration from None + + +def _primary_and_backup_router(**settings: object) -> litellm.Router: + return litellm.Router( + model_list=[ + {"model_name": "primary", "litellm_params": {"model": "openai/primary-model", "api_key": "fake-key"}}, + {"model_name": "backup", "litellm_params": {"model": "openai/backup-model", "api_key": "fake-key"}}, + ], + num_retries=0, + **settings, + ) + + +def _stream_for(**kwargs: object) -> CustomStreamWrapper: + model: Final = str(kwargs["model"]) + return _Answers(model) if "backup" in model else _DiesBeforeFirstChunk(model) + + +def _groups_called(provider_calls: MagicMock) -> list[str]: + return [call.kwargs["metadata"]["model_group"] for call in provider_calls.call_args_list] + + +def _router_internals_reached_the_provider(provider_calls: MagicMock) -> bool: + leaked: Final = frozenset( + ("fallbacks", "context_window_fallbacks", "content_policy_fallbacks", MID_STREAM_FALLBACK_CONTROLS_KEY) + ) + return any(leaked & call.kwargs.keys() for call in provider_calls.call_args_list) + + +def test_completion_mid_stream_fallback_honors_the_per_request_list(): + router: Final = _primary_and_backup_router() + + with patch("litellm.completion", side_effect=_stream_for) as provider_calls: + response: Final = router.completion( + model="primary", + messages=[{"role": "user", "content": "hi"}], + stream=True, + fallbacks=[{"primary": ["backup"]}], + ) + content: Final = "".join(chunk.choices[0].delta.content or "" for chunk in response if chunk is not None) + + assert content == "ok-from-openai/backup-model" + assert _groups_called(provider_calls) == ["primary", "backup"] + assert not _router_internals_reached_the_provider(provider_calls) + + +@pytest.mark.asyncio +async def test_acompletion_mid_stream_fallback_honors_the_per_request_list(): + router: Final = _primary_and_backup_router() + + async def fake_acompletion(**kwargs): + return _stream_for(**kwargs) + + with patch("litellm.acompletion", side_effect=fake_acompletion) as provider_calls: + response: Final = await router.acompletion( + model="primary", + messages=[{"role": "user", "content": "hi"}], + stream=True, + fallbacks=[{"primary": ["backup"]}], + ) + content: Final = "".join( + [chunk.choices[0].delta.content or "" async for chunk in response if chunk is not None] + ) + + assert content == "ok-from-openai/backup-model" + assert _groups_called(provider_calls) == ["primary", "backup"] + assert not _router_internals_reached_the_provider(provider_calls) + + +@pytest.mark.asyncio +async def test_acompletion_mid_stream_fallback_honors_a_per_request_fallbacks_none(): + router: Final = _primary_and_backup_router(fallbacks=[{"primary": ["backup"]}]) + + async def fake_acompletion(**kwargs): + return _stream_for(**kwargs) + + with patch("litellm.acompletion", side_effect=fake_acompletion) as provider_calls: + response: Final = await router.acompletion( + model="primary", messages=[{"role": "user", "content": "hi"}], stream=True, fallbacks=None + ) + with pytest.raises(litellm.InternalServerError, match="provider 500 from openai/primary-model"): + [chunk async for chunk in response] + + assert _groups_called(provider_calls) == ["primary"] + + def test_refusal_on_the_last_fallback_hop_is_returned_instead_of_raised(): """LIT-7400 follow-up: a refusal on the final hop of an exhausted list passes through.""" from litellm.router_utils.fallback_event_handlers import AttemptedFallbackTargets From 42d2158894b8fe99243aa2726a02383469e50988 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:25:35 -0700 Subject: [PATCH 17/65] fix(responses): map input_audio blocks in the chat-to-Responses bridge (#45224) * fix(responses): map input_audio blocks in the chat-to-Responses bridge * fix(responses): honor base_model when deciding to drop input_audio under drop_params * fix(responses): carry a dropped audio part's cache breakpoint to the preceding part Dropping an input_audio part under drop_params lost the prompt_cache_breakpoint riding on it, so an injected system marker landing on a trailing audio block left prompt_cache_options explicit with no breakpoint on the wire. The marker now moves to the nearest preceding kept part when that part has none. Update the CircleCI-only Responses bridge wire tests that pinned input_audio to its stringified input_text shape: video_url becomes the stringified kind so the marker-on-stringified-block coverage survives, and dedicated audio tests assert the forwarded part and the drop_params carry-over. * refactor(responses): carry dropped audio markers in one pass Replace the per-block slice that re-read the rest of the content list for every retained part with a single reverse scan, and skip the copy for a message that carries no audio part. Type the unit test helper's messages as a sequence of mappings instead of bare lists. * test(responses): add integration cells for input_audio parts through the bridge Cover the chat-to-Responses bridge's input_audio handling on the real proxy: the part is forwarded as input_audio without drop_params and dropped under drop_params unless the model or its base_model supports audio input, across httpx, the OpenAI SDK (sync stream and async), the Anthropic SDK, native /v1/responses, tool and assistant messages, hostile input_audio values, duplicate parts, an audio-only message, marker carry-over, a null drop_params, a global litellm_settings.drop_params, a mixed burst, and a burst with dropped upstream connections * test(responses): check every upstream attempt of a dropped connection The dropped-connection cell required exactly one upstream POST per call, which tied it to the retry policy instead of the fix. It now groups the recorded POSTs by marker, requires one attempt per answered call, and checks that every attempt of a doomed call carries the dropped audio shape. --- .../transformation.py | 92 ++- .../_support/prompt_cache_breakpoint.py | 17 +- .../test_responses_bridge_input_audio_wire.py | 536 ++++++++++++++++++ ...ses_bridge_prompt_cache_breakpoint_wire.py | 31 +- ...idge_prompt_cache_breakpoint_wire_chaos.py | 20 + ...responses_transformation_transformation.py | 189 +++++- 6 files changed, 870 insertions(+), 15 deletions(-) create mode 100644 tests/integration/providers/test_responses_bridge_input_audio_wire.py diff --git a/litellm/completion_extras/litellm_responses_transformation/transformation.py b/litellm/completion_extras/litellm_responses_transformation/transformation.py index c8c6dfd106a..41f6a211036 100644 --- a/litellm/completion_extras/litellm_responses_transformation/transformation.py +++ b/litellm/completion_extras/litellm_responses_transformation/transformation.py @@ -5,6 +5,7 @@ Handler for transforming /chat/completions api requests to litellm.responses req import json import os from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Mapping, Sequence +from itertools import accumulate, chain from types import MappingProxyType from typing import TYPE_CHECKING, Any, Final, Literal, TypedDict, TypeVar, Union, cast @@ -114,6 +115,83 @@ def _strip_prompt_cache_breakpoints(input_items: list[object]) -> list[object]: return [_strip_prompt_cache_breakpoints_from_item(item) for item in input_items] +def _is_audio_input_part(part: object) -> bool: + if not isinstance(part, dict): + return False + content_part: Final = cast(dict[str, object], part) # cast-ok: isinstance confirms a content block mapping + return content_part.get("type") == "input_audio" + + +def _breakpoint_of(part: object) -> object: + if not isinstance(part, dict): + return None + content_part: Final = cast(dict[str, object], part) # cast-ok: isinstance confirms a content block mapping + return content_part.get("prompt_cache_breakpoint") + + +def _pending_audio_breakpoint(pending: object, part: object) -> object: + if not _is_audio_input_part(part): + return None + return pending if pending is not None else _breakpoint_of(part) + + +def _carried_breakpoints(content: Sequence[object]) -> tuple[object, ...]: + seeded_from_the_end: Final = chain((None,), reversed(content)) + pending_after_each: Final = tuple(accumulate(seeded_from_the_end, _pending_audio_breakpoint)) + return tuple(reversed(pending_after_each[:-1])) + + +def _with_carried_breakpoint(part: object, marker: object) -> object: + if marker is None or not isinstance(part, dict) or _breakpoint_of(part) is not None: + return part + kept_part: Final = cast(dict[str, object], part) # cast-ok: isinstance confirms a content block mapping + return {**kept_part, "prompt_cache_breakpoint": marker} + + +def _without_audio_input_parts_in_content(value: object) -> object: + if not isinstance(value, list): + return value + content: Final = cast(list[object], value) # cast-ok: isinstance confirms a list of content blocks + if not any(_is_audio_input_part(part) for part in content): + return value + return [ + _with_carried_breakpoint(part, marker) + for part, marker in zip(content, _carried_breakpoints(content)) + if not _is_audio_input_part(part) + ] + + +def _without_audio_input_parts_in_item(value: object) -> object: + if not isinstance(value, dict): + return value + input_item: Final = cast(dict[str, object], value) # cast-ok: isinstance confirms a Responses input item mapping + return { + key: _without_audio_input_parts_in_content(item) if key in ("content", "output") else item + for key, item in input_item.items() + } + + +def _without_audio_input_parts(input_items: list[object]) -> list[object]: + return [_without_audio_input_parts_in_item(item) for item in input_items] + + +def _supports_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool: + custom_llm_provider: Final = litellm_params.get("custom_llm_provider") + provider: Final = custom_llm_provider if isinstance(custom_llm_provider, str) else None + base_model: Final = litellm_params.get("base_model") + return litellm.supports_audio_input(model=model, custom_llm_provider=provider) or ( + isinstance(base_model, str) + and bool(base_model) + and litellm.supports_audio_input(model=base_model, custom_llm_provider=provider) + ) + + +def _drops_audio_input(model: str, litellm_params: Mapping[str, object]) -> bool: + if not (litellm_params.get("drop_params") or litellm.drop_params): + return False + return not _supports_audio_input(model, litellm_params) + + def _provider_metadata(response_fields: Mapping[str, object] | None) -> Mapping[str, object]: return MappingProxyType( { @@ -678,6 +756,11 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): # of instructions, mirroring how non-string system content is already # handled in convert_chat_completion_messages_to_responses_api. is_system_only_request: Final = not converted_input_items and converted_instructions is not None + target_input_items: Final = ( + _without_audio_input_parts(converted_input_items) + if _drops_audio_input(model, litellm_params) + else converted_input_items + ) input_items: Final = ( [ { @@ -687,7 +770,7 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): } ] if is_system_only_request - else converted_input_items + else target_input_items ) instructions: Final = None if is_system_only_request else converted_instructions @@ -1194,6 +1277,13 @@ class LiteLLMResponsesTransformationHandler(CompletionTransformationBridge): ) result.append(converted) verbose_logger.debug("Chat provider: image_url -> %s", converted) + elif original_type == "input_audio": + converted = with_prompt_cache_breakpoint( + {"type": "input_audio", "input_audio": item.get("input_audio")}, + _prompt_cache_breakpoint_for_wire(item.get("prompt_cache_breakpoint"), drop_params), + ) + result.append(converted) + verbose_logger.debug("Chat provider: input_audio -> %s", converted) else: # Try to map other types to responses API format item_type = original_type or "input_text" diff --git a/tests/integration/_support/prompt_cache_breakpoint.py b/tests/integration/_support/prompt_cache_breakpoint.py index ce3b97b86be..bfc7f1efbce 100644 --- a/tests/integration/_support/prompt_cache_breakpoint.py +++ b/tests/integration/_support/prompt_cache_breakpoint.py @@ -32,13 +32,13 @@ _SCRIPTED_FAILURE: Final = re.compile(r"fail-(\d{3})") _MINTED_RESPONSE: Final = re.compile(r"^resp_([0-9a-f]{32})-[0-9a-f]{32}$") _STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") -Kind: TypeAlias = Literal["text", "image_url", "file", "input_audio"] -KINDS: Final[tuple[Kind, ...]] = ("text", "image_url", "file", "input_audio") +Kind: TypeAlias = Literal["text", "image_url", "file", "video_url"] +KINDS: Final[tuple[Kind, ...]] = ("text", "image_url", "file", "video_url") WIRE_TYPE: Final[Mapping[Kind, str]] = { "text": "input_text", "image_url": "input_image", "file": "input_file", - "input_audio": "input_text", + "video_url": "input_text", } @@ -91,12 +91,19 @@ def block(kind: Kind, value: str) -> dict[str, JsonValue]: return {"type": "image_url", "image_url": {"url": "https://example.com/breakpoint.png"}} case "file": return {"type": "file", "file": {"file_id": "file-breakpoint"}} - case "input_audio": - return {"type": "input_audio", "input_audio": {"data": "Zm9v", "format": "wav"}} + case "video_url": + return {"type": "video_url", "video_url": {"url": "https://example.com/clip.mp4"}} case _: assert_never(kind) +AUDIO_PAYLOAD: Final[Mapping[str, JsonValue]] = {"data": "Zm9v", "format": "wav"} + + +def audio() -> dict[str, JsonValue]: + return {"type": "input_audio", "input_audio": dict(AUDIO_PAYLOAD)} + + def drained_posts(wire: Wire) -> tuple[Request, ...]: return tuple(request for request in wire.drain() if request.method == "POST") diff --git a/tests/integration/providers/test_responses_bridge_input_audio_wire.py b/tests/integration/providers/test_responses_bridge_input_audio_wire.py new file mode 100644 index 00000000000..389d31dae3e --- /dev/null +++ b/tests/integration/providers/test_responses_bridge_input_audio_wire.py @@ -0,0 +1,536 @@ +import asyncio +import uuid +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from typing import Final, Literal, TypeAlias + +import anthropic +import httpx +import openai +import pytest +from integration._support import prompt_cache_breakpoint as pcb +from integration._support import responses_vendor as rv +from integration._support.client import Gateway, eventually, gateway_from_environment, string_value +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +pytestmark: Final = pytest.mark.timeout(120) + +Mode: TypeAlias = Literal["on", "off"] + +_MODES: Final[tuple[Mode, ...]] = ("on", "off") +_CAPABLE_MODEL: Final = "openai/responses/gpt-audio-mini" +_CAPABLE_WIRE_MODEL: Final = "gpt-audio-mini" +_HOSTILE: Final[tuple[tuple[str, JsonValue], ...]] = ( + ("int", 7), + ("list", ["Zm9v"]), + ("empty-string", ""), + ("5kb-string", "x" * 5000), +) +_HOSTILE_IDS: Final = tuple(name for name, _ in _HOSTILE) +_HOSTILE_VALUES: Final = tuple(value for _, value in _HOSTILE) + + +@dataclass(frozen=True, slots=True) +class _Bridge: + gateway: Gateway + wire: Wire + on: str + off: str + null: str + capable_on: str + base_model_param_on: str + base_model_info_on: str + injecting_off: str + spend: pcb.SpendLogs + + def model(self, mode: Mode) -> str: + return self.on if mode == "on" else self.off + + +@pytest.fixture(scope="module") +def bridge() -> Iterator[_Bridge]: + with ( + wire_server(pcb.respond) as wire, + gateway_from_environment() as gateway, + gateway.scenario() as scenario, + pcb.spend_logs() as spend, + ): + api_base: Final = f"{wire.url}/v1" + yield _Bridge( + gateway, + wire, + scenario.model(model=pcb.MODEL, api_base=api_base, drop_params=True), + scenario.model(model=pcb.MODEL, api_base=api_base), + scenario.model(model=pcb.MODEL, api_base=api_base, drop_params=None), + scenario.model(model=_CAPABLE_MODEL, api_base=api_base, drop_params=True), + scenario.model(model=pcb.MODEL, api_base=api_base, drop_params=True, base_model=_CAPABLE_WIRE_MODEL), + scenario.model( + model=pcb.MODEL, + api_base=api_base, + drop_params=True, + model_info={"base_model": _CAPABLE_WIRE_MODEL}, + ), + scenario.model(model=pcb.MODEL, api_base=api_base, **pcb.INJECTION), + spend, + ) + + +def _v1(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + "/v1" + + +def _text(marker: str) -> dict[str, JsonValue]: + return {"type": "input_text", "text": pcb.prompt(marker)} + + +def _audio_on_wire(payload: JsonValue) -> dict[str, JsonValue]: + return {"type": "input_audio", "input_audio": payload} + + +def _user(marker: str, *parts: JsonValue) -> dict[str, JsonValue]: + return {"role": "user", "content": [pcb.text(pcb.prompt(marker)), *parts]} + + +def _chat(bridge: _Bridge, model: str, messages: Sequence[JsonValue], *, stream: bool = False) -> httpx.Response: + return bridge.gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": list(messages), "stream": stream, **pcb.NO_CACHE}, + ) + + +def _completion(response: httpx.Response, marker: str) -> str: + assert response.status_code == 200, response.text + body: Final = rv.JSON_OBJECT.validate_json(response.text) + assert pcb.answers(string_value(body["id"]), marker), body + (choice,) = rv.ITEMS.validate_python(body["choices"]) + assert rv.JSON_OBJECT.validate_python(choice["message"])["content"] == rv.answer(marker), body + return response.headers["x-litellm-call-id"] + + +def _wire_request(bridge: _Bridge, marker: str, *, model: str = "gpt-6.1-sol", stream: bool = False) -> Request: + request: Final = pcb.posted(bridge.wire, marker) + body: Final = pcb.body_of(request) + assert body["model"] == model, body + assert (body.get("stream") is True) is stream, body + return request + + +def _user_content_on_wire( + bridge: _Bridge, marker: str, *, model: str = "gpt-6.1-sol", stream: bool = False +) -> list[dict[str, JsonValue]]: + return pcb.content_of(pcb.input_items(_wire_request(bridge, marker, model=model, stream=stream)), "user") + + +def _expected(mode: Mode, marker: str, *forwarded: JsonValue) -> list[JsonValue]: + return [_text(marker)] if mode == "on" else [_text(marker), *forwarded] + + +def test_audio_part_is_dropped_under_drop_params(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + call_id: Final = _completion(_chat(bridge, bridge.on, [_user(marker, pcb.audio())]), marker) + assert _user_content_on_wire(bridge, marker) == [_text(marker)] + bridge.spend.landed(bridge.on, call_id, marker) + + +def test_audio_part_is_forwarded_as_input_audio_without_drop_params(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + call_id: Final = _completion(_chat(bridge, bridge.off, [_user(marker, pcb.audio())]), marker) + assert _user_content_on_wire(bridge, marker) == [_text(marker), _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))] + bridge.spend.landed(bridge.off, call_id, marker) + + +@pytest.mark.parametrize("mode", _MODES) +def test_openai_sdk_stream_shapes_the_audio_part_by_mode(bridge: _Bridge, mode: Mode) -> None: + marker: Final = uuid.uuid4().hex + with openai.OpenAI(api_key=bridge.gateway.key, base_url=_v1(bridge.gateway), max_retries=0) as client: + raw: Final = client.chat.completions.with_raw_response.create( + model=bridge.model(mode), + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": pcb.prompt(marker)}, + {"type": "input_audio", "input_audio": {"data": "Zm9v", "format": "wav"}}, + ], + } + ], + stream=True, + extra_body=dict(pcb.NO_CACHE), + ) + chunks: Final = tuple(raw.parse()) + assert chunks and pcb.answers(chunks[0].id, marker), chunks + streamed: Final = "".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices) + assert streamed == rv.answer(marker), chunks + content: Final = _user_content_on_wire(bridge, marker, stream=True) + assert content == _expected(mode, marker, _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))), content + bridge.spend.landed(bridge.model(mode), raw.headers["x-litellm-call-id"], marker) + + +@pytest.mark.parametrize("mode", _MODES) +async def test_async_openai_sdk_shapes_the_audio_part_by_mode(bridge: _Bridge, mode: Mode) -> None: + marker: Final = uuid.uuid4().hex + async with openai.AsyncOpenAI(api_key=bridge.gateway.key, base_url=_v1(bridge.gateway), max_retries=0) as client: + raw: Final = await client.chat.completions.with_raw_response.create( + model=bridge.model(mode), + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": pcb.prompt(marker)}, + {"type": "input_audio", "input_audio": {"data": "Zm9v", "format": "wav"}}, + ], + } + ], + extra_body=dict(pcb.NO_CACHE), + ) + completion: Final = raw.parse() + assert pcb.answers(completion.id, marker), completion + assert completion.choices[0].message.content == rv.answer(marker), completion + content: Final = _user_content_on_wire(bridge, marker) + assert content == _expected(mode, marker, _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))), content + bridge.spend.landed(bridge.model(mode), raw.headers["x-litellm-call-id"], marker) + + +def test_audio_capable_model_keeps_the_audio_part_under_drop_params(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + call_id: Final = _completion(_chat(bridge, bridge.capable_on, [_user(marker, pcb.audio())]), marker) + content: Final = _user_content_on_wire(bridge, marker, model=_CAPABLE_WIRE_MODEL) + assert content == [_text(marker), _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))], content + bridge.spend.landed(bridge.capable_on, call_id, marker) + + +def test_base_model_in_litellm_params_keeps_the_audio_part_under_drop_params(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + call_id: Final = _completion(_chat(bridge, bridge.base_model_param_on, [_user(marker, pcb.audio())]), marker) + assert _user_content_on_wire(bridge, marker) == [_text(marker), _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))] + bridge.spend.landed(bridge.base_model_param_on, call_id, marker) + + +def test_base_model_in_model_info_keeps_the_audio_part_under_drop_params(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + call_id: Final = _completion(_chat(bridge, bridge.base_model_info_on, [_user(marker, pcb.audio())]), marker) + assert _user_content_on_wire(bridge, marker) == [_text(marker), _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))] + bridge.spend.landed(bridge.base_model_info_on, call_id, marker) + + +@pytest.mark.parametrize("mode", _MODES) +def test_tool_output_audio_part_follows_the_mode(bridge: _Bridge, mode: Mode) -> None: + marker: Final = uuid.uuid4().hex + messages: Final[list[JsonValue]] = [ + {"role": "user", "content": pcb.prompt(marker)}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "record", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [pcb.text("heard it"), pcb.audio()]}, + ] + call_id: Final = _completion(_chat(bridge, bridge.model(mode), messages), marker) + output: Final = pcb.function_output(pcb.input_items(_wire_request(bridge, marker)), "call_1") + heard: Final[dict[str, JsonValue]] = {"type": "input_text", "text": "heard it"} + expected: Final[list[JsonValue]] = [heard] if mode == "on" else [heard, _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))] + assert output == expected, output + bridge.spend.landed(bridge.model(mode), call_id, marker) + + +def test_injected_system_marker_lands_on_a_trailing_audio_part_without_drop_params(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + system: Final[dict[str, JsonValue]] = {"role": "system", "content": [pcb.text("sys"), pcb.audio()]} + call_id: Final = _completion(_chat(bridge, bridge.injecting_off, [system, _user(marker)]), marker) + request: Final = _wire_request(bridge, marker) + assert pcb.body_of(request)["prompt_cache_options"] == {"mode": "explicit"}, request.body + system_content: Final = pcb.content_of(pcb.input_items(request), "system") + assert system_content == [ + {"type": "input_text", "text": "sys"}, + pcb.marked(_audio_on_wire(dict(pcb.AUDIO_PAYLOAD)), pcb.EXPLICIT), + ], system_content + bridge.spend.landed(bridge.injecting_off, call_id, marker) + + +@pytest.mark.parametrize("mode", _MODES) +def test_assistant_audio_part_follows_the_mode(bridge: _Bridge, mode: Mode) -> None: + marker: Final = uuid.uuid4().hex + messages: Final[list[JsonValue]] = [ + {"role": "user", "content": pcb.prompt(marker)}, + {"role": "assistant", "content": [pcb.text("earlier answer"), pcb.audio()]}, + {"role": "user", "content": "and again"}, + ] + call_id: Final = _completion(_chat(bridge, bridge.model(mode), messages), marker) + earlier: Final = pcb.content_of(pcb.input_items(_wire_request(bridge, marker)), "assistant") + spoken: Final[dict[str, JsonValue]] = {"type": "output_text", "text": "earlier answer"} + expected: Final[list[JsonValue]] = [spoken] if mode == "on" else [spoken, _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))] + assert earlier == expected, earlier + bridge.spend.landed(bridge.model(mode), call_id, marker) + + +def test_anthropic_sdk_request_carries_no_audio_part_into_the_bridge(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + client: Final = anthropic.Anthropic( + base_url=str(bridge.gateway.client.base_url), api_key=bridge.gateway.key, max_retries=0 + ) + with client: + raw: Final = client.messages.with_raw_response.create( + model=bridge.on, + max_tokens=64, + messages=[{"role": "user", "content": [pcb.text(pcb.prompt(marker)), pcb.audio()]}], + extra_body=dict(pcb.NO_CACHE), + ) + (content,) = raw.parse().content + assert content.type == "text" and content.text == rv.answer(marker), content + assert _user_content_on_wire(bridge, marker) == [_text(marker)] + bridge.spend.landed(bridge.on, raw.headers["x-litellm-call-id"], None) + + +def test_native_responses_audio_part_never_enters_the_bridge(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + audio: Final = _audio_on_wire(dict(pcb.AUDIO_PAYLOAD)) + response: Final = bridge.gateway.request( + "POST", + "/v1/responses", + { + "model": bridge.on, + "input": [{"type": "message", "role": "user", "content": [_text(marker), audio]}], + **pcb.NO_CACHE, + }, + ) + assert response.status_code == 200, response.text + body: Final = rv.JSON_OBJECT.validate_json(response.text) + assert pcb.answers(string_value(body["id"]), marker), body + assert rv.answer(marker) in response.text, response.text + assert _user_content_on_wire(bridge, marker) == [_text(marker), audio] + bridge.spend.landed(bridge.on, response.headers["x-litellm-call-id"], marker) + + +@pytest.mark.parametrize("value", _HOSTILE_VALUES, ids=_HOSTILE_IDS) +def test_hostile_audio_value_is_forwarded_verbatim_without_drop_params(bridge: _Bridge, value: JsonValue) -> None: + marker: Final = uuid.uuid4().hex + hostile: Final[dict[str, JsonValue]] = {"type": "input_audio", "input_audio": value} + call_id: Final = _completion(_chat(bridge, bridge.off, [_user(marker, hostile)]), marker) + assert _user_content_on_wire(bridge, marker) == [_text(marker), _audio_on_wire(value)] + bridge.spend.landed(bridge.off, call_id, marker) + + +@pytest.mark.parametrize("value", _HOSTILE_VALUES, ids=_HOSTILE_IDS) +def test_hostile_audio_value_is_dropped_under_drop_params(bridge: _Bridge, value: JsonValue) -> None: + marker: Final = uuid.uuid4().hex + hostile: Final[dict[str, JsonValue]] = {"type": "input_audio", "input_audio": value} + call_id: Final = _completion(_chat(bridge, bridge.on, [_user(marker, hostile)]), marker) + assert _user_content_on_wire(bridge, marker) == [_text(marker)] + bridge.spend.landed(bridge.on, call_id, marker) + + +@pytest.mark.parametrize("mode", _MODES) +def test_audio_part_without_a_payload_follows_the_mode(bridge: _Bridge, mode: Mode) -> None: + marker: Final = uuid.uuid4().hex + bare: Final[dict[str, JsonValue]] = {"type": "input_audio"} + call_id: Final = _completion(_chat(bridge, bridge.model(mode), [_user(marker, bare)]), marker) + content: Final = _user_content_on_wire(bridge, marker) + assert content == _expected(mode, marker, _audio_on_wire(None)), content + bridge.spend.landed(bridge.model(mode), call_id, marker) + + +@pytest.mark.parametrize("mode", _MODES) +def test_two_identical_audio_parts_follow_the_mode(bridge: _Bridge, mode: Mode) -> None: + marker: Final = uuid.uuid4().hex + call_id: Final = _completion(_chat(bridge, bridge.model(mode), [_user(marker, pcb.audio(), pcb.audio())]), marker) + audio: Final = _audio_on_wire(dict(pcb.AUDIO_PAYLOAD)) + content: Final = _user_content_on_wire(bridge, marker) + assert content == _expected(mode, marker, audio, audio), content + bridge.spend.landed(bridge.model(mode), call_id, marker) + + +def test_audio_only_message_is_forwarded_with_empty_content_under_drop_params(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + messages: Final[list[JsonValue]] = [ + {"role": "user", "content": [pcb.audio()]}, + {"role": "assistant", "content": "I could not hear that"}, + {"role": "user", "content": pcb.prompt(marker)}, + ] + call_id: Final = _completion(_chat(bridge, bridge.on, messages), marker) + items: Final = pcb.input_items(_wire_request(bridge, marker)) + users: Final = [item for item in items if item.get("role") == "user"] + assert [item["content"] for item in users] == [[], [_text(marker)]], items + bridge.spend.landed(bridge.on, call_id, marker) + + +def test_leading_audio_marker_has_no_preceding_part_to_carry_to(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + content: Final[list[JsonValue]] = [pcb.marked(pcb.audio(), pcb.EXPLICIT), pcb.text(pcb.prompt(marker))] + call_id: Final = _completion(_chat(bridge, bridge.on, [{"role": "user", "content": content}]), marker) + assert _user_content_on_wire(bridge, marker) == [_text(marker)] + bridge.spend.landed(bridge.on, call_id, marker) + + +def test_each_dropped_audio_marker_moves_to_its_own_preceding_text(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + content: Final[list[JsonValue]] = [ + pcb.text(pcb.prompt(marker)), + pcb.marked(pcb.audio(), pcb.EXPLICIT), + pcb.text("second clip follows"), + pcb.marked(pcb.audio(), pcb.EXPLICIT_30M), + ] + call_id: Final = _completion(_chat(bridge, bridge.on, [{"role": "user", "content": content}]), marker) + assert _user_content_on_wire(bridge, marker) == [ + pcb.marked(_text(marker), pcb.EXPLICIT), + {"type": "input_text", "text": "second clip follows", "prompt_cache_breakpoint": pcb.EXPLICIT_30M}, + ] + bridge.spend.landed(bridge.on, call_id, marker) + + +def test_text_marker_wins_over_the_dropped_audio_marker(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + content: Final[list[JsonValue]] = [ + pcb.marked(pcb.text(pcb.prompt(marker)), pcb.EXPLICIT_30M), + pcb.marked(pcb.audio(), pcb.EXPLICIT), + ] + call_id: Final = _completion(_chat(bridge, bridge.on, [{"role": "user", "content": content}]), marker) + assert _user_content_on_wire(bridge, marker) == [pcb.marked(_text(marker), pcb.EXPLICIT_30M)] + bridge.spend.landed(bridge.on, call_id, marker) + + +def test_null_drop_params_forwards_the_audio_part(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + call_id: Final = _completion(_chat(bridge, bridge.null, [_user(marker, pcb.audio())]), marker) + assert _user_content_on_wire(bridge, marker) == [_text(marker), _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))] + bridge.spend.landed(bridge.null, call_id, marker) + + +@dataclass(frozen=True, slots=True) +class _Call: + mode: Mode + stream: bool + marker: str + + +@dataclass(frozen=True, slots=True) +class _Served: + call: _Call + status: int + text: str + call_id: str + + +def _calls(count: int) -> tuple[_Call, ...]: + return tuple(_Call(_MODES[index % 2], index % 4 >= 2, uuid.uuid4().hex) for index in range(count)) + + +async def _send(client: httpx.AsyncClient, bridge: _Bridge, model: str, call: _Call) -> _Served: + body: Final[Mapping[str, JsonValue]] = { + "model": model, + "messages": [_user(call.marker, pcb.audio())], + "stream": call.stream, + **pcb.NO_CACHE, + } + async with client.stream( + "POST", "/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {bridge.gateway.key}"} + ) as response: + raw: Final = await response.aread() + return _Served(call, response.status_code, raw.decode(), response.headers["x-litellm-call-id"]) + + +async def _burst(bridge: _Bridge, calls: Sequence[_Call], model_for: Mapping[Mode, str]) -> tuple[_Served, ...]: + async with httpx.AsyncClient(base_url=str(bridge.gateway.client.base_url), timeout=60, trust_env=False) as client: + return tuple(await asyncio.gather(*(_send(client, bridge, model_for[call.mode], call) for call in calls))) + + +def _answered_in_its_own_shape(served: _Served) -> None: + assert served.status == 200, served.text + assert set(rv.MARKER.findall(served.text)) == {served.call.marker}, served.text + assert served.text.startswith("data:") == served.call.stream, served.text + assert rv.answer(served.call.marker) in served.text, served.text + + +def _attempts_by_marker(bridge: _Bridge, calls: Sequence[_Call]) -> Mapping[str, tuple[Request, ...]]: + posts: Final = pcb.drained_posts(bridge.wire) + marked: Final = tuple((rv.newest_marker(request.body.decode()), request) for request in posts) + attempts: Final = { + call.marker: tuple(request for marker, request in marked if marker == call.marker) for call in calls + } + assert sum(len(group) for group in attempts.values()) == len(posts), [request.body for request in posts] + assert all(attempts.values()), sorted(marker for marker, group in attempts.items() if not group) + return attempts + + +def _posts_by_marker(bridge: _Bridge, calls: Sequence[_Call]) -> Mapping[str, Request]: + attempts: Final = _attempts_by_marker(bridge, calls) + assert all(len(group) == 1 for group in attempts.values()), { + marker: len(group) for marker, group in attempts.items() + } + return {marker: group[0] for marker, group in attempts.items()} + + +def _landed_once(bridge: _Bridge, model: str, served: Sequence[_Served], *, status: str = "success") -> None: + rows: Final = eventually( + lambda: bridge.spend.rows_for(model), + lambda found: {string_value(row["litellm_call_id"]) for row in found} >= {item.call_id for item in served}, + seconds=70, + ) + by_call: Final = {string_value(row["litellm_call_id"]): row for row in rows} + assert len(by_call) == len(rows), rows + for item in served: + assert by_call[item.call_id]["status"] == status, (item.call_id, by_call[item.call_id]) + + +async def test_mixed_audio_burst_shapes_every_upstream_request_by_its_mode(bridge: _Bridge) -> None: + calls: Final = _calls(24) + served: Final = await _burst(bridge, calls, {"on": bridge.on, "off": bridge.off}) + assert len(served) == 24 + for item in served: + _answered_in_its_own_shape(item) + by_marker: Final = _posts_by_marker(bridge, calls) + for call in calls: + body: Final = pcb.body_of(by_marker[call.marker]) + assert (body.get("stream") is True) is call.stream, body + content: Final = pcb.content_of(pcb.input_items(by_marker[call.marker]), "user") + assert content == _expected(call.mode, call.marker, _audio_on_wire(dict(pcb.AUDIO_PAYLOAD))), content + _landed_once(bridge, bridge.on, tuple(item for item in served if item.call.mode == "on")) + _landed_once(bridge, bridge.off, tuple(item for item in served if item.call.mode == "off")) + + +@dataclass(frozen=True, slots=True) +class _Doomed: + markers: frozenset[str] + + def respond(self, request: Request) -> Reply: + marker: Final = rv.newest_marker(request.body.decode()) if request.method == "POST" else None + if marker in self.markers: + return Reply(drop_connection=True) + return pcb.respond(request) + + +async def test_dropped_upstream_connections_fail_only_their_own_calls(bridge: _Bridge) -> None: + calls: Final = tuple(_Call("on", False, uuid.uuid4().hex) for _ in range(16)) + doomed: Final = _Doomed(frozenset(call.marker for index, call in enumerate(calls) if index % 4 == 0)) + with wire_server(doomed.respond) as wire, bridge.gateway.scenario() as scenario: + model: Final = scenario.model(model=pcb.MODEL, api_base=f"{wire.url}/v1", drop_params=True) + rig: Final = _Bridge( + bridge.gateway, + wire, + model, + model, + model, + model, + model, + model, + model, + bridge.spend, + ) + served: Final = await _burst(rig, calls, {"on": model, "off": model}) + failed: Final = tuple(item for item in served if item.call.marker in doomed.markers) + answered: Final = tuple(item for item in served if item.call.marker not in doomed.markers) + assert (len(failed), len(answered)) == (4, 12), [(item.call.marker, item.status) for item in served] + for item in failed: + assert item.status >= 500, (item.status, item.text) + assert "answer marker" not in item.text, item.text + for item in answered: + _answered_in_its_own_shape(item) + attempts: Final = _attempts_by_marker(rig, calls) + for item in answered: + assert len(attempts[item.call.marker]) == 1, attempts[item.call.marker] + for call in calls: + for request in attempts[call.marker]: + assert pcb.content_of(pcb.input_items(request), "user") == [_text(call.marker)], request.body + _landed_once(rig, model, answered) + _landed_once(rig, model, failed, status="failure") diff --git a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py index 75bd9da7795..9fad3a2b191 100644 --- a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py +++ b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire.py @@ -234,6 +234,28 @@ def test_malformed_marker_is_dropped_on_every_block_kind(bridge: _Bridge, kind: bridge.spend.landed(bridge.on, call_id, marker) +def test_input_audio_block_is_forwarded_with_its_marker_without_drop_params(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + content: Final[list[JsonValue]] = [pcb.text(pcb.prompt(marker)), pcb.marked(pcb.audio(), pcb.EXPLICIT)] + call_id: Final = _completion(_chat(bridge, bridge.off, [{"role": "user", "content": content}]), marker) + second: Final = _second_block_on_wire(bridge, marker) + assert second["type"] == "input_audio" and second["input_audio"] == pcb.AUDIO_PAYLOAD, second + pcb.assert_marker(second, pcb.EXPLICIT) + bridge.spend.landed(bridge.off, call_id, marker) + + +def test_input_audio_block_is_dropped_under_drop_params_and_its_marker_moves_to_the_text(bridge: _Bridge) -> None: + marker: Final = uuid.uuid4().hex + content: Final[list[JsonValue]] = [pcb.text(pcb.prompt(marker)), pcb.marked(pcb.audio(), pcb.EXPLICIT)] + call_id: Final = _completion(_chat(bridge, bridge.on, [{"role": "user", "content": content}]), marker) + assert _user_block_on_wire(bridge, marker) == { + "type": "input_text", + "text": pcb.prompt(marker), + "prompt_cache_breakpoint": pcb.EXPLICIT, + } + bridge.spend.landed(bridge.on, call_id, marker) + + @pytest.mark.parametrize(("mode", "breakpoint", "expected"), _CASE_VALUES, ids=_CASE_IDS) def test_tool_output_marker(bridge: _Bridge, mode: Mode, breakpoint: JsonValue, expected: JsonValue) -> None: marker: Final = uuid.uuid4().hex @@ -274,17 +296,14 @@ def test_assistant_list_marker(bridge: _Bridge, mode: Mode, breakpoint: JsonValu def test_injected_system_marker_survives_a_trailing_audio_block(bridge: _Bridge) -> None: marker: Final = uuid.uuid4().hex - system: Final[dict[str, JsonValue]] = {"role": "system", "content": [pcb.text("sys"), pcb.block("input_audio", "")]} + system: Final[dict[str, JsonValue]] = {"role": "system", "content": [pcb.text("sys"), pcb.audio()]} messages: Final[list[JsonValue]] = [system, {"role": "user", "content": pcb.prompt(marker)}] call_id: Final = _completion(_chat(bridge, bridge.injecting_on, messages), marker) request: Final = pcb.posted(bridge.wire, marker) body: Final = _wire_body(request) assert body["prompt_cache_options"] == {"mode": "explicit"}, body - first, audio = pcb.content_of(pcb.input_items(request), "system") - assert first == {"type": "input_text", "text": "sys"}, first - assert audio["type"] == "input_text", audio - assert string_value(audio["text"]).startswith("{'type': 'input_audio'"), audio - pcb.assert_marker(audio, pcb.EXPLICIT) + (system_block,) = pcb.content_of(pcb.input_items(request), "system") + assert system_block == {"type": "input_text", "text": "sys", "prompt_cache_breakpoint": pcb.EXPLICIT}, system_block bridge.spend.landed(bridge.injecting_on, call_id, marker) diff --git a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire_chaos.py b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire_chaos.py index 7742d4b9821..6f9a05d8dfa 100644 --- a/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire_chaos.py +++ b/tests/integration/providers/test_responses_bridge_prompt_cache_breakpoint_wire_chaos.py @@ -252,6 +252,26 @@ def test_global_drop_params_drops_a_malformed_marker(global_rig: _GlobalRig, mod spend.landed(model, control_id, control) +@pytest.mark.parametrize("model", (_GLOBAL_UNSET, _GLOBAL_FALSE), ids=("deployment-unset", "deployment-false")) +def test_global_drop_params_drops_the_audio_part_and_carries_its_marker( + global_rig: _GlobalRig, model: str, spend: pcb.SpendLogs +) -> None: + marker: Final = uuid.uuid4().hex + content: Final[list[JsonValue]] = [pcb.text(pcb.prompt(marker)), pcb.marked(pcb.audio(), pcb.EXPLICIT)] + response: Final = global_rig.gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": content}], **pcb.NO_CACHE}, + ) + call_id: Final = _completion(response, marker) + assert _user_block_on_wire(global_rig.wire, marker) == { + "type": "input_text", + "text": pcb.prompt(marker), + "prompt_cache_breakpoint": pcb.EXPLICIT, + } + spend.landed(model, call_id, marker) + + async def test_mixed_burst_carries_every_marker_once(gateway: Gateway, spend: pcb.SpendLogs) -> None: calls: Final = _calls(24, _ENDPOINTS) with wire_server(pcb.respond) as wire, gateway.scenario() as scenario: diff --git a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py index 5b2187211df..8c6c4059a22 100644 --- a/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py +++ b/tests/unit/completion_extras/litellm_responses_transformation/test_completion_extras_litellm_responses_transformation_transformation.py @@ -3,6 +3,7 @@ import datetime import json import os import unittest +from collections.abc import Mapping, Sequence from typing import TYPE_CHECKING, Final, List, Literal, Optional, Tuple, cast, get_args from unittest.mock import ANY, MagicMock, Mock, patch @@ -4738,6 +4739,188 @@ def test_every_bridged_chunk_after_response_created_carries_the_served_service_t assert relayed == ["default"] * len(events), relayed +_AUDIO_PART: Final = {"type": "input_audio", "input_audio": {"data": "Zm9v", "format": "wav"}} +_TEXT_PART: Final = {"type": "text", "text": "Transcribe this"} + + +def test_convert_chat_completion_messages_to_responses_api_maps_input_audio_block(): + handler: Final = LiteLLMResponsesTransformationHandler() + messages: Final = [ + { + "role": "user", + "content": [_TEXT_PART, {**_AUDIO_PART, "prompt_cache_breakpoint": {"mode": "explicit"}}], + } + ] + + items, _ = handler.convert_chat_completion_messages_to_responses_api(messages, keep_prompt_cache_breakpoints=True) + + assert items[0]["content"] == [ + {"type": "input_text", "text": "Transcribe this"}, + { + "type": "input_audio", + "input_audio": {"data": "Zm9v", "format": "wav"}, + "prompt_cache_breakpoint": {"mode": "explicit"}, + }, + ] + + +def test_convert_chat_completion_messages_to_responses_api_drops_malformed_input_audio_breakpoint_under_drop_params(): + handler: Final = LiteLLMResponsesTransformationHandler() + messages: Final = [{"role": "user", "content": [{**_AUDIO_PART, "prompt_cache_breakpoint": ["explicit"]}]}] + + items, _ = handler.convert_chat_completion_messages_to_responses_api( + messages, drop_params=True, keep_prompt_cache_breakpoints=True + ) + + assert items[0]["content"] == [_AUDIO_PART] + + +@pytest.fixture +def registered_audio_models(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setitem( + litellm.model_cost, + "unit-audio-capable", + {"litellm_provider": "openai", "mode": "chat", "supports_audio_input": True}, + ) + monkeypatch.setitem( + litellm.model_cost, + "unit-text-only", + { + "litellm_provider": "openai", + "mode": "chat", + "supports_audio_input": False, + "supports_prompt_cache_breakpoint": True, + }, + ) + + +def _bridge_input( + model: str, + drop_params: bool | None, + messages: Sequence[Mapping[str, object]] | None = None, + **extra_litellm_params: object, +) -> list[dict[str, object]]: + chat_messages: Final = cast( + List[AllMessageValues], list(messages or [{"role": "user", "content": [_TEXT_PART, _AUDIO_PART]}]) + ) # cast-ok: the tests build chat messages as plain mappings + request: Final = LiteLLMResponsesTransformationHandler().transform_request( + model=model, + messages=chat_messages, + optional_params={}, + litellm_params={"custom_llm_provider": "openai", "drop_params": drop_params, **extra_litellm_params}, + headers={}, + litellm_logging_obj=Mock(), + ) + return cast(list[dict[str, object]], request["input"]) # cast-ok: the bridge emits message item mappings + + +def test_transform_request_forwards_input_audio_without_drop_params( + monkeypatch: pytest.MonkeyPatch, registered_audio_models: None +): + monkeypatch.setattr(litellm, "drop_params", False) + + assert _bridge_input("unit-text-only", drop_params=False)[0]["content"] == [ + {"type": "input_text", "text": "Transcribe this"}, + _AUDIO_PART, + ] + + +def test_transform_request_drops_input_audio_under_drop_params_when_model_lacks_audio_input( + monkeypatch: pytest.MonkeyPatch, registered_audio_models: None +): + monkeypatch.setattr(litellm, "drop_params", False) + + assert _bridge_input("unit-text-only", drop_params=True)[0]["content"] == [ + {"type": "input_text", "text": "Transcribe this"} + ] + + +def test_transform_request_keeps_input_audio_under_drop_params_when_model_supports_audio_input( + monkeypatch: pytest.MonkeyPatch, registered_audio_models: None +): + monkeypatch.setattr(litellm, "drop_params", False) + + assert _bridge_input("unit-audio-capable", drop_params=True)[0]["content"] == [ + {"type": "input_text", "text": "Transcribe this"}, + _AUDIO_PART, + ] + + +def test_transform_request_keeps_input_audio_under_drop_params_when_base_model_supports_audio_input( + monkeypatch: pytest.MonkeyPatch, registered_audio_models: None +): + monkeypatch.setattr(litellm, "drop_params", False) + + assert _bridge_input("my-audio-deployment", drop_params=True, base_model="unit-audio-capable")[0]["content"] == [ + {"type": "input_text", "text": "Transcribe this"}, + _AUDIO_PART, + ] + + +def test_transform_request_drops_input_audio_under_global_drop_params( + monkeypatch: pytest.MonkeyPatch, registered_audio_models: None +): + monkeypatch.setattr(litellm, "drop_params", True) + + assert _bridge_input("unit-text-only", drop_params=None)[0]["content"] == [ + {"type": "input_text", "text": "Transcribe this"} + ] + + +def test_transform_request_drops_input_audio_from_tool_output_under_drop_params( + monkeypatch: pytest.MonkeyPatch, registered_audio_models: None +): + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + {"role": "user", "content": "Describe the recording"}, + { + "role": "assistant", + "content": None, + "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "record", "arguments": "{}"}}], + }, + {"role": "tool", "tool_call_id": "call_1", "content": [_TEXT_PART, _AUDIO_PART]}, + ] + + forwarded: Final = _bridge_input("unit-text-only", drop_params=False, messages=messages) + dropped: Final = _bridge_input("unit-text-only", drop_params=True, messages=messages) + + assert forwarded[-1]["type"] == "function_call_output" + assert forwarded[-1]["output"] == [{"type": "input_text", "text": "Transcribe this"}, _AUDIO_PART] + assert dropped[-1]["output"] == [{"type": "input_text", "text": "Transcribe this"}] + + +def test_transform_request_moves_the_dropped_audio_part_breakpoint_to_the_preceding_part( + monkeypatch: pytest.MonkeyPatch, registered_audio_models: None +): + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + {"role": "user", "content": [_TEXT_PART, {**_AUDIO_PART, "prompt_cache_breakpoint": {"mode": "explicit"}}]} + ] + + assert _bridge_input("unit-text-only", drop_params=True, messages=messages)[0]["content"] == [ + {"type": "input_text", "text": "Transcribe this", "prompt_cache_breakpoint": {"mode": "explicit"}} + ] + + +def test_transform_request_keeps_the_preceding_part_breakpoint_over_the_dropped_audio_part_breakpoint( + monkeypatch: pytest.MonkeyPatch, registered_audio_models: None +): + monkeypatch.setattr(litellm, "drop_params", False) + messages: Final = [ + { + "role": "user", + "content": [ + {**_TEXT_PART, "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}}, + {**_AUDIO_PART, "prompt_cache_breakpoint": {"mode": "explicit"}}, + ], + } + ] + + assert _bridge_input("unit-text-only", drop_params=True, messages=messages)[0]["content"] == [ + {"type": "input_text", "text": "Transcribe this", "prompt_cache_breakpoint": {"mode": "explicit", "ttl": "30m"}} + ] + + def test_convert_chat_completion_messages_to_responses_api_keeps_prompt_cache_breakpoint_on_unknown_block(): """The hook marks the last block of its target message, so a message ending in a block the bridge cannot map reaches the stringify path and has to keep the marker there.""" @@ -4754,8 +4937,8 @@ def test_convert_chat_completion_messages_to_responses_api_keeps_prompt_cache_br "content": [ {"type": "text", "text": "describe this"}, { - "type": "input_audio", - "input_audio": {"data": "Zm9v", "format": "wav"}, + "type": "video_url", + "video_url": {"url": "https://example.com/clip.mp4"}, "prompt_cache_breakpoint": breakpoint_marker, }, ], @@ -4836,4 +5019,4 @@ def test_transform_request_drop_params_in_litellm_params_gates_the_prompt_cache_ litellm_logging_obj=Mock(), ) - assert "prompt_cache_breakpoint" not in result["input"][0]["content"][0] + assert "prompt_cache_breakpoint" not in result["input"][0]["content"][0] \ No newline at end of file From fbbb54c841efbf93f106e1a00f5711447cfa311c Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:27:46 -0700 Subject: [PATCH 18/65] fix(proxy): answer a rejected websocket handshake without crashing the HTTP exception handler (#45251) * fix(proxy): answer a rejected websocket handshake without crashing the HTTP exception handler * fix(proxy): read the OTLP route check's scope keys with .get and drive the websocket auth tests through the real auth path * test(proxy): build the websocket auth test keys without reading the clock --- litellm/proxy/auth/auth_utils.py | 6 + litellm/proxy/auth/user_api_key_auth.py | 3 +- .../proxy/common_utils/http_parsing_utils.py | 7 +- litellm/proxy/proxy_server.py | 37 +++++-- .../unit/proxy/auth/test_user_api_key_auth.py | 93 +++++++++++++++- .../common_utils/test_http_parsing_utils.py | 34 ++++++ .../proxy_server/test_exception_handlers.py | 104 +++++++++++++++++- 7 files changed, 267 insertions(+), 17 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index edd6028f07c..2bd10b90101 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -60,6 +60,12 @@ def is_invalid_virtual_key_error(exception: BaseException | None) -> bool: return getattr(exception, INVALID_VIRTUAL_KEY_ERROR_MARKER, False) is True +def log_model_access_denial(exc: BaseException) -> None: + if not isinstance(exc, ModelAccessDeniedProxyException): + return + verbose_proxy_logger.warning(exc.sanitized_internal_message()) + + def mark_invalid_virtual_key_error(exception: ProxyException, is_invalid_virtual_key: bool) -> ProxyException: """Return an independently marked malformed-key exception after callback transformations.""" if not is_invalid_virtual_key or str(exception.code) != str(status.HTTP_401_UNAUTHORIZED): diff --git a/litellm/proxy/auth/user_api_key_auth.py b/litellm/proxy/auth/user_api_key_auth.py index afe88fe09a3..66bc5ae44d6 100644 --- a/litellm/proxy/auth/user_api_key_auth.py +++ b/litellm/proxy/auth/user_api_key_auth.py @@ -100,6 +100,7 @@ from litellm.proxy.auth.auth_utils import ( get_request_route_template, is_invalid_virtual_key_error, iter_request_fallback_targets, + log_model_access_denial, normalize_request_route, pre_db_read_auth_checks, request_dispatched_to_pass_through_endpoint, @@ -732,7 +733,7 @@ async def user_api_key_auth_websocket_for_model(websocket: WebSocket, model: str except Exception as e: if is_invalid_virtual_key_error(e): raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION) - verbose_proxy_logger.exception(e) + log_model_access_denial(e) await websocket.close(code=status.WS_1008_POLICY_VIOLATION) raise HTTPException(status_code=403, detail=str(e)) diff --git a/litellm/proxy/common_utils/http_parsing_utils.py b/litellm/proxy/common_utils/http_parsing_utils.py index 44360cec2bd..80d80224617 100644 --- a/litellm/proxy/common_utils/http_parsing_utils.py +++ b/litellm/proxy/common_utils/http_parsing_utils.py @@ -7,6 +7,7 @@ from typing import Annotated, Any, Final, Literal, Union, get_args, get_origin import orjson from fastapi import Request, UploadFile, status from starlette._utils import get_route_path +from starlette.requests import HTTPConnection from typing_extensions import NotRequired, ReadOnly, Required, assert_never from litellm._logging import verbose_proxy_logger @@ -182,8 +183,10 @@ def _mark_body_received(byte_count: int | None) -> None: ) -def is_otlp_trace_request(request: Request) -> bool: - return request.method == "POST" and get_route_path(request.scope) in {"/v1/traces", "/v1/logs"} +def is_otlp_trace_request(request: HTTPConnection) -> bool: + if request.scope.get("type") != "http": + return False + return request.scope.get("method") == "POST" and get_route_path(request.scope) in {"/v1/traces", "/v1/logs"} async def read_request_body(request: Request | None) -> dict: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index aa086e25411..b4b23f111d1 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -114,7 +114,6 @@ from litellm.proxy._types import ( LiteLLM_TeamTableCachedObj, LiteLLM_UserTable, LitellmUserRoles, - ModelAccessDeniedProxyException, PassThroughGenericEndpoint, ProxyErrorTypes, ProxyException, @@ -356,6 +355,7 @@ from litellm.proxy.auth.auth_object_prefetch import AUTH_OBJECTS_TARGET from litellm.proxy.auth.auth_utils import ( check_response_size_is_safe, is_request_body_safe, + log_model_access_denial, log_once_if_budget_reservation_disabled, warn_once_if_custom_auth_skips_common_checks, ) @@ -768,6 +768,7 @@ except ImportError: shutdown_billing_metrics_recorder = None from fastapi.exception_handlers import http_exception_handler from starlette.exceptions import HTTPException as StarletteHTTPException +from starlette.websockets import WebSocketState from litellm.proxy import tracing_endpoints from litellm.proxy.middleware.admission_control_middleware import ( @@ -2057,7 +2058,7 @@ class UserAPIKeyCacheTTLEnum(enum.Enum): @app.exception_handler(ProxyException) async def openai_exception_handler(request: Request, exc: ProxyException): # NOTE: DO NOT MODIFY THIS, its crucial to map to Openai exceptions - _log_model_access_denial(exc) + log_model_access_denial(exc) headers: Final = exc.headers error_dict: Final = with_call_id( JSON_OBJECT.validate_python(exc.to_dict()), @@ -2076,18 +2077,32 @@ async def openai_exception_handler(request: Request, exc: ProxyException): @app.exception_handler(StarletteHTTPException) -async def otlp_http_exception_handler(request: Request, exc: StarletteHTTPException) -> Response: - response: Final = tracing_endpoints.otlp_error_response(request, exc.status_code, exc.headers) +async def otlp_http_exception_handler(connection: Request | WebSocket, exc: StarletteHTTPException) -> Response | None: + if isinstance(connection, WebSocket): + return await _websocket_http_exception_response(connection, exc) + response: Final = tracing_endpoints.otlp_error_response(connection, exc.status_code, exc.headers) if response is not None: - _close_dangling_otel_server_span(request, exc.status_code, exc=exc) + _close_dangling_otel_server_span(connection, exc.status_code, exc=exc) return response - return await http_exception_handler(request, exc) + return await http_exception_handler(connection, exc) -def _log_model_access_denial(exc: ProxyException) -> None: - if not isinstance(exc, ModelAccessDeniedProxyException): - return - verbose_proxy_logger.warning(exc.sanitized_internal_message()) +async def _websocket_http_exception_response(websocket: WebSocket, exc: StarletteHTTPException) -> Response | None: + state: Final = websocket.application_state + match state: + case WebSocketState.CONNECTING: + return JSONResponse({"detail": exc.detail}, status_code=exc.status_code, headers=exc.headers) + case WebSocketState.CONNECTED: + await websocket.close(code=_websocket_close_code(exc.status_code)) + return None + case WebSocketState.RESPONSE | WebSocketState.DISCONNECTED: + return None + case _: + assert_never(state) + + +def _websocket_close_code(status_code: int) -> int: + return status.WS_1011_INTERNAL_ERROR if status_code >= 500 else status.WS_1008_POLICY_VIOLATION def _close_dangling_otel_server_span(request: Request, status_code: int, exc: Exception | None = None) -> None: @@ -13247,7 +13262,7 @@ async def realtime_websocket_endpoint( llm_router=llm_router, ) except ProxyException as e: - _log_model_access_denial(e) + log_model_access_denial(e) await _reject_realtime_session(websocket, user_api_key_dict, code=1008, reason=e.message[:120]) return await websocket.accept(**accept_kwargs) diff --git a/tests/unit/proxy/auth/test_user_api_key_auth.py b/tests/unit/proxy/auth/test_user_api_key_auth.py index a58599d28e9..9cc8552efd5 100644 --- a/tests/unit/proxy/auth/test_user_api_key_auth.py +++ b/tests/unit/proxy/auth/test_user_api_key_auth.py @@ -5,11 +5,12 @@ import litellm.proxy import litellm.proxy.proxy_server -from typing import Dict, List, Optional -from unittest.mock import MagicMock, patch, AsyncMock +from typing import Dict, Final, List, Optional +from unittest.mock import AsyncMock, MagicMock, patch import pytest from starlette.datastructures import URL +from starlette.types import Message from litellm._logging import verbose_proxy_logger import logging import litellm @@ -1811,3 +1812,91 @@ def test_mapped_key_jwt_falls_through_to_the_shared_user_budget_attach(): "the mapped-key branch returns before the shared virtual-key checks, so the " "user's per-model budget is never attached and never enforced" ) + + +def _rejected_websocket(sent: list[Message], bearer: str) -> WebSocket: + async def receive() -> Message: + return {"type": "websocket.connect"} + + async def send(message: Message) -> None: + sent.append(message) + + return WebSocket( + { + "type": "websocket", + "path": "/v1/responses", + "query_string": b"model=gpt-5.4", + "headers": [(b"authorization", f"Bearer {bearer}".encode())], + }, + receive, + send, + ) + + +def _serve_virtual_key(monkeypatch: pytest.MonkeyPatch, user_key: str, token: UserAPIKeyAuth) -> None: + from litellm.proxy.proxy_server import hash_token, user_api_key_cache + + user_api_key_cache.set_cache(key=hash_token(user_key), value=token) + monkeypatch.setattr(litellm.proxy.proxy_server, "user_api_key_cache", user_api_key_cache) + monkeypatch.setattr(litellm.proxy.proxy_server, "master_key", MASTER_KEY) + monkeypatch.setattr(litellm.proxy.proxy_server, "prisma_client", "connected") + monkeypatch.setattr(litellm, "log_client_error_tracebacks", False) + monkeypatch.setattr(verbose_proxy_logger, "propagate", True) + + +@pytest.mark.asyncio +async def test_user_api_key_auth_websocket_logs_a_model_access_denial_as_one_warning_without_a_traceback( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + from litellm.proxy.proxy_server import hash_token + + user_key: Final = "sk-websocket-key-limited-to-mini" + _serve_virtual_key( + monkeypatch, + user_key, + UserAPIKeyAuth(token=hash_token(user_key), models=["gpt-5.4-mini"]), + ) + sent: Final[list[Message]] = [] + with ( + caplog.at_level(logging.WARNING, logger=verbose_proxy_logger.name), + pytest.raises(HTTPException) as rejection, + ): + await user_api_key_auth_websocket(_rejected_websocket(sent, user_key)) + + assert rejection.value.status_code == 403 + assert sent == [{"type": "websocket.close", "code": status.WS_1008_POLICY_VIOLATION, "reason": ""}] + proxy_records: Final = [record for record in caplog.records if record.name == verbose_proxy_logger.name] + assert [record.getMessage() for record in proxy_records if record.exc_info is not None] == [] + assert [record.getMessage() for record in proxy_records if record.levelno == logging.WARNING] == [ + "key not allowed to access model. This key can only access models=['gpt-5.4-mini']. Tried to access gpt-5.4" + ] + + +@pytest.mark.asyncio +async def test_user_api_key_auth_websocket_rejection_adds_no_traceback_for_other_auth_errors( + caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch +) -> None: + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth_websocket + from litellm.proxy.proxy_server import hash_token + + user_key: Final = "sk-websocket-key-expired-in-2020" + expired_in_2020: Final = "2020-01-01T00:00:00+00:00" + _serve_virtual_key( + monkeypatch, + user_key, + UserAPIKeyAuth(token=hash_token(user_key), expires=expired_in_2020), + ) + sent: Final[list[Message]] = [] + with ( + caplog.at_level(logging.DEBUG, logger=verbose_proxy_logger.name), + pytest.raises(HTTPException) as rejection, + ): + await user_api_key_auth_websocket(_rejected_websocket(sent, user_key)) + + assert rejection.value.status_code == 403 + assert "expired key" in str(rejection.value.detail).lower() + assert sent == [{"type": "websocket.close", "code": status.WS_1008_POLICY_VIOLATION, "reason": ""}] + proxy_records: Final = [record for record in caplog.records if record.name == verbose_proxy_logger.name] + assert [record.getMessage() for record in proxy_records if record.exc_info is not None] == [] + assert [record.levelno for record in proxy_records if record.levelno >= logging.WARNING] == [logging.ERROR] diff --git a/tests/unit/proxy/common_utils/test_http_parsing_utils.py b/tests/unit/proxy/common_utils/test_http_parsing_utils.py index 7677837aa0f..f6f148f02a8 100644 --- a/tests/unit/proxy/common_utils/test_http_parsing_utils.py +++ b/tests/unit/proxy/common_utils/test_http_parsing_utils.py @@ -34,6 +34,7 @@ from fastapi import Request as Request_http_parsing from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.utils import _invalidate_model_cost_lowercase_map from starlette.types import Message +from starlette.websockets import WebSocket from tests._vcr_conftest_common import install_live_call_probe, record_vcr_outcome @@ -1565,3 +1566,36 @@ async def test_read_request_body_unexpected_error(): result = await read_request_body(_request(receive)) assert result == {} + + +async def _never_receive() -> Message: + raise AssertionError("the OTLP route check never reads the body") + + +async def _never_send(message: Message) -> None: + raise AssertionError("the OTLP route check never sends") + + +@pytest.mark.parametrize( + ("scope", "expected"), + [ + ({"type": "http", "method": "POST", "path": "/v1/traces"}, True), + ({"type": "http", "method": "POST", "path": "/v1/logs"}, True), + ({"type": "http", "method": "GET", "path": "/v1/traces"}, False), + ({"type": "http", "method": "POST", "path": "/v1/responses"}, False), + ({"type": "http", "path": "/v1/traces"}, False), + ({"type": "websocket", "path": "/v1/traces"}, False), + ({"type": "websocket", "path": "/v1/responses"}, False), + ], +) +def test_is_otlp_trace_request_matches_only_http_posts_to_the_otlp_routes( + scope: dict[str, object], expected: bool +) -> None: + full_scope: Final[dict[str, object]] = {**scope, "headers": [], "query_string": b""} + connection: Final = ( + WebSocket(full_scope, _never_receive, _never_send) + if scope["type"] == "websocket" + else Request(full_scope, _never_receive) + ) + + assert http_parsing_utils.is_otlp_trace_request(connection) is expected diff --git a/tests/unit/proxy/proxy_server/test_exception_handlers.py b/tests/unit/proxy/proxy_server/test_exception_handlers.py index 0aff43057f9..f6efdecc884 100644 --- a/tests/unit/proxy/proxy_server/test_exception_handlers.py +++ b/tests/unit/proxy/proxy_server/test_exception_handlers.py @@ -5,6 +5,7 @@ Pins covered: - ``_close_dangling_otel_server_span`` - ``otel_request_validation_exception_handler`` - ``otel_unhandled_exception_handler`` +- ``otlp_http_exception_handler`` """ from __future__ import annotations @@ -16,8 +17,12 @@ from unittest.mock import MagicMock import httpx import pytest -from fastapi import HTTPException, Request +from fastapi import Depends, FastAPI, HTTPException, Request from fastapi.exceptions import RequestValidationError +from fastapi.testclient import TestClient +from starlette.testclient import WebSocketDenialResponse +from starlette.types import Message +from starlette.websockets import WebSocket, WebSocketDisconnect from litellm.proxy._types import ProxyException from litellm.proxy.proxy_server import ( @@ -25,6 +30,7 @@ from litellm.proxy.proxy_server import ( openai_exception_handler, otel_request_validation_exception_handler, otel_unhandled_exception_handler, + otlp_http_exception_handler, ) from .conftest import normalize @@ -507,6 +513,7 @@ async def test_otlp_auth_errors_hide_internal_details_and_survive_missing_native if isinstance(error, ProxyException) else await otlp_http_exception_handler(request, error) ) + assert response is not None assert response.status_code == (401 if isinstance(error, ProxyException) else 403) assert response.headers["content-type"].startswith(media_type) message: Final = ( @@ -516,3 +523,98 @@ async def test_otlp_auth_errors_hide_internal_details_and_survive_missing_native ) expected: Final = "Unauthorized" if isinstance(error, ProxyException) else "Forbidden" assert message == (expected if native_available or media_type == "application/json" else "") + + +def _websocket(sent: list[Message]) -> WebSocket: + async def receive() -> Message: + return {"type": "websocket.connect"} + + async def send(message: Message) -> None: + sent.append(message) + + return WebSocket({"type": "websocket", "path": "/v1/responses", "headers": [], "query_string": b""}, receive, send) + + +@pytest.mark.asyncio +async def test_http_exception_on_a_plain_http_request_keeps_the_default_json_body() -> None: + request: Final = _make_request(path="/v1/models") + + response: Final = await otlp_http_exception_handler(request, HTTPException(404, "not found")) + + assert response is not None + assert response.status_code == 404 + assert json.loads(bytes(response.body)) == {"detail": "not found"} + + +@pytest.mark.asyncio +async def test_http_exception_on_a_websocket_closed_before_accept_sends_nothing_more() -> None: + sent: Final[list[Message]] = [] + websocket: Final = _websocket(sent) + await websocket.close(code=1008) + + response: Final = await otlp_http_exception_handler(websocket, HTTPException(403, "No API key provided")) + + assert response is None + assert sent == [{"type": "websocket.close", "code": 1008, "reason": ""}] + + +@pytest.mark.asyncio +async def test_http_exception_on_a_connecting_websocket_denies_the_upgrade_with_its_status() -> None: + websocket: Final = _websocket([]) + + response: Final = await otlp_http_exception_handler(websocket, HTTPException(403, "No API key provided")) + + assert response is not None + assert response.status_code == 403 + assert json.loads(bytes(response.body)) == {"detail": "No API key provided"} + + +@pytest.mark.asyncio +@pytest.mark.parametrize(("status_code", "close_code"), [(403, 1008), (429, 1008), (500, 1011), (503, 1011)]) +async def test_http_exception_on_an_accepted_websocket_closes_it_with_a_matching_code( + status_code: int, close_code: int +) -> None: + sent: Final[list[Message]] = [] + websocket: Final = _websocket(sent) + await websocket.accept() + + response: Final = await otlp_http_exception_handler(websocket, HTTPException(status_code, "late failure")) + + assert response is None + assert sent[-1] == {"type": "websocket.close", "code": close_code, "reason": ""} + + +def test_websocket_auth_rejection_reaches_the_client_as_a_denial_instead_of_a_server_error() -> None: + async def reject_like_user_api_key_auth_websocket(websocket: WebSocket) -> None: + await websocket.close(code=1008) + raise HTTPException(status_code=403, detail="No API key provided") + + async def responses(websocket: WebSocket, _: None = Depends(reject_like_user_api_key_auth_websocket)) -> None: + await websocket.accept() + + app: Final = FastAPI() + app.exception_handler(HTTPException)(otlp_http_exception_handler) + app.add_api_websocket_route("/v1/responses", responses) + + with pytest.raises(WebSocketDisconnect) as disconnect, TestClient(app).websocket_connect("/v1/responses"): + pass + + assert disconnect.value.code == 1008 + + +def test_websocket_rejected_before_any_close_reaches_the_client_as_an_http_denial_with_the_status() -> None: + async def reject_without_closing(websocket: WebSocket) -> None: + raise HTTPException(status_code=403, detail="No API key provided") + + async def responses(websocket: WebSocket, _: None = Depends(reject_without_closing)) -> None: + await websocket.accept() + + app: Final = FastAPI() + app.exception_handler(HTTPException)(otlp_http_exception_handler) + app.add_api_websocket_route("/v1/responses", responses) + + with pytest.raises(WebSocketDenialResponse) as denial, TestClient(app).websocket_connect("/v1/responses"): + pass + + assert denial.value.status_code == 403 + assert denial.value.json() == {"detail": "No API key provided"} From d6e453f74b45ab10ee30276091eb40773f3322af Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 05:39:29 +0000 Subject: [PATCH 19/65] ci(lint): gate every rule on its merge-base count and cap Anys at a fixed total (#40193) * ci(lint): derive gate ceilings from merge-base counts and drop the budget files The four lint gates (ruff strict, type discipline, basedpyright, test quality) now fail a branch only when a rule's codebase count grows past its count at the merge-base with litellm_internal_staging plus a fixed per-rule headroom, which is zero everywhere except the LIT010/LIT011 and reportAny/reportExplicitAny seeds. Base counts come from a disk cache, then the CI artifact the renamed publish-lint-base-counts workflow uploads for every staging push (all four checkers, one artifact per checker and sha), then a scan of the base worktree. The four *-budget.json files, make lint-budget-update, budget_ratchet_check.py, and the unratcheted check are gone, so no PR carries a budget edit again. * test(lint-gates): assert checker identities by behavior and type the new gate tests * test(lint-gates): resolve base points over an injected git so unit tests stay in-process --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../publish-basedpyright-base-counts.yml | 65 --- .../workflows/publish-lint-base-counts.yml | 96 ++++ .github/workflows/test-linting.yml | 54 +- AGENTS.md | 8 +- Makefile | 39 +- basedpyright-code-budget.json | 146 ----- pyproject.toml | 1 + ruff-strict-budget.json | 260 --------- ruff.toml | 2 +- scripts/budget_ratchet_check.py | 260 --------- scripts/check_test_quality.py | 4 +- scripts/gate_slot_lock.py | 2 +- scripts/lint_base_counts.py | 327 +++++++++++ scripts/pre_commit_lint.sh | 16 +- scripts/ruff_strict_gate.py | 196 +++---- scripts/test_quality_gate.py | 174 ++---- scripts/type_check_gate.py | 497 +++------------- scripts/type_discipline_gate.py | 250 +++----- test-quality-budget.json | 26 - tests/unit/test_budget_ratchet_check.py | 189 ------ tests/unit/test_check_type_discipline.py | 16 - tests/unit/test_default_branch.py | 32 +- tests/unit/test_lint_base_counts.py | 454 +++++++++++++++ tests/unit/test_pre_commit_lint.py | 9 +- tests/unit/test_ruff_strict_gate.py | 175 +----- tests/unit/test_test_quality_gate.py | 97 +--- tests/unit/test_type_check_gate.py | 544 ++---------------- tests/unit/test_type_discipline_gate.py | 138 ++--- type-discipline-budget.json | 44 -- 29 files changed, 1319 insertions(+), 2802 deletions(-) delete mode 100644 .github/workflows/publish-basedpyright-base-counts.yml create mode 100644 .github/workflows/publish-lint-base-counts.yml delete mode 100644 basedpyright-code-budget.json delete mode 100644 ruff-strict-budget.json delete mode 100644 scripts/budget_ratchet_check.py create mode 100644 scripts/lint_base_counts.py delete mode 100644 test-quality-budget.json delete mode 100644 tests/unit/test_budget_ratchet_check.py create mode 100644 tests/unit/test_lint_base_counts.py delete mode 100644 type-discipline-budget.json diff --git a/.github/workflows/publish-basedpyright-base-counts.yml b/.github/workflows/publish-basedpyright-base-counts.yml deleted file mode 100644 index 34f60d25980..00000000000 --- a/.github/workflows/publish-basedpyright-base-counts.yml +++ /dev/null @@ -1,65 +0,0 @@ -name: Publish basedpyright base counts - -# Every commit on main can become a future merge-base. -# Publishing its per-rule basedpyright counts as an artifact lets -# scripts/type_check_gate.py download them in seconds instead of paying a -# 60-110s second basedpyright pass on every fresh worktree or moved merge-base. -# No concurrency group on purpose: runs must never cancel each other, because -# every sha's artifact matters (any of them can become a merge-base). - -on: - push: - branches: - - main - workflow_dispatch: - inputs: - ref: - description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)" - required: false - -permissions: - contents: read - -jobs: - publish: - runs-on: ubuntu-latest - timeout-minutes: 20 - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - ref: ${{ inputs.ref || github.sha }} - clean: true - persist-credentials: false - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Set up uv - uses: ./.github/actions/setup-uv-with-retries - with: - version: "0.10.9" - - - name: Cache the Rust build - uses: ./.github/actions/cache-cargo-build - - - name: Cache Prisma binaries - uses: ./.github/actions/cache-prisma-binaries - - # The gate provisions its own measurement env (.venv-typecheck: a frozen - # uv sync of its canonical dependency groups plus a generated Prisma - # client), so no install step here can drift from what local runs measure. - - name: Emit basedpyright counts for HEAD - run: | - python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/basedpyright-counts" - counts_file=$(ls "$RUNNER_TEMP"/basedpyright-counts/basedpyright-counts-*.json) - echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV" - - - name: Upload counts artifact - uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 - with: - name: ${{ env.COUNTS_ARTIFACT_NAME }} - path: ${{ runner.temp }}/basedpyright-counts/ - if-no-files-found: error diff --git a/.github/workflows/publish-lint-base-counts.yml b/.github/workflows/publish-lint-base-counts.yml new file mode 100644 index 00000000000..7af00e9c842 --- /dev/null +++ b/.github/workflows/publish-lint-base-counts.yml @@ -0,0 +1,96 @@ +name: Publish lint base counts + +# Every commit on main can become a future merge-base. +# Publishing its per-rule counts for each lint gate (strict ruff, type discipline, +# test quality, basedpyright) as an artifact lets the gates download them through +# scripts/lint_base_counts.py in seconds instead of scanning the merge-base tree in +# a throwaway worktree on every fresh checkout or moved merge-base. +# No concurrency group on purpose: runs must never cancel each other, because +# every sha's artifact matters (any of them can become a merge-base). + +on: + push: + branches: + - main + workflow_dispatch: + inputs: + ref: + description: "Ref to compute and publish base counts for (defaults to the workflow run's commit)" + required: false + +permissions: + contents: read + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 20 + strategy: + fail-fast: false + matrix: + include: + - checker: ruff-strict + script: scripts/ruff_strict_gate.py + - checker: type-discipline + script: scripts/type_discipline_gate.py + - checker: test-quality + script: scripts/test_quality_gate.py + - checker: basedpyright + script: scripts/type_check_gate.py + + steps: + - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 + with: + ref: ${{ inputs.ref || github.sha }} + clean: true + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Set up uv + uses: ./.github/actions/setup-uv-with-retries + with: + version: "0.10.9" + + - name: Cache the Rust build + if: matrix.checker == 'basedpyright' + uses: ./.github/actions/cache-cargo-build + + - name: Cache Prisma binaries + if: matrix.checker == 'basedpyright' + uses: ./.github/actions/cache-prisma-binaries + + # The three source scanners only need the pinned dev tools (ruff and the + # stdlib checkers), the same versions test-linting.yml's lint job runs. + - name: Install the dev tools + if: matrix.checker != 'basedpyright' + run: | + uv sync --frozen --only-group dev --no-install-project + + - name: Emit ${{ matrix.checker }} counts for HEAD + if: matrix.checker != 'basedpyright' + run: | + uv run --no-sync python ${{ matrix.script }} --emit-counts-dir "$RUNNER_TEMP/lint-counts" + + # The basedpyright gate provisions its own measurement env (.venv-typecheck: + # a frozen uv sync of its canonical dependency groups plus a generated Prisma + # client), so no install step here can drift from what local runs measure. + - name: Emit basedpyright counts for HEAD + if: matrix.checker == 'basedpyright' + run: | + python scripts/type_check_gate.py --emit-counts-dir "$RUNNER_TEMP/lint-counts" + + - name: Name the counts artifact + run: | + counts_file=$(ls "$RUNNER_TEMP"/lint-counts/${{ matrix.checker }}-counts-*.json) + echo "COUNTS_ARTIFACT_NAME=$(basename "$counts_file" .json)" >> "$GITHUB_ENV" + + - name: Upload counts artifact + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1 + with: + name: ${{ env.COUNTS_ARTIFACT_NAME }} + path: ${{ runner.temp }}/lint-counts/ + if-no-files-found: error diff --git a/.github/workflows/test-linting.yml b/.github/workflows/test-linting.yml index bed94a69873..db2fc969ace 100644 --- a/.github/workflows/test-linting.yml +++ b/.github/workflows/test-linting.yml @@ -17,9 +17,9 @@ jobs: lint: runs-on: ubuntu-latest timeout-minutes: 15 - # actions: read lets scripts/type_check_gate.py download the base-counts - # artifact published by publish-basedpyright-base-counts.yml instead of - # re-running basedpyright over the merge-base tree. + # actions: read lets the four lint gates download the base-counts artifacts + # published by publish-lint-base-counts.yml instead of re-scanning the + # merge-base tree in a throwaway worktree. permissions: contents: read pull-requests: read @@ -144,18 +144,24 @@ jobs: run: | uv run --no-sync ruff check --config ruff-tests.toml tests - - name: Check strict-rule budget (delta vs base) + - name: Check strict ruff rules (delta vs merge-base counts) if: steps.changes.outputs.decision != 'skip' + env: + GH_TOKEN: ${{ github.token }} run: | uv run --no-sync python scripts/ruff_strict_gate.py --base "$GATE_BASE_SHA" - - name: Check type-discipline budget (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs base) + - name: Check type discipline (mutable collections / casts / type guards / kwargs / unexplained suppressions, delta vs merge-base counts) if: steps.changes.outputs.decision != 'skip' + env: + GH_TOKEN: ${{ github.token }} run: | uv run --no-sync python scripts/type_discipline_gate.py --base "$GATE_BASE_SHA" - - name: Check test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs base) + - name: Check test quality (zero-assert / mock-echo tests, sys.path.insert, raw env writes, litellm global mutation, credential-gated skips, conftest snapshot inventory, delta vs merge-base counts) if: steps.changes.outputs.decision != 'skip' + env: + GH_TOKEN: ${{ github.token }} run: | uv run --no-sync python scripts/test_quality_gate.py --base "$GATE_BASE_SHA" @@ -164,7 +170,7 @@ jobs: run: | uv run --no-sync python -c "import openai; print(f'OpenAI version: {openai.__version__}')" - - name: Check basedpyright budget (delta vs base) + - name: Check basedpyright (delta vs merge-base counts) if: steps.changes.outputs.decision != 'skip' env: GH_TOKEN: ${{ github.token }} @@ -206,40 +212,6 @@ jobs: run: | uv run --no-sync python -c "from litellm import *" || (echo '🚨 import failed, this means you introduced unprotected imports! 🚨'; exit 1) - # Intentionally NON-GATING. This job turns red when a *-budget.json ceiling is - # raised (or a rule/budget is dropped) so a loosening is obvious in review, but it - # must be kept OUT of the branch-protection required-checks list so a justified - # bump can still be merged by a human who has seen and accepted the red. - budget-ratchet: - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - - steps: - - uses: actions/checkout@08eba0b27e820071cde6df949e0beb9ba4906955 # v4.3.0 - with: - fetch-depth: 1 - persist-credentials: false - - - name: Fetch ratchet base - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - retry() { "$@" || { sleep 15; "$@"; } || { sleep 30; "$@"; }; } - retry git fetch --no-tags --depth=1 origin "$BASE_SHA" - - - name: Set up Python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 - with: - python-version: "3.12" - - - name: Ratchet check (budgets may only decrease; non-gating) - env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - run: | - python scripts/budget_ratchet_check.py --base "$BASE_SHA" - secret-scan: runs-on: ubuntu-latest timeout-minutes: 5 diff --git a/AGENTS.md b/AGENTS.md index ce3bfa94a0d..f0d1a083abb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,13 +54,13 @@ Don't hesitate to use values in .env to get needed API keys and other secrets, a Python max line length is 120, not 88 -Never edit or commit `ruff-strict-budget.json`, `type-discipline-budget.json`, `basedpyright-code-budget.json`, or `test-quality-budget.json` on a PR branch, and don't run `make lint-budget-update` there. A scheduled Devin automation lowers the limits on the default branch in its own PR by exactly what landed since the last ratchet, so concurrent PRs don't fight over the same `"limit"` lines. Keep the hosted automation's target in sync when the repository default changes. If your branch already carries a budget edit, drop it before opening the PR +The four lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/test_quality_gate.py`, `scripts/type_check_gate.py`) compare each rule's codebase count on your branch against the count at its merge-base with the default branch, and a rule may not grow. There are no budget files to edit or ratchet: when a gate fails, fix the violations the branch introduced or remove at least as many of that rule elsewhere in the tree. The one exception is `reportAny` / `reportExplicitAny`, which share a fixed codebase-wide cap in `ANY_CAPS` in `scripts/type_check_gate.py` because Any spreads past the lines you touch. A branch may add Anys while the total stays under that cap. Only lower `ANY_CAPS`, and only in its own PR to the default branch, never raise it on a feature branch `make check` (f.k.a. `make pre-commit`, which still works identically as an alias) saves its complete output to a log file in .git (overwriting previous logs) and prints that path as its first and last output lines. To inspect a run, read or grep that log instead of re-running the multi-minute checks just to see a different slice -`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone budget gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0` +`make check`, `make lint`, `scripts/pre_commit_lint.sh`, and the standalone lint gates (`scripts/ruff_strict_gate.py`, `scripts/type_discipline_gate.py`, `scripts/type_check_gate.py`) each hold one of 2 machine-wide slots, so when other sessions or worktrees on the same box are already running heavy work, yours prints "all N machine-wide slots are busy; queueing" and then stays quiet until a slot frees. Give the command a long timeout and let it wait rather than killing it, retrying it, or assuming it hung. Don't change the # of machine-wide slots or make it unlimited by setting `LITELLM_GATE_SLOTS=0` -If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their basedpyright ceilings, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in +If you're trying to create a new function that relies on untyped stuff, instead of adding more Any's and pushing `reportAny` / `reportExplicitAny` closer to their cap in `ANY_CAPS`, just validate it in the caller with Pydantic (a model or `TypeAdapter` that returns the typed thing or raises will do) and then pass the now typed variable in If you get an LIT001 fail, refactor the code to follow functional programming best practices rather than introducing mutable data structures. For example, build values in one shot with comprehensions or generators wrapped in `tuple()` / `MappingProxyType()` / `frozenset()` instead of seeding an empty `list`/`dict`/`set` and mutating it over time. Ideally, `# mutable-ok` is never used; reach for it only as a genuine last resort when an immutable rewrite is truly impossible, and always pair it with a real reason @@ -130,4 +130,4 @@ Before implementing: Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify -Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or lint budgets to satisfy a check +Before requesting maintainer review, verify the current PR tip passes required CI and code coverage, meets Greptile confidence of at least 4/5, and has acceptable Veria and Bugbot reviews. Inspect warnings and findings, fix actionable issues, and rerun the affected checks and reviewers after changes. Record evidence for any false positive or unavailable review; never treat a pending or missing bot result as a pass. Do not lower coverage thresholds or raise `ANY_CAPS` to satisfy a check diff --git a/Makefile b/Makefile index 25b966e2b9a..b8614ca4eec 100644 --- a/Makefile +++ b/Makefile @@ -6,9 +6,8 @@ test-proxy-unit-a test-proxy-unit-b test-integration test-unit-helm \ test-rust-extension rust-sqlx-prepare lens-dev \ info lint lint-inner lint-dev lint-checks format \ - lint-basedpyright lint-e2e-basedpyright lint-basedpyright-budget-update lint-type-discipline lint-type-discipline-budget-update \ - lint-ruff-budget lint-ruff-budget-update lint-budget-update lint-gate \ - lint-test-quality lint-test-quality-budget-update \ + lint-basedpyright lint-e2e-basedpyright lint-type-discipline \ + lint-ruff-strict lint-gate lint-test-quality \ install-dev install-proxy-dev install-test-deps install-hooks \ install-helm-unittest check-circular-imports check-import-safety check check-inner pre-commit \ lint-install lint-fetch-base bootstrap @@ -32,13 +31,10 @@ help: @echo " make lint-ruff - Run Ruff linting only" @echo " make lint-basedpyright - Run basedpyright strict, gated by per-rule error counts" @echo " make lint-e2e-basedpyright - Run basedpyright over tests/e2e and tests/e2e_harness (zero errors allowed)" - @echo " make lint-basedpyright-budget-update - Ratchet basedpyright limits down by what this branch fixed" @echo " make lint-format - Check ruff format formatting (matches CI)" - @echo " make lint-ruff-budget - Gate the codebase total of each strict ruff rule against its limit" + @echo " make lint-ruff-strict - Gate each strict ruff rule's codebase total against its merge-base count" @echo " make lint-gate - Strict ruff gate in CI-parity mode (fetches the default branch, simulates the merge)" - @echo " make lint-ruff-budget-update - Ratchet ruff-strict-budget.json limits down by what this branch fixed" - @echo " make lint-test-quality - Gate the test suite against test-quality-budget.json" - @echo " make lint-budget-update - Ratchet all budgets down (ruff + type-discipline + test quality + basedpyright)" + @echo " make lint-test-quality - Gate the test suite's TQ counts against their merge-base counts" @echo " make check-circular-imports - Check for circular imports" @echo " make check-import-safety - Check import safety" @echo " make test - Run all tests" @@ -143,7 +139,7 @@ lint-fetch-base: # Mirror test-linting.yml's lint job environment: the proxy-dev group plus a generated # Prisma client, so `basedpyright tests/e2e` resolves the same modules CI does. The -# budget gate itself no longer measures here (scripts/type_check_gate.py provisions its +# basedpyright gate itself no longer measures here (scripts/type_check_gate.py provisions its # own .venv-typecheck). --inexact tops up the venv instead of pruning the proxy extras # gen:api and the running proxy need. lint-install: @@ -213,25 +209,20 @@ lint-basedpyright: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) lint-e2e-basedpyright: $(LINT_E2E_DEP_INSTALL) $(UV_RUN) basedpyright tests/e2e tests/e2e_harness -# Type-discipline budget (mutable collections / casts / type guards / kwargs / +# Type-discipline gate (mutable collections / casts / type guards / kwargs / # unexplained suppressions), the test-linting.yml step `make lint` used to omit. lint-type-discipline: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) $(UV_RUN) python scripts/type_discipline_gate.py --base "$(BASE_REF)" -# Test-quality budget (zero-assert / mock-echo tests, sys.path.insert, raw env writes, +# Test-quality gate (zero-assert / mock-echo tests, sys.path.insert, raw env writes, # litellm module-global mutation, credential-gated skips, conftest snapshot # inventory), counted across tests/ the same delta-vs-base way. lint-test-quality: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) $(UV_RUN) python scripts/test_quality_gate.py --base "$(BASE_REF)" -# --update lowers each limit by what this branch fixed since its branch point, so -# it needs the base ref fetched to resolve the merge-base. -lint-basedpyright-budget-update: install-dev - $(UV_RUN) python scripts/type_check_gate.py --update --base "$(BASE_REF)" - lint-format: format-check -lint-ruff-budget: install-dev +lint-ruff-strict: install-dev $(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)" # Strict gate, invoked the same way CI does in test-linting.yml so a local pass @@ -239,18 +230,6 @@ lint-ruff-budget: install-dev lint-gate: $(LINT_DEP_INSTALL) $(LINT_DEP_BASE) $(UV_RUN) python scripts/ruff_strict_gate.py --base "$(BASE_REF)" -lint-ruff-budget-update: install-dev - $(UV_RUN) python scripts/ruff_strict_gate.py --update --base "$(BASE_REF)" - -lint-type-discipline-budget-update: install-dev - $(UV_RUN) python scripts/type_discipline_gate.py --update --base "$(BASE_REF)" - -lint-test-quality-budget-update: install-dev - $(UV_RUN) python scripts/test_quality_gate.py --update --base "$(BASE_REF)" - -# Ratchet all budgets in one shot (ruff strict + type-discipline + test quality + basedpyright) -lint-budget-update: lint-ruff-budget-update lint-type-discipline-budget-update lint-test-quality-budget-update lint-basedpyright-budget-update - check-circular-imports: $(LINT_DEP_INSTALL) cd litellm && $(UV_RUN) python ../tests/documentation_tests/test_circular_imports.py && cd .. @@ -260,7 +239,7 @@ check-import-safety: $(LINT_DEP_INSTALL) # Combined linting, isomorphic to test-linting.yml's lint job so a local pass means a # green CI lint: it installs the same env (proxy-dev + generated Prisma client) and then # runs the diff-scoped ruff format check, whole-tree ruff check, the strict-rule / -# type-discipline / basedpyright budgets as a delta vs the base, then the circular-import +# type-discipline / basedpyright gates as a delta vs the base, then the circular-import # and import-safety checks. Steps that compare against the base resolve it the same way CI # does (merge-base with origin's current default branch). Setup (env sync, Prisma client, # base fetch) runs once up front; the checks themselves are independent, so a sub-make diff --git a/basedpyright-code-budget.json b/basedpyright-code-budget.json deleted file mode 100644 index 92dc89eb0b8..00000000000 --- a/basedpyright-code-budget.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "reportAny": { - "limit": 13429 - }, - "reportArgumentType": { - "limit": 2198 - }, - "reportAssignmentType": { - "limit": 319 - }, - "reportAttributeAccessIssue": { - "limit": 480 - }, - "reportCallIssue": { - "limit": 112 - }, - "reportConstantRedefinition": { - "limit": 40 - }, - "reportDeprecated": { - "limit": 209 - }, - "reportDuplicateImport": { - "limit": 19 - }, - "reportExplicitAny": { - "limit": 3369 - }, - "reportFunctionMemberAccess": { - "limit": 7 - }, - "reportGeneralTypeIssues": { - "limit": 101 - }, - "reportIncompatibleMethodOverride": { - "limit": 56 - }, - "reportIncompatibleVariableOverride": { - "limit": 8 - }, - "reportInconsistentOverload": { - "limit": 12 - }, - "reportIndexIssue": { - "limit": 24 - }, - "reportInvalidTypeForm": { - "limit": 30 - }, - "reportInvalidTypeVarUse": { - "limit": 1 - }, - "reportMatchNotExhaustive": { - "limit": 0 - }, - "reportMissingParameterType": { - "limit": 5570 - }, - "reportMissingTypeArgument": { - "limit": 15281 - }, - "reportMissingTypeStubs": { - "limit": 40 - }, - "reportOperatorIssue": { - "limit": 0 - }, - "reportOptionalCall": { - "limit": 0 - }, - "reportOptionalIterable": { - "limit": 0 - }, - "reportOptionalMemberAccess": { - "limit": 0 - }, - "reportOptionalOperand": { - "limit": 0 - }, - "reportOptionalSubscript": { - "limit": 0 - }, - "reportPossiblyUnboundVariable": { - "limit": 56 - }, - "reportPrivateUsage": { - "limit": 1804 - }, - "reportRedeclaration": { - "limit": 8 - }, - "reportReturnType": { - "limit": 180 - }, - "reportTypedDictNotRequiredAccess": { - "limit": 22 - }, - "reportUndefinedVariable": { - "limit": 0 - }, - "reportUnknownArgumentType": { - "limit": 44802 - }, - "reportUnknownLambdaType": { - "limit": 109 - }, - "reportUnknownMemberType": { - "limit": 38269 - }, - "reportUnknownParameterType": { - "limit": 19584 - }, - "reportUnknownVariableType": { - "limit": 29814 - }, - "reportUnnecessaryCast": { - "limit": 110 - }, - "reportUnnecessaryComparison": { - "limit": 687 - }, - "reportUnnecessaryContains": { - "limit": 4 - }, - "reportUnnecessaryIsInstance": { - "limit": 816 - }, - "reportUntypedBaseClass": { - "limit": 0 - }, - "reportUntypedFunctionDecorator": { - "limit": 27 - }, - "reportUnusedClass": { - "limit": 21 - }, - "reportUnusedFunction": { - "limit": 136 - }, - "reportUnusedImport": { - "limit": 542 - }, - "reportUnusedVariable": { - "limit": 137 - } -} diff --git a/pyproject.toml b/pyproject.toml index d9ec313471e..f4d89cf512b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -378,6 +378,7 @@ version_files = [ ] [tool.pytest.ini_options] +pythonpath = ["scripts"] asyncio_mode = "auto" asyncio_default_fixture_loop_scope = "session" markers = [ diff --git a/ruff-strict-budget.json b/ruff-strict-budget.json deleted file mode 100644 index d63a69de76f..00000000000 --- a/ruff-strict-budget.json +++ /dev/null @@ -1,260 +0,0 @@ -{ - "ANN001": { - "limit": 2918 - }, - "ANN002": { - "limit": 71 - }, - "ANN003": { - "limit": 806 - }, - "ANN201": { - "limit": 1965 - }, - "ANN202": { - "limit": 829 - }, - "ANN204": { - "limit": 683 - }, - "ANN205": { - "limit": 112 - }, - "ANN206": { - "limit": 133 - }, - "ANN401": { - "limit": 119 - }, - "ASYNC230": { - "limit": 11 - }, - "B004": { - "limit": 2 - }, - "B006": { - "limit": 176 - }, - "B008": { - "limit": 503 - }, - "B009": { - "limit": 52 - }, - "B010": { - "limit": 187 - }, - "B018": { - "limit": 2 - }, - "B019": { - "limit": 1 - }, - "B021": { - "limit": 1 - }, - "B026": { - "limit": 3 - }, - "BLE001": { - "limit": 2914 - }, - "C401": { - "limit": 8 - }, - "C404": { - "limit": 1 - }, - "C405": { - "limit": 19 - }, - "C408": { - "limit": 11 - }, - "C414": { - "limit": 4 - }, - "C419": { - "limit": 1 - }, - "C901": { - "limit": 306 - }, - "D419": { - "limit": 6 - }, - "DTZ001": { - "limit": 2 - }, - "DTZ003": { - "limit": 24 - }, - "DTZ005": { - "limit": 233 - }, - "DTZ006": { - "limit": 10 - }, - "DTZ007": { - "limit": 6 - }, - "DTZ011": { - "limit": 3 - }, - "EXE001": { - "limit": 4 - }, - "EXE002": { - "limit": 3 - }, - "F401": { - "limit": 12 - }, - "LOG015": { - "limit": 5 - }, - "N999": { - "limit": 1 - }, - "PERF102": { - "limit": 21 - }, - "PERF401": { - "limit": 12 - }, - "PERF403": { - "limit": 33 - }, - "PIE804": { - "limit": 18 - }, - "PIE810": { - "limit": 43 - }, - "PLC0206": { - "limit": 26 - }, - "PLC0414": { - "limit": 46 - }, - "PLR0124": { - "limit": 1 - }, - "PLR0206": { - "limit": 1 - }, - "PLR1704": { - "limit": 1 - }, - "PLR1714": { - "limit": 253 - }, - "PLW0127": { - "limit": 57 - }, - "PLW0602": { - "limit": 215 - }, - "PLW0603": { - "limit": 190 - }, - "PLW1508": { - "limit": 190 - }, - "PLW1510": { - "limit": 2 - }, - "PYI036": { - "limit": 3 - }, - "RET504": { - "limit": 173 - }, - "RUF012": { - "limit": 239 - }, - "RUF015": { - "limit": 8 - }, - "RUF019": { - "limit": 27 - }, - "RUF046": { - "limit": 4 - }, - "RUF059": { - "limit": 66 - }, - "RUF100": { - "limit": 0 - }, - "S110": { - "limit": 207 - }, - "S112": { - "limit": 22 - }, - "SIM101": { - "limit": 56 - }, - "SIM102": { - "limit": 310 - }, - "SIM103": { - "limit": 119 - }, - "SIM113": { - "limit": 3 - }, - "SIM115": { - "limit": 2 - }, - "SIM117": { - "limit": 6 - }, - "SIM201": { - "limit": 1 - }, - "SIM210": { - "limit": 8 - }, - "SIM211": { - "limit": 1 - }, - "SIM222": { - "limit": 1 - }, - "SIM401": { - "limit": 11 - }, - "TC004": { - "limit": 5 - }, - "TID251": { - "limit": 1035 - }, - "TRY002": { - "limit": 524 - }, - "TRY004": { - "limit": 96 - }, - "TRY201": { - "limit": 401 - }, - "TRY203": { - "limit": 109 - }, - "TRY300": { - "limit": 852 - }, - "UP028": { - "limit": 2 - }, - "UP031": { - "limit": 2 - }, - "UP036": { - "limit": 1 - } -} diff --git a/ruff.toml b/ruff.toml index fab3fe27aed..93396a723e7 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,6 +1,6 @@ lint.ignore = ["F405", "E402", "F403"] # The second group is the strict gate's graduates: rules the codebase already has zero -# violations of, so they hard-fail here instead of being ratcheted in ruff-strict-budget.json. +# violations of, so they hard-fail here instead of being counted by the strict gate. # That gives editors and `ruff check --fix` the diagnostic, which the gate script cannot. lint.extend-select = [ "T20", "PGH004", "RUF008", "RUF009", "RUF100", diff --git a/scripts/budget_ratchet_check.py b/scripts/budget_ratchet_check.py deleted file mode 100644 index 3ca5e9f3e9d..00000000000 --- a/scripts/budget_ratchet_check.py +++ /dev/null @@ -1,260 +0,0 @@ -#!/usr/bin/env python3 -"""Non-gating ratchet guard: budget limits may only fall, never rise. - -Every `*-budget.json` file (ruff-strict, type-discipline, basedpyright-code) is a -one-way ratchet: each rule's ceiling is its `limit`, and that limit is meant to be -driven DOWN over time. This check compares every budget file against its own -content at the merge-base with the target branch and fails (exits 1, red) if: - - * a rule's `limit` went up, - * a rule was dropped from a budget (its ceiling effectively became infinite) while - its checker still emits it, or - * an entire budget file was deleted. - -New rules and lowered/equal limits are fine. So is a rule that graduated: once a -paired config (ruff.toml for the ruff-strict budget) selects the rule outright it -hard-fails at the first violation, which is stricter than any ceiling the budget -could hold, so dropping its entry tightens the guard rather than removing it. -Likewise a retired rule: once the paired checker (check_test_quality.py for the -test-quality budget) no longer emits a code, its entry has no ceiling left to -loosen. - -This is deliberately NOT a gating check. It should turn the run red so that a -loosening is impossible to miss in review, but it must stay OUT of the -branch-protection required-checks list: a justified bump (e.g. banning a new API, -which mechanically raises a baseline) can then still be merged by a human who has -seen the red and accepted it. - -Usage: - python scripts/budget_ratchet_check.py [--base REF] [budget.json ...] - -""" - -from __future__ import annotations - -import argparse -import importlib.util -import json -import subprocess -import sys -from pathlib import Path -from types import MappingProxyType, ModuleType -from typing import Final, NamedTuple - -if sys.version_info >= (3, 11): - import tomllib -else: - import tomli as tomllib - -REPO_ROOT = Path(__file__).resolve().parent.parent -DEFAULT_BUDGETS: tuple[str, ...] = ( - "ruff-strict-budget.json", - "type-discipline-budget.json", - "basedpyright-code-budget.json", - "test-quality-budget.json", -) -GRADUATION_CONFIGS = MappingProxyType({"ruff-strict-budget.json": "ruff.toml"}) -RETIREMENT_SOURCES = MappingProxyType({"test-quality-budget.json": "check_test_quality"}) - - -class Regression(NamedTuple): - budget: str - rule: str - detail: str - - -def _run(cmd: list[str]) -> subprocess.CompletedProcess[str]: - return subprocess.run(cmd, cwd=REPO_ROOT, capture_output=True, text=True) - - -def _merge_base(base: str) -> str: - """The common ancestor of `base` and HEAD, so unrelated base drift is ignored.""" - proc = _run(["git", "merge-base", base, "HEAD"]) - return proc.stdout.strip() or base - - -def _load_head(rel: str) -> dict | None: - path = REPO_ROOT / rel - if not path.exists(): - return None - return json.loads(path.read_text()) - - -def _ref_is_commit(ref: str) -> bool: - return ( - _run( - ["git", "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"] - ).returncode - == 0 - ) - - -def _load_base(rel: str, ref: str) -> dict | None: - """Budget content at `ref`, or None when the file did not exist there. - - `ref` is verified as a real commit by the caller, so a non-zero `git show` here means - the path was absent at that commit, not that the ref itself is unresolvable. - """ - proc = _run(["git", "show", f"{ref}:{rel}"]) - if proc.returncode != 0: - return None - return json.loads(proc.stdout) - - -def _ceiling(spec: dict) -> int: - """A rule's ceiling: its `limit`, or legacy `baseline + slack`. - - The base side of the diff can predate the `limit` migration, so a spec is read - under either schema and the two are compared on the same footing. - """ - if "limit" in spec: - return int(spec["limit"]) - return int(spec.get("baseline", 0)) + int(spec.get("slack", 0)) - - -def _limits(budget: dict) -> dict[str, int]: - """Map each rule to its ceiling; skip malformed specs.""" - return { - rule: _ceiling(spec) - for rule, spec in budget.items() - if isinstance(spec, dict) - } - - -def selectors_hard_failed_by(lint: dict) -> tuple[str, ...]: - """A ruff `[lint]` table's selected codes, minus anything `ignore` turns back off. - - `lint.ignore` wins over `lint.extend-select` in ruff, so an ignored code is not - actually enforced and must not count as a graduation. - """ - ignored = tuple(lint.get("ignore", ())) - return tuple( - selector - for selector in lint.get("extend-select", ()) - if not (ignored and selector.startswith(ignored)) - ) - - -def graduated_selectors(rel: str) -> tuple[str, ...]: - """Selectors the budget's paired ruff config hard-fails, so its ceiling is moot.""" - config = GRADUATION_CONFIGS.get(rel) - if config is None or not (REPO_ROOT / config).exists(): - return () - return selectors_hard_failed_by( - tomllib.loads((REPO_ROOT / config).read_text()).get("lint", {}) - ) - - -def _load_script(name: str) -> ModuleType: - if name in sys.modules: - return sys.modules[name] - spec: Final = importlib.util.spec_from_file_location(name, REPO_ROOT / "scripts" / f"{name}.py") - assert spec is not None and spec.loader is not None - module: Final = importlib.util.module_from_spec(spec) - sys.modules[name] = module - spec.loader.exec_module(module) - return module - - -def retired_rules(rel: str, base: dict[str, object]) -> frozenset[str]: - """Rules in the base budget that the paired checker can no longer emit, so there is no ceiling to loosen.""" - source: Final = RETIREMENT_SOURCES.get(rel) - if source is None: - return frozenset() - return frozenset(_limits(base)) - _load_script(source).RULE_CODES - - -def _regression_detail( - rule: str, - base_limits: dict[str, int], - head_limits: dict[str, int], - graduated: tuple[str, ...], - retired: frozenset[str] = frozenset(), -) -> str | None: - """Why `rule` regressed vs base, or None when it held flat, fell, or left the budget legitimately. - - A dropped rule is terminal unless it graduated or retired; otherwise the only - loosening left is a raised limit. - """ - base_limit = base_limits[rule] - if rule not in head_limits: - if rule in retired or (graduated and rule.startswith(graduated)): - return None - return f"rule dropped (limit {base_limit} -> removed)" - if head_limits[rule] > base_limit: - return f"limit raised {base_limit} -> {head_limits[rule]}" - return None - - -def regressions_for( - rel: str, - base: dict | None, - head: dict | None, - graduated: tuple[str, ...] = (), - retired: frozenset[str] = frozenset(), -) -> list[Regression]: - if base is None: - return [] # new budget file: nothing to ratchet against yet - if head is None: - return [Regression(rel, "*", "budget file was deleted (every limit removed)")] - - base_limits, head_limits = _limits(base), _limits(head) - return [ - Regression(rel, rule, detail) - for rule in sorted(base_limits) - if (detail := _regression_detail(rule, base_limits, head_limits, graduated, retired)) is not None - ] - - -def main() -> int: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--base", help="Comparison ref (default: origin's current default branch)") - parser.add_argument("budgets", nargs="*", help="budget files to check") - args = parser.parse_args() - from default_branch import resolve_base_ref - - base_ref: Final = resolve_base_ref(args.base, REPO_ROOT) - budgets = args.budgets or list(DEFAULT_BUDGETS) - - ref = _merge_base(base_ref) - if not _ref_is_commit(ref): - print( - f"FAIL: base ref {ref!r} does not resolve to a commit, so the ratchet has nothing " - f"to compare against; refusing to pass vacuously (check the --base / BASE_SHA value)", - file=sys.stderr, - ) - return 1 - - regressions: list[Regression] = [] - checked: list[str] = [] - for rel in budgets: - base = _load_base(rel, ref) - head = _load_head(rel) - if base is None and head is None: - continue - if base is None: - print(f"skip {rel}: new file (no base at {base_ref} to ratchet against)") - continue - checked.append(rel) - regressions.extend(regressions_for(rel, base, head, graduated_selectors(rel), retired_rules(rel, base))) - - if regressions: - print( - f"FAIL: budget limit(s) loosened vs base {base_ref} (merge-base {ref[:12]}):" - ) - for reg in regressions: - print(f" {reg.budget} {reg.rule}: {reg.detail}") - print( - "Budgets are one-way ratchets and may only go down or stay flat. This " - "check is non-gating: if the increase is justified (e.g. a newly banned " - "API), a human can merge over the red after acknowledging it." - ) - return 1 - - suffix = f" ({', '.join(checked)})" if checked else "" - print(f"OK: no budget limit increased vs base {base_ref}{suffix}") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/check_test_quality.py b/scripts/check_test_quality.py index 9f93023cd53..6f40dd6ea84 100644 --- a/scripts/check_test_quality.py +++ b/scripts/check_test_quality.py @@ -4,8 +4,8 @@ Sibling of scripts/check_type_discipline.py, same output contract (``path:line: CODE message``) and same stdlib-only constraint, aimed at the test tree instead of the package. Each rule is a shape the testing-strategy audit -measured and named; scripts/test_quality_gate.py caps the codebase total of each -one against test-quality-budget.json so the counts can only ratchet down. +measured and named; scripts/test_quality_gate.py fails any change that grows the +codebase total of one past its merge-base count. Rules ----- diff --git a/scripts/gate_slot_lock.py b/scripts/gate_slot_lock.py index 999b28ced1c..c1ab00461e0 100644 --- a/scripts/gate_slot_lock.py +++ b/scripts/gate_slot_lock.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Machine-wide slot lock for this repo's heavy entrypoints. -`make check`, `make lint`, and the standalone budget gates +`make check`, `make lint`, and the standalone lint gates (scripts/ruff_strict_gate.py, scripts/type_discipline_gate.py, scripts/type_check_gate.py) each hold one of N machine-wide slots while they run, so however many sessions and worktrees share one machine, at most N of diff --git a/scripts/lint_base_counts.py b/scripts/lint_base_counts.py new file mode 100644 index 00000000000..2b86d4ce55b --- /dev/null +++ b/scripts/lint_base_counts.py @@ -0,0 +1,327 @@ +#!/usr/bin/env python3 +"""Merge-base counts for the delta-vs-base lint gates. + +Each gate (scripts/ruff_strict_gate.py, scripts/type_discipline_gate.py, +scripts/type_check_gate.py, scripts/test_quality_gate.py) counts its rules +across the whole tree at HEAD and at the merge-base with the branch the change +merges into, and fails only when a rule grew past its ceiling: the merge-base +count, or the rule's fixed codebase-wide cap when the gate sets one and it is +higher. There is no committed budget, so a ceiling moves only when the base +branch does or when someone lowers a cap on it. + +The merge-base counts come from, in order, the disk cache under the git common +dir, the CI artifact publish-lint-base-counts.yml uploads for every push to the +default branch, and a scan of the base tree in a temporary worktree. Every +entry is keyed by the merge-base commit plus the checker's fingerprints (its +config, its rule logic, its tool version), so counts measured under a different +rule set are never matched, only recomputed. +""" + +from __future__ import annotations + +import hashlib +import io +import json +import os +import re +import subprocess +import sys +import zipfile +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final, NamedTuple, TypeAlias + +REPO_ROOT: Final = Path(__file__).resolve().parents[1] +CACHE_DIR_NAME: Final = "litellm-lint-cache" +CACHE_KEEP_ENTRIES: Final = 8 +GH_TIMEOUT_SECONDS: Final = 10 + +_ORIGIN_SLUG: Final = re.compile(r"(?:git@github\.com:|https://github\.com/)([^/]+/[^/]+?)(?:\.git)?/?") + +Counts: TypeAlias = Mapping[str, int] +GhOutput: TypeAlias = Callable[[Sequence[str]], bytes | None] +NO_CAPS: Final[Counts] = MappingProxyType({}) + + +class Breach(NamedTuple): + rule: str + total: int + ceiling: int + added: int + + +@dataclass(frozen=True, slots=True) +class Checker: + name: str + fingerprints: tuple[str, ...] + + def key(self, base_point: str) -> str: + return cache_key(base_point, self.fingerprints) + + def artifact_name(self, base_point: str) -> str: + return f"{self.name}-counts-{self.key(base_point)}" + + def cache_file_name(self, base_point: str) -> str: + return f"{self.name}-base-{self.key(base_point)}.json" + + def cache_glob(self) -> str: + return f"{self.name}-base-*.json" + + +Fetch: TypeAlias = Callable[[Checker, str], Counts | None] + + +def sha256_of(path: Path) -> str: + return hashlib.sha256(path.read_bytes()).hexdigest() + + +def cache_key(base_point: str, fingerprints: Sequence[str]) -> str: + return hashlib.sha256("|".join((base_point, *fingerprints)).encode()).hexdigest()[:16] + + +Git: TypeAlias = Callable[[Sequence[str]], str] + + +def git_in(cwd: Path) -> Git: + def run(args: Sequence[str]) -> str: + proc: Final = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) + if proc.returncode not in (0, 1): + sys.stderr.write(proc.stderr) + raise SystemExit(f"git exited {proc.returncode}") + return proc.stdout + + return run + + +REPO_GIT: Final = git_in(REPO_ROOT) + + +def head_sha(git: Git = REPO_GIT) -> str: + return git(["rev-parse", "HEAD"]).strip() + + +def resolve_base_point(base_ref: str, git: Git = REPO_GIT) -> str: + """The snapshot commit base counts are measured at: merge-base(base_ref, HEAD), + made aware of an in-progress merge. Mid-merge, HEAD is still the pre-merge tip, + so its merge-base is the old branch point and every violation the base gained + since then would be blamed on this change. While MERGE_HEAD exists, prefer + merge-base(base_ref, MERGE_HEAD) whenever it is the newer of the two.""" + head_point: Final = git(["merge-base", base_ref, "HEAD"]).strip() + if not head_point: + return base_ref + merge_head: Final = git(["rev-parse", "--verify", "--quiet", "MERGE_HEAD"]).strip() + if not merge_head: + return head_point + merge_point: Final = git(["merge-base", base_ref, merge_head]).strip() + if not merge_point: + return head_point + older: Final = git(["merge-base", head_point, merge_point]).strip() + return merge_point if older == head_point else head_point + + +def default_cache_dir(git: Git = REPO_GIT) -> Path: + return Path(git(["rev-parse", "--path-format=absolute", "--git-common-dir"]).strip()) / CACHE_DIR_NAME + + +def validated_counts(data: object) -> Counts | None: + counts: Final = data.get("counts") if isinstance(data, dict) else None + if not isinstance(counts, dict): + return None + if not all( + isinstance(code, str) and isinstance(total, int) and not isinstance(total, bool) + for code, total in counts.items() + ): + return None + return counts + + +def load_cached_counts(path: Path) -> Counts | None: + try: + data: Final = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return None + return validated_counts(data) + + +def scratch_path(path: Path) -> Path: + return path.with_name(f".{path.name}.{os.getpid()}.tmp") + + +def counts_payload(base_point: str, counts: Counts) -> str: + return json.dumps({"base_point": base_point, "counts": dict(sorted(counts.items()))}, indent=2) + "\n" + + +def entry_recency(path: Path) -> float: + try: + return path.stat().st_mtime + except OSError: + return 0.0 + + +def evicted_beyond_cap(entries: Sequence[Path], keep: int) -> tuple[Path, ...]: + newest_first: Final = sorted(entries, key=entry_recency, reverse=True) + return tuple(newest_first[keep:]) + + +def store_counts(directory: Path, checker: Checker, base_point: str, counts: Counts) -> Path: + directory.mkdir(parents=True, exist_ok=True) + path: Final = directory / checker.cache_file_name(base_point) + scratch: Final = scratch_path(path) + scratch.write_text(counts_payload(base_point, counts)) + scratch.replace(path) + siblings: Final = tuple(entry for entry in directory.glob(checker.cache_glob()) if entry != path) + for stale in evicted_beyond_cap(siblings, CACHE_KEEP_ENTRIES - 1): + stale.unlink(missing_ok=True) + return path + + +def parse_origin_slug(url: str) -> str | None: + match: Final = _ORIGIN_SLUG.fullmatch(url.strip()) + return match.group(1) if match else None + + +def origin_slug(cwd: Path = REPO_ROOT) -> str | None: + proc: Final = subprocess.run(["git", "remote", "get-url", "origin"], cwd=cwd, capture_output=True, text=True) + return parse_origin_slug(proc.stdout) if proc.returncode == 0 else None + + +def gh_output(args: Sequence[str]) -> bytes | None: + try: + proc: Final = subprocess.run(["gh", *args], capture_output=True, timeout=GH_TIMEOUT_SECONDS) + except (OSError, subprocess.SubprocessError): + return None + return proc.stdout if proc.returncode == 0 else None + + +def _parsed_json(raw: bytes) -> object | None: + try: + return json.loads(raw) + except ValueError: + return None + + +def _artifact_download_url(listing: object) -> str | None: + artifacts: Final = listing.get("artifacts") if isinstance(listing, dict) else None + if not isinstance(artifacts, list) or not artifacts: + return None + newest: Final = artifacts[0] + if not isinstance(newest, dict) or newest.get("expired"): + return None + url: Final = newest.get("archive_download_url") + return url if isinstance(url, str) else None + + +def _counts_json_from_zip(zip_bytes: bytes) -> object | None: + try: + with zipfile.ZipFile(io.BytesIO(zip_bytes)) as archive: + members: Final = tuple(name for name in archive.namelist() if name.endswith(".json")) + if len(members) != 1: + return None + return json.loads(archive.read(members[0])) + except (zipfile.BadZipFile, ValueError, OSError): + return None + + +def counts_for_base(payload: object, base_point: str) -> Counts | None: + if not isinstance(payload, dict) or payload.get("base_point") != base_point: + return None + counts: Final = validated_counts(payload) + return counts if counts else None + + +def _fetch_fallback(reason: str) -> None: + sys.stderr.write(f"{reason}; computing base counts locally\n") + + +def fetch_ci_base_counts( + checker: Checker, + base_point: str, + gh: GhOutput = gh_output, + cwd: Path = REPO_ROOT, +) -> Counts | None: + """Base counts from the CI artifact published for `base_point`, or None. + + Every failure mode (no gh, no auth, offline, expired or missing artifact, + malformed payload, counts for a different commit) returns None so the + caller falls back to the local base scan; the fetch is an optimization and + must never make the gate less available than local compute alone.""" + slug: Final = origin_slug(cwd) + if slug is None: + return _fetch_fallback("origin remote is not a github.com URL") + name: Final = checker.artifact_name(base_point) + listing: Final = gh(["api", f"repos/{slug}/actions/artifacts?name={name}&per_page=1"]) + if listing is None: + return _fetch_fallback(f"could not list CI artifacts named {name}") + url: Final = _artifact_download_url(_parsed_json(listing)) + if url is None: + return _fetch_fallback(f"no usable CI artifact named {name}") + zip_bytes: Final = gh(["api", url]) + if zip_bytes is None: + return _fetch_fallback(f"download failed for CI artifact {name}") + counts: Final = counts_for_base(_counts_json_from_zip(zip_bytes), base_point) + if counts is None: + return _fetch_fallback(f"CI artifact {name} is not valid base counts for {base_point[:12]}") + sys.stderr.write(f"base counts fetched from CI artifact {name}\n") + return counts + + +def base_counts_cached( + checker: Checker, + base_point: str, + compute: Callable[[str], Counts], + cache_dir: Path | None = None, + fetch: Fetch = fetch_ci_base_counts, +) -> Counts: + """`compute` memoized on disk. The base tree at a given commit is immutable, + so its counts are a pure function of the merge-base plus the checker's + fingerprints in the cache key; an empty result is never stored because it is + the signature of a crashed pass, not a clean tree. On a disk miss the counts + CI already published for the merge-base are fetched before the expensive + local base scan; a fetch miss of any kind computes locally.""" + directory: Final = default_cache_dir() if cache_dir is None else cache_dir + cached: Final = load_cached_counts(directory / checker.cache_file_name(base_point)) + if cached is not None: + return cached + fetched: Final = fetch(checker, base_point) + if fetched: + store_counts(directory, checker, base_point, fetched) + return fetched + counts: Final = compute(base_point) + if counts: + store_counts(directory, checker, base_point, counts) + return counts + + +def emit_counts(checker: Checker, counts: Counts, directory: Path, head_point: str) -> Path: + """Write HEAD's per-rule counts as the file the publisher workflow uploads. + + The filename stem is exactly the artifact name `fetch_ci_base_counts` will + later look up for this commit, so emit and fetch cannot drift apart. Empty + counts are refused: a pass that produced nothing almost certainly crashed, + and publishing it would poison every branch that fetches it.""" + if not counts: + print( + f"FAIL: {checker.name} produced no violations; refusing to publish empty base " + "counts because the pass almost certainly crashed or emitted nothing." + ) + raise SystemExit(1) + name: Final = checker.artifact_name(head_point) + directory.mkdir(parents=True, exist_ok=True) + path: Final = directory / f"{name}.json" + path.write_text(counts_payload(head_point, counts)) + print(f"Emitted base counts for {head_point} as {name}.json ({sum(counts.values())} violations total)") + return path + + +def ceiling(rule: str, base: Counts, caps: Counts) -> int: + return max(base.get(rule, 0), caps.get(rule, 0)) + + +def evaluate(head: Counts, base: Counts, caps: Counts = NO_CAPS) -> tuple[Breach, ...]: + return tuple( + Breach(rule, total, ceiling(rule, base, caps), total - base.get(rule, 0)) + for rule, total in sorted(head.items()) + if total > ceiling(rule, base, caps) + ) diff --git a/scripts/pre_commit_lint.sh b/scripts/pre_commit_lint.sh index bc5341d265c..1c793705dc6 100755 --- a/scripts/pre_commit_lint.sh +++ b/scripts/pre_commit_lint.sh @@ -13,10 +13,10 @@ # - tests/e2e and tests/e2e_harness Python # -> `make lint-e2e-basedpyright` (test-linting.yml's e2e type-check step) # + raw HTTP client ban (test-code-quality.yml's check_e2e_no_raw_requests) -# - tests/ Python, ruff-tests.toml, test-quality-budget.json, scripts/check_test_quality.py, +# - tests/ Python, ruff-tests.toml, scripts/check_test_quality.py, # scripts/test_quality_gate.py # -> ruff over ruff-tests.toml + `make lint-test-quality` (test-linting.yml's -# test-tree ruff and test-quality budget steps) +# test-tree ruff and test-quality gate steps) # - dashboard -> prettier + eslint + lint budgets (test-litellm-ui-build.yml's frontend-lint) # - proxy/types -> regenerate the lazy OpenAPI snapshot and dashboard API types, fail on drift (check-ui-api-types.yml) # @@ -33,7 +33,7 @@ set -eu # before anything else, so N parallel `make check` runs across worktrees execute two # at a time instead of thrashing the machine. The wrapper exports # LITELLM_GATE_SLOT_HELD, so this re-exec happens exactly once and everything this -# script spawns (make lint, the budget gates) skips its own acquisition. +# script spawns (make lint, the lint gates) skips its own acquisition. script_dir=$(python3 -c 'import os, sys; print(os.path.dirname(os.path.realpath(sys.argv[1])))' "$0") if [ -z "${LITELLM_GATE_SLOT_HELD:-}" ]; then exec python3 "$script_dir/gate_slot_lock.py" "$0" "$@" @@ -97,7 +97,7 @@ existing_files() { litellm_py_pattern='^litellm/.*\.py$' e2e_py_pattern='^tests/e2e(_harness)?/.*\.py$' -test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|test-quality-budget\.json|scripts/(check_test_quality|test_quality_gate)\.py)$' +test_tree_pattern='^(tests/.*\.py|ruff-tests\.toml|scripts/(check_test_quality|test_quality_gate)\.py)$' spec_pattern='^(litellm/(proxy|types)/.*|ui/litellm-dashboard/(scripts/gen-api-types\.mjs|package\.json|package-lock\.json|src/lib/http/schema\.d\.ts))$' ui_prettier_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs|json|css|scss|md|mdx|yml|yaml|html)$' ui_eslint_pattern='^ui/litellm-dashboard/.*\.(js|jsx|ts|tsx|mjs|cjs)$' @@ -147,7 +147,7 @@ if [ -n "$staged" ]; then } warn_skipped "Python lint (make lint)" "$litellm_py_pattern" "$litellm_py_files" warn_skipped "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_pattern" "$e2e_py_files" - warn_skipped "test-tree lint (ruff-tests.toml + test-quality budget)" "$test_tree_pattern" "$test_tree_files" + warn_skipped "test-tree lint (ruff-tests.toml + test-quality gate)" "$test_tree_pattern" "$test_tree_files" warn_skipped "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_pattern" "$ui_prettier_changed" warn_skipped "dashboard API-type sync (npm run gen:api)" "$spec_pattern" "$spec_files" fi @@ -304,9 +304,9 @@ if [ -n "$test_tree_files" ] && [ -z "$litellm_py_files" ]; then echo "check: linting the test tree (ruff check --config ruff-tests.toml tests)" uv run --no-sync ruff check --config ruff-tests.toml tests \ || { echo "✗ Test-tree ruff failed. Fix the errors above, then re-run make check." >&2; status=1; } - echo "check: checking the test-quality budget (make lint-test-quality)" + echo "check: checking the test-quality gate (make lint-test-quality)" make lint-test-quality \ - || { echo "✗ Test-quality budget failed. Fix the errors above, then re-run make check." >&2; status=1; } + || { echo "✗ Test-quality gate failed. Fix the errors above, then re-run make check." >&2; status=1; } fi if [ -n "${python_pid:-}" ]; then @@ -334,7 +334,7 @@ summary_item() { echo "check: summary" summary_item "Python lint (make lint)" "$litellm_py_files" "no litellm/ Python files in scope" summary_item "tests/e2e checks (basedpyright + raw HTTP client ban)" "$e2e_py_files" "no tests/e2e Python files in scope" -summary_item "test-tree lint (ruff-tests.toml + test-quality budget)" "$test_tree_files" \ +summary_item "test-tree lint (ruff-tests.toml + test-quality gate)" "$test_tree_files" \ "no tests/ Python files or test-tree lint inputs in scope" summary_item "dashboard lint (prettier + eslint + lint budgets)" "$ui_prettier_changed$ui_eslint_changed" "no dashboard files in scope" summary_item "dashboard API-type sync (npm run gen:api)" "$spec_files" "no litellm/proxy, litellm/types, or generator files in scope" diff --git a/scripts/ruff_strict_gate.py b/scripts/ruff_strict_gate.py index 8da10dd76f0..61ae0236b53 100644 --- a/scripts/ruff_strict_gate.py +++ b/scripts/ruff_strict_gate.py @@ -1,12 +1,17 @@ #!/usr/bin/env python3 -"""Total-count gate for the strict ruff rules in ruff-strict.toml. +"""Delta-vs-base gate for the strict ruff rules in ruff-strict.toml. -Each rule has a hard ``limit`` in ruff-strict-budget.json. The gate counts each -rule across the whole tree and fails when a rule is both over its limit and -higher than the base it merges into, so a change is blamed for the violations it -adds, never for drift that already exists in the base. ``--update`` ratchets each -rule's limit down by the number of violations this branch fixed relative to its -branch point (the merge-base). +Each rule is counted across the whole tree at HEAD and at the merge-base with +the branch this change merges into, and the gate fails only when a rule grew +past the merge-base count, so a change is blamed for the violations it adds, +never for drift that already sits in the base. There is no committed budget: the merge-base count is the +ceiling, so it moves only when the base branch does. + +The merge-base counts come from scripts/lint_base_counts.py: the disk cache, +then the CI artifact published for that commit, then a ruff pass over a +detached worktree at the merge-base under the current ruff configs. +``--emit-counts-dir`` writes HEAD's counts as the file that artifact is built +from. """ import argparse @@ -17,15 +22,26 @@ import subprocess import sys import tempfile from collections import Counter +from collections.abc import Callable, Mapping, Sequence from pathlib import Path from typing import Final, NamedTuple -REPO_ROOT = Path(__file__).resolve().parent.parent -STRICT_CONFIG = REPO_ROOT / "ruff-strict.toml" -BUDGET_PATH = REPO_ROOT / "ruff-strict-budget.json" -TARGET = "litellm" +from lint_base_counts import ( + Checker, + base_counts_cached, + emit_counts, + evaluate, + head_sha, + resolve_base_point, + sha256_of, +) -_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") +REPO_ROOT: Final = Path(__file__).resolve().parent.parent +STRICT_CONFIG: Final = REPO_ROOT / "ruff-strict.toml" +BASE_CONFIG: Final = REPO_ROOT / "ruff.toml" +TARGET: Final = "litellm" + +_HUNK: Final = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") class Violation(NamedTuple): @@ -34,38 +50,22 @@ class Violation(NamedTuple): code: str -class Breach(NamedTuple): - rule: str - total: int - cap: int - added: int - - -def _run(cmd: list, cwd: Path = REPO_ROOT) -> str: - proc = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) +def _run(cmd: Sequence[str], cwd: Path = REPO_ROOT) -> str: + proc: Final = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) if proc.returncode not in (0, 1): sys.stderr.write(proc.stderr) raise SystemExit(f"{cmd[0]} exited {proc.returncode}") return proc.stdout -def resolve_base_point(base_ref: str, cwd: Path = REPO_ROOT) -> str: - """The snapshot commit base counts are measured at: merge-base(base_ref, HEAD), - made aware of an in-progress merge. Mid-merge, HEAD is still the pre-merge tip, - so its merge-base is the old branch point and every violation the base gained - since then would be blamed on this change. While MERGE_HEAD exists, prefer - merge-base(base_ref, MERGE_HEAD) whenever it is the newer of the two.""" - head_point: Final = _run(["git", "merge-base", base_ref, "HEAD"], cwd=cwd).strip() - if not head_point: - return base_ref - merge_head: Final = _run(["git", "rev-parse", "--verify", "--quiet", "MERGE_HEAD"], cwd=cwd).strip() - if not merge_head: - return head_point - merge_point: Final = _run(["git", "merge-base", base_ref, merge_head], cwd=cwd).strip() - if not merge_point: - return head_point - older: Final = _run(["git", "merge-base", head_point, merge_point], cwd=cwd).strip() - return merge_point if older == head_point else head_point +def ruff_version() -> str: + return _run(["ruff", "--version"]).strip() + + +def checker_identity( + strict_config: Path = STRICT_CONFIG, base_config: Path = BASE_CONFIG, version: Callable[[], str] = ruff_version +) -> Checker: + return Checker("ruff-strict", (sha256_of(strict_config), sha256_of(base_config), version())) def _ruff_json(cwd: Path, config: Path) -> list: @@ -76,7 +76,7 @@ def _ruff_json(cwd: Path, config: Path) -> list: return json.loads(raw or "[]") -def head_violations() -> list: +def head_violations() -> list[Violation]: out = [] for item in _ruff_json(REPO_ROOT, STRICT_CONFIG): name = Path(item["filename"]) @@ -90,47 +90,26 @@ def head_violations() -> list: return out -def count_by_rule(violations: list) -> dict: +def count_by_rule(violations: Sequence[Violation]) -> dict[str, int]: return dict(Counter(v.code for v in violations)) -def base_counts(ref: str) -> dict: - parent = Path(tempfile.mkdtemp(prefix="ruff_base_")) - worktree = parent / "wt" +def base_counts(ref: str) -> dict[str, int]: + parent: Final = Path(tempfile.mkdtemp(prefix="ruff_base_")) + worktree: Final = parent / "wt" try: _run(["git", "worktree", "add", "--detach", str(worktree), ref]) - shutil.copy(STRICT_CONFIG, worktree / "ruff-strict.toml") - items = _ruff_json(worktree, worktree / "ruff-strict.toml") + shutil.copy(BASE_CONFIG, worktree / BASE_CONFIG.name) + shutil.copy(STRICT_CONFIG, worktree / STRICT_CONFIG.name) + items: Final = _ruff_json(worktree, worktree / STRICT_CONFIG.name) return dict(Counter(item["code"] for item in items)) finally: _run(["git", "worktree", "remove", "--force", str(worktree)]) shutil.rmtree(parent, ignore_errors=True) -def over_ceiling(head: dict, budget: dict) -> frozenset: - """Rules whose head count already exceeds their limit. - - A rule can only breach when it is over its limit, so when none are the base - comparison cannot change the verdict and the base worktree scan can be skipped. - """ - return frozenset( - rule for rule, spec in budget.items() - if head.get(rule, 0) > spec["limit"] - ) - - -def evaluate(head: dict, base: dict, budget: dict) -> list: - breaches = [] - for rule, spec in budget.items(): - cap = spec["limit"] - total = head.get(rule, 0) - if total > cap and total > base.get(rule, 0): - breaches.append(Breach(rule, total, cap, total - base.get(rule, 0))) - return sorted(breaches) - - -def parse_changed_lines(diff_text: str) -> dict: - changed: dict = {} +def parse_changed_lines(diff_text: str) -> dict[str, set[int]]: + changed: dict[str, set[int]] = {} path = None for line in diff_text.splitlines(): if line.startswith("+++ b/"): @@ -142,84 +121,49 @@ def parse_changed_lines(diff_text: str) -> dict: return changed -def introduced(violations: list, changed: dict) -> list: +def introduced(violations: Sequence[Violation], changed: Mapping[str, set[int]]) -> list[Violation]: return [v for v in violations if v.line in changed.get(v.file, set())] def cmd_check(base: str) -> None: - budget = json.loads(BUDGET_PATH.read_text()) - head = head_violations() - head_counts = count_by_rule(head) - if not over_ceiling(head_counts, budget): - print(f"OK: every strict rule is within its codebase ceiling (base {base})") - return - base_point = resolve_base_point(base) - breaches = evaluate(head_counts, base_counts(base_point), budget) + head: Final = head_violations() + base_point: Final = resolve_base_point(base) + breaches: Final = evaluate(count_by_rule(head), base_counts_cached(checker_identity(), base_point, base_counts)) if not breaches: - print(f"OK: every strict rule is within its codebase ceiling (base {base})") + print(f"OK: no strict rule grew past its merge-base count (base {base})") return - new = introduced( - head, - parse_changed_lines( - _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) - ), - ) - print(f"FAIL: strict-rule totals exceed their limit (base {base}):") + diff: Final = _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) + new: Final = introduced(head, parse_changed_lines(diff)) + print(f"FAIL: strict-rule totals grew past their merge-base count (base {base}):") for breach in breaches: - print( - f" {breach.rule}: total {breach.total} over limit {breach.cap} (this change added {breach.added})" - ) + print(f" {breach.rule}: total {breach.total} over ceiling {breach.ceiling} (this change added {breach.added})") for violation in sorted(v for v in new if v.code == breach.rule): print(f" {violation.file}:{violation.line}") print( - "Reduce the new violations or remove an equal number elsewhere; the ceiling is the limit in ruff-strict-budget.json." + "Reduce the new violations or remove an equal number elsewhere; the ceiling is the merge-base count." ) raise SystemExit(1) -def ratcheted_budget(budget: dict, current: dict, base: dict) -> dict: - """Each rule's limit lowered by the violations `current` fixed vs `base`. - - `base` is the count at the branch point (the commit this branch diverged - from). The drop is clamped to what was actually cleared (a rule that grew - stays put), so the limit only ever falls. - """ - return { - rule: { - "limit": max(0, spec["limit"] - max(0, base.get(rule, 0) - current.get(rule, 0))) - } - for rule, spec in sorted(budget.items()) - } - - -def cmd_update(base_ref: str) -> None: - """Ratchet each rule's limit down by the violations this branch fixed. - - The working-tree count is compared against a ruff pass over a detached - worktree at the branch point (the merge-base with `base_ref`), so a branch's - fixes tighten its own ceilings by exactly what they cleared since it diverged. - """ - budget = json.loads(BUDGET_PATH.read_text()) - base_point = resolve_base_point(base_ref) - updated = ratcheted_budget( - budget, count_by_rule(head_violations()), base_counts(base_point) - ) - BUDGET_PATH.write_text(json.dumps(updated, indent=2, sort_keys=True) + "\n") - cleared = sum(budget[rule]["limit"] - updated[rule]["limit"] for rule in updated) - print(f"Ratcheted strict-rule limits down by {cleared} violations this branch fixed") - - def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) + parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", help="Comparison ref (default: origin's current default branch)") - parser.add_argument("--update", action="store_true") - args = parser.parse_args() + parser.add_argument( + "--emit-counts-dir", + type=Path, + help="Write HEAD's per-rule counts to this directory as a base-counts artifact instead of gating", + ) + args: Final = parser.parse_args() from default_branch import resolve_base_ref from gate_slot_lock import held_slot + if args.emit_counts_dir is not None: + with held_slot(): + emit_counts(checker_identity(), count_by_rule(head_violations()), args.emit_counts_dir, head_sha()) + return base_ref: Final = resolve_base_ref(args.base, REPO_ROOT) with held_slot(): - cmd_update(base_ref) if args.update else cmd_check(base_ref) + cmd_check(base_ref) if __name__ == "__main__": diff --git a/scripts/test_quality_gate.py b/scripts/test_quality_gate.py index e486324c741..a9ad60f088c 100644 --- a/scripts/test_quality_gate.py +++ b/scripts/test_quality_gate.py @@ -1,32 +1,24 @@ #!/usr/bin/env python3 -"""Total-count gate for the TQ* rules in scripts/check_test_quality.py. +"""Delta-vs-base gate for the TQ* rules in scripts/check_test_quality.py. Sibling of scripts/type_discipline_gate.py, pointed at the test tree instead of -the package. Each rule listed in test-quality-budget.json has a hard ``limit``. -The gate counts each rule across the whole `tests` tree and fails when a rule is -both over its limit and higher than the base it merges into, so a change is -blamed for the violations it adds, never for drift that already exists in the -base. +the package. Each rule is counted across the whole `tests` tree at HEAD and at +the merge-base with the branch this change merges into, and the gate fails only +when a rule grew past the merge-base count, so a change is blamed for the +violations it adds, never for drift that already exists in the base. There is +no committed budget: the merge-base count is the ceiling, so it moves only when +the base branch does. -Every rule is seeded at exactly its count on the day the gate landed, so the -suite's existing debt is grandfathered and any net-new violation trips the gate -immediately. ``--update`` ratchets a limit down by the violations fixed relative -to ``--base``, so the ceilings only ever fall. Base counts are measured with the -*current* checker, so a rule introduced on this branch is counted at the base too -and ratchets like every other one. The ratchet runs as a scheduled automation -against the repository's default branch, not on PR branches, so concurrent PRs never -race to edit the same limit. - -The deliberate difference from its sibling: this gate has no headroom anywhere. -Type discipline seeded LIT010/LIT011 at 1.5x to leave room for an in-flight -sweep; a test-quality violation has no such transition to absorb, so the line is -today's count and the only legal direction is down. +The merge-base counts come from scripts/lint_base_counts.py: the disk cache, +then the CI artifact published for that commit, then a pass of the current +checker over a detached worktree at the merge-base, so a rule introduced on +this branch is counted at the base too. ``--emit-counts-dir`` writes HEAD's +counts as the file that artifact is built from. """ from __future__ import annotations import argparse -import json import re import shutil import signal @@ -39,9 +31,18 @@ from pathlib import Path from types import FrameType, MappingProxyType from typing import Final, NamedTuple +from lint_base_counts import ( + Checker, + base_counts_cached, + emit_counts, + evaluate, + head_sha, + resolve_base_point, + sha256_of, +) + REPO_ROOT: Final = Path(__file__).resolve().parent.parent CHECKER: Final = REPO_ROOT / "scripts" / "check_test_quality.py" -BUDGET_PATH: Final = REPO_ROOT / "test-quality-budget.json" TARGET: Final = "tests" TERMINATION_SIGNALS: Final = (signal.SIGTERM, signal.SIGHUP) @@ -56,13 +57,6 @@ class Violation(NamedTuple): code: str -class Breach(NamedTuple): - rule: str - total: int - cap: int - added: int - - def _run(cmd: Sequence[str], cwd: Path = REPO_ROOT) -> str: proc: Final = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) if proc.returncode not in (0, 1): @@ -71,22 +65,8 @@ def _run(cmd: Sequence[str], cwd: Path = REPO_ROOT) -> str: return proc.stdout -def resolve_base_point(base_ref: str, cwd: Path = REPO_ROOT) -> str: - """The snapshot commit base counts are measured at: merge-base(base_ref, HEAD), - made aware of an in-progress merge. Mid-merge, HEAD is still the pre-merge tip, - so its merge-base is the old branch point and every violation the base gained - since then would be blamed on this change.""" - head_point: Final = _run(["git", "merge-base", base_ref, "HEAD"], cwd=cwd).strip() - if not head_point: - return base_ref - merge_head: Final = _run(["git", "rev-parse", "--verify", "--quiet", "MERGE_HEAD"], cwd=cwd).strip() - if not merge_head: - return head_point - merge_point: Final = _run(["git", "merge-base", base_ref, merge_head], cwd=cwd).strip() - if not merge_point: - return head_point - older: Final = _run(["git", "merge-base", head_point, merge_point], cwd=cwd).strip() - return merge_point if older == head_point else head_point +def checker_identity(checker: Path = CHECKER) -> Checker: + return Checker("test-quality", (sha256_of(checker),)) def _check(root: Path, checker: Path) -> tuple[Violation, ...]: @@ -143,26 +123,6 @@ def base_counts(ref: str, repo_root: Path = REPO_ROOT, checker: Path = CHECKER) shutil.rmtree(parent, ignore_errors=True) -def over_ceiling(head: Mapping[str, int], budget: Mapping[str, Mapping[str, int]]) -> frozenset[str]: - """Rules whose head count already exceeds their limit. When none are, the base - comparison cannot change the verdict and the base worktree scan is skipped.""" - return frozenset( - rule for rule, spec in budget.items() if head.get(rule, 0) > spec["limit"] - ) - - -def evaluate( - head: Mapping[str, int], - base: Mapping[str, int], - budget: Mapping[str, Mapping[str, int]], -) -> tuple[Breach, ...]: - return tuple(sorted( - Breach(rule, head.get(rule, 0), spec["limit"], head.get(rule, 0) - base.get(rule, 0)) - for rule, spec in budget.items() - if head.get(rule, 0) > spec["limit"] and head.get(rule, 0) > base.get(rule, 0) - )) - - def _hunk_lines(body: str) -> frozenset[int]: return frozenset( line @@ -191,92 +151,46 @@ def introduced( def cmd_check(base: str) -> None: - budget: Final = json.loads(BUDGET_PATH.read_text()) head: Final = head_violations() - head_counts: Final = count_by_rule(head) - if not over_ceiling(head_counts, budget): - print(f"OK: every TQ rule is within its test-suite ceiling (base {base})") - return base_point: Final = resolve_base_point(base) - base_at_point: Final = base_counts(base_point) - breaches: Final = evaluate(head_counts, base_at_point, budget) + breaches: Final = evaluate(count_by_rule(head), base_counts_cached(checker_identity(), base_point, base_counts)) if not breaches: - print(f"OK: every TQ rule is within its test-suite ceiling (base {base})") + print(f"OK: no TQ rule grew past its merge-base count (base {base})") return - new: Final = introduced( - head, - parse_changed_lines( - _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) - ), - ) - print(f"FAIL: TQ-rule totals exceed their limit (base {base}):") + diff: Final = _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) + new: Final = introduced(head, parse_changed_lines(diff)) + print(f"FAIL: TQ-rule totals grew past their merge-base count (base {base}):") for breach in breaches: - print( - f" {breach.rule}: total {breach.total} over limit {breach.cap} " - f"(this change added {breach.added})" - ) + print(f" {breach.rule}: total {breach.total} over ceiling {breach.ceiling} (this change added {breach.added})") for violation in sorted(v for v in new if v.code == breach.rule): print(f" {violation.file}:{violation.line}") print( - "Fix the new violations, or give each one a reason " - "(`# test-quality-ok: `), or remove an equal number elsewhere; " - "the ceiling is the limit in test-quality-budget.json. " - "Run `python scripts/check_test_quality.py tests/` to see every finding." + "Fix the new violations, or give each one a reason (`# test-quality-ok: `), or remove an " + "equal number elsewhere; the ceiling is the merge-base count. Run " + "`python scripts/check_test_quality.py tests/` to see every finding." ) raise SystemExit(1) -def ratcheted_budget( - budget: Mapping[str, Mapping[str, int]], - current: Mapping[str, int], - base: Mapping[str, int], -) -> Mapping[str, Mapping[str, int]]: - """Each rule's limit lowered by the violations `current` fixed vs `base`. The drop - is clamped to what was actually cleared, so a limit only ever falls.""" - return MappingProxyType({ - rule: {"limit": max(0, spec["limit"] - max(0, base.get(rule, 0) - current.get(rule, 0)))} - for rule, spec in sorted(budget.items()) - }) - - -def cmd_update(base_ref: str) -> None: - """Ratchet each rule's limit down by the violations this branch fixed.""" - budget: Final = json.loads(BUDGET_PATH.read_text()) - base_point: Final = resolve_base_point(base_ref) - updated: Final = ratcheted_budget( - budget, count_by_rule(head_violations()), base_counts(base_point) - ) - BUDGET_PATH.write_text(json.dumps(dict(updated), indent=2, sort_keys=True) + "\n") - cleared: Final = sum(budget[rule]["limit"] - updated[rule]["limit"] for rule in updated) - print(f"Ratcheted TQ-rule limits down by {cleared} violations this branch fixed") - - -def cmd_seed() -> None: - """Write the budget from the working tree's current counts. Used once, to land - the gate; afterwards `--update` is the only thing that may move a limit.""" - counts: Final = count_by_rule(head_violations()) - BUDGET_PATH.write_text( - json.dumps({rule: {"limit": counts[rule]} for rule in sorted(counts)}, indent=2) + "\n" - ) - print(f"Seeded {BUDGET_PATH.name} at " + ", ".join(f"{r}={counts[r]}" for r in sorted(counts))) - - def main() -> None: parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", help="Comparison ref (default: origin's current default branch)") - parser.add_argument("--update", action="store_true") - parser.add_argument("--seed", action="store_true") + parser.add_argument( + "--emit-counts-dir", + type=Path, + help="Write HEAD's per-rule counts to this directory as a base-counts artifact instead of gating", + ) args: Final = parser.parse_args() from default_branch import resolve_base_ref from gate_slot_lock import held_slot + if args.emit_counts_dir is not None: + with held_slot(): + emit_counts(checker_identity(), count_by_rule(head_violations()), args.emit_counts_dir, head_sha()) + return + base_ref: Final = resolve_base_ref(args.base, REPO_ROOT) with held_slot(): - if args.seed: - cmd_seed() - elif args.update: - cmd_update(resolve_base_ref(args.base, REPO_ROOT)) - else: - cmd_check(resolve_base_ref(args.base, REPO_ROOT)) + cmd_check(base_ref) if __name__ == "__main__": diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py index 023b35af8fe..2a9f1c1229f 100644 --- a/scripts/type_check_gate.py +++ b/scripts/type_check_gate.py @@ -2,15 +2,17 @@ """Delta-vs-base per-rule gate for basedpyright. basedpyright's ``--outputjson`` is reduced to a count of errors per *rule* -(``reportAny``, ``reportArgumentType``, ...) and checked against a committed -budget of the form ``{rule: {limit}}``, the same shape as -``ruff-strict-budget.json``. A rule fails only when its codebase-wide total is -both over its ``limit`` *and* higher than the count on the base it merges into, -so a change is blamed for the errors it adds, never for drift that already sits -in the base. That ``> base`` guard is what stops an unrelated PR from inheriting -a red once two PRs each land near the limit and their sum crosses it: the -bystander's count equals its base, so it is spared, while any PR that actually -grows the rule past its limit still fails. +(``reportAny``, ``reportArgumentType``, ...) at HEAD and at the merge-base with +the branch this change merges into. A rule fails only when its codebase-wide +total grew past the merge-base count, so a change is blamed for the errors it +adds, never for drift that already sits in the base, and an unrelated PR never +inherits a red from what landed next to it: its count equals its base. + +``reportAny`` and ``reportExplicitAny`` are the exception, because Any spreads: +a correct change can surface new ones far from the lines it touched. Their +ceiling is the larger of the merge-base count and a fixed codebase-wide cap in +ANY_CAPS, so a change may add some while the total stays under the cap, and the +total can never pass it. The cap moves only when someone lowers it on main. Installed packages are part of the measurement: a typed dependency that is present changes what basedpyright can prove (and therefore which diagnostics @@ -28,21 +30,16 @@ The gate runs basedpyright itself, for both the head and the base pass, with ``NODE_OPTIONS`` raised to the heap this repo needs: basedpyright's node process OOMs at the ~4 GB default, and when callers had to remember the flag, every hand-copied pipeline (Makefile, CI, a dev running the recipe by hand) -was one forgotten env line away from an 80-second crash. The base count only -matters once some rule is over its limit, so when none is the base pass is -skipped outright. When it is needed, it is a second basedpyright pass over a -detached worktree at the merge-base, run under the same environment so import -resolution matches, and its per-rule counts are cached under the repo's git +was one forgotten env line away from an 80-second crash. The base pass is a +second basedpyright run over a detached worktree at the merge-base, under the +same environment so import resolution matches, and scripts/lint_base_counts.py +spares it whenever it can: the per-rule counts are cached under the repo's git common dir keyed by merge-base commit, ``pyrightconfig.json``, ``uv.lock``, -the Prisma schema, and the dependency-group set, so re-runs against the same -branch point pay for it once. A CI workflow publishes every main commit's counts as -an artifact (``--emit-counts-dir`` is its entry point), and on a disk-cache miss -the gate first tries to download the merge-base's artifact through the ``gh`` -CLI; any fetch failure falls back silently to the local base pass, so the gate -never gets worse than it was without CI. ``--update`` ratchets each rule's ``limit`` down by the -number of errors this branch fixed relative to its branch point (the merge-base), -so the headroom you were granted shrinks by exactly what you cleared and never -grows. +the Prisma schema, and the dependency-group set, and on a disk-cache miss the +artifact publish-lint-base-counts.yml uploaded for the merge-base is +downloaded through the ``gh`` CLI (``--emit-counts-dir`` is the publisher's +entry point); any fetch failure falls back silently to the local base pass, so +the gate never gets worse than it was without CI. ``--outputjson`` is used rather than text diagnostics because the latter wrap across lines, leaving the ``(reportRule)`` on a continuation line away from the @@ -53,33 +50,36 @@ carries an unambiguous ``rule`` field. import argparse import contextlib import hashlib -import io import json import os -import re import shutil import subprocess import sys import tempfile -import zipfile from collections import Counter -from collections.abc import Callable, Iterator, Mapping, Sequence +from collections.abc import Callable, Iterator, Mapping from pathlib import Path -from typing import Final, NamedTuple +from types import MappingProxyType +from typing import Final + +from lint_base_counts import ( + Checker, + base_counts_cached, + emit_counts, + evaluate, + head_sha, + resolve_base_point, +) REPO_ROOT = Path(__file__).resolve().parent.parent -BUDGET_PATH = REPO_ROOT / "basedpyright-code-budget.json" PYRIGHT_CONFIG = REPO_ROOT / "pyrightconfig.json" UV_LOCK = REPO_ROOT / "uv.lock" -CACHE_FILE_PREFIX = "basedpyright-base-" -CACHE_KEEP_ENTRIES = 8 -ARTIFACT_NAME_PREFIX = "basedpyright-counts-" -GH_TIMEOUT_SECONDS = 10 # The one environment every basedpyright pass measures in. The group set is # the slim one the CI publisher has always installed (not bootstrap's fatter -# --extra proxy env), so the committed budgets stay valid; changing it re-keys -# every cache and artifact fingerprint, so stale counts can never be matched. +# --extra proxy env), so published and cached counts stay comparable; changing +# it re-keys every cache and artifact fingerprint, so stale counts can never be +# matched. TYPECHECK_ENV_DIR = REPO_ROOT / ".venv-typecheck" TYPECHECK_DEP_GROUPS = ("proxy-dev", "e2e-dev") PRISMA_GENERATE_SCRIPT = REPO_ROOT / "scripts" / "prisma_generate_if_needed.py" @@ -93,17 +93,7 @@ NODE_HEAP_OPTION = "--max-old-space-size=8192" # Bucket for a basedpyright diagnostic with no `rule`. Counted so it's gated. UNCODED = "" -# Limit for a rule that shows up at HEAD but isn't in the budget at all -- a -# brand-new error category (new construct, or a tool/version change). The rule -# fails once it clears this many errors. -DEFAULT_LIMIT = 10 - - -class Breach(NamedTuple): - code: str - total: int - cap: int - added: int +ANY_CAPS: Final[Mapping[str, int]] = MappingProxyType({"reportAny": 6150, "reportExplicitAny": 1440}) def _to_relative(raw: str, root: Path) -> str | None: @@ -238,25 +228,6 @@ def run_basedpyright(cwd: Path = REPO_ROOT, env_dir: Path = TYPECHECK_ENV_DIR) - return proc.stdout -def resolve_base_point(base_ref: str, cwd: Path = REPO_ROOT) -> str: - """The snapshot commit base counts are measured at: merge-base(base_ref, HEAD), - made aware of an in-progress merge. Mid-merge, HEAD is still the pre-merge tip, - so its merge-base is the old branch point and every violation the base gained - since then would be blamed on this change. While MERGE_HEAD exists, prefer - merge-base(base_ref, MERGE_HEAD) whenever it is the newer of the two.""" - head_point: Final = _run(["git", "merge-base", base_ref, "HEAD"], cwd=cwd).strip() - if not head_point: - return base_ref - merge_head: Final = _run(["git", "rev-parse", "--verify", "--quiet", "MERGE_HEAD"], cwd=cwd).strip() - if not merge_head: - return head_point - merge_point: Final = _run(["git", "merge-base", base_ref, merge_head], cwd=cwd).strip() - if not merge_point: - return head_point - older: Final = _run(["git", "merge-base", head_point, merge_point], cwd=cwd).strip() - return merge_point if older == head_point else head_point - - @contextlib.contextmanager def _temp_worktree(ref: str) -> Iterator[Path]: parent = Path(tempfile.mkdtemp(prefix="bpr_base_")) @@ -283,21 +254,6 @@ def base_counts(ref: str) -> dict[str, int]: return count_basedpyright(run_basedpyright(worktree), root=worktree) -def over_ceiling( - head: Mapping[str, int], budget: Mapping[str, Mapping[str, int]] -) -> frozenset[str]: - """Rules whose head count already exceeds their limit. - - A rule can only breach when it is over its limit, so when none are the base - comparison cannot change the verdict and the base worktree pass can be skipped. - """ - return frozenset( - code - for code, total in head.items() - if total > (budget[code]["limit"] if code in budget else DEFAULT_LIMIT) - ) - - def environment_fingerprints( dep_groups: tuple[str, ...] = TYPECHECK_DEP_GROUPS, ) -> tuple[str, ...]: @@ -311,377 +267,70 @@ def environment_fingerprints( ) -def cache_key(base_point: str, fingerprints: tuple[str, ...]) -> str: - return hashlib.sha256("|".join((base_point, *fingerprints)).encode()).hexdigest()[ - :16 - ] - - -def cache_path( - directory: Path, base_point: str, fingerprints: tuple[str, ...] -) -> Path: - return directory / f"{CACHE_FILE_PREFIX}{cache_key(base_point, fingerprints)}.json" - - -def default_cache_dir() -> Path: - common = Path(_run(["git", "rev-parse", "--git-common-dir"]).strip()) - resolved = common if common.is_absolute() else REPO_ROOT / common - return resolved / "litellm-lint-cache" - - -def validated_counts(data: object) -> dict[str, int] | None: - counts: Final = data.get("counts") if isinstance(data, dict) else None - if not isinstance(counts, dict): - return None - if not all( - isinstance(code, str) and isinstance(total, int) and not isinstance(total, bool) - for code, total in counts.items() - ): - return None - return counts - - -def load_cached_counts(path: Path) -> dict[str, int] | None: - try: - data = json.loads(path.read_text()) - except (OSError, json.JSONDecodeError): - return None - return validated_counts(data) - - -def scratch_path(path: Path) -> Path: - """In-flight scratch for the tmp+rename write. Dot-prefixed so the prune - glob in `store_counts` can never match it (a concurrent run would otherwise - unlink it between write and rename), and pid-suffixed so two concurrent - writers of the same entry never share a scratch.""" - return path.with_name(f".{path.name}.{os.getpid()}.tmp") - - -def counts_payload(base_point: str, counts: Mapping[str, int]) -> str: - return ( - json.dumps( - {"base_point": base_point, "counts": dict(sorted(counts.items()))}, - indent=2, - ) - + "\n" - ) - - -def entry_recency(path: Path) -> float: - try: - return path.stat().st_mtime - except OSError: - return 0.0 - - -def evicted_beyond_cap(entries: Sequence[Path], keep: int) -> tuple[Path, ...]: - newest_first: Final = sorted(entries, key=entry_recency, reverse=True) - return tuple(newest_first[keep:]) - - -def store_counts( - directory: Path, path: Path, base_point: str, counts: Mapping[str, int] -) -> None: - directory.mkdir(parents=True, exist_ok=True) - scratch = scratch_path(path) - scratch.write_text(counts_payload(base_point, counts)) - scratch.replace(path) - siblings: Final = tuple( - entry for entry in directory.glob(f"{CACHE_FILE_PREFIX}*.json") if entry != path - ) - for stale in evicted_beyond_cap(siblings, CACHE_KEEP_ENTRIES - 1): - stale.unlink(missing_ok=True) - - -def parse_origin_slug(url: str) -> str | None: - match: Final = re.fullmatch( - r"(?:git@github\.com:|https://github\.com/)([^/]+/[^/]+?)(?:\.git)?/?", - url.strip(), - ) - return match.group(1) if match else None - - -def origin_slug() -> str | None: - proc: Final = subprocess.run( - ["git", "remote", "get-url", "origin"], - cwd=REPO_ROOT, - capture_output=True, - text=True, - ) - if proc.returncode != 0: - return None - return parse_origin_slug(proc.stdout) - - -def artifact_name(base_point: str) -> str: - return f"{ARTIFACT_NAME_PREFIX}{cache_key(base_point, environment_fingerprints())}" - - -def _gh_output(args: list[str]) -> bytes | None: - try: - proc = subprocess.run( - ["gh", *args], capture_output=True, timeout=GH_TIMEOUT_SECONDS - ) - except (OSError, subprocess.SubprocessError): - return None - return proc.stdout if proc.returncode == 0 else None - - -def _parsed_json(raw: bytes) -> object | None: - try: - return json.loads(raw) - except ValueError: - return None - - -def _artifact_download_url(listing: object) -> str | None: - artifacts: Final = listing.get("artifacts") if isinstance(listing, dict) else None - if not isinstance(artifacts, list) or not artifacts: - return None - newest: Final = artifacts[0] - if not isinstance(newest, dict) or newest.get("expired"): - return None - url: Final = newest.get("archive_download_url") - return url if isinstance(url, str) else None - - -def _counts_json_from_zip(zip_bytes: bytes) -> object | None: - try: - with zipfile.ZipFile(io.BytesIO(zip_bytes)) as archive: - members: Final = [ - name for name in archive.namelist() if name.endswith(".json") - ] - if len(members) != 1: - return None - return json.loads(archive.read(members[0])) - except (zipfile.BadZipFile, ValueError, OSError): - return None - - -def counts_for_base(payload: object, base_point: str) -> dict[str, int] | None: - if not isinstance(payload, dict) or payload.get("base_point") != base_point: - return None - counts: Final = validated_counts(payload) - return counts if counts else None - - -def _fetch_fallback(reason: str) -> None: - sys.stderr.write(f"{reason}; computing base counts locally\n") - - -def fetch_ci_base_counts( - base_point: str, - gh_output: Callable[[list[str]], bytes | None] = _gh_output, -) -> dict[str, int] | None: - """Base counts from the CI artifact published for `base_point`, or None. - - Every failure mode (no gh, no auth, offline, expired or missing artifact, - malformed payload, counts for a different commit) returns None so the - caller falls back to the local base pass; the fetch is an optimization and - must never make the gate less available than local compute alone.""" - slug: Final = origin_slug() - if slug is None: - return _fetch_fallback("origin remote is not a github.com URL") - name: Final = artifact_name(base_point) - listing: Final = gh_output( - ["api", f"repos/{slug}/actions/artifacts?name={name}&per_page=1"] - ) - if listing is None: - return _fetch_fallback(f"could not list CI artifacts named {name}") - url: Final = _artifact_download_url(_parsed_json(listing)) - if url is None: - return _fetch_fallback(f"no usable CI artifact named {name}") - zip_bytes: Final = gh_output(["api", url]) - if zip_bytes is None: - return _fetch_fallback(f"download failed for CI artifact {name}") - counts: Final = counts_for_base(_counts_json_from_zip(zip_bytes), base_point) - if counts is None: - return _fetch_fallback( - f"CI artifact {name} is not valid base counts for {base_point[:12]}" - ) - sys.stderr.write(f"base counts fetched from CI artifact {name}\n") - return counts - - -def base_counts_cached( - base_point: str, - cache_dir: Path | None = None, - compute: Callable[[str], dict[str, int]] = base_counts, - fetch: Callable[[str], dict[str, int] | None] = fetch_ci_base_counts, -) -> dict[str, int]: - """`base_counts` memoized on disk. The base tree at a given commit is - immutable, so its counts are a pure function of the merge-base plus the - environment fingerprints in the cache key; an empty result is never stored - because it is the signature of a crashed pass, not a clean tree. On a disk - miss the counts CI already published for the merge-base are fetched before - the expensive local base pass; a fetch miss of any kind computes locally.""" - directory = default_cache_dir() if cache_dir is None else cache_dir - path = cache_path(directory, base_point, environment_fingerprints()) - cached = load_cached_counts(path) - if cached is not None: - return cached - fetched: Final = fetch(base_point) - if fetched: - store_counts(directory, path, base_point, fetched) - return fetched - counts = compute(base_point) - if counts: - store_counts(directory, path, base_point, counts) - return counts - - -def evaluate( - head: Mapping[str, int], - base: Mapping[str, int], - budget: Mapping[str, Mapping[str, int]], -) -> list[Breach]: - breaches = [] - for code, total in head.items(): - spec = budget.get(code) - cap = spec["limit"] if spec else DEFAULT_LIMIT - prior = base.get(code, 0) - if total > cap and total > prior: - breaches.append(Breach(code, total, cap, total - prior)) - return sorted(breaches) - - -def is_vacuous_run( - counts: Mapping[str, int], budget: Mapping[str, Mapping[str, int]] -) -> bool: - """True when nothing was parsed but the budget expects errors -- the - signature of a type checker that produced no output. `run_basedpyright` - already fails crash exit codes, so this guards the remaining case: a run - that exits cleanly while emitting nothing, which would otherwise clear - every limit and pass silently.""" - return not counts and any(spec["limit"] for spec in budget.values()) - - -def ratcheted_budget( - budget: Mapping[str, Mapping[str, int]], - current: Mapping[str, int], - base: Mapping[str, int], -) -> dict[str, dict[str, int]]: - """Each rule's limit lowered by the errors `current` fixed vs `base`. - - `base` is the count at the branch point (the commit this branch diverged - from). The drop is clamped to what was actually cleared (a rule that grew - stays put), so the limit only ever falls. Rules absent from the budget are - dropped: a genuinely new error category is added to the JSON deliberately, - not on update. - """ - return { - code: { - "limit": max(0, spec["limit"] - max(0, base.get(code, 0) - current.get(code, 0))) - } - for code, spec in sorted(budget.items()) - } - - -def cmd_update(current: Mapping[str, int], base_ref: str) -> None: - """Ratchet each rule's limit down by the errors this branch fixed. - - `current` is the working-tree count; the reference count comes - from a second basedpyright pass over a detached worktree at the branch point - (the merge-base with `base_ref`), so a branch's fixes tighten its own ceilings - by exactly what they cleared since it diverged, and limits never rise. - """ - budget = json.loads(BUDGET_PATH.read_text()) if BUDGET_PATH.exists() else {} - base_point = resolve_base_point(base_ref) - updated = ratcheted_budget(budget, current, base_counts_cached(base_point)) - BUDGET_PATH.write_text(json.dumps(updated, indent=2, sort_keys=True) + "\n") - cleared = sum(budget[code]["limit"] - updated[code]["limit"] for code in updated) - print( - f"Ratcheted basedpyright limits down by {cleared} errors this branch fixed " - f"across {len(updated)} rules" - ) - - -def cmd_emit_counts(head: Mapping[str, int], directory: Path, head_sha: str) -> None: - """Write HEAD's per-rule counts as the file the publisher workflow uploads. - - The filename stem is exactly the artifact name `fetch_ci_base_counts` will - later look up for this commit, so emit and fetch cannot drift apart. Empty - counts are refused for the same reason `is_vacuous_run` exists: a pass that - produced nothing almost certainly crashed, and publishing it would poison - every branch that fetches it.""" - if not head: - print( - "FAIL: basedpyright produced no errors; refusing to publish empty base " - "counts because the pass almost certainly crashed or emitted nothing." - ) - raise SystemExit(1) - name: Final = artifact_name(head_sha) - directory.mkdir(parents=True, exist_ok=True) - (directory / f"{name}.json").write_text(counts_payload(head_sha, head)) - print( - f"Emitted base counts for {head_sha} as {name}.json " - f"({sum(head.values())} errors total)" - ) +def checker_identity(dep_groups: tuple[str, ...] = TYPECHECK_DEP_GROUPS) -> Checker: + return Checker("basedpyright", environment_fingerprints(dep_groups)) def cmd_check(head: Mapping[str, int], base_ref: str) -> None: - budget = json.loads(BUDGET_PATH.read_text()) - if is_vacuous_run(head, budget): - expected = sum(spec["limit"] for spec in budget.values()) + if not head: print( - f"FAIL: basedpyright produced no errors, but {BUDGET_PATH.name} allows " - f"up to ~{expected}. The type checker almost certainly crashed or emitted " - f"nothing; refusing to certify a vacuous run." + "FAIL: basedpyright produced no errors. The type checker almost certainly " + "crashed or emitted nothing; refusing to certify a vacuous run." ) raise SystemExit(1) - if not over_ceiling(head, budget): + base_point: Final = resolve_base_point(base_ref) + base: Final = base_counts_cached(checker_identity(), base_point, base_counts) + if not base: print( - f"OK: every rule is within its basedpyright limit ({sum(head.values())} errors total)" - ) - return - base_point = resolve_base_point(base_ref) - base = base_counts_cached(base_point) - if is_vacuous_run(base, budget): - print( - f"FAIL: basedpyright produced no errors for the base tree at " - f"{base_point[:12]}, so every rule would look freshly added. The base " - f"pass almost certainly crashed; refusing to blame this change for it." + f"FAIL: basedpyright produced no errors for the base tree at {base_point[:12]}, " + "so every rule would look freshly added. The base pass almost certainly " + "crashed; refusing to blame this change for it." ) raise SystemExit(1) - breaches = evaluate(head, base, budget) + judge(head, base, base_point) + + +def judge(head: Mapping[str, int], base: Mapping[str, int], base_point: str) -> None: + breaches: Final = evaluate(head, base, ANY_CAPS) if not breaches: print( - f"OK: every rule is within its basedpyright limit or no higher than base ({sum(head.values())} errors total)" + f"OK: every basedpyright rule is within its ceiling " + f"({sum(head.values())} errors total, base {base_point[:12]})" ) return - print("FAIL: basedpyright errors exceed the per-rule limit:") + print(f"FAIL: basedpyright errors grew past their ceiling (base {base_point[:12]}):") for breach in breaches: - print( - f" {breach.code}: total {breach.total} over limit {breach.cap} (this change added {breach.added})" - ) + print(f" {breach.rule}: total {breach.total} over ceiling {breach.ceiling} (this change added {breach.added})") print( - "Reduce the new errors or remove an equal number elsewhere; the ceiling is " - "the limit in basedpyright-code-budget.json." + "Reduce the new errors or remove an equal number elsewhere; the ceiling is the merge-base " + "count, or the cap in ANY_CAPS (scripts/type_check_gate.py) when that is higher." ) - summary = "; ".join(f"{b.code} {b.total}/{b.cap} (+{b.added})" for b in breaches) + summary: Final = "; ".join(f"{b.rule} {b.total}/{b.ceiling} (+{b.added})" for b in breaches) print(f"BREACHED RULES: {summary}") raise SystemExit(1) def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) + parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", help="Comparison ref (default: origin's current default branch)") - parser.add_argument("--update", action="store_true") - parser.add_argument("--emit-counts-dir", type=Path) - args = parser.parse_args() + parser.add_argument( + "--emit-counts-dir", + type=Path, + help="Write HEAD's per-rule counts to this directory as a base-counts artifact instead of gating", + ) + args: Final = parser.parse_args() from default_branch import resolve_base_ref from gate_slot_lock import held_slot - base_ref: Final = None if args.emit_counts_dir is not None else resolve_base_ref(args.base, REPO_ROOT) + if args.emit_counts_dir is not None: + with held_slot(): + ensure_typecheck_env() + emit_counts(checker_identity(), count_basedpyright(run_basedpyright()), args.emit_counts_dir, head_sha()) + return + base_ref: Final = resolve_base_ref(args.base, REPO_ROOT) with held_slot(): ensure_typecheck_env() - head = count_basedpyright(run_basedpyright()) - if args.emit_counts_dir is not None: - cmd_emit_counts( - head, args.emit_counts_dir, _run(["git", "rev-parse", "HEAD"]).strip() - ) - elif base_ref is not None: - cmd_update(head, base_ref) if args.update else cmd_check(head, base_ref) + cmd_check(count_basedpyright(run_basedpyright()), base_ref) if __name__ == "__main__": diff --git a/scripts/type_discipline_gate.py b/scripts/type_discipline_gate.py index 3a0564c3928..5e6745c041d 100644 --- a/scripts/type_discipline_gate.py +++ b/scripts/type_discipline_gate.py @@ -1,57 +1,62 @@ #!/usr/bin/env python3 -"""Total-count gate for the LIT* rules in scripts/check_type_discipline.py. +"""Delta-vs-base gate for the LIT* rules in scripts/check_type_discipline.py. -Sibling of scripts/ruff_strict_gate.py. Each rule listed in -type-discipline-budget.json has a hard ``limit``. The gate counts each rule -across the whole `litellm` tree and fails when a rule is both over its limit and -higher than the base it merges into, so a change is blamed for the violations it -adds, never for drift that already exists in the base. +Sibling of scripts/ruff_strict_gate.py. Each rule is counted across the whole +`litellm` tree at HEAD and at the merge-base with the branch this change merges +into, and the gate fails only when a rule grew past the merge-base count, so a +change is blamed for the violations it adds, never for drift that already exists +in the base. There is no committed budget: the merge-base count is the ceiling, +so it moves only when the base branch does. -Rules not present in the budget are ignored, but today every rule the checker -emits is gated: LIT001 (mutable collection in any annotation), LIT003/LIT004 -(noqa / pyright-mypy ignore without codes or reason), LIT006 (cast), LIT008 (`**kwargs`), LIT009 (inert -`# type: ignore`, dead syntax while enableTypeIgnoreComments is false), LIT010 -(assignment without a Final declaration; suppress deliberate rebinding with -`# rebind-ok: `), LIT011 (parameter rebinding or in-place mutation), and -LIT012 (TypedDict field without a `ReadOnly[...]` qualifier; suppress with -`# writable-ok: `), and LIT014 (comprehension with more than one `for` -or `if` clause; suppress with `# comprehension-ok: ` on a spanned -line, which belongs to the innermost violating comprehension spanning it and -to any single-line violating comprehension on that line), and LIT015 -(pydantic model not frozen; suppress with `# frozen-ok: `) carry limits at -or above their current count to ratchet down; LIT005 (`*-ok` suppression -without a reason) is frozen at limit 0 -so any net-new reasonless suppression trips the gate; LIT013 (`*-ok` suppression -that suppresses nothing) is frozen at 0 for the same reason; and LIT007 -(TypeGuard/TypeIs) is a hard zero. -LIT010 and LIT011 were seeded at 1.5x the count left after the sweep that -annotated every never-rebound name with Final, so that headroom is the hard -line new code cannot cross. -``--update`` ratchets a limit down by the violations this branch fixed relative -to its branch point (the merge-base). A rule absent from the budget at the -merge-base was seeded on this branch; ``--update`` leaves its limit untouched, -because the base tree predates the rule and its whole grandfathered count would -otherwise be misread as "fixed", collapsing the deliberate headroom to zero. +Every rule the checker emits is gated: LIT001 (mutable collection in any +annotation), LIT003/LIT004 (noqa / pyright-mypy ignore without codes or +reason), LIT005 (`*-ok` suppression without a reason), LIT006 (cast), LIT007 +(TypeGuard/TypeIs), LIT008 (`**kwargs`), LIT009 (inert `# type: ignore`, dead +syntax while enableTypeIgnoreComments is false), LIT010 (assignment without a +Final declaration; suppress deliberate rebinding with `# rebind-ok: `), +LIT011 (parameter rebinding or in-place mutation), LIT012 (TypedDict field +without a `ReadOnly[...]` qualifier; suppress with `# writable-ok: `), +LIT013 (`*-ok` suppression that suppresses nothing), LIT014 (comprehension +with more than one `for` or `if` clause; suppress with +`# comprehension-ok: ` on a spanned line, which belongs to the innermost +violating comprehension spanning it and to any single-line violating +comprehension on that line), and LIT015 (pydantic model not frozen; suppress +with `# frozen-ok: `). + +The merge-base counts come from scripts/lint_base_counts.py: the disk cache, +then the CI artifact published for that commit, then a pass of the current +checker over a detached worktree at the merge-base, so a rule change on this +branch is measured on both sides. ``--emit-counts-dir`` writes HEAD's counts +as the file that artifact is built from. """ import argparse -import json import re import shutil import subprocess import sys import tempfile from collections import Counter +from collections.abc import Mapping, Sequence from pathlib import Path from typing import Final, NamedTuple -REPO_ROOT = Path(__file__).resolve().parent.parent -CHECKER = REPO_ROOT / "scripts" / "check_type_discipline.py" -BUDGET_PATH = REPO_ROOT / "type-discipline-budget.json" -TARGET = "litellm" +from lint_base_counts import ( + Checker, + base_counts_cached, + emit_counts, + evaluate, + head_sha, + resolve_base_point, + sha256_of, +) -_HUNK = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") -_LINE = re.compile(r"^(?P.+?):(?P\d+): (?PLIT\d+) ") +REPO_ROOT: Final = Path(__file__).resolve().parent.parent +CHECKER: Final = REPO_ROOT / "scripts" / "check_type_discipline.py" +TARGET: Final = "litellm" + +_HUNK: Final = re.compile(r"^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@") +_LINE: Final = re.compile(r"^(?P.+?):(?P\d+): (?PLIT\d+) ") class Violation(NamedTuple): @@ -60,41 +65,19 @@ class Violation(NamedTuple): code: str -class Breach(NamedTuple): - rule: str - total: int - cap: int - added: int - - -def _run(cmd: list, cwd: Path = REPO_ROOT) -> str: - proc = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) +def _run(cmd: Sequence[str], cwd: Path = REPO_ROOT) -> str: + proc: Final = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True) if proc.returncode not in (0, 1): sys.stderr.write(proc.stderr) raise SystemExit(f"{cmd[0]} exited {proc.returncode}") return proc.stdout -def resolve_base_point(base_ref: str, cwd: Path = REPO_ROOT) -> str: - """The snapshot commit base counts are measured at: merge-base(base_ref, HEAD), - made aware of an in-progress merge. Mid-merge, HEAD is still the pre-merge tip, - so its merge-base is the old branch point and every violation the base gained - since then would be blamed on this change. While MERGE_HEAD exists, prefer - merge-base(base_ref, MERGE_HEAD) whenever it is the newer of the two.""" - head_point: Final = _run(["git", "merge-base", base_ref, "HEAD"], cwd=cwd).strip() - if not head_point: - return base_ref - merge_head: Final = _run(["git", "rev-parse", "--verify", "--quiet", "MERGE_HEAD"], cwd=cwd).strip() - if not merge_head: - return head_point - merge_point: Final = _run(["git", "merge-base", base_ref, merge_head], cwd=cwd).strip() - if not merge_point: - return head_point - older: Final = _run(["git", "merge-base", head_point, merge_point], cwd=cwd).strip() - return merge_point if older == head_point else head_point +def checker_identity(checker: Path = CHECKER) -> Checker: + return Checker("type-discipline", (sha256_of(checker),)) -def _check(root: Path, checker: Path) -> list: +def _check(root: Path, checker: Path) -> list[Violation]: # Resolve root first: on macOS tempfile dirs (/var/...) resolve to /private/var/..., # and the checker prints already-resolved absolute paths, so relative_to would fail. root = root.resolve() @@ -111,22 +94,22 @@ def _check(root: Path, checker: Path) -> list: return found -def head_violations() -> list: +def head_violations() -> list[Violation]: return _check(REPO_ROOT, CHECKER) -def count_by_rule(violations: list) -> dict: +def count_by_rule(violations: Sequence[Violation]) -> dict[str, int]: return dict(Counter(v.code for v in violations)) -def base_counts(ref: str) -> dict: - parent = Path(tempfile.mkdtemp(prefix="lit_base_")) - worktree = parent / "wt" +def base_counts(ref: str) -> dict[str, int]: + parent: Final = Path(tempfile.mkdtemp(prefix="lit_base_")) + worktree: Final = parent / "wt" try: _run(["git", "worktree", "add", "--detach", str(worktree), ref]) # Measure the base with the *current* rule logic, not whatever shipped at base. (worktree / "scripts").mkdir(parents=True, exist_ok=True) - checker = worktree / "scripts" / "check_type_discipline.py" + checker: Final = worktree / "scripts" / "check_type_discipline.py" shutil.copy(CHECKER, checker) return count_by_rule(_check(worktree, checker)) finally: @@ -141,27 +124,8 @@ def base_counts(ref: str) -> dict: shutil.rmtree(parent, ignore_errors=True) -def over_ceiling(head: dict, budget: dict) -> frozenset: - """Rules whose head count already exceeds their limit. - - A rule can only breach when it is over its limit, so when none are the base - comparison cannot change the verdict and the base worktree scan can be skipped. - """ - return frozenset(rule for rule, spec in budget.items() if head.get(rule, 0) > spec["limit"]) - - -def evaluate(head: dict, base: dict, budget: dict) -> list: - breaches = [] - for rule, spec in budget.items(): - cap = spec["limit"] - total = head.get(rule, 0) - if total > cap and total > base.get(rule, 0): - breaches.append(Breach(rule, total, cap, total - base.get(rule, 0))) - return sorted(breaches) - - -def parse_changed_lines(diff_text: str) -> dict: - changed: dict = {} +def parse_changed_lines(diff_text: str) -> dict[str, set[int]]: + changed: dict[str, set[int]] = {} path = None for line in diff_text.splitlines(): if line.startswith("+++ b/"): @@ -173,104 +137,54 @@ def parse_changed_lines(diff_text: str) -> dict: return changed -def introduced(violations: list, changed: dict) -> list: +def introduced(violations: Sequence[Violation], changed: Mapping[str, set[int]]) -> list[Violation]: return [v for v in violations if v.line in changed.get(v.file, set())] def cmd_check(base: str) -> None: - budget = json.loads(BUDGET_PATH.read_text()) - head = head_violations() - head_counts = count_by_rule(head) - if not over_ceiling(head_counts, budget): - print(f"OK: every LIT rule is within its codebase ceiling (base {base})") - return - base_point = resolve_base_point(base) - breaches = evaluate(head_counts, base_counts(base_point), budget) + head: Final = head_violations() + base_point: Final = resolve_base_point(base) + breaches: Final = evaluate(count_by_rule(head), base_counts_cached(checker_identity(), base_point, base_counts)) if not breaches: - print(f"OK: every LIT rule is within its codebase ceiling (base {base})") + print(f"OK: no LIT rule grew past its merge-base count (base {base})") return - new = introduced( - head, - parse_changed_lines(_run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET])), - ) - print(f"FAIL: LIT-rule totals exceed their limit (base {base}):") + diff: Final = _run(["git", "diff", base_point, "--unified=0", "--no-color", "--", TARGET]) + new: Final = introduced(head, parse_changed_lines(diff)) + print(f"FAIL: LIT-rule totals grew past their merge-base count (base {base}):") for breach in breaches: - print(f" {breach.rule}: total {breach.total} over limit {breach.cap} (this change added {breach.added})") + print(f" {breach.rule}: total {breach.total} over ceiling {breach.ceiling} (this change added {breach.added})") for violation in sorted(v for v in new if v.code == breach.rule): print(f" {violation.file}:{violation.line}") print( "Remove the new violations, give each a reason (`# noqa: XXX # `, " - "`# pyright: ignore[rule] # `, `# mutable-ok: `, " - "`# cast-ok: `, `# guard-ok: `, `# kwargs-ok: `, " - "`# rebind-ok: `, `# writable-ok: `, " - "`# comprehension-ok: `, `# frozen-ok: `), or remove an equal " - "number elsewhere; the ceiling " - "is the limit in type-discipline-budget.json." + "`# pyright: ignore[rule] # `, `# mutable-ok: `, `# cast-ok: `, " + "`# guard-ok: `, `# kwargs-ok: `, `# rebind-ok: `, " + "`# writable-ok: `, `# comprehension-ok: `, `# frozen-ok: `), " + "or remove an equal number " + "elsewhere; the ceiling is the merge-base count." ) raise SystemExit(1) -def ratcheted_budget(budget: dict, current: dict, base: dict, seeded: frozenset = frozenset()) -> dict: - """Each rule's limit lowered by the violations `current` fixed vs `base`. - - `base` is the count at the branch point (the commit this branch diverged - from). The drop is clamped to what was actually cleared (a rule that grew - stays put), so the limit only ever falls. Rules in `seeded` were introduced - on this branch with deliberate grandfathered headroom; their limits pass - through untouched, since the base predates the rule and comparing against it - would misread the entire grandfathered count as fixed. - """ - return { - rule: { - "limit": spec["limit"] - if rule in seeded - else max(0, spec["limit"] - max(0, base.get(rule, 0) - current.get(rule, 0))) - } - for rule, spec in sorted(budget.items()) - } - - -def _base_budget_rules(base_point: str) -> frozenset: - proc = subprocess.run( - ["git", "show", f"{base_point}:{BUDGET_PATH.name}"], - cwd=REPO_ROOT, - capture_output=True, - text=True, - ) - if proc.returncode != 0: - return frozenset() - return frozenset(json.loads(proc.stdout)) - - -def cmd_update(base_ref: str) -> None: - """Ratchet each rule's limit down by the violations this branch fixed. - - The working-tree count is compared against a checker pass over a detached - worktree at the branch point (the merge-base with `base_ref`), so a branch's - fixes tighten its own ceilings by exactly what they cleared since it diverged. - """ - budget = json.loads(BUDGET_PATH.read_text()) - base_point = resolve_base_point(base_ref) - seeded = frozenset(budget) - _base_budget_rules(base_point) - updated = ratcheted_budget(budget, count_by_rule(head_violations()), base_counts(base_point), seeded) - BUDGET_PATH.write_text(json.dumps(updated, indent=2, sort_keys=True) + "\n") - cleared = sum(budget[rule]["limit"] - updated[rule]["limit"] for rule in updated) - print(f"Ratcheted LIT-rule limits down by {cleared} violations this branch fixed") - if seeded: - print("Left untouched (seeded on this branch, absent from the base budget): " + ", ".join(sorted(seeded))) - - def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) + parser: Final = argparse.ArgumentParser(description=__doc__) parser.add_argument("--base", help="Comparison ref (default: origin's current default branch)") - parser.add_argument("--update", action="store_true") - args = parser.parse_args() + parser.add_argument( + "--emit-counts-dir", + type=Path, + help="Write HEAD's per-rule counts to this directory as a base-counts artifact instead of gating", + ) + args: Final = parser.parse_args() from default_branch import resolve_base_ref from gate_slot_lock import held_slot + if args.emit_counts_dir is not None: + with held_slot(): + emit_counts(checker_identity(), count_by_rule(head_violations()), args.emit_counts_dir, head_sha()) + return base_ref: Final = resolve_base_ref(args.base, REPO_ROOT) with held_slot(): - cmd_update(base_ref) if args.update else cmd_check(base_ref) + cmd_check(base_ref) if __name__ == "__main__": diff --git a/test-quality-budget.json b/test-quality-budget.json deleted file mode 100644 index 6f3dde8461b..00000000000 --- a/test-quality-budget.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "TQ001": { - "limit": 733 - }, - "TQ002": { - "limit": 737 - }, - "TQ003": { - "limit": 62 - }, - "TQ004": { - "limit": 469 - }, - "TQ005": { - "limit": 2399 - }, - "TQ006": { - "limit": 34 - }, - "TQ007": { - "limit": 117 - }, - "TQ009": { - "limit": 59 - } -} diff --git a/tests/unit/test_budget_ratchet_check.py b/tests/unit/test_budget_ratchet_check.py deleted file mode 100644 index b359b8b42e5..00000000000 --- a/tests/unit/test_budget_ratchet_check.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Tests for scripts/budget_ratchet_check.py. - -The guard's contract is "limits may only fall": a raised limit, a dropped rule, or -a deleted file is a regression, while a lowered/equal limit, a brand-new rule, or a -brand-new budget file is fine. Each branch is pinned here. -""" - -import importlib.util -import subprocess -import sys -from pathlib import Path -from typing import Final - -_MODULE_PATH = ( - Path(__file__).resolve().parents[2] / "scripts" / "budget_ratchet_check.py" -) -_spec = importlib.util.spec_from_file_location("budget_ratchet_check", _MODULE_PATH) -ratchet = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(ratchet) - - -def _spec_of(limit): - return {"limit": limit} - - -def test_limits_read_the_limit_and_skip_malformed(): - limits = ratchet._limits({"LIT006": _spec_of(1023), "junk": 5}) - assert limits == {"LIT006": 1023} # malformed (non-dict) spec ignored - - -def test_limits_fall_back_to_legacy_baseline_plus_slack(): - # The base side of a diff can predate the `limit` migration; its ceiling is - # baseline + slack, read on the same footing as a new-schema `limit`. - assert ratchet._limits({"LIT006": {"baseline": 1013, "slack": 10}}) == {"LIT006": 1023} - - -def test_migration_from_legacy_schema_to_equal_limit_is_clean(): - # baseline+slack (1023) -> limit 1023 is the same ceiling, so no regression. - base = {"LIT006": {"baseline": 1013, "slack": 10}} - assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1023)}) == [] - # ...and a genuine raise across the migration is still caught. - regs = ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1024)}) - assert [r.rule for r in regs] == ["LIT006"] and "1023 -> 1024" in regs[0].detail - - -def test_raised_limit_is_a_regression(): - base = {"LIT006": _spec_of(1023)} - head = {"LIT006": _spec_of(1024)} - regs = ratchet.regressions_for("b.json", base, head) - assert [r.rule for r in regs] == ["LIT006"] - assert "1023 -> 1024" in regs[0].detail - - -def test_lowered_or_equal_limit_is_clean(): - base = {"LIT006": _spec_of(1023)} - # limit drops - assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1000)}) == [] - # nothing changes - assert ratchet.regressions_for("b.json", base, {"LIT006": _spec_of(1023)}) == [] - - -def test_dropped_rule_is_a_regression(): - regs = ratchet.regressions_for("b.json", {"LIT007": _spec_of(0)}, {}) - assert [r.rule for r in regs] == ["LIT007"] - assert "dropped" in regs[0].detail - - -def test_new_rule_in_head_is_clean(): - assert ratchet.regressions_for("b.json", {}, {"new-rule": _spec_of(5)}) == [] - - -def test_dropped_rule_that_graduated_to_a_hard_failing_config_is_clean(): - base = {"UP006": _spec_of(0)} - assert ratchet.regressions_for("b.json", base, {}, graduated=("UP006",)) == [] - - -def test_graduation_matches_by_prefix_like_ruff_selectors_do(): - base = {"ANN202": _spec_of(865)} - assert ratchet.regressions_for("b.json", base, {}, graduated=("ANN",)) == [] - - -def test_an_unrelated_graduation_does_not_excuse_a_dropped_rule(): - base = {"C901": _spec_of(3)} - regs = ratchet.regressions_for("b.json", base, {}, graduated=("UP006", "SIM118")) - assert [r.rule for r in regs] == ["C901"] - assert "dropped" in regs[0].detail - - -def test_graduation_never_excuses_a_raised_limit(): - base = {"UP006": _spec_of(0)} - regs = ratchet.regressions_for("b.json", base, {"UP006": _spec_of(7)}, graduated=("UP006",)) - assert [r.rule for r in regs] == ["UP006"] - assert "0 -> 7" in regs[0].detail - - -def test_dropped_rule_the_checker_retired_is_clean(): - base: Final = {"TQ008": _spec_of(10993)} - assert ratchet.regressions_for("b.json", base, {}, retired=frozenset({"TQ008"})) == [] - - -def test_dropped_rule_the_checker_still_emits_is_a_regression(): - base: Final = {"TQ001": _spec_of(5), "TQ008": _spec_of(10993)} - regs: Final = ratchet.regressions_for("b.json", base, {}, retired=frozenset({"TQ008"})) - assert [r.rule for r in regs] == ["TQ001"] - assert "dropped" in regs[0].detail - - -def test_retirement_never_excuses_a_raised_limit(): - base: Final = {"TQ008": _spec_of(0)} - regs: Final = ratchet.regressions_for("b.json", base, {"TQ008": _spec_of(7)}, retired=frozenset({"TQ008"})) - assert [r.rule for r in regs] == ["TQ008"] - assert "0 -> 7" in regs[0].detail - - -def test_retired_rules_come_from_the_paired_checker(): - base: Final = {"TQ001": _spec_of(5), "TQ008": _spec_of(10993)} - assert ratchet.retired_rules("test-quality-budget.json", base) == frozenset({"TQ008"}) - - -def test_budgets_without_a_paired_checker_never_retire(): - base: Final = {"TQ008": _spec_of(1)} - for rel in ("ruff-strict-budget.json", "type-discipline-budget.json", "basedpyright-code-budget.json"): - assert ratchet.retired_rules(rel, base) == frozenset() - - -def test_graduated_selectors_come_from_the_paired_ruff_config(): - selectors = ratchet.graduated_selectors("ruff-strict-budget.json") - assert "UP006" in selectors - assert "ANN" not in selectors - - -def test_budgets_without_a_paired_config_can_never_graduate(): - assert ratchet.graduated_selectors("type-discipline-budget.json") == () - assert ratchet.graduated_selectors("basedpyright-code-budget.json") == () - - -def test_a_selector_the_config_also_ignores_does_not_count_as_graduated(): - lint = {"ignore": ["UP006"], "extend-select": ["UP006", "SIM118"]} - assert ratchet.selectors_hard_failed_by(lint) == ("SIM118",) - - -def test_selectors_hard_failed_by_reads_a_config_with_no_ignore_list(): - assert ratchet.selectors_hard_failed_by({"extend-select": ["UP006"]}) == ("UP006",) - - -def test_deleted_budget_file_is_a_regression(): - regs = ratchet.regressions_for("b.json", {"LIT006": _spec_of(1)}, None) - assert [r.rule for r in regs] == ["*"] - assert "deleted" in regs[0].detail - - -def test_new_budget_file_has_nothing_to_ratchet(): - assert ratchet.regressions_for("b.json", None, {"LIT006": _spec_of(1)}) == [] - - -def test_default_budgets_watch_every_budget_file_in_the_repo(): - # This job is the repo's only ceiling-raise alarm, so every *-budget.json on disk must be - # watched; a budget left out of DEFAULT_BUDGETS (e.g. basedpyright-code-budget.json) can be - # loosened with no signal. Equality also catches a phantom entry that no longer exists. - repo_root = _MODULE_PATH.parents[1] - on_disk = frozenset(p.name for p in repo_root.glob("*budget*.json")) - assert on_disk == frozenset(ratchet.DEFAULT_BUDGETS) - - -# --------------------------------------------------------------------------- # -# Base-ref resolution: a bad ref must fail loudly, never pass vacuously -# --------------------------------------------------------------------------- # - - -def test_ref_is_commit_distinguishes_real_from_bogus(): - assert ratchet._ref_is_commit("HEAD") is True - assert ratchet._ref_is_commit("definitely-not-a-real-ref-zzz") is False - - -def test_load_base_reads_a_present_file_and_none_for_an_absent_one(): - # A real budget file exists at HEAD; a made-up path is absent at the same (valid) ref. - assert ratchet._load_base("type-discipline-budget.json", "HEAD") is not None - assert ratchet._load_base("scripts/no-such-budget-xyz.json", "HEAD") is None - - -def test_unresolvable_base_ref_exits_nonzero_instead_of_skipping(): - proc = subprocess.run( - [sys.executable, str(_MODULE_PATH), "--base", "definitely-not-a-real-ref-zzz"], - cwd=_MODULE_PATH.parents[1], - capture_output=True, - text=True, - ) - assert proc.returncode == 1 - assert "does not resolve to a commit" in proc.stderr diff --git a/tests/unit/test_check_type_discipline.py b/tests/unit/test_check_type_discipline.py index abb7ea1408c..abb8c0d6a9f 100644 --- a/tests/unit/test_check_type_discipline.py +++ b/tests/unit/test_check_type_discipline.py @@ -7,9 +7,7 @@ a test fail. The comment-scanner cases are the regression for the readline path: """ import importlib.util -import json import os -import re import subprocess import sys from pathlib import Path @@ -1003,20 +1001,6 @@ def test_violation_message_names_the_clause_counts(tmp_path: Path): assert "2 `for` clauses and 1 `if` clause" in messages[0] -# --------------------------------------------------------------------------- # -# Budget integrity: every emittable LIT rule (bar the LIT000 read/parse error) is gated -# --------------------------------------------------------------------------- # - - -def test_budget_covers_exactly_the_checker_rules(): - budget = json.loads((_REPO_ROOT / "type-discipline-budget.json").read_text()) - emitted = set(re.findall(r"LIT\d{3}", _MODULE_PATH.read_text(encoding="utf-8"))) - {"LIT000"} - assert set(budget) == emitted - for spec in budget.values(): - assert isinstance(spec["limit"], int) - assert spec["limit"] >= 0 - - _FANS_OUT = checker._worker_count(checker.PARALLEL_MIN_PATHS) > 1 _SERIAL_ONLY = "one usable core, so scan_paths stays serial and there is no fan-out to compare" diff --git a/tests/unit/test_default_branch.py b/tests/unit/test_default_branch.py index 0401bec324f..bb04ea2a0fa 100644 --- a/tests/unit/test_default_branch.py +++ b/tests/unit/test_default_branch.py @@ -28,7 +28,7 @@ def remote_and_clone(tmp_path: Path) -> tuple[Path, Path]: (seed / "scripts").mkdir() for name in ( "default_branch.py", - "budget_ratchet_check.py", + "lint_base_counts.py", "ruff_strict_gate.py", "type_discipline_gate.py", "test_quality_gate.py", @@ -39,11 +39,9 @@ def remote_and_clone(tmp_path: Path) -> tuple[Path, Path]: shutil.copyfile(ROOT / "Makefile", seed / "Makefile") (seed / "litellm").mkdir() (seed / "litellm" / "example.py").write_text("value = 0\n") - (seed / "ruff-strict-budget.json").write_text('{"C901": {"limit": 1}}\n') _commit(seed, "staging base") _git(seed, "checkout", "-qb", "main") (seed / "litellm" / "example.py").write_text("value = 1\n") - (seed / "ruff-strict-budget.json").write_text('{"C901": {"limit": 0}}\n') _commit(seed, "main base") remote: Final = tmp_path / "remote.git" _git(tmp_path, "clone", "-q", "--bare", str(seed), str(remote)) @@ -121,28 +119,6 @@ def test_explicit_base_works_without_remote_access( assert "No changed litellm Python files" in checked.stdout -def test_budget_ratchet_compares_against_new_default(remote_and_clone: tuple[Path, Path]) -> None: - remote, repo = remote_and_clone - _git(remote, "symbolic-ref", "HEAD", "refs/heads/main") - resolved: Final = _resolve(repo) - assert resolved.returncode == 0, resolved.stderr - _git(repo, "checkout", "-qb", "litellm_feature", "origin/main") - (repo / "ruff-strict-budget.json").write_text('{"C901": {"limit": 1}}\n') - command: Final = [sys.executable, "scripts/budget_ratchet_check.py"] - checked: Final = subprocess.run(command, cwd=repo, capture_output=True, text=True, check=False) - assert checked.returncode == 1 - assert "limit raised 0 -> 1" in checked.stdout - assert "base origin/main" in checked.stdout - overridden: Final = subprocess.run( - [*command, "--base", "origin/release_branch"], - cwd=repo, - capture_output=True, - text=True, - check=False, - ) - assert overridden.returncode == 0, overridden.stdout + overridden.stderr - - def _freshness(repo: Path, *args: str) -> subprocess.CompletedProcess[str]: return subprocess.run( [ @@ -192,7 +168,6 @@ def test_migration_freshness_refuses_unavailable_remote(remote_and_clone: tuple[ @pytest.mark.parametrize( "gate", [ - "budget_ratchet_check", "ruff_strict_gate", "type_discipline_gate", "test_quality_gate", @@ -213,14 +188,11 @@ def test_each_gate_refuses_an_unverifiable_default(remote_and_clone: tuple[Path, assert "Cannot verify the base branch against origin" in result.stderr -@pytest.mark.parametrize( - "target", ["lint-format-check-changed", "lint-test-quality", "lint-test-quality-budget-update"] -) +@pytest.mark.parametrize("target", ["lint-format-check-changed", "lint-test-quality"]) def test_direct_make_target_fetches_default_once(remote_and_clone: tuple[Path, Path], target: str) -> None: _, repo = remote_and_clone trace: Final = repo.parent / "git-trace.jsonl" shutil.copyfile(ROOT / "scripts" / "check_test_quality.py", repo / "scripts" / "check_test_quality.py") - shutil.copyfile(ROOT / "test-quality-budget.json", repo / "test-quality-budget.json") (repo / "tests").mkdir() result: Final = subprocess.run( ["make", "-o", "install-dev", target, "LINT_DEP_INSTALL=", "UV_RUN=env"], diff --git a/tests/unit/test_lint_base_counts.py b/tests/unit/test_lint_base_counts.py new file mode 100644 index 00000000000..84f8a0936c0 --- /dev/null +++ b/tests/unit/test_lint_base_counts.py @@ -0,0 +1,454 @@ +"""Tests for scripts/lint_base_counts.py, the merge-base counting shared by the +four lint gates: the ceiling rule with its optional per-rule caps, the on-disk cache and +its eviction, the CI artifact fetch, the artifact emit, and the merge-base +resolution.""" + +import fnmatch +import io +import json +import os +import zipfile +from collections.abc import Callable, Mapping, Sequence +from pathlib import Path +from typing import Final, NamedTuple, NoReturn + +import pytest + +import lint_base_counts as counts + +_CHECKER: Final = counts.Checker("basedpyright", ("f1", "f2")) +_OTHER_CHECKER: Final = counts.Checker("ruff-strict", ("f1", "f2")) + + +def test_evaluate_passes_a_rule_that_did_not_grow() -> None: + assert counts.evaluate({"LIT006": 12}, {"LIT006": 12}) == () + + +def test_evaluate_blames_one_new_violation_of_an_uncapped_rule() -> None: + assert counts.evaluate({"LIT006": 13}, {"LIT006": 12}) == (counts.Breach("LIT006", 13, 12, 1),) + + +def test_evaluate_lets_a_capped_rule_grow_up_to_its_cap_and_no_further() -> None: + caps: Final = {"reportAny": 110} + assert counts.evaluate({"reportAny": 110}, {"reportAny": 100}, caps) == () + assert counts.evaluate({"reportAny": 111}, {"reportAny": 100}, caps) == (counts.Breach("reportAny", 111, 110, 11),) + + +def test_evaluate_never_blames_a_bystander_for_a_base_already_over_the_cap() -> None: + caps: Final = {"reportAny": 110} + assert counts.evaluate({"reportAny": 120}, {"reportAny": 120}, caps) == () + assert counts.evaluate({"reportAny": 119}, {"reportAny": 120}, caps) == () + + +def test_evaluate_holds_a_base_over_its_cap_to_no_growth() -> None: + caps: Final = {"reportAny": 110} + assert counts.evaluate({"reportAny": 121}, {"reportAny": 120}, caps) == (counts.Breach("reportAny", 121, 120, 1),) + + +def test_evaluate_cap_applies_only_to_the_rule_it_names() -> None: + caps: Final = {"reportAny": 110} + assert counts.evaluate({"LIT006": 13}, {"LIT006": 12}, caps) == (counts.Breach("LIT006", 13, 12, 1),) + + +def test_evaluate_counts_a_rule_absent_from_the_base_as_zero() -> None: + assert counts.evaluate({"NEW99": 1}, {}) == (counts.Breach("NEW99", 1, 0, 1),) + + +def test_evaluate_never_blames_a_change_that_reduced_a_rule() -> None: + assert counts.evaluate({"LIT006": 11}, {"LIT006": 12}) == () + + +def test_evaluate_reports_every_grown_rule_sorted_by_name() -> None: + head: Final = {"TQ008": 3, "TQ001": 2, "TQ003": 5} + base: Final = {"TQ008": 2, "TQ001": 1, "TQ003": 5} + assert [b.rule for b in counts.evaluate(head, base)] == ["TQ001", "TQ008"] + + +def test_evaluate_ignores_a_base_rule_the_head_fixed_entirely() -> None: + assert counts.evaluate({}, {"LIT006": 12}) == () + + +def test_cache_key_changes_with_base_point_and_each_fingerprint() -> None: + key: Final = counts.cache_key("abc", ("cfg", "lock")) + assert counts.cache_key("abc", ("cfg", "lock")) == key + assert counts.cache_key("def", ("cfg", "lock")) != key + assert counts.cache_key("abc", ("cfg2", "lock")) != key + assert counts.cache_key("abc", ("cfg", "lock2")) != key + + +def test_checker_names_its_artifact_and_cache_file_by_the_same_key() -> None: + key: Final = counts.cache_key("abc123", ("f1", "f2")) + assert _CHECKER.artifact_name("abc123") == f"basedpyright-counts-{key}" + assert _CHECKER.cache_file_name("abc123") == f"basedpyright-base-{key}.json" + assert fnmatch.fnmatch(_CHECKER.cache_file_name("abc123"), _CHECKER.cache_glob()) + + +def test_checkers_with_the_same_fingerprints_never_share_a_name() -> None: + assert _CHECKER.artifact_name("abc123") != _OTHER_CHECKER.artifact_name("abc123") + assert not fnmatch.fnmatch(_OTHER_CHECKER.cache_file_name("abc123"), _CHECKER.cache_glob()) + + +def test_cached_counts_round_trip(tmp_path: Path) -> None: + path: Final = counts.store_counts(tmp_path, _CHECKER, "abc123", {"reportAny": 3, "reportCall": 1}) + assert path == tmp_path / _CHECKER.cache_file_name("abc123") + assert counts.load_cached_counts(path) == {"reportAny": 3, "reportCall": 1} + + +@pytest.mark.parametrize( + "content", + [ + None, + "{not json", + json.dumps(["counts"]), + json.dumps({"base_point": "abc"}), + json.dumps({"counts": {"reportAny": "three"}}), + json.dumps({"counts": {"reportAny": True}}), + ], +) +def test_missing_corrupt_or_misshapen_cache_reads_as_none(tmp_path: Path, content: str | None) -> None: + path: Final = tmp_path / "cache.json" + if content is not None: + path.write_text(content) + assert counts.load_cached_counts(path) is None + + +def test_scratch_is_invisible_to_the_prune_glob() -> None: + scratch: Final = counts.scratch_path(Path("/c") / _CHECKER.cache_file_name("abc")) + assert not fnmatch.fnmatch(scratch.name, _CHECKER.cache_glob()) + + +def test_store_prune_spares_a_concurrent_runs_in_flight_scratch(tmp_path: Path) -> None: + foreign: Final = counts.scratch_path(tmp_path / _CHECKER.cache_file_name("other")) + foreign.write_text("{}") + mine: Final = counts.store_counts(tmp_path, _CHECKER, "mine", {"reportAny": 1}) + assert foreign.exists() + assert counts.load_cached_counts(mine) == {"reportAny": 1} + + +def test_store_keeps_a_concurrent_worktrees_entry_for_another_branch_point(tmp_path: Path) -> None: + old: Final = counts.store_counts(tmp_path, _CHECKER, "old", {"reportAny": 1}) + new: Final = counts.store_counts(tmp_path, _CHECKER, "new", {"reportAny": 2}) + assert counts.load_cached_counts(old) == {"reportAny": 1} + assert counts.load_cached_counts(new) == {"reportAny": 2} + + +def test_store_evicts_only_the_oldest_entries_beyond_the_cap(tmp_path: Path) -> None: + aged: Final = tuple( + counts.store_counts(tmp_path, _CHECKER, f"base{age}", {"reportAny": age}) + for age in range(counts.CACHE_KEEP_ENTRIES) + ) + for age, path in enumerate(aged): + os.utime(path, (age, age)) + newest: Final = counts.store_counts(tmp_path, _CHECKER, "newest", {"reportAny": 99}) + assert not aged[0].exists() + assert all(path.exists() for path in aged[1:]) + assert counts.load_cached_counts(newest) == {"reportAny": 99} + + +def test_store_never_evicts_the_entry_it_just_wrote_even_on_mtime_ties(tmp_path: Path) -> None: + for index in range(counts.CACHE_KEEP_ENTRIES + 2): + os.utime(counts.store_counts(tmp_path, _CHECKER, f"base{index}", {"reportAny": 1}), (9_999_999_999,) * 2) + mine: Final = counts.store_counts(tmp_path, _CHECKER, "mine", {"reportAny": 2}) + assert counts.load_cached_counts(mine) == {"reportAny": 2} + assert len(list(tmp_path.glob(_CHECKER.cache_glob()))) == counts.CACHE_KEEP_ENTRIES + + +def test_store_eviction_never_touches_another_checkers_entries(tmp_path: Path) -> None: + other: Final = counts.store_counts(tmp_path, _OTHER_CHECKER, "base", {"E501": 1}) + os.utime(other, (1, 1)) + for index in range(counts.CACHE_KEEP_ENTRIES + 1): + counts.store_counts(tmp_path, _CHECKER, f"base{index}", {"reportAny": 1}) + assert counts.load_cached_counts(other) == {"E501": 1} + + +def _no_fetch(checker: counts.Checker, base_point: str) -> None: + return None + + +def _never(reason: str) -> Callable[..., NoReturn]: + def callback(*args: object) -> NoReturn: + raise AssertionError(reason) + + return callback + + +def test_base_counts_cached_returns_the_hit_without_recomputing(tmp_path: Path) -> None: + counts.store_counts(tmp_path, _CHECKER, "abc123", {"reportAny": 7}) + assert counts.base_counts_cached( + _CHECKER, + "abc123", + _never("a cache hit must not re-run the base pass"), + cache_dir=tmp_path, + fetch=_never("a cache hit must not reach for CI"), + ) == {"reportAny": 7} + + +def test_base_counts_cached_computes_once_then_hits(tmp_path: Path) -> None: + calls: Final[list[str]] = [] + + def fake(ref: str) -> counts.Counts: + calls.append(ref) + return {"reportAny": 4} + + first: Final = counts.base_counts_cached(_CHECKER, "abc123", fake, cache_dir=tmp_path, fetch=_no_fetch) + second: Final = counts.base_counts_cached(_CHECKER, "abc123", fake, cache_dir=tmp_path, fetch=_no_fetch) + assert first == second == {"reportAny": 4} + assert calls == ["abc123"] + + +def test_base_counts_cached_keeps_each_checker_apart(tmp_path: Path) -> None: + counts.store_counts(tmp_path, _OTHER_CHECKER, "abc123", {"E501": 7}) + assert counts.base_counts_cached( + _CHECKER, "abc123", lambda ref: {"reportAny": 4}, cache_dir=tmp_path, fetch=_no_fetch + ) == {"reportAny": 4} + + +def test_an_empty_base_pass_is_never_cached(tmp_path: Path) -> None: + calls: Final[list[str]] = [] + + def crashed(ref: str) -> counts.Counts: + calls.append(ref) + return {} + + assert counts.base_counts_cached(_CHECKER, "abc123", crashed, cache_dir=tmp_path, fetch=_no_fetch) == {} + assert counts.base_counts_cached(_CHECKER, "abc123", crashed, cache_dir=tmp_path, fetch=_no_fetch) == {} + assert calls == ["abc123", "abc123"] + assert list(tmp_path.iterdir()) == [] + + +def test_base_counts_cached_uses_fetched_counts_and_persists_them(tmp_path: Path) -> None: + fetched: Final = counts.base_counts_cached( + _CHECKER, + "abc123", + _never("fetched counts must skip the local base pass"), + cache_dir=tmp_path, + fetch=lambda checker, base_point: {"reportAny": 9}, + ) + assert fetched == {"reportAny": 9} + assert counts.load_cached_counts(tmp_path / _CHECKER.cache_file_name("abc123")) == {"reportAny": 9} + assert counts.base_counts_cached( + _CHECKER, + "abc123", + _never("the persisted fetch must satisfy later runs"), + cache_dir=tmp_path, + fetch=_never("the persisted fetch must satisfy later runs"), + ) == {"reportAny": 9} + + +def test_base_counts_cached_hands_the_fetcher_the_checker_and_base_point(tmp_path: Path) -> None: + seen: Final[list[tuple[counts.Checker, str]]] = [] + + def fetch(checker: counts.Checker, base_point: str) -> None: + seen.append((checker, base_point)) + + counts.base_counts_cached(_CHECKER, "abc123", lambda ref: {"reportAny": 4}, cache_dir=tmp_path, fetch=fetch) + assert seen == [(_CHECKER, "abc123")] + + +def test_base_counts_cached_falls_back_to_compute_on_a_fetch_miss(tmp_path: Path) -> None: + calls: Final[list[str]] = [] + + def local(ref: str) -> counts.Counts: + calls.append(ref) + return {"reportAny": 4} + + assert counts.base_counts_cached(_CHECKER, "abc123", local, cache_dir=tmp_path, fetch=_no_fetch) == { + "reportAny": 4 + } + assert calls == ["abc123"] + + +def test_base_counts_cached_treats_empty_fetched_counts_as_a_miss(tmp_path: Path) -> None: + assert counts.base_counts_cached( + _CHECKER, + "abc123", + lambda ref: {"reportAny": 2}, + cache_dir=tmp_path, + fetch=lambda checker, base_point: {}, + ) == {"reportAny": 2} + assert counts.load_cached_counts(tmp_path / _CHECKER.cache_file_name("abc123")) == {"reportAny": 2} + + +def test_origin_slug_parsing_supports_ssh_and_https_github_forms() -> None: + assert counts.parse_origin_slug("git@github.com:BerriAI/litellm.git") == "BerriAI/litellm" + assert counts.parse_origin_slug("git@github.com:BerriAI/litellm") == "BerriAI/litellm" + assert counts.parse_origin_slug("https://github.com/BerriAI/litellm.git") == "BerriAI/litellm" + assert counts.parse_origin_slug("https://github.com/BerriAI/litellm") == "BerriAI/litellm" + assert counts.parse_origin_slug("https://github.com/BerriAI/litellm/") == "BerriAI/litellm" + + +def test_origin_slug_parsing_rejects_non_github_urls() -> None: + assert counts.parse_origin_slug("https://gitlab.com/BerriAI/litellm.git") is None + assert counts.parse_origin_slug("git@bitbucket.org:BerriAI/litellm.git") is None + assert counts.parse_origin_slug("not a url") is None + assert counts.parse_origin_slug("") is None + + +def _artifact_zip(payload: Mapping[str, object]) -> bytes: + buffer: Final = io.BytesIO() + with zipfile.ZipFile(buffer, "w") as archive: + archive.writestr("counts.json", json.dumps(payload)) + return buffer.getvalue() + + +def _gh_stub( + listing: Mapping[str, object], zip_bytes: bytes, seen: list[tuple[str, ...]] | None = None +) -> counts.GhOutput: + def gh_output(args: Sequence[str]) -> bytes: + if seen is not None: + seen.append(tuple(args)) + if args[-1].startswith("repos/"): + return json.dumps(listing).encode() + return zip_bytes + + return gh_output + + +def _live_listing() -> Mapping[str, object]: + return {"artifacts": [{"expired": False, "archive_download_url": "https://api.github.com/x/zip"}]} + + +def test_fetcher_returns_counts_from_a_matching_artifact(capsys: pytest.CaptureFixture[str]) -> None: + payload: Final = {"base_point": "abc123", "counts": {"reportAny": 3}} + fetched: Final = counts.fetch_ci_base_counts( + _CHECKER, "abc123", gh=_gh_stub(_live_listing(), _artifact_zip(payload)) + ) + assert fetched == {"reportAny": 3} + assert "fetched from CI artifact" in capsys.readouterr().err + + +def test_fetcher_asks_for_the_artifact_named_by_the_checker_and_base_point() -> None: + seen: Final[list[tuple[str, ...]]] = [] + payload: Final = {"base_point": "abc123", "counts": {"reportAny": 3}} + counts.fetch_ci_base_counts(_CHECKER, "abc123", gh=_gh_stub(_live_listing(), _artifact_zip(payload), seen)) + listing_request: Final = seen[0][-1] + assert f"name={_CHECKER.artifact_name('abc123')}" in listing_request + assert seen[1][-1] == "https://api.github.com/x/zip" + + +def test_fetcher_rejects_an_artifact_for_a_different_base_point() -> None: + payload: Final = {"base_point": "someothersha", "counts": {"reportAny": 3}} + assert ( + counts.fetch_ci_base_counts(_CHECKER, "abc123", gh=_gh_stub(_live_listing(), _artifact_zip(payload))) is None + ) + + +@pytest.mark.parametrize("bad_counts", [{}, {"reportAny": "three"}, {"reportAny": True}]) +def test_fetcher_rejects_empty_or_misshapen_artifact_counts(bad_counts: Mapping[str, object]) -> None: + payload: Final = {"base_point": "abc123", "counts": bad_counts} + assert ( + counts.fetch_ci_base_counts(_CHECKER, "abc123", gh=_gh_stub(_live_listing(), _artifact_zip(payload))) is None + ) + + +def test_fetcher_rejects_an_expired_artifact() -> None: + listing: Final = {"artifacts": [{"expired": True, "archive_download_url": "https://api.github.com/x/zip"}]} + payload: Final = {"base_point": "abc123", "counts": {"reportAny": 3}} + assert counts.fetch_ci_base_counts(_CHECKER, "abc123", gh=_gh_stub(listing, _artifact_zip(payload))) is None + + +def test_fetcher_misses_when_no_artifact_is_published() -> None: + assert counts.fetch_ci_base_counts(_CHECKER, "abc123", gh=_gh_stub({"artifacts": []}, b"")) is None + + +def test_fetcher_misses_when_gh_is_unusable(capsys: pytest.CaptureFixture[str]) -> None: + assert counts.fetch_ci_base_counts(_CHECKER, "abc123", gh=lambda args: None) is None + assert "computing base counts locally" in capsys.readouterr().err + + +def test_fetcher_misses_on_a_corrupt_artifact_archive() -> None: + assert counts.fetch_ci_base_counts(_CHECKER, "abc123", gh=_gh_stub(_live_listing(), b"not a zip")) is None + + +def test_emit_writes_the_artifact_json_named_by_the_head_key( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + counts.emit_counts(_CHECKER, {"reportAny": 3, "aRule": 1}, tmp_path, "deadbeef") + name: Final = _CHECKER.artifact_name("deadbeef") + assert json.loads((tmp_path / f"{name}.json").read_text()) == { + "base_point": "deadbeef", + "counts": {"aRule": 1, "reportAny": 3}, + } + summary: Final = capsys.readouterr().out + assert "deadbeef" in summary + assert name in summary + assert "4" in summary + + +def test_emit_refuses_to_publish_empty_counts(tmp_path: Path) -> None: + with pytest.raises(SystemExit): + counts.emit_counts(_CHECKER, {}, tmp_path, "deadbeef") + assert list(tmp_path.iterdir()) == [] + + +def test_emitted_file_is_the_one_the_fetcher_looks_up(tmp_path: Path) -> None: + written: Final = counts.emit_counts(_CHECKER, {"reportAny": 3}, tmp_path, "deadbeef") + payload: Final = json.loads(written.read_text()) + assert counts.counts_for_base(payload, "deadbeef") == {"reportAny": 3} + assert counts.counts_for_base(payload, "someothersha") is None + listing_zip: Final = _artifact_zip(payload) + assert counts.fetch_ci_base_counts(_CHECKER, "deadbeef", gh=_gh_stub(_live_listing(), listing_zip)) == { + "reportAny": 3 + } + + +class _History(NamedTuple): + parents: Mapping[str, tuple[str, ...]] + refs: Mapping[str, str] + + def ancestry(self, commit: str) -> frozenset[str]: + return frozenset((commit,)).union(*(self.ancestry(parent) for parent in self.parents[commit])) + + def merge_base(self, left: str, right: str) -> str: + common: Final = self.ancestry(self.refs.get(left, left)) & self.ancestry(self.refs.get(right, right)) + return next(c for c in common if not any(c != other and c in self.ancestry(other) for other in common)) + + +def _git_over(history: _History) -> counts.Git: + def git(args: Sequence[str]) -> str: + match tuple(args): + case ("merge-base", left, right): + return f"{history.merge_base(left, right)}\n" + case ("rev-parse", "HEAD"): + return f"{history.refs['HEAD']}\n" + case ("rev-parse", "--verify", "--quiet", ref): + return f"{history.refs[ref]}\n" if ref in history.refs else "" + case ("rev-parse", "--path-format=absolute", "--git-common-dir"): + return "/repo/.git\n" + case ("rev-parse", "--path-format=absolute", "--git-dir"): + return "/repo/.git/worktrees/feature\n" + case _: + raise AssertionError(f"unexpected git call: {args}") + + return git + + +_FEATURE_OFF_MAIN: Final = _History( + parents={"shared": (), "feature": ("shared",), "drift": ("shared",)}, + refs={"main": "drift", "HEAD": "feature"}, +) + + +def test_base_point_is_the_branch_point_when_no_merge_is_in_progress() -> None: + assert counts.resolve_base_point("main", _git_over(_FEATURE_OFF_MAIN)) == "shared" + + +def test_base_point_mid_merge_advances_to_the_merged_in_base_tip() -> None: + merging_main: Final = _FEATURE_OFF_MAIN._replace(refs={**_FEATURE_OFF_MAIN.refs, "MERGE_HEAD": "drift"}) + assert counts.resolve_base_point("main", _git_over(merging_main)) == "drift" + + +def test_base_point_mid_merge_of_an_older_side_branch_keeps_the_newer_branch_point() -> None: + merging_old_side: Final = _History( + parents={"shared": (), "old": ("shared",), "drift": ("shared",), "feature": ("drift",)}, + refs={"main": "drift", "HEAD": "feature", "MERGE_HEAD": "old"}, + ) + assert counts.resolve_base_point("main", _git_over(merging_old_side)) == "drift" + + +def test_head_sha_is_the_checked_out_commit() -> None: + assert counts.head_sha(_git_over(_FEATURE_OFF_MAIN)) == "feature" + + +def test_default_cache_dir_is_shared_by_every_worktree() -> None: + assert counts.default_cache_dir(_git_over(_FEATURE_OFF_MAIN)) == Path("/repo/.git") / counts.CACHE_DIR_NAME diff --git a/tests/unit/test_pre_commit_lint.py b/tests/unit/test_pre_commit_lint.py index 471c8b41b5c..7c9b98c1c94 100644 --- a/tests/unit/test_pre_commit_lint.py +++ b/tests/unit/test_pre_commit_lint.py @@ -14,9 +14,9 @@ from tests._process_helpers import process_is_gone ROOT = Path(__file__).resolve().parents[2] SCRIPT = ROOT / "scripts" / "pre_commit_lint.sh" WHOLE_TREE_RUFF = "run --no-sync ruff check --config ruff-tests.toml tests" -TEST_TREE_RAN = "ran: test-tree lint (ruff-tests.toml + test-quality budget)" +TEST_TREE_RAN = "ran: test-tree lint (ruff-tests.toml + test-quality gate)" TEST_TREE_SKIPPED = ( - "skipped: test-tree lint (ruff-tests.toml + test-quality budget) " + "skipped: test-tree lint (ruff-tests.toml + test-quality gate) " "(no tests/ Python files or test-tree lint inputs in scope)" ) @@ -470,7 +470,6 @@ def test_tests_only_change_runs_the_whole_test_tree_ruff_and_the_quality_gate(tm "changed", [ "ruff-tests.toml", - "test-quality-budget.json", "scripts/check_test_quality.py", "scripts/test_quality_gate.py", "tests/e2e/test_x.py", @@ -523,7 +522,7 @@ def test_a_failing_quality_gate_fails_a_tests_only_run(tmp_path: Path) -> None: _stage_file(repo, "tests/test_a.py", "def test_a() -> None: ...\n") proc = _run(repo, bin_dir, {"STUB_FAIL": "test-quality"}) assert proc.returncode == 1 - assert "Test-quality budget failed" in proc.stdout + proc.stderr + assert "Test-quality gate failed" in proc.stdout + proc.stderr assert "check: FAIL" in proc.stdout @@ -566,7 +565,7 @@ def test_partial_staging_warns_when_test_files_are_left_unstaged(tmp_path: Path) (repo / "tests" / "test_a.py").write_text("def test_a() -> None:\n assert True\n") proc = _run(repo, bin_dir, {"STUB_ARGS_DIR": str(args_dir)}) assert proc.returncode == 0, proc.stdout + proc.stderr - assert "SKIPPED test-tree lint (ruff-tests.toml + test-quality budget)" in proc.stdout + assert "SKIPPED test-tree lint (ruff-tests.toml + test-quality gate)" in proc.stdout assert "tests/test_a.py" in proc.stdout assert _recorded(args_dir, "ruff_tests.args") == [] assert _recorded(args_dir, "make.args") == [] diff --git a/tests/unit/test_ruff_strict_gate.py b/tests/unit/test_ruff_strict_gate.py index 8fa9a18cf53..852ebef7436 100644 --- a/tests/unit/test_ruff_strict_gate.py +++ b/tests/unit/test_ruff_strict_gate.py @@ -1,87 +1,27 @@ -import importlib.util import json import re import shutil import subprocess import sys from pathlib import Path +from typing import Final import pytest +import ruff_strict_gate as gate + if sys.version_info >= (3, 11): import tomllib else: import tomli as tomllib _REPO_ROOT = Path(__file__).resolve().parents[2] -_MODULE_PATH = _REPO_ROOT / "scripts" / "ruff_strict_gate.py" -_spec = importlib.util.spec_from_file_location("ruff_strict_gate", _MODULE_PATH) -gate = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(gate) Violation = gate.Violation _ENABLED_BY_RUFF_DEFAULTS = frozenset({"F401"}) -def rule(name, limit): - return {name: {"limit": limit}} - - -def test_under_ceiling_passes(): - assert gate.evaluate({"ANN001": 100}, {"ANN001": 100}, rule("ANN001", 110)) == [] - - -def test_ceiling_is_the_limit_boundary(): - budget = rule("ANN001", 110) - at = gate.evaluate({"ANN001": 110}, {"ANN001": 90}, budget) - over = gate.evaluate({"ANN001": 111}, {"ANN001": 90}, budget) - assert at == [] - assert [b.rule for b in over] == ["ANN001"] - assert over[0].cap == 110 - assert over[0].added == 21 - - -def test_over_ceiling_and_change_added_fails(): - breaches = gate.evaluate({"C901": 11}, {"C901": 9}, rule("C901", 10)) - assert [b.rule for b in breaches] == ["C901"] - assert breaches[0].added == 2 - - -def test_base_already_over_ceiling_change_added_nothing_is_not_blamed(): - # drift safety: base is over limit, this change leaves the count where it is - assert gate.evaluate({"C901": 15}, {"C901": 15}, rule("C901", 10)) == [] - - -def test_change_that_reduces_an_over_ceiling_rule_is_not_blamed(): - # still over limit, but moving the right direction - assert gate.evaluate({"C901": 14}, {"C901": 16}, rule("C901", 10)) == [] - - -def test_rules_are_independent(): - budget = {**rule("ANN001", 150), **rule("C901", 10)} - breaches = gate.evaluate( - {"ANN001": 130, "C901": 11}, {"ANN001": 100, "C901": 10}, budget - ) - assert [b.rule for b in breaches] == ["C901"] # ANN001 130 <= 150, C901 11 > 10 - - -def test_missing_rule_counts_as_zero(): - assert gate.evaluate({}, {}, rule("C901", 0)) == [] - - -def test_update_ratchets_limit_down_by_what_the_branch_fixed_never_up(): - budget = {**rule("ANN001", 150), **rule("C901", 10)} - # ANN001 fixed 20 (100 -> 80) so its limit falls 150 -> 130; C901 grew, so its - # limit holds flat at 10 (a fix must never loosen a ceiling). - current = {"ANN001": 80, "C901": 12} - base = {"ANN001": 100, "C901": 9} - assert gate.ratcheted_budget(budget, current, base) == { - "ANN001": {"limit": 130}, - "C901": {"limit": 10}, - } - - def test_parse_changed_lines_maps_added_lines_per_file(): diff = ( "+++ b/litellm/a.py\n" @@ -109,62 +49,6 @@ def test_parse_changed_lines_handles_single_and_ranged_hunks(hunk): assert gate.parse_changed_lines(f"+++ b/litellm/a.py\n{hunk}\n")["litellm/a.py"] -def test_over_ceiling_flags_only_counts_above_the_limit(): - budget = rule("C901", 10) - assert gate.over_ceiling({"C901": 10}, budget) == frozenset() - assert gate.over_ceiling({"C901": 11}, budget) == frozenset({"C901"}) - assert gate.over_ceiling({}, budget) == frozenset() - - -def test_over_ceiling_ignores_rules_missing_from_the_budget(): - assert gate.over_ceiling({"NEW99": 100}, rule("C901", 10)) == frozenset() - - -def test_over_ceiling_is_independent_across_rules(): - budget = {**rule("ANN001", 150), **rule("C901", 10)} - assert gate.over_ceiling({"ANN001": 130, "C901": 11}, budget) == frozenset({"C901"}) - - -def _git(cwd, *args): - proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) - assert proc.returncode == 0, proc.stderr - return proc.stdout.strip() - - -def _commit(cwd, name): - (cwd / name).write_text(name) - _git(cwd, "add", "-A") - _git(cwd, "commit", "-q", "-m", name) - return _git(cwd, "rev-parse", "HEAD") - - -def _branched_repo(tmp_path): - repo = tmp_path / "repo" - repo.mkdir() - _git(repo, "init", "-q", "-b", "main") - _git(repo, "config", "user.email", "gate@example.com") - _git(repo, "config", "user.name", "gate") - _git(repo, "config", "commit.gpgsign", "false") - branch_point = _commit(repo, "shared.txt") - _git(repo, "checkout", "-q", "-b", "feature") - _commit(repo, "feature.txt") - _git(repo, "checkout", "-q", "main") - base_tip = _commit(repo, "drift.txt") - _git(repo, "checkout", "-q", "feature") - return repo, branch_point, base_tip - - -def test_base_point_is_the_branch_point_when_no_merge_is_in_progress(tmp_path): - repo, branch_point, _ = _branched_repo(tmp_path) - assert gate.resolve_base_point("main", cwd=repo) == branch_point - - -def test_base_point_mid_merge_advances_to_the_merged_in_base_tip(tmp_path): - repo, _, base_tip = _branched_repo(tmp_path) - _git(repo, "merge", "--no-commit", "--no-ff", "main") - assert gate.resolve_base_point("main", cwd=repo) == base_tip - - def _lint_section(config_name: str) -> dict: return tomllib.loads((_REPO_ROOT / config_name).read_text())["lint"] @@ -189,10 +73,6 @@ def _selected_by_the_normal_config() -> frozenset: return frozenset(_lint_section("ruff.toml")["extend-select"]) | _ENABLED_BY_RUFF_DEFAULTS -def _budgeted_rules() -> frozenset: - return frozenset(json.loads((_REPO_ROOT / "ruff-strict-budget.json").read_text())) - - def _ruff_binary() -> str | None: beside_interpreter = Path(sys.executable).with_name("ruff") return str(beside_interpreter) if beside_interpreter.exists() else shutil.which("ruff") @@ -302,37 +182,6 @@ def test_every_base_owned_rule_is_external_or_selected_in_the_strict_config(all_ ) -def test_every_budgeted_rule_is_one_the_gate_actually_measures(): - selectors = tuple(_lint_section("ruff-strict.toml")["select"]) - unmeasured = frozenset(code for code in _budgeted_rules() if not code.startswith(selectors)) - assert unmeasured == frozenset(), ( - f"the gate never counts {sorted(unmeasured)}, so their ceilings are dead config that reads " - "as coverage. Either select them in ruff-strict.toml or drop them from the budget." - ) - - -@_needs_ruff -def test_every_strict_selected_rule_is_budgeted_or_hard_failed_by_the_base_config(all_ruff_rule_codes): - strict_enabled = frozenset( - code - for code in all_ruff_rule_codes - if code.startswith(tuple(_lint_section("ruff-strict.toml")["select"])) - ) - base_hard_failed = tuple(_lint_section("ruff.toml")["extend-select"]) - unpoliced = frozenset( - code - for code in strict_enabled - if code not in _budgeted_rules() - and not code.startswith(base_hard_failed) - and code not in _ENABLED_BY_RUFF_DEFAULTS - ) - assert unpoliced == frozenset(), ( - f"nothing enforces {sorted(unpoliced)}: the gate skips rules missing from the budget, and " - "the base config does not hard-fail them. Re-add a budget ceiling or graduate them into " - "ruff.toml's lint.extend-select." - ) - - @_needs_ruff def test_a_noqa_for_a_strict_gate_rule_survives_the_normal_ruff_run(): assert "RUF100" not in _ruff_output_for_noqa("ANN202") @@ -373,3 +222,21 @@ def test_a_graduated_rule_can_still_be_suppressed_without_tripping_unused_noqa() output = _ruff_output_for_source(suppressed) assert "UP006" not in output assert "RUF100" not in output + + +def test_editing_either_ruff_config_or_upgrading_ruff_rekeys_the_base_counts(tmp_path: Path) -> None: + strict: Final = tmp_path / "ruff-strict.toml" + base: Final = tmp_path / "ruff.toml" + strict.write_text("strict v1\n") + base.write_text("base v1\n") + + def artifact(version: str) -> str: + return gate.checker_identity(strict, base, lambda: version).artifact_name("abc123") + + before: Final = artifact("ruff 0.1.0") + assert artifact("ruff 0.1.0") == before + assert artifact("ruff 0.2.0") != before + strict.write_text("strict v2\n") + after_strict_edit: Final = artifact("ruff 0.1.0") + base.write_text("base v2\n") + assert len({before, after_strict_edit, artifact("ruff 0.1.0")}) == 3 diff --git a/tests/unit/test_test_quality_gate.py b/tests/unit/test_test_quality_gate.py index 873e7b9b336..b98e539d4b0 100644 --- a/tests/unit/test_test_quality_gate.py +++ b/tests/unit/test_test_quality_gate.py @@ -1,12 +1,11 @@ """Tests for scripts/test_quality_gate.py. -The gate's whole value is that it blames a change only for what it adds and that a -limit can never rise. Both live in pure functions, so they are tested directly: -`evaluate` for the blame rule, `ratcheted_budget` for the one-way ratchet, and -`parse_changed_lines` for the diff scan that turns a breach into file:line. +The blame rule itself lives in scripts/lint_base_counts.py and is tested there; +what is tested here is `parse_changed_lines`, the diff scan that turns a breach into +file:line. The base scan spawns a worktree and a checker subprocess, so its cleanup +on termination is driven end to end. """ -import importlib.util import os import signal import subprocess @@ -15,77 +14,28 @@ import time from collections.abc import Callable from contextlib import suppress from pathlib import Path -from typing import NamedTuple +from typing import Final, NamedTuple + +import test_quality_gate as gate _REPO_ROOT = Path(__file__).resolve().parents[2] _MODULE_PATH = _REPO_ROOT / "scripts" / "test_quality_gate.py" -_spec = importlib.util.spec_from_file_location("test_quality_gate", _MODULE_PATH) -gate = importlib.util.module_from_spec(_spec) -# @dataclass(slots=True) rebuilds its class through sys.modules[__module__], so the -# module has to be registered before exec_module runs or Scope fails to construct. -sys.modules[_spec.name] = gate -_spec.loader.exec_module(gate) - -_BUDGET = {"TQ001": {"limit": 10}, "TQ003": {"limit": 5}} _SCAN_BASE = ( - "import importlib.util, pathlib, sys\n" - "spec = importlib.util.spec_from_file_location('test_quality_gate', sys.argv[1])\n" - "gate = importlib.util.module_from_spec(spec)\n" - "sys.modules[spec.name] = gate\n" - "spec.loader.exec_module(gate)\n" - "gate.base_counts('HEAD', repo_root=pathlib.Path(sys.argv[2]), checker=pathlib.Path(sys.argv[3]))\n" + "import pathlib, sys\n" + "import test_quality_gate as gate\n" + "gate.base_counts('HEAD', repo_root=pathlib.Path(sys.argv[1]), checker=pathlib.Path(sys.argv[2]))\n" ) _SCAN_BASE_WITH_SIGHUP_IGNORED = "import signal\nsignal.signal(signal.SIGHUP, signal.SIG_IGN)\n" + _SCAN_BASE -def test_a_rule_within_its_limit_is_not_a_breach(): - assert gate.evaluate({"TQ001": 10}, {"TQ001": 10}, _BUDGET) == () - - -def test_a_rule_over_its_limit_that_the_change_added_is_a_breach(): - breaches = gate.evaluate({"TQ001": 12}, {"TQ001": 10}, _BUDGET) - assert [(b.rule, b.total, b.cap, b.added) for b in breaches] == [("TQ001", 12, 10, 2)] - - -def test_drift_already_in_the_base_is_not_blamed_on_the_change(): - assert gate.evaluate({"TQ001": 14}, {"TQ001": 14}, _BUDGET) == () - - -def test_a_change_that_reduces_an_over_limit_rule_is_not_blamed(): - assert gate.evaluate({"TQ001": 13}, {"TQ001": 14}, _BUDGET) == () - - -def test_a_rule_absent_from_head_counts_as_zero(): - assert gate.evaluate({}, {}, _BUDGET) == () - - -def test_over_ceiling_names_only_the_rules_above_their_limit(): - assert gate.over_ceiling({"TQ001": 11, "TQ003": 5}, _BUDGET) == frozenset({"TQ001"}) - - -def test_over_ceiling_is_empty_when_everything_fits(): - assert gate.over_ceiling({"TQ001": 10, "TQ003": 4}, _BUDGET) == frozenset() - - -def test_ratchet_lowers_a_limit_by_what_the_branch_fixed(): - updated = gate.ratcheted_budget(_BUDGET, {"TQ001": 6}, {"TQ001": 10}) - assert updated["TQ001"]["limit"] == 6 - - -def test_ratchet_never_raises_a_limit_when_violations_grew(): - updated = gate.ratcheted_budget(_BUDGET, {"TQ001": 20}, {"TQ001": 10}) - assert updated["TQ001"]["limit"] == 10 - - -def test_ratchet_never_goes_below_zero(): - updated = gate.ratcheted_budget({"TQ001": {"limit": 2}}, {"TQ001": 0}, {"TQ001": 100}) - assert updated["TQ001"]["limit"] == 0 - - -def test_ratchet_lowers_a_rule_introduced_on_this_branch_like_any_other(): - updated = gate.ratcheted_budget(_BUDGET, {"TQ001": 4}, {"TQ001": 10}) - assert updated["TQ001"]["limit"] == 4 +def test_editing_the_checker_rekeys_the_base_counts(tmp_path: Path) -> None: + checker: Final = tmp_path / "check.py" + checker.write_text("print('v1')\n") + before: Final = gate.checker_identity(checker).artifact_name("abc123") + assert gate.checker_identity(checker).artifact_name("abc123") == before + checker.write_text("print('v2')\n") + assert gate.checker_identity(checker).artifact_name("abc123") != before def test_parse_changed_lines_groups_hunks_under_their_own_file(): @@ -132,16 +82,6 @@ def test_introduced_keeps_only_violations_on_changed_lines(): assert kept == (gate.Violation("tests/a.py", 3, "TQ001"),) -def test_the_shipped_budget_covers_every_rule_the_checker_can_emit(): - import json - - budget = json.loads((_REPO_ROOT / "test-quality-budget.json").read_text()) - assert set(budget) == { - "TQ001", "TQ002", "TQ003", "TQ004", "TQ005", "TQ006", "TQ007", "TQ009" - } - assert all(spec["limit"] >= 0 for spec in budget.values()) - - def _git(cwd: Path, *args: str) -> str: proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) assert proc.returncode == 0, proc.stderr @@ -204,7 +144,8 @@ def _base_scan_stalled_in_its_checker(tmp_path: Path, driver: str) -> _StalledSc temp_dir = tmp_path / "tmp" temp_dir.mkdir() scan = subprocess.Popen( - [sys.executable, "-c", driver, str(_MODULE_PATH), str(repo), str(slow_checker)], + [sys.executable, "-c", driver, str(repo), str(slow_checker)], + cwd=_MODULE_PATH.parent, env={**os.environ, "TMPDIR": str(temp_dir)}, ) if not _wait_until(scanning.exists, 30): diff --git a/tests/unit/test_type_check_gate.py b/tests/unit/test_type_check_gate.py index d104e4ca0c8..76003a3d296 100644 --- a/tests/unit/test_type_check_gate.py +++ b/tests/unit/test_type_check_gate.py @@ -1,14 +1,11 @@ import hashlib -import importlib.util import json -import os -import subprocess from pathlib import Path +from typing import Final -_MODULE_PATH = Path(__file__).resolve().parents[2] / "scripts" / "type_check_gate.py" -_spec = importlib.util.spec_from_file_location("type_check_gate", _MODULE_PATH) -gate = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(gate) +import pytest + +import type_check_gate as gate ROOT = gate.REPO_ROOT @@ -132,99 +129,6 @@ def test_run_basedpyright_fails_loudly_on_a_crash_exit_code(tmp_path): gate.run_basedpyright(cwd=tmp_path, env_dir=env_dir) -def test_at_or_under_ceiling_passes(): - budget = {"no-any-return": {"limit": 5}} - assert gate.evaluate({"no-any-return": 5}, {}, budget) == [] - - -def test_one_more_error_than_ceiling_fails(): - budget = {"no-any-return": {"limit": 5}} - assert gate.evaluate({"no-any-return": 6}, {}, budget) == [ - gate.Breach("no-any-return", 6, 5, 6) - ] - - -def test_limit_absorbs_increase_up_to_it_then_fails_past_it(): - budget = {"arg-type": {"limit": 10}} - assert gate.evaluate({"arg-type": 10}, {}, budget) == [] - assert gate.evaluate({"arg-type": 11}, {}, budget) == [ - gate.Breach("arg-type", 11, 10, 11) - ] - - -def test_unbudgeted_new_code_uses_default_limit(): - assert gate.evaluate({"brand-new": gate.DEFAULT_LIMIT}, {}, {}) == [] - assert gate.evaluate({"brand-new": gate.DEFAULT_LIMIT + 1}, {}, {}) == [ - gate.Breach( - "brand-new", - gate.DEFAULT_LIMIT + 1, - gate.DEFAULT_LIMIT, - gate.DEFAULT_LIMIT + 1, - ) - ] - - -def test_drift_already_over_cap_in_base_is_not_blamed_on_a_flat_change(): - # The bystander case: a rule sits over its limit because two earlier PRs - # summed past it. A PR that branches off that base and adds nothing must pass - # -- total > limit but total == base, so the `> base` guard spares it. - budget = {"arg-type": {"limit": 10}} - assert gate.evaluate({"arg-type": 12}, {"arg-type": 12}, budget) == [] - - -def test_change_that_grows_an_over_cap_rule_is_blamed_for_only_what_it_added(): - # Over limit AND above base: blamed, and `added` is the delta vs base, not the - # whole overage, so the message points at this change's contribution. - budget = {"arg-type": {"limit": 10}} - assert gate.evaluate({"arg-type": 14}, {"arg-type": 12}, budget) == [ - gate.Breach("arg-type", 14, 10, 2) - ] - - -def test_reducing_an_over_cap_rule_below_base_passes(): - budget = {"arg-type": {"limit": 10}} - assert gate.evaluate({"arg-type": 11}, {"arg-type": 12}, budget) == [] - - -def test_no_output_against_a_nonempty_budget_is_a_vacuous_run(): - # A crashed type checker emits nothing; the gate must not certify it as clean. - budget = {"no-untyped-def": {"limit": 4898}} - assert gate.is_vacuous_run({}, budget) is True - - -def test_genuine_zero_and_empty_budget_are_not_vacuous(): - assert gate.is_vacuous_run({}, {}) is False - assert gate.is_vacuous_run({}, {"no-untyped-def": {"limit": 0}}) is False - assert ( - gate.is_vacuous_run({"arg-type": 1}, {"arg-type": {"limit": 10}}) is False - ) - - -def test_update_ratchets_a_limit_down_by_what_the_branch_fixed(): - # A rule that dropped from 40 (branch point) to 30 (current) fixed 10, so its - # limit of 100 falls to 90 -- the granted headroom (60) is preserved, not the - # raw count. - budget = {"reportAny": {"limit": 100}} - assert gate.ratcheted_budget(budget, {"reportAny": 30}, {"reportAny": 40}) == { - "reportAny": {"limit": 90} - } - - -def test_update_never_raises_a_limit_when_a_rule_grows(): - # Adding violations must not loosen the ceiling; the limit holds flat. - budget = {"reportAny": {"limit": 100}} - assert gate.ratcheted_budget(budget, {"reportAny": 55}, {"reportAny": 40}) == { - "reportAny": {"limit": 100} - } - - -def test_update_clamps_a_limit_at_zero_never_negative(): - budget = {"reportAny": {"limit": 5}} - assert gate.ratcheted_budget(budget, {"reportAny": 0}, {"reportAny": 40}) == { - "reportAny": {"limit": 0} - } - - def test_malformed_basedpyright_json_exits_loudly_not_as_zero_errors(): import pytest @@ -238,35 +142,6 @@ def test_empty_basedpyright_payload_counts_zero(): assert gate.count_basedpyright("") == {} -def test_over_ceiling_flags_only_rules_above_their_limit(): - budget = {"reportAny": {"limit": 10}} - assert gate.over_ceiling({"reportAny": 10}, budget) == frozenset() - assert gate.over_ceiling({"reportAny": 11}, budget) == frozenset({"reportAny"}) - assert gate.over_ceiling({}, budget) == frozenset() - - -def test_over_ceiling_holds_unbudgeted_rules_to_the_default_limit(): - assert gate.over_ceiling({"brand-new": gate.DEFAULT_LIMIT}, {}) == frozenset() - assert gate.over_ceiling({"brand-new": gate.DEFAULT_LIMIT + 1}, {}) == frozenset( - {"brand-new"} - ) - - -def test_over_ceiling_is_independent_across_rules(): - budget = {"reportAny": {"limit": 10}, "reportArgumentType": {"limit": 5}} - assert gate.over_ceiling( - {"reportAny": 9, "reportArgumentType": 6}, budget - ) == frozenset({"reportArgumentType"}) - - -def test_cache_key_changes_with_base_point_and_each_fingerprint(): - key = gate.cache_key("abc", ("cfg", "lock")) - assert gate.cache_key("abc", ("cfg", "lock")) == key - assert gate.cache_key("def", ("cfg", "lock")) != key - assert gate.cache_key("abc", ("cfg2", "lock")) != key - assert gate.cache_key("abc", ("cfg", "lock2")) != key - - def test_fingerprints_carry_the_dependency_group_set(): # Counts measured under one group set must never be compared against # another's: the fingerprint difference re-keys every cache entry and @@ -350,381 +225,42 @@ def test_ensure_env_is_silent_when_the_env_already_exists(tmp_path, capsys): assert capsys.readouterr().err == "" -def test_cached_counts_round_trip(tmp_path): - path = gate.cache_path(tmp_path, "abc123", ("f1", "f2")) - gate.store_counts(tmp_path, path, "abc123", {"reportAny": 3, "reportCall": 1}) - assert gate.load_cached_counts(path) == {"reportAny": 3, "reportCall": 1} - - -def test_missing_corrupt_or_misshapen_cache_reads_as_none(tmp_path): - path = tmp_path / "cache.json" - assert gate.load_cached_counts(path) is None - path.write_text("{not json") - assert gate.load_cached_counts(path) is None - path.write_text(json.dumps(["counts"])) - assert gate.load_cached_counts(path) is None - path.write_text(json.dumps({"base_point": "abc"})) - assert gate.load_cached_counts(path) is None - path.write_text(json.dumps({"counts": {"reportAny": "three"}})) - assert gate.load_cached_counts(path) is None - path.write_text(json.dumps({"counts": {"reportAny": True}})) - assert gate.load_cached_counts(path) is None - - -def test_scratch_is_invisible_to_the_prune_glob(): - import fnmatch - - scratch = gate.scratch_path(gate.cache_path(Path("/c"), "abc", ("f",))) - assert not fnmatch.fnmatch(scratch.name, f"{gate.CACHE_FILE_PREFIX}*") - - -def test_store_prune_spares_a_concurrent_runs_in_flight_scratch(tmp_path): - foreign = gate.scratch_path(gate.cache_path(tmp_path, "other", ("f",))) - foreign.parent.mkdir(parents=True, exist_ok=True) - foreign.write_text("{}") - mine = gate.cache_path(tmp_path, "mine", ("f",)) - gate.store_counts(tmp_path, mine, "mine", {"reportAny": 1}) - assert foreign.exists() - assert gate.load_cached_counts(mine) == {"reportAny": 1} - - -def test_store_keeps_a_concurrent_worktrees_entry_for_another_branch_point(tmp_path): - old = gate.cache_path(tmp_path, "old", ("f",)) - gate.store_counts(tmp_path, old, "old", {"reportAny": 1}) - new = gate.cache_path(tmp_path, "new", ("f",)) - gate.store_counts(tmp_path, new, "new", {"reportAny": 2}) - assert gate.load_cached_counts(old) == {"reportAny": 1} - assert gate.load_cached_counts(new) == {"reportAny": 2} - - -def test_store_evicts_only_the_oldest_entries_beyond_the_cap(tmp_path): - aged = [ - gate.cache_path(tmp_path, f"base{i}", ("f",)) - for i in range(gate.CACHE_KEEP_ENTRIES) - ] - for age, path in enumerate(aged): - gate.store_counts(tmp_path, path, f"base{age}", {"reportAny": age}) - os.utime(path, (age, age)) - newest = gate.cache_path(tmp_path, "newest", ("f",)) - gate.store_counts(tmp_path, newest, "newest", {"reportAny": 99}) - assert not aged[0].exists() - assert all(path.exists() for path in aged[1:]) - assert gate.load_cached_counts(newest) == {"reportAny": 99} - - -def test_store_never_evicts_the_entry_it_just_wrote_even_on_mtime_ties(tmp_path): - others = [ - gate.cache_path(tmp_path, f"base{i}", ("f",)) - for i in range(gate.CACHE_KEEP_ENTRIES + 2) - ] - for path in others: - gate.store_counts(tmp_path, path, path.name, {"reportAny": 1}) - os.utime(path, (9_999_999_999, 9_999_999_999)) - mine = gate.cache_path(tmp_path, "mine", ("f",)) - gate.store_counts(tmp_path, mine, "mine", {"reportAny": 2}) - assert gate.load_cached_counts(mine) == {"reportAny": 2} - survivors = list(tmp_path.glob(f"{gate.CACHE_FILE_PREFIX}*.json")) - assert len(survivors) == gate.CACHE_KEEP_ENTRIES - - -def _no_fetch(ref): - return None - - -def _never(reason): - def callback(ref): - raise AssertionError(reason) - - return callback - - -def test_base_counts_cached_returns_the_hit_without_recomputing(tmp_path): - path = gate.cache_path(tmp_path, "abc123", gate.environment_fingerprints()) - gate.store_counts(tmp_path, path, "abc123", {"reportAny": 7}) - - assert gate.base_counts_cached( - "abc123", - cache_dir=tmp_path, - compute=_never("a cache hit must not re-run the base pass"), - fetch=_never("a cache hit must not reach for CI"), - ) == {"reportAny": 7} - - -def test_base_counts_cached_computes_once_then_hits(tmp_path): - calls = [] - - def fake(ref): - calls.append(ref) - return {"reportAny": 4} - - first = gate.base_counts_cached( - "abc123", cache_dir=tmp_path, compute=fake, fetch=_no_fetch - ) - second = gate.base_counts_cached( - "abc123", cache_dir=tmp_path, compute=fake, fetch=_no_fetch - ) - assert first == second == {"reportAny": 4} - assert calls == ["abc123"] - - -def test_an_empty_base_pass_is_never_cached(tmp_path): - calls = [] - - def crashed(ref): - calls.append(ref) - return {} - - assert ( - gate.base_counts_cached( - "abc123", cache_dir=tmp_path, compute=crashed, fetch=_no_fetch - ) - == {} - ) - assert ( - gate.base_counts_cached( - "abc123", cache_dir=tmp_path, compute=crashed, fetch=_no_fetch - ) - == {} - ) - assert calls == ["abc123", "abc123"] - assert list(tmp_path.iterdir()) == [] - - -def test_base_counts_cached_uses_fetched_counts_and_persists_them(tmp_path): - counts = gate.base_counts_cached( - "abc123", - cache_dir=tmp_path, - compute=_never("fetched counts must skip the local base pass"), - fetch=lambda ref: {"reportAny": 9}, - ) - assert counts == {"reportAny": 9} - path = gate.cache_path(tmp_path, "abc123", gate.environment_fingerprints()) - assert gate.load_cached_counts(path) == {"reportAny": 9} - assert gate.base_counts_cached( - "abc123", - cache_dir=tmp_path, - compute=_never("the persisted fetch must satisfy later runs"), - fetch=_never("the persisted fetch must satisfy later runs"), - ) == {"reportAny": 9} - - -def test_base_counts_cached_falls_back_to_compute_on_a_fetch_miss(tmp_path): - calls = [] - - def local(ref): - calls.append(ref) - return {"reportAny": 4} - - assert gate.base_counts_cached( - "abc123", cache_dir=tmp_path, compute=local, fetch=_no_fetch - ) == {"reportAny": 4} - assert calls == ["abc123"] - - -def test_base_counts_cached_treats_empty_fetched_counts_as_a_miss(tmp_path): - assert gate.base_counts_cached( - "abc123", - cache_dir=tmp_path, - compute=lambda ref: {"reportAny": 2}, - fetch=lambda ref: {}, - ) == {"reportAny": 2} - path = gate.cache_path(tmp_path, "abc123", gate.environment_fingerprints()) - assert gate.load_cached_counts(path) == {"reportAny": 2} - - -def test_origin_slug_parsing_supports_ssh_and_https_github_forms(): - assert gate.parse_origin_slug("git@github.com:BerriAI/litellm.git") == "BerriAI/litellm" - assert gate.parse_origin_slug("git@github.com:BerriAI/litellm") == "BerriAI/litellm" - assert gate.parse_origin_slug("https://github.com/BerriAI/litellm.git") == "BerriAI/litellm" - assert gate.parse_origin_slug("https://github.com/BerriAI/litellm") == "BerriAI/litellm" - assert gate.parse_origin_slug("https://github.com/BerriAI/litellm/") == "BerriAI/litellm" - - -def test_origin_slug_parsing_rejects_non_github_urls(): - assert gate.parse_origin_slug("https://gitlab.com/BerriAI/litellm.git") is None - assert gate.parse_origin_slug("git@bitbucket.org:BerriAI/litellm.git") is None - assert gate.parse_origin_slug("not a url") is None - assert gate.parse_origin_slug("") is None - - -def _artifact_zip(payload): - import io - import zipfile - - buffer = io.BytesIO() - with zipfile.ZipFile(buffer, "w") as archive: - archive.writestr("basedpyright-counts.json", json.dumps(payload)) - return buffer.getvalue() - - -def _gh_stub(listing, zip_bytes): - def gh_output(args): - if args[-1].startswith("repos/"): - return json.dumps(listing).encode() - return zip_bytes - - return gh_output - - -def _live_listing(): - return { - "artifacts": [ - {"expired": False, "archive_download_url": "https://api.github.com/x/zip"} - ] - } - - -def test_fetcher_returns_counts_from_a_matching_artifact(capsys): - payload = {"base_point": "abc123", "counts": {"reportAny": 3}} - fetched = gate.fetch_ci_base_counts( - "abc123", gh_output=_gh_stub(_live_listing(), _artifact_zip(payload)) - ) - assert fetched == {"reportAny": 3} - assert "fetched from CI artifact" in capsys.readouterr().err - - -def test_fetcher_rejects_an_artifact_for_a_different_base_point(): - payload = {"base_point": "someothersha", "counts": {"reportAny": 3}} - assert ( - gate.fetch_ci_base_counts( - "abc123", gh_output=_gh_stub(_live_listing(), _artifact_zip(payload)) - ) - is None - ) - - -def test_fetcher_rejects_empty_or_misshapen_artifact_counts(): - for counts in ({}, {"reportAny": "three"}, {"reportAny": True}): - payload = {"base_point": "abc123", "counts": counts} - assert ( - gate.fetch_ci_base_counts( - "abc123", gh_output=_gh_stub(_live_listing(), _artifact_zip(payload)) - ) - is None - ) - - -def test_fetcher_rejects_an_expired_artifact(): - listing = { - "artifacts": [ - {"expired": True, "archive_download_url": "https://api.github.com/x/zip"} - ] - } - payload = {"base_point": "abc123", "counts": {"reportAny": 3}} - assert ( - gate.fetch_ci_base_counts( - "abc123", gh_output=_gh_stub(listing, _artifact_zip(payload)) - ) - is None - ) - - -def test_fetcher_misses_when_no_artifact_is_published(): - assert ( - gate.fetch_ci_base_counts( - "abc123", gh_output=_gh_stub({"artifacts": []}, b"") - ) - is None - ) - - -def test_fetcher_misses_when_gh_is_unusable(capsys): - assert gate.fetch_ci_base_counts("abc123", gh_output=lambda args: None) is None - assert "computing base counts locally" in capsys.readouterr().err - - -def test_fetcher_misses_on_a_corrupt_artifact_archive(): - assert ( - gate.fetch_ci_base_counts( - "abc123", gh_output=_gh_stub(_live_listing(), b"not a zip") - ) - is None - ) - - -def test_emit_writes_the_artifact_json_named_by_the_head_key(tmp_path, capsys): - gate.cmd_emit_counts({"reportAny": 3, "aRule": 1}, tmp_path, "deadbeef") - key = gate.cache_key("deadbeef", gate.environment_fingerprints()) - path = tmp_path / f"basedpyright-counts-{key}.json" - assert json.loads(path.read_text()) == { - "base_point": "deadbeef", - "counts": {"aRule": 1, "reportAny": 3}, - } - summary = capsys.readouterr().out - assert "deadbeef" in summary - assert key in summary - assert "4" in summary - - -def test_emit_refuses_to_publish_empty_counts(tmp_path): - import pytest - +def test_changing_the_dependency_groups_rekeys_the_base_counts() -> None: + default: Final = gate.checker_identity().artifact_name("abc123") + assert gate.checker_identity().artifact_name("abc123") == default + assert gate.checker_identity(("proxy-dev",)).artifact_name("abc123") != default + + +@pytest.mark.parametrize("rule", ["reportAny", "reportExplicitAny"]) +def test_any_rules_may_grow_past_their_base_up_to_the_cap_and_no_further( + rule: str, capsys: pytest.CaptureFixture[str] +) -> None: + cap: Final = gate.ANY_CAPS[rule] + base: Final = {rule: cap - 5, "reportArgumentType": 3} + gate.judge({rule: cap, "reportArgumentType": 3}, base, "a" * 40) + assert "OK" in capsys.readouterr().out + with pytest.raises(SystemExit) as exit_info: + gate.judge({rule: cap + 1, "reportArgumentType": 3}, base, "a" * 40) + assert exit_info.value.code == 1 + assert f"BREACHED RULES: {rule} {cap + 1}/{cap} (+6)" in capsys.readouterr().out + + +def test_an_any_rule_already_over_its_cap_at_base_does_not_fail_a_bystander( + capsys: pytest.CaptureFixture[str], +) -> None: + over: Final = gate.ANY_CAPS["reportAny"] + 50 + gate.judge({"reportAny": over}, {"reportAny": over}, "a" * 40) + assert "OK" in capsys.readouterr().out + + +def test_rules_without_a_cap_may_not_grow_past_their_base(capsys: pytest.CaptureFixture[str]) -> None: with pytest.raises(SystemExit): - gate.cmd_emit_counts({}, tmp_path, "deadbeef") - assert list(tmp_path.iterdir()) == [] + gate.judge({"reportArgumentType": 4}, {"reportArgumentType": 3}, "a" * 40) + assert "BREACHED RULES: reportArgumentType 4/3 (+1)" in capsys.readouterr().out -def test_emitted_file_round_trips_through_the_fetch_validation(tmp_path): - gate.cmd_emit_counts({"reportAny": 3}, tmp_path, "deadbeef") - key = gate.cache_key("deadbeef", gate.environment_fingerprints()) - payload = json.loads((tmp_path / f"basedpyright-counts-{key}.json").read_text()) - assert gate.counts_for_base(payload, "deadbeef") == {"reportAny": 3} - assert gate.counts_for_base(payload, "someothersha") is None - - -def _git(cwd, *args): - proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) - assert proc.returncode == 0, proc.stderr - return proc.stdout.strip() - - -def _commit(cwd, name): - (cwd / name).write_text(name) - _git(cwd, "add", "-A") - _git(cwd, "commit", "-q", "-m", name) - return _git(cwd, "rev-parse", "HEAD") - - -def _init_repo(tmp_path): - repo = tmp_path / "repo" - repo.mkdir() - _git(repo, "init", "-q", "-b", "main") - _git(repo, "config", "user.email", "gate@example.com") - _git(repo, "config", "user.name", "gate") - _git(repo, "config", "commit.gpgsign", "false") - return repo - - -def _branched_repo(tmp_path): - repo = _init_repo(tmp_path) - branch_point = _commit(repo, "shared.txt") - _git(repo, "checkout", "-q", "-b", "feature") - _commit(repo, "feature.txt") - _git(repo, "checkout", "-q", "main") - base_tip = _commit(repo, "drift.txt") - _git(repo, "checkout", "-q", "feature") - return repo, branch_point, base_tip - - -def test_base_point_is_the_branch_point_when_no_merge_is_in_progress(tmp_path): - repo, branch_point, _ = _branched_repo(tmp_path) - assert gate.resolve_base_point("main", cwd=repo) == branch_point - - -def test_base_point_mid_merge_advances_to_the_merged_in_base_tip(tmp_path): - repo, _, base_tip = _branched_repo(tmp_path) - _git(repo, "merge", "--no-commit", "--no-ff", "main") - assert gate.resolve_base_point("main", cwd=repo) == base_tip - - -def test_base_point_mid_merge_of_an_older_side_branch_keeps_the_newer_branch_point(tmp_path): - repo = _init_repo(tmp_path) - _commit(repo, "shared.txt") - _git(repo, "checkout", "-q", "-b", "old-side") - _commit(repo, "old.txt") - _git(repo, "checkout", "-q", "main") - newer_point = _commit(repo, "drift.txt") - _git(repo, "checkout", "-q", "-b", "feature") - _commit(repo, "feature.txt") - _git(repo, "merge", "--no-commit", "--no-ff", "old-side") - assert gate.resolve_base_point("main", cwd=repo) == newer_point +def test_no_head_output_is_refused_as_vacuous_before_any_base_lookup(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as exit_info: + gate.cmd_check({}, "irrelevant-base-ref") + assert exit_info.value.code == 1 + assert "vacuous" in capsys.readouterr().out diff --git a/tests/unit/test_type_discipline_gate.py b/tests/unit/test_type_discipline_gate.py index 1832668e7c3..e99a7be4359 100644 --- a/tests/unit/test_type_discipline_gate.py +++ b/tests/unit/test_type_discipline_gate.py @@ -1,106 +1,64 @@ """Tests for scripts/type_discipline_gate.py. -The gate's correctness lives in two pure functions: `over_ceiling` (which decides -whether the expensive base worktree scan is even needed) and `evaluate` (the -drift-safe breach check). Both are pinned here. +The gate compares each LIT rule's codebase count against the merge-base count, so +what is pinned here is the identity that keys those base counts and the diff scan +that turns a breach into file:line. """ -import importlib.util -import subprocess from pathlib import Path +from typing import Final -_MODULE_PATH = Path(__file__).resolve().parents[2] / "scripts" / "type_discipline_gate.py" -_spec = importlib.util.spec_from_file_location("type_discipline_gate", _MODULE_PATH) -gate = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(gate) +import type_discipline_gate as gate -def _budget(limit): - return {"LIT006": {"limit": limit}} +def test_editing_the_checker_rekeys_the_base_counts(tmp_path: Path) -> None: + checker: Final = tmp_path / "check.py" + checker.write_text("print('v1')\n") + before: Final = gate.checker_identity(checker).artifact_name("abc123") + assert gate.checker_identity(checker).artifact_name("abc123") == before + checker.write_text("print('v2')\n") + assert gate.checker_identity(checker).artifact_name("abc123") != before -def test_over_ceiling_flags_only_counts_above_the_limit(): - budget = _budget(12) - assert gate.over_ceiling({"LIT006": 12}, budget) == frozenset() # at limit - assert gate.over_ceiling({"LIT006": 13}, budget) == frozenset({"LIT006"}) # over limit - assert gate.over_ceiling({}, budget) == frozenset() # missing rule counts as zero +def test_parse_changed_lines_groups_hunks_under_their_own_file() -> None: + diff: Final = ( + "diff --git a/litellm/a.py b/litellm/a.py\n" + "--- a/litellm/a.py\n" + "+++ b/litellm/a.py\n" + "@@ -0,0 +3,2 @@\n" + "+one\n" + "+two\n" + "diff --git a/litellm/b.py b/litellm/b.py\n" + "--- a/litellm/b.py\n" + "+++ b/litellm/b.py\n" + "@@ -0,0 +10 @@\n" + "+only\n" + ) + changed: Final = gate.parse_changed_lines(diff) + assert changed["litellm/a.py"] == {3, 4} + assert changed["litellm/b.py"] == {10} -def test_over_ceiling_is_independent_across_rules(): - budget = {"LIT001": {"limit": 5}, "LIT006": {"limit": 10}} - assert gate.over_ceiling({"LIT001": 6, "LIT006": 10}, budget) == frozenset({"LIT001"}) +def test_parse_changed_lines_handles_several_hunks_in_one_file() -> None: + diff: Final = ( + "+++ b/litellm/a.py\n" + "@@ -0,0 +1,2 @@\n" + "+a\n" + "@@ -9,0 +20,1 @@\n" + "+b\n" + ) + assert gate.parse_changed_lines(diff)["litellm/a.py"] == {1, 2, 20} -def test_evaluate_blames_only_a_rule_over_limit_and_over_base(): - budget = _budget(10) - # over limit and grown vs base -> breach - assert [b.rule for b in gate.evaluate({"LIT006": 12}, {"LIT006": 9}, budget)] == ["LIT006"] - # over limit but flat vs base (pre-existing drift) -> not blamed - assert gate.evaluate({"LIT006": 12}, {"LIT006": 12}, budget) == [] - # within limit -> not blamed regardless of base - assert gate.evaluate({"LIT006": 10}, {"LIT006": 0}, budget) == [] +def test_parse_changed_lines_on_an_empty_diff_is_empty() -> None: + assert gate.parse_changed_lines("") == {} -def test_update_ratchets_limit_down_by_what_the_branch_fixed_never_up(): - budget = {"LIT001": {"limit": 100}, "LIT006": {"limit": 10}} - # LIT001 fixed 15 (60 -> 45) so its limit falls 100 -> 85; LIT006 grew, so its - # limit holds flat at 10. - current = {"LIT001": 45, "LIT006": 12} - base = {"LIT001": 60, "LIT006": 9} - assert gate.ratcheted_budget(budget, current, base) == { - "LIT001": {"limit": 85}, - "LIT006": {"limit": 10}, - } - - -def test_update_leaves_rules_seeded_on_this_branch_untouched(): - # A rule absent from the base budget was seeded with grandfathered headroom on - # this branch; the base tree predates the rule (e.g. no Final annotations yet), - # so ratcheting against it would collapse the deliberate headroom. - budget = {"LIT001": {"limit": 100}, "LIT010": {"limit": 24600}} - current = {"LIT001": 45, "LIT010": 16400} - base = {"LIT001": 60, "LIT010": 40000} - assert gate.ratcheted_budget(budget, current, base, frozenset({"LIT010"})) == { - "LIT001": {"limit": 85}, - "LIT010": {"limit": 24600}, - } - - -def _git(cwd, *args): - proc = subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True) - assert proc.returncode == 0, proc.stderr - return proc.stdout.strip() - - -def _commit(cwd, name): - (cwd / name).write_text(name) - _git(cwd, "add", "-A") - _git(cwd, "commit", "-q", "-m", name) - return _git(cwd, "rev-parse", "HEAD") - - -def _branched_repo(tmp_path): - repo = tmp_path / "repo" - repo.mkdir() - _git(repo, "init", "-q", "-b", "main") - _git(repo, "config", "user.email", "gate@example.com") - _git(repo, "config", "user.name", "gate") - _git(repo, "config", "commit.gpgsign", "false") - branch_point = _commit(repo, "shared.txt") - _git(repo, "checkout", "-q", "-b", "feature") - _commit(repo, "feature.txt") - _git(repo, "checkout", "-q", "main") - base_tip = _commit(repo, "drift.txt") - _git(repo, "checkout", "-q", "feature") - return repo, branch_point, base_tip - - -def test_base_point_is_the_branch_point_when_no_merge_is_in_progress(tmp_path): - repo, branch_point, _ = _branched_repo(tmp_path) - assert gate.resolve_base_point("main", cwd=repo) == branch_point - - -def test_base_point_mid_merge_advances_to_the_merged_in_base_tip(tmp_path): - repo, _, base_tip = _branched_repo(tmp_path) - _git(repo, "merge", "--no-commit", "--no-ff", "main") - assert gate.resolve_base_point("main", cwd=repo) == base_tip +def test_introduced_keeps_only_violations_on_changed_lines() -> None: + violations: Final = ( + gate.Violation("litellm/a.py", 3, "LIT006"), + gate.Violation("litellm/a.py", 99, "LIT006"), + gate.Violation("litellm/b.py", 3, "LIT001"), + ) + kept: Final = gate.introduced(violations, {"litellm/a.py": {3}}) + assert kept == [gate.Violation("litellm/a.py", 3, "LIT006")] diff --git a/type-discipline-budget.json b/type-discipline-budget.json deleted file mode 100644 index aeb77212e8e..00000000000 --- a/type-discipline-budget.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "LIT001": { - "limit": 22174 - }, - "LIT003": { - "limit": 261 - }, - "LIT004": { - "limit": 38 - }, - "LIT005": { - "limit": 0 - }, - "LIT006": { - "limit": 1035 - }, - "LIT007": { - "limit": 0 - }, - "LIT008": { - "limit": 945 - }, - "LIT009": { - "limit": 0 - }, - "LIT010": { - "limit": 16398 - }, - "LIT011": { - "limit": 5504 - }, - "LIT012": { - "limit": 4486 - }, - "LIT013": { - "limit": 0 - }, - "LIT014": { - "limit": 369 - }, - "LIT015": { - "limit": 1448 - } -} From da2bb5a6b1c35be03ae7226322df1a0bb6444585 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:48:29 -0700 Subject: [PATCH 20/65] test(e2e): cover responses API gaps on gemini, azure, compact and context management (#45248) * test(e2e): cover responses API gaps on gemini, azure, compact and context management * test(e2e): assert streamed usage cost and split anthropic strict schema test * test(e2e): assert compaction items and final stream event in responses e2e --------- Co-authored-by: yuneng --- .../e2e/llm_translation/responses_helpers.py | 17 + .../e2e/llm_translation/test_responses_e2e.py | 414 +++++++++++++++++- .../test_responses_retrieve_e2e.py | 133 ++++-- 3 files changed, 515 insertions(+), 49 deletions(-) create mode 100644 tests/e2e/llm_translation/responses_helpers.py diff --git a/tests/e2e/llm_translation/responses_helpers.py b/tests/e2e/llm_translation/responses_helpers.py new file mode 100644 index 00000000000..be0fac74bb6 --- /dev/null +++ b/tests/e2e/llm_translation/responses_helpers.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from typing import Final + +from models import LiteLLMParamsBody + +AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" +AZURE_OPENAI_API_VERSION: Final = "v1" + + +def azure_openai_params(api_version: str = AZURE_OPENAI_API_VERSION) -> LiteLLMParamsBody: + return LiteLLMParamsBody( + model=AZURE_OPENAI_BACKEND, + api_base="os.environ/AZURE_API_BASE", + api_key="os.environ/AZURE_API_KEY", + api_version=api_version, + ) diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 00a1d3611f9..95d01ec71f1 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -42,6 +42,7 @@ from provider_edge import LiveEdge, start_provider_edge from provider_edge_bedrock import bedrock_signer from proxy_client import ProxyClient from pydantic import BaseModel, TypeAdapter +from responses_helpers import AZURE_OPENAI_BACKEND, azure_openai_params from sdk_clients import NO_PROXY_CACHE, SdkClients pytestmark = pytest.mark.e2e @@ -58,8 +59,8 @@ OPENAI_VISION_BACKEND: Final = "openai/gpt-4o" ANTHROPIC_BACKEND: Final = "anthropic/claude-haiku-4-5" BEDROCK_CONVERSE_BACKEND: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" VERTEX_BACKEND: Final = "vertex_ai/gemini-2.5-flash" -AZURE_OPENAI_BACKEND: Final = "azure/gpt-5.4-nano" -AZURE_OPENAI_API_VERSION: Final = "v1" +GEMINI_BACKEND: Final = "gemini/gemini-2.5-flash" +OPENAI_RESPONSES_BACKEND: Final = "openai/gpt-5.5" INSTRUCTIONS = "You are a helpful assistant" CAT_IMAGE_URL = "https://upload.wikimedia.org/wikipedia/commons/3/3a/Cat03.jpg" BEDROCK_EDGE_REGION: Final = "us-east-1" @@ -102,11 +103,36 @@ WEATHER_TOOL: FunctionToolParam = { "strict": False, } +LOCATIONS_TOOL: Final[FunctionToolParam] = { + "type": "function", + "name": "get_locations", + "description": "Return locations that need weather information", + "parameters": { + "type": "object", + "properties": {"locations": {"type": "array", "items": {"type": "string"}}}, + "required": ["locations"], + "additionalProperties": False, + }, + "strict": True, +} + + +class LocationsArguments(BaseModel): + locations: list[str] + + +class ResponseUsageCost(BaseModel): + cost: float | None = None + def _openai_params() -> LiteLLMParamsBody: return LiteLLMParamsBody(model=OPENAI_MINI_BACKEND, api_key="os.environ/OPENAI_API_KEY") +def _openai_responses_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody(model=OPENAI_RESPONSES_BACKEND, api_key="os.environ/OPENAI_API_KEY") + + def _anthropic_params() -> LiteLLMParamsBody: return LiteLLMParamsBody(model=ANTHROPIC_BACKEND, api_key="os.environ/ANTHROPIC_API_KEY") @@ -128,13 +154,8 @@ def _vertex_params() -> LiteLLMParamsBody: ) -def _azure_openai_params() -> LiteLLMParamsBody: - return LiteLLMParamsBody( - model=AZURE_OPENAI_BACKEND, - api_base="os.environ/AZURE_API_BASE", - api_key="os.environ/AZURE_API_KEY", - api_version=AZURE_OPENAI_API_VERSION, - ) +def _gemini_params() -> LiteLLMParamsBody: + return LiteLLMParamsBody(model=GEMINI_BACKEND, api_key="os.environ/GEMINI_API_KEY") def _register( @@ -197,22 +218,147 @@ class TestResponses: def test_responses_streaming_returns_completion( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = _register(proxy, resources, _openai_params()) - client = sdk.openai(resources.key()) + model: Final = _register(proxy, resources, _openai_params()) + client: Final = sdk.openai(resources.key()) - stream = client.responses.create( + stream: Final = client.responses.create( model=model, input="reply with one word", instructions=INSTRUCTIONS, stream=True, extra_body=NO_PROXY_CACHE, ) - events = tuple(stream) + events: Final = tuple(stream) assert events, "responses stream returned no events" - deltas = tuple(event.delta for event in events if event.type == "response.output_text.delta") + deltas: Final = tuple(event.delta for event in events if event.type == "response.output_text.delta") assert any(delta for delta in deltas), "responses stream returned no text deltas" - assert events[-1].type == "response.completed", ( - f"responses stream did not terminate with response.completed: {events[-1].type}" + completed: Final = events[-1] + assert isinstance(completed, ResponseCompletedEvent), ( + f"responses stream did not terminate with response.completed: {completed.type}" + ) + usage: Final = completed.response.usage + assert usage is not None, f"response.completed had no usage: {completed.response!r}" + assert usage.input_tokens > 0, f"response.completed had no input tokens: {usage!r}" + assert usage.output_tokens > 0, f"response.completed had no output tokens: {usage!r}" + assert usage.total_tokens == usage.input_tokens + usage.output_tokens, ( + f"response.completed token totals were inconsistent: {usage!r}" + ) + usage_cost: Final = TypeAdapter(ResponseUsageCost).validate_python( + cast(object, usage.model_extra if usage.model_extra is not None else {}) + ) + assert usage_cost.cost is not None, f"response.completed usage had no cost: {usage.model_extra!r}" + assert usage_cost.cost > 0, f"response.completed cost was not positive: {usage_cost.cost}" + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.GEMINI,), + models=(GEMINI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_gemini_returns_completion( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register(proxy, resources, _gemini_params(), prefix="e2e-responses-gemini") + client: Final = sdk.openai(resources.key()) + + response: Final = client.responses.create( + model=model, input="reply with one word", instructions=INSTRUCTIONS, extra_body=NO_PROXY_CACHE + ) + assert response.output_text.strip(), f"/responses over gemini returned no output text: {response.output!r}" + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.GEMINI,), + models=(GEMINI_BACKEND,), + mode=Mode.STREAM, + ) + ) + def test_responses_gemini_streaming_returns_completion( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register(proxy, resources, _gemini_params(), prefix="e2e-responses-gemini") + client: Final = sdk.openai(resources.key()) + + stream: Final = client.responses.create( + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + stream=True, + extra_body=NO_PROXY_CACHE, + ) + events: Final = tuple(stream) + deltas: Final = tuple(event.delta for event in events if event.type == "response.output_text.delta") + assert any(deltas), "responses stream over gemini returned no text deltas" + assert isinstance(events[-1], ResponseCompletedEvent), ( + f"responses stream over gemini did not end with response.completed: {events[-1].type}" + ) + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.GEMINI,), + models=(GEMINI_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_gemini_replays_legacy_function_call_output( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register(proxy, resources, _gemini_params(), prefix="e2e-responses-gemini-tool") + client: Final = sdk.openai(resources.key()) + function_call_id: Final = f"fc_{unique_marker()}" + input_items: Final[ResponseInputParam] = [ + { + "type": "message", + "role": "user", + "content": "What is the temperature in Paris today?", + }, + { + "type": "function_call", + "arguments": '{"location": "Paris, France"}', + "call_id": function_call_id, + "name": "get_temperature", + "id": function_call_id, + "status": "completed", + }, + { + "type": "function_call_output", + "call_id": function_call_id, + "output": "Temperature is exactly 31 Celsius.", + }, + ] + tools: Final[tuple[FunctionToolParam, ...]] = ( + { + "type": "function", + "name": "get_temperature", + "description": "Get the current temperature for a location", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + "required": ["location"], + "additionalProperties": False, + }, + "strict": False, + }, + ) + + response: Final = client.responses.create( + model=model, + input=input_items, + tools=tools, + store=False, + extra_body=NO_PROXY_CACHE, + ) + assert response.status == "completed", f"legacy tool replay was not completed: {response.status}" + assert "31" in response.output_text, ( + f"legacy tool result was missing from output text: {response.output_text!r}" ) @pytest.mark.covers("llm.responses.openai.basic.nonstream.cost_logged") @@ -361,6 +507,94 @@ class TestResponses: ) _assert_weather_call(response) + @pytest.mark.covers("llm.responses.anthropic.tool_use.nonstream.works") + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_anthropic_strict_array_schema_tool_call( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register(proxy, resources, _anthropic_params()) + client: Final = sdk.openai(resources.key()) + + response: Final = client.responses.create( + model=model, + input="Find the weather locations for Tokyo and Paris using get_locations.", + instructions=INSTRUCTIONS, + tools=[LOCATIONS_TOOL], + tool_choice="required", + extra_body=NO_PROXY_CACHE, + ) + function_call: Final = next( + (call for call in _function_calls(response) if call.name == "get_locations"), + None, + ) + assert function_call is not None, f"response had no get_locations call: {response.output!r}" + arguments: Final = LocationsArguments.model_validate_json(function_call.arguments) + assert arguments.locations, f"get_locations returned no locations: {function_call.arguments}" + + @pytest.mark.covers("llm.responses.anthropic.multi_turn.nonstream.works") + @pytest.mark.skip( + reason="stage red: product gap, Anthropic previous_response_id continuation sends invalid unmatched tool_use history" + ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.ANTHROPIC,), + models=(ANTHROPIC_BACKEND,), + capabilities=(Capability.FUNCTION_CALLING,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_anthropic_tool_output_continues_with_previous_response_id( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register(proxy, resources, _anthropic_params()) + client: Final = sdk.openai(resources.key()) + + first: Final = client.responses.create( + model=model, + input="Find the weather locations for Tokyo and Paris using get_locations.", + instructions=INSTRUCTIONS, + tools=[LOCATIONS_TOOL], + tool_choice="required", + extra_body=NO_PROXY_CACHE, + ) + function_call: Final = next( + (call for call in _function_calls(first) if call.name == "get_locations"), + None, + ) + assert function_call is not None, f"response had no get_locations call: {first.output!r}" + assert function_call.call_id, f"get_locations call had no call_id: {function_call!r}" + arguments: Final = LocationsArguments.model_validate_json(function_call.arguments) + assert arguments.locations, f"get_locations call had no locations: {function_call.arguments}" + + tool_result: Final = "Distinctive forecast: 47 degrees Celsius" + follow_up_input: Final[ResponseInputParam] = [ + { + "type": "function_call_output", + "call_id": function_call.call_id, + "output": tool_result, + } + ] + second: Final = client.responses.create( + model=model, + previous_response_id=first.id, + input=follow_up_input, + instructions=INSTRUCTIONS, + tools=[LOCATIONS_TOOL], + extra_body=NO_PROXY_CACHE, + ) + assert tool_result in second.output_text, f"follow-up omitted tool result: {second.output_text!r}" + @pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works") @meta( Subject( @@ -469,7 +703,7 @@ class TestResponses: def test_responses_azure_openai_returns_completion( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = _register(proxy, resources, _azure_openai_params(), prefix="e2e-responses-azure-openai") + model = _register(proxy, resources, azure_openai_params(), prefix="e2e-responses-azure-openai") client = sdk.openai(resources.key()) response = client.responses.create( @@ -479,6 +713,150 @@ class TestResponses: f"/responses over azure openai returned no output text: {response.output!r}" ) + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + mode=Mode.STREAM, + ) + ) + def test_responses_azure_openai_streaming_returns_completion( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register(proxy, resources, azure_openai_params(), prefix="e2e-responses-azure-stream") + client: Final = sdk.openai(resources.key()) + + stream: Final = client.responses.create( + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + stream=True, + extra_body=NO_PROXY_CACHE, + ) + events: Final = tuple(stream) + deltas: Final = tuple(event.delta for event in events if event.type == "response.output_text.delta") + assert any(deltas), "responses stream over azure openai returned no text deltas" + assert isinstance(events[-1], ResponseCompletedEvent), ( + f"responses stream over azure openai did not end with response.completed: {events[-1].type}" + ) + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.AZURE,), + models=(AZURE_OPENAI_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_azure_openai_preview_api_version_accepts_truncation( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register( + proxy, + resources, + azure_openai_params(api_version="preview"), + prefix="e2e-responses-azure-preview", + ) + client: Final = sdk.openai(resources.key()) + + response: Final = client.responses.create( + model=model, + input="reply with one word", + instructions=INSTRUCTIONS, + truncation="auto", + extra_body=NO_PROXY_CACHE, + ) + assert response.output_text.strip(), ( + f"/responses over azure openai preview returned no output text: {response.output!r}" + ) + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_RESPONSES_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_compact_returns_compacted_conversation( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register( + proxy, + resources, + _openai_responses_params(), + prefix="e2e-responses-compact", + ) + client: Final = sdk.openai(resources.key()) + conversation: Final[ResponseInputParam] = [ + {"role": "user", "content": "Remember that my favorite color is blue."}, + {"role": "assistant", "content": "I will remember that your favorite color is blue."}, + ] + + compacted: Final = client.responses.compact( + model=model, + input=conversation, + extra_body=NO_PROXY_CACHE, + ) + assert compacted.id, f"/responses/compact returned no id: {compacted!r}" + assert any(item.type == "compaction" for item in compacted.output), ( + f"/responses/compact returned no compaction item: {compacted.output!r}" + ) + compacted_input: Final[ResponseInputParam] = TypeAdapter(ResponseInputParam).validate_python( + [item.model_dump(exclude_none=True) for item in compacted.output] + + [{"role": "user", "content": "What is my favorite color?"}] + ) + + response: Final = client.responses.create( + model=model, + input=compacted_input, + extra_body=NO_PROXY_CACHE, + ) + assert "blue" in response.output_text.lower(), ( + f"compacted conversation did not retain the favorite color: {response.output_text!r}" + ) + + @meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(Provider.OPENAI,), + models=(OPENAI_RESPONSES_BACKEND,), + mode=Mode.NONSTREAM, + ) + ) + def test_responses_context_management_compacts_server_side( + self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + ) -> None: + model: Final = _register( + proxy, + resources, + _openai_responses_params(), + prefix="e2e-responses-context-compaction", + ) + client: Final = sdk.openai(resources.key()) + filler: Final = "The archive record has a blue marker beside every stored entry. " * 350 + conversation: Final[ResponseInputParam] = [ + {"role": "user", "content": filler}, + {"role": "assistant", "content": "I have read the archive and retained its details."}, + {"role": "user", "content": "Reply with one word to verify server-side compaction."}, + ] + + response: Final = client.responses.create( + model=model, + input=conversation, + context_management=[{"type": "compaction", "compact_threshold": 1000}], + extra_body=NO_PROXY_CACHE, + ) + assert response.status == "completed", f"context management did not complete: {response.status}" + assert any(item.type == "compaction" for item in response.output), ( + f"context management returned no compaction item: {response.output!r}" + ) + @pytest.mark.covers("llm.responses.azure_openai.tool_use.nonstream.works") @meta( Subject( @@ -493,7 +871,7 @@ class TestResponses: def test_responses_azure_openai_returns_function_call( self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients ) -> None: - model = _register(proxy, resources, _azure_openai_params(), prefix="e2e-responses-azure-openai-tool") + model = _register(proxy, resources, azure_openai_params(), prefix="e2e-responses-azure-openai-tool") client = sdk.openai(resources.key()) response = client.responses.create( diff --git a/tests/e2e/llm_translation/test_responses_retrieve_e2e.py b/tests/e2e/llm_translation/test_responses_retrieve_e2e.py index b4a9634fbef..dd94d5cab72 100644 --- a/tests/e2e/llm_translation/test_responses_retrieve_e2e.py +++ b/tests/e2e/llm_translation/test_responses_retrieve_e2e.py @@ -6,7 +6,7 @@ Creates a stored response, retrieves it by id, and pins invalid-id error handlin from __future__ import annotations import time -from typing import Final +from typing import Final, Literal import openai import pytest @@ -15,7 +15,9 @@ from e2e_http import NoBody, Success, UnknownApiError, unwrap from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta from lifecycle import ResourceManager from models import LiteLLMParamsBody +from responses_helpers import AZURE_OPENAI_BACKEND, azure_openai_params from openai.types.responses import ( + ResponseCompletedEvent, ResponseCreatedEvent, ResponseInputMessageItem, ResponseInputText, @@ -138,12 +140,42 @@ class TestResponsesRetrieve: def _register_openai(proxy: ProxyClient, resources: ResourceManager, prefix: str) -> str: - model = f"{prefix}-{unique_marker()}" - model_id = proxy.create_model(model, LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY")) + return _register_response_deployment(proxy, resources, "openai", prefix) + + +def _register_response_deployment( + proxy: ProxyClient, + resources: ResourceManager, + deployment: Literal["openai", "azure"], + prefix: str, +) -> str: + model: Final = f"{prefix}-{unique_marker()}" + params: Final = ( + azure_openai_params() + if deployment == "azure" + else LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY") + ) + model_id: Final = proxy.create_model(model, params) resources.defer(lambda: proxy.delete_model(model_id)) return model +def _deployment_param(deployment: Literal["openai", "azure"], provider: Provider, backend: str, mode: Mode) -> object: + return pytest.param( + deployment, + id=deployment, + marks=meta( + Subject( + domain=Domain.LLM_TRANSLATION, + route=Route.RESPONSES, + providers=(provider,), + models=(backend,), + mode=mode, + ) + ), + ) + + def _input_texts(item: object) -> tuple[str, ...]: if not isinstance(item, ResponseInputMessageItem): return () @@ -176,57 +208,96 @@ class TestStoredResponseLifecycle: texts = tuple(text for item in items for text in _input_texts(item)) assert any(marker in text for text in texts), f"input_items did not list the stored prompt: {items!r}" - @meta( - Subject( - domain=Domain.LLM_TRANSLATION, - route=Route.RESPONSES, - providers=(Provider.OPENAI,), - models=(OPENAI_BACKEND,), - mode=Mode.NONSTREAM, - ) + @pytest.mark.parametrize( + "deployment", + [ + _deployment_param("openai", Provider.OPENAI, OPENAI_BACKEND, Mode.NONSTREAM), + _deployment_param("azure", Provider.AZURE, AZURE_OPENAI_BACKEND, Mode.NONSTREAM), + ], ) def test_deleted_response_is_no_longer_retrievable( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + self, + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + deployment: Literal["openai", "azure"], ) -> None: - model = _register_openai(proxy, resources, "e2e-resp-delete") - client = sdk.openai(resources.key()) + model: Final = _register_response_deployment(proxy, resources, deployment, "e2e-resp-delete") + client: Final = sdk.openai(resources.key()) - created = client.responses.create( + created: Final = client.responses.create( model=model, input=f"Reply with one word. {unique_marker()}", store=True, extra_body=NO_PROXY_CACHE ) - retrieved = client.responses.retrieve(created.id) + retrieved: Final = client.responses.retrieve(created.id) assert retrieved.status == "completed", f"stored response not retrievable as completed: {retrieved!r}" + assert retrieved.output_text == created.output_text, ( + f"retrieved output changed: created={created.output_text!r}, retrieved={retrieved.output_text!r}" + ) client.responses.delete(created.id) - with pytest.raises(openai.APIStatusError) as gone: - client.responses.retrieve(created.id) + gone: Final = pytest.raises(openai.APIStatusError, client.responses.retrieve, created.id) assert 400 <= gone.value.status_code < 500, f"retrieve after delete expected a 4xx: {gone.value!r}" + @pytest.mark.parametrize( + "deployment", + [ + _deployment_param("openai", Provider.OPENAI, OPENAI_BACKEND, Mode.STREAM), + _deployment_param("azure", Provider.AZURE, AZURE_OPENAI_BACKEND, Mode.STREAM), + ], + ) + def test_streamed_response_can_be_deleted( + self, + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + deployment: Literal["openai", "azure"], + ) -> None: + model: Final = _register_response_deployment(proxy, resources, deployment, "e2e-resp-delete-stream") + client: Final = sdk.openai(resources.key()) + events: Final = tuple( + client.responses.create( + model=model, + input=f"Reply with one word. {unique_marker()}", + store=True, + stream=True, + extra_body=NO_PROXY_CACHE, + ) + ) + completed: Final = next((event for event in events if isinstance(event, ResponseCompletedEvent)), None) + assert completed is not None, f"stream did not complete: {events!r}" + response_id: Final = completed.response.id + + client.responses.delete(response_id) + gone: Final = pytest.raises(openai.APIStatusError, client.responses.retrieve, response_id) + assert 400 <= gone.value.status_code < 500, f"retrieve after streamed delete expected a 4xx: {gone.value!r}" + @pytest.mark.provider_live class TestBackgroundResponseCancel: - @meta( - Subject( - domain=Domain.LLM_TRANSLATION, - route=Route.RESPONSES, - providers=(Provider.OPENAI,), - models=(OPENAI_BACKEND,), - mode=Mode.NONSTREAM, - ) + @pytest.mark.parametrize( + "deployment", + [ + _deployment_param("openai", Provider.OPENAI, OPENAI_BACKEND, Mode.NONSTREAM), + _deployment_param("azure", Provider.AZURE, AZURE_OPENAI_BACKEND, Mode.NONSTREAM), + ], ) def test_cancel_background_response( - self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients + self, + proxy: ProxyClient, + resources: ResourceManager, + sdk: SdkClients, + deployment: Literal["openai", "azure"], ) -> None: - model = _register_openai(proxy, resources, "e2e-resp-cancel") - client = sdk.openai(resources.key()) + model: Final = _register_response_deployment(proxy, resources, deployment, "e2e-resp-cancel") + client: Final = sdk.openai(resources.key()) - created = client.responses.create( + created: Final = client.responses.create( model=model, input=f"{LONG_TASK} {unique_marker()}", background=True, extra_body=NO_PROXY_CACHE ) assert created.status in CANCELLABLE_STATUSES, f"background response was not queued: {created.status}" - cancelled = client.responses.cancel(created.id) + cancelled: Final = client.responses.cancel(created.id) assert cancelled.status == "cancelled", f"cancel did not stop the response: {cancelled.status}" @meta( From 91f36f17740cea08c21f27e63e60f97ad8c03c84 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:03:01 -0700 Subject: [PATCH 21/65] chore: bump litellm-proxy-extras to 0.4.107 and litellm-enterprise to 0.1.75 (#45291) Co-authored-by: yuneng --- enterprise/pyproject.toml | 4 ++-- litellm-proxy-extras/pyproject.toml | 4 ++-- pyproject.toml | 4 ++-- uv.lock | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/enterprise/pyproject.toml b/enterprise/pyproject.toml index 08b049f0157..6b55abae873 100644 --- a/enterprise/pyproject.toml +++ b/enterprise/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-enterprise" -version = "0.1.74" +version = "0.1.75" description = "Package for LiteLLM Enterprise features" readme = "README.md" requires-python = ">=3.9" @@ -26,7 +26,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.1.74" +version = "0.1.75" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-enterprise==", diff --git a/litellm-proxy-extras/pyproject.toml b/litellm-proxy-extras/pyproject.toml index 00984dbbf82..221d5012360 100644 --- a/litellm-proxy-extras/pyproject.toml +++ b/litellm-proxy-extras/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "litellm-proxy-extras" -version = "0.4.106" +version = "0.4.107" description = "Additional files for the LiteLLM Proxy. Reduces the size of the main litellm package." readme = "README.md" requires-python = ">=3.9" @@ -30,7 +30,7 @@ required-version = ">=0.10.9" module-root = "" [tool.commitizen] -version = "0.4.106" +version = "0.4.107" version_files = [ "pyproject.toml:^version", "../pyproject.toml:litellm-proxy-extras==", diff --git a/pyproject.toml b/pyproject.toml index f4d89cf512b..e670b10547e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,8 +77,8 @@ proxy = [ "mcp>=2.2.0,<3", "httpx2>=2.5.0,<3", "pydantic>=2.12.0,<3", - "litellm-proxy-extras==0.4.106", - "litellm-enterprise==0.1.74", + "litellm-proxy-extras==0.4.107", + "litellm-enterprise==0.1.75", "RestrictedPython>=8.5,<9.0", "rich>=13.9.4,<14.0", "InquirerPy>=0.3.4,<1.0", diff --git a/uv.lock b/uv.lock index 231e8cb0d06..a481ecb2a2c 100644 --- a/uv.lock +++ b/uv.lock @@ -4996,12 +4996,12 @@ provides-extras = ["test"] [[package]] name = "litellm-enterprise" -version = "0.1.74" +version = "0.1.75" source = { editable = "enterprise" } [[package]] name = "litellm-proxy-extras" -version = "0.4.106" +version = "0.4.107" source = { editable = "litellm-proxy-extras" } dependencies = [ { name = "psycopg" }, From 9df3d4c96b0c5b1d6391744e919160e0236a11bd Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:40:26 -0700 Subject: [PATCH 22/65] test(integration): boot owned proxies with a readiness-sized worker healthcheck budget (#45292) * test(integration): boot owned proxies with a readiness-sized worker healthcheck budget * test(integration): keep owned_proxy_process's extra_arguments for cells that pass their own flags --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- tests/integration/_support/process.py | 8 ++-- .../test_multi_worker_slow_import_boot.py | 45 +++++++++++++++++++ .../test_priority_scheduler_queue_cleanup.py | 25 +++-------- 3 files changed, 55 insertions(+), 23 deletions(-) create mode 100644 tests/integration/configuration/test_multi_worker_slow_import_boot.py diff --git a/tests/integration/_support/process.py b/tests/integration/_support/process.py index 12780540380..23297b28584 100644 --- a/tests/integration/_support/process.py +++ b/tests/integration/_support/process.py @@ -260,6 +260,8 @@ def owned_proxy_process( "127.0.0.1", "--num_workers", str(workers), + "--timeout_worker_healthcheck", + str(int(graceful_stop_seconds())), *database_setup, *extra_arguments, ) @@ -294,6 +296,8 @@ def owned_gateway_image( str(workers), "--host", "127.0.0.1", + "--timeout-worker-healthcheck", + str(int(graceful_stop_seconds())), ) launch: Final = _launch_until_bound(command, root, environment, output, _PORT_ATTEMPTS) try: @@ -397,9 +401,7 @@ class UpstreamSlot: __slots__ = ("certificate", "directory", "port", "process", "root") - def __init__( - self, directory: Path, port: int, root: Path, certificate: UpstreamCertificate | None = None - ) -> None: + def __init__(self, directory: Path, port: int, root: Path, certificate: UpstreamCertificate | None = None) -> None: self.directory = directory self.port = port self.root = root diff --git a/tests/integration/configuration/test_multi_worker_slow_import_boot.py b/tests/integration/configuration/test_multi_worker_slow_import_boot.py new file mode 100644 index 00000000000..a90160e5e80 --- /dev/null +++ b/tests/integration/configuration/test_multi_worker_slow_import_boot.py @@ -0,0 +1,45 @@ +import os +import re +from pathlib import Path +from typing import Final + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.process import graceful_stop_seconds, owned_proxy_process + +WORKERS: Final = 2 +DEPLOYMENT_HEALTHCHECK_SECONDS: Final = 5 +WORKER_IMPORT_DELAY_SECONDS: Final = 3 * DEPLOYMENT_HEALTHCHECK_SECONDS +STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +DIED_WORKER: Final = re.compile(r"Child process \[(\d+)\] died") +SLOW_WORKER_HOOK: Final = f"""\ +import sys +import time + +if "--multiprocessing-fork" in sys.argv: + time.sleep({WORKER_IMPORT_DELAY_SECONDS}) +""" + + +def _slow_worker_environment(directory: Path) -> dict[str, str]: + hook: Final = directory / "slow_worker" + hook.mkdir() + (hook / "sitecustomize.py").write_text(SLOW_WORKER_HOOK) + return { + "PYTHONPATH": os.pathsep.join((str(hook), os.environ.get("PYTHONPATH", ""))), + "TIMEOUT_WORKER_HEALTHCHECK": str(DEPLOYMENT_HEALTHCHECK_SECONDS), + } + + +@pytest.mark.timeout(2 * graceful_stop_seconds() + 60) +def test_owned_proxy_workers_outlast_the_deployments_healthcheck_default(gateway: Gateway, tmp_path: Path) -> None: + with owned_proxy_process(gateway, tmp_path, _slow_worker_environment(tmp_path), workers=WORKERS) as owned: + started: Final = eventually( + lambda: STARTED_WORKER.findall(owned.log.read_text()), + lambda pids: len(pids) >= WORKERS, + seconds=graceful_stop_seconds(), + ) + assert owned.gateway.request("GET", "/health/readiness").status_code == 200 + log: Final = owned.log.read_text() + assert len(started) == WORKERS, log + assert DIED_WORKER.findall(log) == [], log diff --git a/tests/integration/routing/test_priority_scheduler_queue_cleanup.py b/tests/integration/routing/test_priority_scheduler_queue_cleanup.py index a4606554d34..3c38acadaaf 100644 --- a/tests/integration/routing/test_priority_scheduler_queue_cleanup.py +++ b/tests/integration/routing/test_priority_scheduler_queue_cleanup.py @@ -46,7 +46,6 @@ USAGE: Final[dict[str, JsonValue]] = {"prompt_tokens": 5, "completion_tokens": 3 STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") OWNED_CELL_TIMEOUT: Final = 2 * graceful_stop_seconds() + 120 PAIR_CELL_TIMEOUT: Final = 3 * graceful_stop_seconds() + 120 -WORKER_HEALTHCHECK_ARGUMENTS: Final = ("--timeout_worker_healthcheck", str(int(graceful_stop_seconds()))) PINNED_CONNECTION_TIMEOUT_SECONDS: Final = 30 PINNED_LIMITS: Final = httpx.Limits(max_connections=1, max_keepalive_connections=1, keepalive_expiry=30) ENDPOINTS: Final[tuple[Endpoint, ...]] = ( @@ -541,11 +540,7 @@ def test_in_memory_queue_forgets_served_requests_before_a_cooldown(gateway: Gate with ExitStack() as stack: wire: Final = stack.enter_context(wire_server(answering_model_discovery(upstream.respond))) config: Final = owned_config(tmp_path, wire, (INMEM_GROUP,), cooldown_settings(None)) - owned: Final = stack.enter_context( - owned_proxy_process( - gateway, tmp_path, {}, config=config, workers=2, extra_arguments=WORKER_HEALTHCHECK_ARGUMENTS - ) - ) + owned: Final = stack.enter_context(owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2)) worker: Final = pinned(owned.gateway, stack) served: Final = new_marker() assert_served(post(worker, "/v1/chat/completions", chat_body(INMEM_GROUP, served, priority=1)), served) @@ -580,15 +575,9 @@ def pair(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Pair]: wire: Final = stack.enter_context(wire_server(answering_model_discovery(upstream.respond))) config: Final = owned_config(directory, wire, PAIR_GROUPS, cooldown_settings(cache), cancel_on_disconnect=True) overrides: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)} - first: Final = stack.enter_context( - owned_proxy_process( - gateway, directory, overrides, config=config, workers=2, extra_arguments=WORKER_HEALTHCHECK_ARGUMENTS - ) - ) + first: Final = stack.enter_context(owned_proxy_process(gateway, directory, overrides, config=config, workers=2)) second: Final = stack.enter_context( - owned_proxy_process( - gateway, directory, overrides, config=config, workers=2, extra_arguments=WORKER_HEALTHCHECK_ARGUMENTS - ) + owned_proxy_process(gateway, directory, overrides, config=config, workers=2) ) yield Pair(first.gateway, second.gateway, cache, wire, upstream) @@ -797,9 +786,7 @@ def test_prioritized_requests_survive_a_redis_outage(gateway: Gateway, tmp_path: "REDIS_PORT": str(cache.port), "REDIS_CIRCUIT_BREAKER_RECOVERY_TIMEOUT": "1", } - with owned_proxy_process( - gateway, tmp_path, overrides, config=config, workers=2, extra_arguments=WORKER_HEALTHCHECK_ARGUMENTS - ) as owned: + with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned: before: Final = assert_all_served(burst(owned.gateway, OUTAGE_GROUP, 12)) cache.stop() during: Final = assert_all_served(burst(owned.gateway, OUTAGE_GROUP, 12)) @@ -826,9 +813,7 @@ def test_sibling_worker_keeps_serving_prioritized_requests_after_a_worker_is_kil with owned_redis(tmp_path) as cache, wire_server(answering_model_discovery(upstream.respond)) as wire: config: Final = owned_config(tmp_path, wire, (KILL_GROUP,), redis_settings(cache)) overrides: Final = {"REDIS_HOST": cache.host, "REDIS_PORT": str(cache.port)} - with owned_proxy_process( - gateway, tmp_path, overrides, config=config, workers=2, extra_arguments=WORKER_HEALTHCHECK_ARGUMENTS - ) as owned: + with owned_proxy_process(gateway, tmp_path, overrides, config=config, workers=2) as owned: workers: Final = eventually( lambda: tuple(int(found.group(1)) for found in STARTED_WORKER.finditer(owned.log.read_text())), lambda pids: len(pids) == 2, From 8683f1418ee57ec42a6b7cea64e75c9fad526e45 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Wed, 7 Oct 2026 23:54:38 -0700 Subject: [PATCH 23/65] fix(utils): make supports_audio_output read the supports_audio_output cost-map key (#45294) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/utils.py | 2 +- tests/unit/test_utils.py | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/litellm/utils.py b/litellm/utils.py index 89544cf31e2..9ded4ff1131 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -2962,7 +2962,7 @@ def supports_pdf_input(model: str, custom_llm_provider: str | None = None) -> bo def supports_audio_output(model: str, custom_llm_provider: str | None = None) -> bool: """Check if a given model supports audio output in a chat completion call""" - return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_audio_input") + return supports_factory(model=model, custom_llm_provider=custom_llm_provider, key="supports_audio_output") def supports_prompt_caching(model: str, custom_llm_provider: str | None = None) -> bool: diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index 7551a310977..cf3e749df95 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -325,6 +325,25 @@ def test_supports_function_calling_unknown_github_alias_returns_false(): assert litellm.utils.supports_function_calling(model="github/non-existent-model-for-capability-check") is False +@pytest.mark.parametrize(("audio_input", "audio_output"), [(True, False), (False, True)]) +def test_supports_audio_output_reads_its_own_cost_map_flag( + local_model_cost_map: None, monkeypatch: pytest.MonkeyPatch, audio_input: bool, audio_output: bool +) -> None: + model: Final = "openai/audio-flags-disagree-model" + monkeypatch.setitem( + litellm.model_cost, + model, + { + "litellm_provider": "openai", + "mode": "chat", + "supports_audio_input": audio_input, + "supports_audio_output": audio_output, + }, + ) + assert litellm.supports_audio_input(model) is audio_input + assert litellm.supports_audio_output(model) is audio_output + + def test_get_optional_params_image_gen(): from litellm.llms.azure.image_generation import AzureGPTImageGenerationConfig From 057034d0d37ba790750f1aebe5cb298d19b66557 Mon Sep 17 00:00:00 2001 From: tin-berri Date: Thu, 8 Oct 2026 00:24:41 -0700 Subject: [PATCH 24/65] fix(lens): refresh runs until gateway costs are complete (#45105) --- litellm-rust/crates/lens/tests/receiver.rs | 106 +++++++++++++++++- litellm-rust/crates/traces-cache/src/cache.rs | 59 ++++++++-- litellm-rust/crates/traces-cache/src/list.rs | 6 +- .../crates/traces-cache/src/reader.rs | 7 +- .../crates/traces-cache/tests/read.rs | 104 +++++++++++++++++ .../crates/traces/src/resolve/resolution.rs | 51 +++++---- .../crates/traces/src/resolve/spend.rs | 4 + .../crates/traces/src/resolve/view.rs | 5 +- litellm-rust/crates/traces/src/view.rs | 3 + litellm-rust/crates/traces/tests/resolve.rs | 62 +++++++++- 10 files changed, 363 insertions(+), 44 deletions(-) diff --git a/litellm-rust/crates/lens/tests/receiver.rs b/litellm-rust/crates/lens/tests/receiver.rs index bda1cecbbb3..2af3c67955a 100644 --- a/litellm-rust/crates/lens/tests/receiver.rs +++ b/litellm-rust/crates/lens/tests/receiver.rs @@ -10,7 +10,10 @@ use rstest::rstest; use serde_json::json; use sha2::{Digest, Sha256}; use std::{ - sync::{Arc, atomic::Ordering}, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, time::Duration, }; use wiremock::{ @@ -116,6 +119,107 @@ async fn agent_picker_query_preserves_scope_through_the_internal_read_route() { assert_eq!(response.json::().await.unwrap(), result); } +#[rstest] +#[case::list_paid("list", None, 0.75)] +#[case::list_zero("list", None, 0.0)] +#[case::detail_paid("trace", None, 0.75)] +#[case::paged_zero("trace", Some(1), 0.0)] +#[tokio::test] +async fn internal_reads_refresh_delayed_gateway_amounts( + #[case] operation: &str, + #[case] page_size: Option, + #[case] cost: f64, +) { + let store = MockServer::start().await; + let start_ms = (unix_seconds() as i64 - 600) * 1000; + let available = Arc::new(AtomicBool::new(false)); + Mock::given(body_string_contains("FROM agent_traces_by_key")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": [{ + "trace_id": "trace", "trace_ref": "ref", "team_id": "team", + "api_key_hash": "key", "user_id": "owner", "name": "model call", + "service": "agent", "input_preview": "", "status": "STATUS_CODE_OK", + "start_ms": start_ms, "duration_ms": 1, "span_count": 1, + "agent_count": 0, "agent_invocations": 0, "llm_calls": 1, + "tool_calls": 0, "input_tokens": 1, "output_tokens": 1, + "models": ["test-model"], "error_count": 0, "request_ids": [] + }]}))) + .mount(&store) + .await; + Mock::given(body_string_contains("o.SpanId AS span_id")) + .respond_with(ResponseTemplate::new(200).set_body_json(json!({"data": [{ + "trace_id": "trace", "span_id": "span", "parent_span_id": "", + "name": "model call", "type": "llm", "agent": "", + "status": "STATUS_CODE_OK", "status_message": "", "error_truncated": 0, + "start_ns": start_ms * 1_000_000, "duration_ns": 1_000_000, + "service": "agent", "input_preview": "", "model": "test-model", + "input_tokens": 1, "output_tokens": 1, "litellm_request_id": "", + "call_keys": ["provider_response:response"], "call_evidence": "complete", + "team_id": "team", "api_key_hash": "key", "user_id": "owner" + }]}))) + .expect(2) + .mount(&store) + .await; + let spend_available = available.clone(); + Mock::given(body_string_contains("FROM spend_logs FINAL")) + .respond_with(move |_: &wiremock::Request| { + let rows = if spend_available.load(Ordering::Acquire) { + json!([{ + "request_id": "request", "litellm_call_id": "", + "response_id": "response", "upstream_response_id": "", + "trace_id": "", "span_id": "", "team_id": "team", + "api_key": "key", "user": "owner", "spend": cost, + "start_ms": start_ms + }]) + } else { + json!([]) + }; + ResponseTemplate::new(200).set_body_json(json!({"data": rows})) + }) + .expect(2) + .mount(&store) + .await; + let server = serve(&store.uri(), true).await; + let scope = json!({"all_teams": 0, "user_id": "owner", "team_ids": []}); + let (request, summary_path) = if operation == "list" { + ( + json!({ + "operation": operation, "scope": scope, "start_ms": start_ms, + "end_ms": start_ms + 1000, "cursor": null, "limit": 50 + }), + "/data/0", + ) + } else { + ( + json!({ + "operation": operation, "scope": scope, "trace_id": "trace", + "trace_ref": "ref", "cursor": null, "page_size": page_size + }), + "/summary", + ) + }; + let client = http_client().unwrap(); + for expected in [None, Some(cost)] { + if expected.is_some() { + available.store(true, Ordering::Release); + tokio::time::sleep(litellm_traces_cache::LIVE_TTL + Duration::from_millis(200)).await; + } + let response = client + .post(format!("{}/internal/read", server.url)) + .bearer_auth(SERVICE_TOKEN) + .json(&request) + .send() + .await + .unwrap(); + assert_eq!(response.status(), 200); + let body = response.json::().await.unwrap(); + let summary = body.pointer(summary_path).unwrap(); + assert_eq!(summary["spend"], json!(expected)); + assert_eq!(summary["priced_calls"], u64::from(expected.is_some())); + assert_eq!(summary["llm_calls"], 1); + assert!(!body.to_string().contains("gateway_spend_pending")); + } +} + #[rstest] #[tokio::test] async fn feedback_summary_query_preserves_scope_through_the_internal_read_route() { diff --git a/litellm-rust/crates/traces-cache/src/cache.rs b/litellm-rust/crates/traces-cache/src/cache.rs index eda054e6c7d..e326c0c34d4 100644 --- a/litellm-rust/crates/traces-cache/src/cache.rs +++ b/litellm-rust/crates/traces-cache/src/cache.rs @@ -59,7 +59,7 @@ impl SnapshotKey { } /// Native sessions can resume without a terminal record, so their reads retain `LIVE_TTL`. -/// Other traces with known spend settle after `SETTLED_AFTER_MS` of inactivity. +/// Other traces settle after `SETTLED_AFTER_MS` without activity or recoverable missing spend. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum Freshness { Live, @@ -67,7 +67,7 @@ pub enum Freshness { } impl Freshness { - pub fn of(rows: &[TraceSpansRow], spend_known: bool, snapshot_ms: u64) -> Self { + pub fn of(rows: &[TraceSpansRow], trace: &Trace, snapshot_ms: u64) -> Self { if rows .iter() .any(|row| matches!(row.framework.as_str(), "claude-code" | "claude-agent-sdk")) @@ -82,7 +82,7 @@ impl Freshness { let quiet_ms = i64::try_from(snapshot_ms) .unwrap_or(i64::MAX) .saturating_sub(last_end_ms); - if spend_known && quiet_ms >= SETTLED_AFTER_MS as i64 { + if !trace.gateway_spend_pending && quiet_ms >= SETTLED_AFTER_MS as i64 { Self::Settled } else { Self::Live @@ -388,21 +388,54 @@ mod tests { const LAST_END_MS: u64 = 1_790_742_989_010; #[rstest] - #[case::just_ended(LAST_END_MS, true, Freshness::Live)] - #[case::quiet_just_under(LAST_END_MS + SETTLED_AFTER_MS - 1, true, Freshness::Live)] - #[case::quiet_long_enough(LAST_END_MS + SETTLED_AFTER_MS, true, Freshness::Settled)] - #[case::spend_unknown(LAST_END_MS + SETTLED_AFTER_MS, false, Freshness::Live)] - fn freshness_settles_once_spans_stop_and_spend_is_known( + #[case::just_ended(LAST_END_MS, Freshness::Live)] + #[case::quiet_just_under(LAST_END_MS + SETTLED_AFTER_MS - 1, Freshness::Live)] + #[case::quiet_long_enough(LAST_END_MS + SETTLED_AFTER_MS, Freshness::Settled)] + fn freshness_settles_traces_without_model_calls_once_spans_stop( #[case] snapshot_ms: u64, - #[case] spend_known: bool, #[case] expected: Freshness, ) { assert_eq!( - Freshness::of(&[row("root")], spend_known, snapshot_ms), + Freshness::of(&[row("root")], &trace("root"), snapshot_ms), expected ); } + #[rstest] + #[case::missing_log(None)] + #[case::catalog_estimate(Some(0.25))] + fn estimated_amounts_do_not_settle_pending_gateway_spend(#[case] amount: Option) { + let model = TraceSpansRow { + kind: litellm_traces::ObservationType::Llm, + call_keys: vec![litellm_traces::CallKey::ProviderResponse("response".into())], + call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), + ..row("model") + }; + let rows = [model]; + let original = resolve_trace("trace", "ref", &rows, &[]).unwrap(); + let trace = Trace { + summary: TraceSummary { + llm_calls: 1, + spend: amount, + priced_calls: u64::from(amount.is_some()), + ..original.summary + }, + spans: original + .spans + .into_iter() + .map(|span| litellm_traces::Span { + spend: amount, + ..span + }) + .collect(), + ..original + }; + assert_eq!( + Freshness::of(&rows, &trace, LAST_END_MS + SETTLED_AFTER_MS), + Freshness::Live + ); + } + #[rstest] #[case::claude_code("claude-code")] #[case::claude_agent_sdk("claude-agent-sdk")] @@ -412,7 +445,11 @@ mod tests { ..row("native") }; assert_eq!( - Freshness::of(&[row("root"), native], true, LAST_END_MS + SETTLED_AFTER_MS), + Freshness::of( + &[row("root"), native], + &trace("root"), + LAST_END_MS + SETTLED_AFTER_MS, + ), Freshness::Live ); } diff --git a/litellm-rust/crates/traces-cache/src/list.rs b/litellm-rust/crates/traces-cache/src/list.rs index 1b1dcbbefc3..288f6e19c63 100644 --- a/litellm-rust/crates/traces-cache/src/list.rs +++ b/litellm-rust/crates/traces-cache/src/list.rs @@ -162,10 +162,8 @@ async fn resolve_runs( let spend = spend_window(spans).map_or(&[][..], |window| spend_within(&spend_rows, window)); resolve_trace(&row.trace_id, &row.trace_ref, spans, spend).map(|trace| { - ListedRun::Resolved( - Box::new(trace.summary), - Freshness::of(spans, true, snapshot_ms), - ) + let freshness = Freshness::of(spans, &trace, snapshot_ms); + ListedRun::Resolved(Box::new(trace.summary), freshness) }) }) .collect()) diff --git a/litellm-rust/crates/traces-cache/src/reader.rs b/litellm-rust/crates/traces-cache/src/reader.rs index b0f83a89118..00abbd78601 100644 --- a/litellm-rust/crates/traces-cache/src/reader.rs +++ b/litellm-rust/crates/traces-cache/src/reader.rs @@ -211,14 +211,16 @@ impl TraceReader { .await .map_err(|error| Miss::Read(map_store_error(error)))?; let spend_rows = spend(store, access, &rows).await; - let freshness = Freshness::of(&rows, spend_rows.is_some(), snapshot_ms); resolve_trace( trace_id, trace_ref, &rows, spend_rows.as_deref().unwrap_or_default(), ) - .map(|trace| (trace, freshness)) + .map(|trace| { + let freshness = Freshness::of(&rows, &trace, snapshot_ms); + (trace, freshness) + }) .ok_or(Miss::Absent) }) .await @@ -325,6 +327,7 @@ fn page( let create_page = |count: usize| { let end = position.offset.saturating_add(count).min(spans.len()); Trace { + gateway_spend_pending: snapshot.trace().gateway_spend_pending, summary: snapshot.trace().summary.clone(), agents: snapshot.trace().agents.clone(), spans: spans[position.offset..end].to_vec(), diff --git a/litellm-rust/crates/traces-cache/tests/read.rs b/litellm-rust/crates/traces-cache/tests/read.rs index 22293db98e8..b206f63e58e 100644 --- a/litellm-rust/crates/traces-cache/tests/read.rs +++ b/litellm-rust/crates/traces-cache/tests/read.rs @@ -539,6 +539,110 @@ fn spend_row(response_id: &str, cost: f64) -> SpendByResponseIdsRow { } } +#[rstest] +#[case::missing(&[], None, 0, false, 0.5, None)] +#[case::partial(&[Some(0.25)], Some(0.25), 1, false, 0.5, None)] +#[case::delayed_zero(&[Some(0.25)], Some(0.25), 1, false, 0.0, None)] +#[case::null_amount(&[Some(0.25), None], Some(0.25), 1, false, 0.5, None)] +#[case::complete_zero(&[Some(0.25), Some(0.0)], Some(0.25), 2, true, 0.5, None)] +#[case::no_call_id(&[Some(0.25)], Some(0.25), 1, true, 0.5, Some(CallEvidenceKind::Unknown))] +#[case::incomplete_identity(&[Some(0.25)], Some(0.25), 1, true, 0.5, Some(CallEvidenceKind::Partial))] +#[tokio::test] +async fn gateway_cost_refreshes_until_every_model_call_is_priced( + #[case] initial_costs: &[Option], + #[case] initial_total: Option, + #[case] initial_priced: u64, + #[case] settled: bool, + #[case] final_second: f64, + #[case] terminal: Option, +) { + let rows: Vec<_> = std::iter::once(span(0)) + .chain((1..=2).map(|index| TraceSpansRow { + kind: ObservationType::Llm, + call_keys: vec![CallKey::ProviderResponse(format!("response-{index}"))], + call_evidence: Some(if index == 2 { + terminal.unwrap_or(CallEvidenceKind::Complete) + } else { + CallEvidenceKind::Complete + }), + ..span(index) + })) + .collect(); + let store = FakeStore::with_spans("ref", rows.clone()); + store.set_list_runs(vec![run("trace", "ref")]); + store.set_run_spans(rows); + store.state.lock().unwrap().spend = initial_costs + .iter() + .enumerate() + .map(|(index, cost)| SpendByResponseIdsRow { + spend: *cost, + ..spend_row(&format!("response-{}", index + 1), 0.0) + }) + .collect(); + let reader = TraceReader::new(usize::MAX); + let access = access(); + let detail = reader + .get_trace_page(&store, &access, "trace", "ref", None, 1) + .await + .unwrap() + .unwrap(); + let list = reader + .list_traces(&store, &access, 0, i64::MAX, None, 2) + .await + .unwrap(); + assert_eq!(detail.summary.spend, initial_total); + assert_eq!(detail.summary.priced_calls, initial_priced); + assert_eq!(list.data[0].spend, initial_total); + assert_eq!(list.data[0].priced_calls, initial_priced); + + store.state.lock().unwrap().spend = vec![ + spend_row("response-1", 0.25), + spend_row("response-2", final_second), + ]; + tokio::time::sleep(LIVE_TTL + Duration::from_millis(200)).await; + let refreshed = reader + .get_trace(&store, &access, "trace", "ref") + .await + .unwrap() + .unwrap(); + let listed = reader + .list_traces(&store, &access, 0, i64::MAX, None, 2) + .await + .unwrap(); + let expected = if settled { + initial_total + } else { + Some(0.25 + final_second) + }; + assert_eq!(refreshed.summary.spend, expected); + assert_eq!(listed.data[0].spend, expected); + let expected_priced = if settled { initial_priced } else { 2 }; + assert_eq!(refreshed.summary.priced_calls, expected_priced); + assert_eq!(listed.data[0].priced_calls, expected_priced); + assert_eq!( + store.calls(Operation::TraceSpans), + if settled { 1 } else { 2 } + ); + assert_eq!( + store.calls(Operation::RunSpans), + if settled { 1 } else { 2 } + ); + let pinned = reader + .get_trace_page( + &store, + &access, + "trace", + "ref", + detail.next_cursor.as_deref(), + 1, + ) + .await + .unwrap() + .unwrap(); + assert_eq!(pinned.summary.spend, initial_total); + assert_eq!(pinned.summary.priced_calls, initial_priced); +} + #[rstest] #[tokio::test] async fn failed_batch_spend_lookup_falls_back_to_each_run_instead_of_losing_every_cost() { diff --git a/litellm-rust/crates/traces/src/resolve/resolution.rs b/litellm-rust/crates/traces/src/resolve/resolution.rs index e19a98a1ede..007fc10edf8 100644 --- a/litellm-rust/crates/traces/src/resolve/resolution.rs +++ b/litellm-rust/crates/traces/src/resolve/resolution.rs @@ -31,7 +31,11 @@ pub(super) struct Resolution<'a> { call_matches: HashMap>, } -pub(super) type CallMatch<'a> = (Option>, SpendMatch); +pub(super) struct CallMatch<'a> { + pub(super) requests: Option>, + pub(super) state: SpendMatch, + pub(super) spend_pending: bool, +} impl<'a> Resolution<'a> { pub(super) fn new(rows: &'a [TraceSpansRow], spend: &'a [SpendRow]) -> Self { @@ -138,18 +142,16 @@ impl<'a> Resolution<'a> { pub(super) fn call_requests(&self, call: usize) -> Option> { self.call_match(call) - .and_then(|(requests, _)| requests.clone()) + .and_then(|matched| matched.requests.clone()) + } + + pub(super) fn gateway_spend_pending(&self) -> bool { + self.call_matches + .values() + .any(|matched| matched.spend_pending) } fn resolve_call_match(&self, call: usize) -> CallMatch<'a> { - if let Some(requests) = self.resolve_call_requests(call) { - return (Some(requests), SpendMatch::Matched); - } - let evidence = self.requests(call); - (None, evidence.unmatched_reason()) - } - - fn resolve_call_requests(&self, call: usize) -> Option> { let wrappers = self.graph.ancestors(call).into_iter().filter(|ancestor| { self.kind(*ancestor) == ObservationType::Llm && self @@ -178,7 +180,7 @@ impl<'a> Resolution<'a> { .collect() }) .flatten(); - let selected: Requests<'a> = transport_requests + let selected = transport_requests .map(|requests| requests.into_iter().flatten().collect()) .into_iter() .chain(sources.iter().filter_map(SpendEvidence::complete_requests)) @@ -187,15 +189,24 @@ impl<'a> Resolution<'a> { .iter() .chain(&transports) .all(|source| source.agrees_with(selected)) - })?; - Some( - selected - .into_iter() - .map(|request| (request.identity(), request)) - .collect::>() - .into_values() - .collect(), - ) + }); + if let Some(selected) = selected { + let requests = spend::unique(selected); + return CallMatch { + spend_pending: spend::request_cost(&requests).is_none(), + requests: Some(requests), + state: SpendMatch::Matched, + }; + } + // A leaf without complete identifiers may still be priced from a wrapper or transport + // once its spend arrives. Only the absence of every complete source is terminal. + let spend_pending = sources.iter().any(SpendEvidence::has_complete_keys) + || (!transports.is_empty() && transports.iter().all(SpendEvidence::has_complete_keys)); + CallMatch { + requests: None, + state: sources[0].unmatched_reason(), + spend_pending, + } } fn transports(&self, call: usize) -> Vec { diff --git a/litellm-rust/crates/traces/src/resolve/spend.rs b/litellm-rust/crates/traces/src/resolve/spend.rs index a6b2fc0dbc5..e74cc3704af 100644 --- a/litellm-rust/crates/traces/src/resolve/spend.rs +++ b/litellm-rust/crates/traces/src/resolve/spend.rs @@ -162,6 +162,10 @@ pub(super) enum SpendEvidence<'a> { } impl<'a> SpendEvidence<'a> { + pub(super) fn has_complete_keys(&self) -> bool { + matches!(self, Self::Complete(keys) if !keys.is_empty()) + } + pub(super) fn unmatched_reason(&self) -> SpendMatch { match self { Self::Unknown => SpendMatch::NoCallId, diff --git a/litellm-rust/crates/traces/src/resolve/view.rs b/litellm-rust/crates/traces/src/resolve/view.rs index 62c27676605..1b1ae7a783e 100644 --- a/litellm-rust/crates/traces/src/resolve/view.rs +++ b/litellm-rust/crates/traces/src/resolve/view.rs @@ -25,8 +25,8 @@ fn optional(value: &str) -> Option { fn span(resolution: &Resolution<'_>, index: usize, trace_start_ns: i64) -> Span { let row = resolution.row(index); let status = resolution.status_source(index); - let (requests, spend_match) = if let Some((requests, matched)) = resolution.call_match(index) { - (requests.clone(), Some(*matched)) + let (requests, spend_match) = if let Some(matched) = resolution.call_match(index) { + (matched.requests.clone(), Some(matched.state)) } else { (resolution.requests(index).complete_requests(), None) }; @@ -249,6 +249,7 @@ pub fn resolve_trace( }), }; Some(Trace { + gateway_spend_pending: resolution.gateway_spend_pending(), summary, agents, spans, diff --git a/litellm-rust/crates/traces/src/view.rs b/litellm-rust/crates/traces/src/view.rs index 729ab5ef1c3..345e1454855 100644 --- a/litellm-rust/crates/traces/src/view.rs +++ b/litellm-rust/crates/traces/src/view.rs @@ -131,6 +131,9 @@ pub struct TraceSummary { #[macro_rules_attribute::apply(response_type)] #[derive(Clone, Debug, PartialEq)] pub struct Trace { + /// Read-cache metadata from gateway resolution; display estimates must not clear it. + #[serde(skip)] + pub gateway_spend_pending: bool, pub summary: TraceSummary, pub agents: Vec, pub spans: Vec, diff --git a/litellm-rust/crates/traces/tests/resolve.rs b/litellm-rust/crates/traces/tests/resolve.rs index 73f691cb2a6..d81aed59a76 100644 --- a/litellm-rust/crates/traces/tests/resolve.rs +++ b/litellm-rust/crates/traces/tests/resolve.rs @@ -494,21 +494,70 @@ fn model_calls_link_the_spend_log_they_were_priced_from() { } #[rstest] -fn model_call_span_cost_agrees_with_the_run_total_when_priced_from_a_wrapper() { +#[case::wrapper_without_leaf_id(false, litellm_traces::CallEvidenceKind::Unknown)] +#[case::wrapper_with_partial_leaf(false, litellm_traces::CallEvidenceKind::Partial)] +#[case::transport_without_leaf_id(true, litellm_traces::CallEvidenceKind::Unknown)] +#[case::transport_with_partial_leaf(true, litellm_traces::CallEvidenceKind::Partial)] +fn model_call_span_cost_agrees_with_the_run_total_when_priced_from_related_evidence( + #[case] transport: bool, + #[case] leaf_evidence: litellm_traces::CallEvidenceKind, +) { let rows = [ TraceSpansRow { - call_keys: vec![litellm_traces::CallKey::LiteLlmRequest("gateway".into())], + trace_id: "trace".into(), + parent_span_id: if transport { "call" } else { "" }.into(), + kind: if transport { + litellm_traces::ObservationType::Chain + } else { + litellm_traces::ObservationType::Llm + }, + call_keys: vec![if transport { + litellm_traces::CallKey::Transport + } else { + litellm_traces::CallKey::LiteLlmRequest("gateway".into()) + }], call_evidence: Some(litellm_traces::CallEvidenceKind::Complete), - ..llm("wrapper", "", "agent", "") + ..llm("evidence", "", "agent", "") + }, + TraceSpansRow { + trace_id: "trace".into(), + call_evidence: Some(leaf_evidence), + ..llm( + "call", + if transport { "" } else { "evidence" }, + "agent", + "chatcmpl-request", + ) }, - llm("call", "wrapper", "agent", ""), ]; + let pending = resolve_trace("trace", "ref", &rows, &[]).unwrap(); + assert_eq!( + pending.spans[1].spend_match, + Some( + if leaf_evidence == litellm_traces::CallEvidenceKind::Partial { + SpendMatch::IncompleteEvidence + } else { + SpendMatch::NoCallId + } + ) + ); + assert!(pending.gateway_spend_pending); + assert!( + serde_json::to_value(&pending) + .unwrap() + .get("gateway_spend_pending") + .is_none() + ); + assert_eq!(pending.summary.spend, None); let logs = [SpendByResponseIdsRow { + trace_id: "trace".into(), + span_id: "evidence".into(), litellm_call_id: "gateway".into(), ..spend("request", "chatcmpl-request", 0.25) }]; let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, Some(0.25)); + assert!(!trace.gateway_spend_pending); assert_eq!( ( trace.spans[1].spend, @@ -1568,7 +1617,12 @@ fn complete_wrapper_reconciles_ambiguous_response( owned_spend("request-a", "response", "team", "", "key", 0.25), owned_spend("request-b", "response", "team", "", "key", 0.5), owned_spend("request-c", "other-response", "team", "", "key", 0.75), + owned_spend("request-d", "response", "team", "", "key", 0.0), ]; + let pending = resolve_trace("trace", "ref", &rows, &logs[1..]).unwrap(); + assert_eq!(pending.spans[1].spend_match, Some(SpendMatch::Ambiguous)); + assert!(pending.gateway_spend_pending); + assert_eq!(pending.summary.spend, None); let trace = resolve_trace("trace", "ref", &rows, &logs).unwrap(); assert_eq!(trace.summary.spend, expected); assert_eq!(trace.agents[0].spend, expected); From 7c7efd4d55b331ea2c6f688ca1ca4db727fc1b69 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:06:18 +0000 Subject: [PATCH 25/65] fix(lint-gates): fail on an empty base scan instead of blaming the change for every violation (#45314) Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- scripts/lint_base_counts.py | 15 +++++++++++---- scripts/type_check_gate.py | 10 +--------- tests/unit/test_lint_base_counts.py | 19 +++++++++---------- 3 files changed, 21 insertions(+), 23 deletions(-) diff --git a/scripts/lint_base_counts.py b/scripts/lint_base_counts.py index 2b86d4ce55b..b710ac0ae1b 100644 --- a/scripts/lint_base_counts.py +++ b/scripts/lint_base_counts.py @@ -276,8 +276,9 @@ def base_counts_cached( ) -> Counts: """`compute` memoized on disk. The base tree at a given commit is immutable, so its counts are a pure function of the merge-base plus the checker's - fingerprints in the cache key; an empty result is never stored because it is - the signature of a crashed pass, not a clean tree. On a disk miss the counts + fingerprints in the cache key. An empty local result fails the gate instead of + being returned, because it is the signature of a crashed pass, not a clean + tree, and would make every existing violation look new. On a disk miss the counts CI already published for the merge-base are fetched before the expensive local base scan; a fetch miss of any kind computes locally.""" directory: Final = default_cache_dir() if cache_dir is None else cache_dir @@ -289,8 +290,14 @@ def base_counts_cached( store_counts(directory, checker, base_point, fetched) return fetched counts: Final = compute(base_point) - if counts: - store_counts(directory, checker, base_point, counts) + if not counts: + print( + f"FAIL: {checker.name} produced no violations for the base tree at {base_point[:12]}, " + "so every rule would look freshly added. The base pass almost certainly " + "crashed; refusing to blame this change for it." + ) + raise SystemExit(1) + store_counts(directory, checker, base_point, counts) return counts diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py index 2a9f1c1229f..dbcce091ac6 100644 --- a/scripts/type_check_gate.py +++ b/scripts/type_check_gate.py @@ -279,15 +279,7 @@ def cmd_check(head: Mapping[str, int], base_ref: str) -> None: ) raise SystemExit(1) base_point: Final = resolve_base_point(base_ref) - base: Final = base_counts_cached(checker_identity(), base_point, base_counts) - if not base: - print( - f"FAIL: basedpyright produced no errors for the base tree at {base_point[:12]}, " - "so every rule would look freshly added. The base pass almost certainly " - "crashed; refusing to blame this change for it." - ) - raise SystemExit(1) - judge(head, base, base_point) + judge(head, base_counts_cached(checker_identity(), base_point, base_counts), base_point) def judge(head: Mapping[str, int], base: Mapping[str, int], base_point: str) -> None: diff --git a/tests/unit/test_lint_base_counts.py b/tests/unit/test_lint_base_counts.py index 84f8a0936c0..29fa57a6b4b 100644 --- a/tests/unit/test_lint_base_counts.py +++ b/tests/unit/test_lint_base_counts.py @@ -203,16 +203,15 @@ def test_base_counts_cached_keeps_each_checker_apart(tmp_path: Path) -> None: ) == {"reportAny": 4} -def test_an_empty_base_pass_is_never_cached(tmp_path: Path) -> None: - calls: Final[list[str]] = [] - - def crashed(ref: str) -> counts.Counts: - calls.append(ref) - return {} - - assert counts.base_counts_cached(_CHECKER, "abc123", crashed, cache_dir=tmp_path, fetch=_no_fetch) == {} - assert counts.base_counts_cached(_CHECKER, "abc123", crashed, cache_dir=tmp_path, fetch=_no_fetch) == {} - assert calls == ["abc123", "abc123"] +def test_an_empty_base_pass_fails_the_gate_instead_of_blaming_the_change( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + with pytest.raises(SystemExit) as exit_info: + counts.base_counts_cached( + _CHECKER, "abc123def4567", lambda ref: {}, cache_dir=tmp_path, fetch=lambda checker, base_point: {} + ) + assert exit_info.value.code == 1 + assert f"FAIL: {_CHECKER.name} produced no violations for the base tree at abc123def456," in capsys.readouterr().out assert list(tmp_path.iterdir()) == [] From 7dd9aff2443e758b200ab9a0f07d8ecf9a2da268 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 01:31:59 -0700 Subject: [PATCH 26/65] refactor(proxy): type fresh Moyai connect helpers and drop MCP rpm getattr (#45315) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../mcp_server/mcp_server_manager.py | 2 +- litellm/proxy/moyai_endpoints.py | 36 +++++++++++++------ 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index a0a83339d8d..701a8f74813 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -3267,7 +3267,7 @@ class MCPServerManager: or "rfc8693", timeout=getattr(mcp_server, "timeout", None), max_concurrent_requests=getattr(mcp_server, "max_concurrent_requests", None), - rpm=getattr(mcp_server, "rpm", None), + rpm=mcp_server.rpm, ) _warn_legacy_delegate_auth_if_applicable(new_server, source="database") if register_oauth_discovery: diff --git a/litellm/proxy/moyai_endpoints.py b/litellm/proxy/moyai_endpoints.py index a2335c0a49b..c4d8ad139f7 100644 --- a/litellm/proxy/moyai_endpoints.py +++ b/litellm/proxy/moyai_endpoints.py @@ -14,6 +14,8 @@ import json import os import secrets import time +from collections.abc import Mapping +from dataclasses import dataclass from typing import TYPE_CHECKING, Annotated, Final from urllib.parse import urlencode, urlparse @@ -63,6 +65,14 @@ class MoyaiConnectExchangeResponse(BaseModel): api_base: str +@dataclass(frozen=True, slots=True) +class _MoyaiConnectCode: + moyai_origin: str + nonce: str + exp: int + user_id: str | None + + def _b64url(data: bytes) -> str: return base64.urlsafe_b64encode(data).decode().rstrip("=") @@ -104,7 +114,7 @@ def _sign_connect_code(master_key: str, moyai_url: str, user_id: str | None) -> return f"{_b64url(payload)}.{_b64url(signature)}" -def _decode_connect_code(master_key: str, code: str) -> dict: +def _decode_connect_code(master_key: str, code: str) -> _MoyaiConnectCode: try: payload_b64, signature_b64 = code.split(".", 1) payload_raw: Final = _b64url_decode(payload_b64) @@ -122,7 +132,13 @@ def _decode_connect_code(master_key: str, code: str) -> dict: raise HTTPException(status_code=400, detail="Invalid Moyai connect code") if not isinstance(payload.get("moyai_origin"), str) or not isinstance(payload.get("nonce"), str): raise HTTPException(status_code=400, detail="Invalid Moyai connect code") - return payload + user_id: Final = payload.get("user_id") + return _MoyaiConnectCode( + moyai_origin=payload["moyai_origin"], + nonce=payload["nonce"], + exp=payload["exp"], + user_id=user_id if isinstance(user_id, str) else None, + ) @router.post( @@ -178,7 +194,7 @@ async def _claim_connect_nonce(prisma_client: "PrismaClient", nonce: str, exp: i raise HTTPException(status_code=400, detail="Invalid Moyai connect code") -async def _moyai_key_alias(prisma_client, moyai_url: str) -> str: +async def _moyai_key_alias(prisma_client: "PrismaClient", moyai_url: str) -> str: from litellm.repositories.verification_token_repository import ( VerificationTokenRepository, ) @@ -192,16 +208,14 @@ async def _moyai_key_alias(prisma_client, moyai_url: str) -> str: @with_service_target(CONFIG_PARAMS_TARGET) -async def _persist_moyai_url(prisma_client, moyai_url: str) -> None: +async def _persist_moyai_url(prisma_client: "PrismaClient", moyai_url: str) -> None: from litellm.proxy.proxy_server import user_api_key_cache - existing: dict = {} db_existing: Final = await _ui_settings_db(UISettingsRepository(prisma_client)).find_unique( where={"id": "ui_settings"} ) - if db_existing and db_existing.ui_settings: - raw: Final = db_existing.ui_settings - existing = json.loads(raw) if isinstance(raw, str) else dict(raw) + raw: Final = db_existing.ui_settings if db_existing else None + existing: Final[Mapping[str, object]] = (json.loads(raw) if isinstance(raw, str) else dict(raw)) if raw else {} ui_settings: Final = {**existing, "moyai_url": moyai_url} await _ui_settings_db(UISettingsRepository(prisma_client)).upsert( @@ -232,13 +246,13 @@ async def moyai_connect_exchange(request: Request, body: MoyaiConnectExchangeReq moyai_url: Final = normalize_moyai_url(body.moyai_url) except ValueError: raise HTTPException(status_code=400, detail="Invalid Moyai connect code") - if moyai_url is None or _origin(moyai_url) != payload["moyai_origin"]: + if moyai_url is None or _origin(moyai_url) != payload.moyai_origin: raise HTTPException(status_code=400, detail="Invalid Moyai connect code") if prisma_client is None: raise HTTPException(status_code=400, detail="Moyai quick connect needs a database connected to the proxy") - await _claim_connect_nonce(prisma_client, payload["nonce"], payload["exp"]) + await _claim_connect_nonce(prisma_client, payload.nonce, payload.exp) alias: Final = await _moyai_key_alias(prisma_client, moyai_url) key_response: Final = await generate_key_helper_fn( @@ -248,7 +262,7 @@ async def moyai_connect_exchange(request: Request, body: MoyaiConnectExchangeReq metadata={ "created_via": "moyai_quick_connect", "moyai_url": moyai_url, - "connected_by": payload.get("user_id"), + "connected_by": payload.user_id, }, table_name="key", llm_router=llm_router, From 7a659973e362673d130e33f51b2a4332644b8260 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 08:40:08 +0000 Subject: [PATCH 27/65] revert(lint-gates): accept an empty base scan again, since zero violations is a legitimate count (#45319) This reverts commit 7c7efd4d55b331ea2c6f688ca1ca4db727fc1b69 (#45314). Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- scripts/lint_base_counts.py | 15 ++++----------- scripts/type_check_gate.py | 10 +++++++++- tests/unit/test_lint_base_counts.py | 19 ++++++++++--------- 3 files changed, 23 insertions(+), 21 deletions(-) diff --git a/scripts/lint_base_counts.py b/scripts/lint_base_counts.py index b710ac0ae1b..2b86d4ce55b 100644 --- a/scripts/lint_base_counts.py +++ b/scripts/lint_base_counts.py @@ -276,9 +276,8 @@ def base_counts_cached( ) -> Counts: """`compute` memoized on disk. The base tree at a given commit is immutable, so its counts are a pure function of the merge-base plus the checker's - fingerprints in the cache key. An empty local result fails the gate instead of - being returned, because it is the signature of a crashed pass, not a clean - tree, and would make every existing violation look new. On a disk miss the counts + fingerprints in the cache key; an empty result is never stored because it is + the signature of a crashed pass, not a clean tree. On a disk miss the counts CI already published for the merge-base are fetched before the expensive local base scan; a fetch miss of any kind computes locally.""" directory: Final = default_cache_dir() if cache_dir is None else cache_dir @@ -290,14 +289,8 @@ def base_counts_cached( store_counts(directory, checker, base_point, fetched) return fetched counts: Final = compute(base_point) - if not counts: - print( - f"FAIL: {checker.name} produced no violations for the base tree at {base_point[:12]}, " - "so every rule would look freshly added. The base pass almost certainly " - "crashed; refusing to blame this change for it." - ) - raise SystemExit(1) - store_counts(directory, checker, base_point, counts) + if counts: + store_counts(directory, checker, base_point, counts) return counts diff --git a/scripts/type_check_gate.py b/scripts/type_check_gate.py index dbcce091ac6..2a9f1c1229f 100644 --- a/scripts/type_check_gate.py +++ b/scripts/type_check_gate.py @@ -279,7 +279,15 @@ def cmd_check(head: Mapping[str, int], base_ref: str) -> None: ) raise SystemExit(1) base_point: Final = resolve_base_point(base_ref) - judge(head, base_counts_cached(checker_identity(), base_point, base_counts), base_point) + base: Final = base_counts_cached(checker_identity(), base_point, base_counts) + if not base: + print( + f"FAIL: basedpyright produced no errors for the base tree at {base_point[:12]}, " + "so every rule would look freshly added. The base pass almost certainly " + "crashed; refusing to blame this change for it." + ) + raise SystemExit(1) + judge(head, base, base_point) def judge(head: Mapping[str, int], base: Mapping[str, int], base_point: str) -> None: diff --git a/tests/unit/test_lint_base_counts.py b/tests/unit/test_lint_base_counts.py index 29fa57a6b4b..84f8a0936c0 100644 --- a/tests/unit/test_lint_base_counts.py +++ b/tests/unit/test_lint_base_counts.py @@ -203,15 +203,16 @@ def test_base_counts_cached_keeps_each_checker_apart(tmp_path: Path) -> None: ) == {"reportAny": 4} -def test_an_empty_base_pass_fails_the_gate_instead_of_blaming_the_change( - tmp_path: Path, capsys: pytest.CaptureFixture[str] -) -> None: - with pytest.raises(SystemExit) as exit_info: - counts.base_counts_cached( - _CHECKER, "abc123def4567", lambda ref: {}, cache_dir=tmp_path, fetch=lambda checker, base_point: {} - ) - assert exit_info.value.code == 1 - assert f"FAIL: {_CHECKER.name} produced no violations for the base tree at abc123def456," in capsys.readouterr().out +def test_an_empty_base_pass_is_never_cached(tmp_path: Path) -> None: + calls: Final[list[str]] = [] + + def crashed(ref: str) -> counts.Counts: + calls.append(ref) + return {} + + assert counts.base_counts_cached(_CHECKER, "abc123", crashed, cache_dir=tmp_path, fetch=_no_fetch) == {} + assert counts.base_counts_cached(_CHECKER, "abc123", crashed, cache_dir=tmp_path, fetch=_no_fetch) == {} + assert calls == ["abc123", "abc123"] assert list(tmp_path.iterdir()) == [] From 7c7b0ea85bc4c9138a457013b5b7170ef1f4184a Mon Sep 17 00:00:00 2001 From: Priyansh Nandwana <136072927+msdianprince-7@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:37:22 +0530 Subject: [PATCH 28/65] fix(bedrock_mantle): send OpenAI explicit prompt cache breakpoints for GPT-5.6 and newer (#38729) * fix(cache_control): let a hosted deployment opt into the OpenAI cache dialect _targets_openai_prompt_cache_breakpoint gated on custom_llm_provider == "openai" unconditionally, so an OpenAI-shaped model served by another provider could never qualify, even with supports_prompt_cache_breakpoint set explicitly on its own cost-map entry. bedrock_mantle honours prompt_cache_breakpoint end to end, and had no way to say so. model_cost is keyed per exact deployment string, so a flag on the deployment's own entry states the dialect more precisely than a provider name can. Consult it before the provider check, and require the entry's litellm_provider to match the serving provider so an openai entry cannot license another provider. Entries for the openai provider keep their api_base check, so an OpenAI-compatible third-party host is still not assumed to speak the dialect. Fixes #38666 * test: drop a laziness assertion the existing suite already makes test_provider_lookup_skipped_for_models_below_gpt_5_6 already pins that _resolve_provider is not called for an unflagged model, and it covers the new flag lookup unchanged. The duplicate patched a litellm internal for no added coverage, which the test-quality gate counts against TQ008. * fix(bedrock_mantle): flag GPT-5.6 and newer rows for OpenAI explicit prompt cache breakpoints * fix(responses): predict the chat-completions bridge with the model name the dispatch resolves * fix(cache_control): read a region-prefixed Mantle GPT deployment through its region-free price-map row * fix(cache_control): let a bare hosted deployment name read its own provider's price-map row * fix(responses): hand the hook the provider the router resolved for a Foundry GPT deployment * fix(cache_control): read the deployment breakpoint flag strictly and default a null prompt_cache_options A cost-map or deployment `supports_prompt_cache_breakpoint` that is not the boolean `true` (the string "true", the integer 1, "false", 0, a long string) no longer opts a deployment into the OpenAI prompt cache dialect; only `true` does, the same reading Bedrock Converse applies to its own flag. A client that sends `"prompt_cache_options": null` on `/v1/chat/completions` or `/v1/messages` now gets the implicit default the hook already applied on `/v1/responses` when it placed a breakpoint; before, the null suppressed the default and the request left with a breakpoint and no options. * test(integration): audit cells for the Mantle GPT prompt cache breakpoint dialect Deterministic cells for the configured-breakpoint path on Bedrock Mantle GPT and Azure AI Foundry GPT-6 deployments: every endpoint, streaming and not, sync and async SDKs and raw httpx, the hostile option shapes, flag precedence, the response cache twin, a two-worker burst, a worker kill and a graceful restart. * fix(cache_control): ignore a malformed cache_control_injection_points value instead of failing the request A deployment or client `cache_control_injection_points` that is not a list of points (a string, an integer, a bare point dict, a list of strings) raised inside the prompt hook (`'str' object has no attribute 'get'`, `'int' object is not iterable`) and turned every request to that deployment into a 500 on `/v1/chat/completions`, `/v1/messages` and `/v1/responses`. Every entry point now reads such a value as no configured points, the way a `null` already read, and the request leaves without a breakpoint; entries of a list that are not points are dropped and the point entries kept. * test(integration): cover int, dict and string-list injection point shapes in the Mantle audit cells The D9 cell now also sends an integer, a bare point dict and a list of strings as the deployment's `cache_control_injection_points`, which the merge base answered with a 500 on every request. * fix(cache_control): stamp the OpenAI dialect for a bare deployment name served by a flagged provider A deployment written as `model: openai.gpt-5.6-sol` with `custom_llm_provider: bedrock_mantle` has no cost-map row of its own and no openai row of the same name, so the dialect stamp's cheap gate (`supports_openai_prompt_cache_breakpoint`) returned early and the configured point was never stamped. On `/v1/responses` and on the chat seeding path the hook sees no provider, so it fell back to the Anthropic `cache_control` marker, which Bedrock Mantle strips. The gate now also passes a model whose serving provider is already known and whose provider-keyed row carries the flag, which is the same row the dialect resolution reads and costs no provider lookup. A bare name without a provider is still left alone, so models below GPT-5.6 keep their points untouched and resolve nothing. * test(integration): cover a bare Mantle deployment name with its provider in the audit cells A deployment configured as `model: openai.gpt-5.6-sol` plus `custom_llm_provider: bedrock_mantle` sends the breakpoint and the implicit options on all three endpoints; the merge base leaves it on the Anthropic dialect. * fix(cache_control): keep a configured point that sits beside junk entries on every chat seed path `configured_injection_points` kept the point dicts of a mixed `cache_control_injection_points` list as a tuple, but only read a `list` back. The chat seed and the `/v1/responses` dialect stamp write the normalized value back onto the request, and on a deployment the OpenAI dialect does not stamp (an Anthropic model, Claude on Bedrock Mantle) that value is the tuple itself, so the hook then read it as no configured points and the valid point was silently dropped. The stamped OpenAI dialect and the `/v1/messages` path kept it only because they build a new list. The normalizer now reads back the tuple it wrote, so the point reaches the wire on every path; an all-dict list still passes through as the same object. * test(integration): cover a configured point beside junk entries on a Claude Mantle deployment A deployment configured with `cache_control_injection_points: ["system", {system point}, 3]` marks the system block on the Anthropic dialect on all three endpoints; the merge base answers 500 on the junk entry. --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../anthropic_cache_control_hook.py | 107 +- ...odel_prices_and_context_window_backup.json | 8 + litellm/responses/main.py | 55 +- model_prices_and_context_window.json | 8 + .../_mantle_gpt_prompt_cache_support.py | 959 ++++++++++++++++++ ...mantle_gpt_prompt_cache_breakpoint_wire.py | 531 ++++++++++ ...ect_prompt_cache_breakpoint_owned_proxy.py | 385 +++++++ .../test_anthropic_cache_control_hook.py | 360 +++++++ .../test_responses_api_request_body.py | 56 + 9 files changed, 2450 insertions(+), 19 deletions(-) create mode 100644 tests/integration/providers/_mantle_gpt_prompt_cache_support.py create mode 100644 tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py create mode 100644 tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py diff --git a/litellm/integrations/anthropic_cache_control_hook.py b/litellm/integrations/anthropic_cache_control_hook.py index 22d76242f2f..6047c625665 100644 --- a/litellm/integrations/anthropic_cache_control_hook.py +++ b/litellm/integrations/anthropic_cache_control_hook.py @@ -12,7 +12,7 @@ Supported for both `v1/chat/completions` (via the prompt-management hook) and import copy import os import re -from collections.abc import Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, Final, cast from urllib.parse import urlparse @@ -90,6 +90,14 @@ def _validated_object_list(value: object) -> list[object] | None: return None +def configured_injection_points(value: object) -> Sequence[CacheControlInjectionPoint]: + if not isinstance(value, (list, tuple)): + return () + if all(isinstance(entry, dict) for entry in value): + return cast(Sequence[CacheControlInjectionPoint], value) + return tuple(cast(CacheControlInjectionPoint, entry) for entry in value if isinstance(entry, dict)) + + def supports_openai_prompt_cache_breakpoint(model: str) -> bool: model_map_flag: Final = _model_map_prompt_cache_breakpoint_flag(model) if model_map_flag is not None: @@ -106,7 +114,60 @@ def _model_map_prompt_cache_breakpoint_flag(model: str) -> bool | None: entries: Final = (litellm.model_cost.get(key) for key in (model, model.rsplit("/", 1)[-1])) flags: Final = (entry.get("supports_prompt_cache_breakpoint") for entry in entries if isinstance(entry, dict)) - return next((bool(flag) for flag in flags if flag is not None), None) + return next((flag is True for flag in flags if flag is not None), None) + + +def _hosted_entry_flag(entry: Mapping[str, object], resolve_provider: Callable[[], str | None]) -> bool | None: + flag: Final = entry.get("supports_prompt_cache_breakpoint") + entry_provider: Final = entry.get("litellm_provider") + if flag is None or entry_provider is None or entry_provider == "openai": + return None + return (flag is True) if entry_provider == resolve_provider() else None + + +def _hosted_openai_dialect_flag( + model: str, custom_llm_provider: str | None, resolve_provider: Callable[[str], str | None] +) -> bool | None: + """ + Explicit opt-in for an OpenAI-shaped deployment served by another provider. + + ``model_cost`` is keyed per deployment string, so a flag on the deployment's own + entry states the dialect directly, which a provider name cannot express. A routing + form the map does not key verbatim (``bedrock_mantle/us-east-1/openai.gpt-5.6-sol``) + is read through the candidate keys ``get_model_info`` resolves it with, an entry + keyed with the region outranking the region-free one. A bare name the map does not + key has no such candidates, so it costs no provider lookup. Entries for the openai + provider are left to the caller's api_base check, so an OpenAI-compatible + third-party host is still not assumed to speak the dialect. A bare deployment name + another provider serves (``gpt-6-astra`` on azure_ai) may collide with the openai + row of the same name, so an exact entry only speaks for a deployment when it is + keyed for that deployment's provider. + """ + import litellm + + exact_entry: Final = litellm.model_cost.get(model) + exact_provider: Final = exact_entry.get("litellm_provider") if isinstance(exact_entry, dict) else None + if custom_llm_provider is None and (exact_provider == "openai" or ("/" not in model and exact_provider is None)): + return None + provider: Final = custom_llm_provider or resolve_provider(model) + if provider is None or provider == "openai": + return None + if isinstance(exact_entry, dict) and exact_provider == provider: + return _hosted_entry_flag(exact_entry, lambda: provider) + from litellm.utils import get_potential_model_names + + names: Final = get_potential_model_names(model, provider) + candidates: Final = ( + names["combined_model_name"], + names["region_free_combined_model_name"], + names["split_model"], + names["combined_stripped_model_name"], + names["stripped_model_name"], + names["provider_prefixed_model_name"], + ) + entries: Final = (litellm.model_cost.get(candidate) for candidate in candidates) + flags: Final = (_hosted_entry_flag(entry, lambda: provider) for entry in entries if isinstance(entry, dict)) + return next((flag for flag in flags if flag is not None), None) def targets_openai_api(api_base: object) -> bool: @@ -227,8 +288,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): """ # Extract cache control injection points carry_unmatched: Final = bool(non_default_params.pop(CARRY_UNMATCHED_MESSAGE_POINTS, False)) - injection_points: Final[list[CacheControlInjectionPoint]] = non_default_params.pop( - "cache_control_injection_points", [] + injection_points: Final = configured_injection_points( + non_default_params.pop("cache_control_injection_points", None) ) if not injection_points: return model, messages, non_default_params @@ -282,8 +343,9 @@ class AnthropicCacheControlHook(CustomPromptManagement): if ( openai_dialect and AnthropicCacheControlHook.count_request_cache_breakpoints(processed_messages) > breakpoints_before + and non_default_params.get("prompt_cache_options") is None ): - non_default_params.setdefault("prompt_cache_options", PromptCacheOptions(mode="implicit")) + non_default_params["prompt_cache_options"] = PromptCacheOptions(mode="implicit") # Points this pass did not place: non-message ones for the provider transform, and # the deferred role-targeted ones. Deferring is what reaches the Responses API's @@ -310,7 +372,14 @@ class AnthropicCacheControlHook(CustomPromptManagement): api_base: object = None, prompt_cache_options: object = None, ) -> bool: - if model is None or not supports_openai_prompt_cache_breakpoint(model): + if model is None: + return False + hosted_flag: Final = _hosted_openai_dialect_flag( + model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider + ) + if hosted_flag is not None: + return hosted_flag + if not supports_openai_prompt_cache_breakpoint(model): return False if (custom_llm_provider or AnthropicCacheControlHook._resolve_provider(model)) != "openai": return False @@ -701,7 +770,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): api_base: object, prompt_cache_options: object, ) -> Sequence[Mapping[str, object]]: - if not supports_openai_prompt_cache_breakpoint(model): + if not AnthropicCacheControlHook._may_target_openai_prompt_cache_breakpoint(model, custom_llm_provider): return points return AnthropicCacheControlHook._stamped( points, @@ -711,6 +780,20 @@ class AnthropicCacheControlHook(CustomPromptManagement): ), ) + @staticmethod + def _may_target_openai_prompt_cache_breakpoint(model: str, custom_llm_provider: str | None) -> bool: + """Cheap gate before the dialect is resolved: the model's own row or version, or, when the + serving provider is already known, a row keyed for that provider (``openai.gpt-5.6-sol`` + served by ``bedrock_mantle``), which costs no provider lookup.""" + if supports_openai_prompt_cache_breakpoint(model): + return True + if custom_llm_provider is None: + return False + return ( + _hosted_openai_dialect_flag(model, custom_llm_provider, AnthropicCacheControlHook._resolve_provider) + is not None + ) + @staticmethod def _stamped(points: Sequence[Mapping[str, object]], key: str, value: object) -> Sequence[Mapping[str, object]]: return [{**point, key: value} for point in points] @@ -859,7 +942,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): """ import litellm - configured: Final = non_default_params.get("cache_control_injection_points") + configured: Final = configured_injection_points(non_default_params.get("cache_control_injection_points")) if configured: tools_keeping_marks: Final = tuple( tool for tool in tools or () if not _chat_transform_drops_tool_cache_control(tool) @@ -990,9 +1073,7 @@ class AnthropicCacheControlHook(CustomPromptManagement): bool | None, kwargs.pop("enable_prompt_caching", None) ) cache_control: Final = kwargs.get("cache_control") - configured: Final = cast( # cast-ok: kwargs is untyped; this key only holds the documented injection-point list - list[CacheControlInjectionPoint] | None, kwargs.pop("cache_control_injection_points", None) - ) + configured: Final = configured_injection_points(kwargs.pop("cache_control_injection_points", None)) injection_points: Final[Sequence[CacheControlInjectionPoint]] = configured or ( AnthropicCacheControlHook.get_default_injection_points( messages=typed_messages, @@ -1028,8 +1109,8 @@ class AnthropicCacheControlHook(CustomPromptManagement): AnthropicCacheControlHook.count_request_cache_breakpoints(messages, system) - breakpoints_before ) AnthropicCacheControlHook.record_gateway_injection(kwargs, breakpoints_added) - if openai_dialect and breakpoints_added > 0: - kwargs.setdefault("prompt_cache_options", PromptCacheOptions(mode="implicit")) + if openai_dialect and breakpoints_added > 0 and kwargs.get("prompt_cache_options") is None: + kwargs["prompt_cache_options"] = PromptCacheOptions(mode="implicit") if remaining: kwargs["cache_control_injection_points"] = remaining return messages, system diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index f34606ce127..32fd15de9e7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -58344,6 +58344,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58387,6 +58388,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58419,6 +58421,7 @@ "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58495,6 +58498,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58882,6 +58886,7 @@ "supports_minimal_reasoning_effort": false, "supports_none_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58920,6 +58925,7 @@ "supports_minimal_reasoning_effort": false, "supports_none_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58958,6 +58964,7 @@ "supports_minimal_reasoning_effort": false, "supports_none_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -80349,6 +80356,7 @@ "supports_minimal_reasoning_effort": false, "supports_none_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/litellm/responses/main.py b/litellm/responses/main.py index c4632f09723..7298f4b1a36 100644 --- a/litellm/responses/main.py +++ b/litellm/responses/main.py @@ -18,7 +18,11 @@ from litellm.completion_extras.litellm_responses_transformation.transformation i LiteLLMResponsesTransformationHandler, ) from litellm.constants import DEFAULT_CHAT_COMPLETION_PARAM_VALUES, request_timeout -from litellm.integrations.anthropic_cache_control_hook import CARRY_UNMATCHED_MESSAGE_POINTS +from litellm.integrations.anthropic_cache_control_hook import ( + CARRY_UNMATCHED_MESSAGE_POINTS, + AnthropicCacheControlHook, + configured_injection_points, +) from litellm.litellm_core_utils.asyncify import run_async_function from litellm.litellm_core_utils.core_helpers import normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -526,6 +530,13 @@ def _api_base_kwarg(kwargs: Mapping[str, object]) -> str | None: return api_base if isinstance(api_base, str) else None +def _dispatched_model_name(model: str, custom_llm_provider: str, api_base: str | None) -> str: + provider_model, _, _, _ = litellm.get_llm_provider( + model=model, custom_llm_provider=custom_llm_provider, api_base=api_base + ) + return _strip_responses_routing_prefix(provider_model) + + def _will_bridge_to_chat_completions( model: str, custom_llm_provider: str | None, @@ -536,20 +547,50 @@ def _will_bridge_to_chat_completions( """``_bridges_to_chat_completions`` for callers running before the provider config is resolved. Resolving the config is a pure lookup, so this asks the same question the dispatch - asks rather than restating its condition. Both callers resolve the provider before - this runs, so the only way to be wrong is a prompt manager that moves the model - across the bridge boundary, which would leave the deferred points to a pass that - never comes. + asks rather than restating its condition, with the model name the dispatch hands the + lookup: a provider whose config is keyed by model name (Bedrock Mantle reads the + price map) answers nothing for ``bedrock_mantle/openai.gpt-5.6-sol`` and would read + as bridged. Both callers resolve the provider before this runs, so the only way to be + wrong is a prompt manager that moves the model across the bridge boundary, which + would leave the deferred points to a pass that never comes. """ normalized_model: Final = _normalize_openai_chat_completions_responses_model(model) if custom_llm_provider is None: return True return _bridges_to_chat_completions( - _resolve_responses_api_provider_config(normalized_model[0], custom_llm_provider, model_info, api_base), + _resolve_responses_api_provider_config( + _dispatched_model_name(normalized_model[0], custom_llm_provider, api_base), + custom_llm_provider, + model_info, + api_base, + ), use_chat_completions_api or normalized_model[1], ) +def _stamp_injection_points_with_dialect( + kwargs: dict[str, object], # mutable-ok: the points are rewritten in the caller's own kwargs for the hook to read + model: str, + custom_llm_provider: str | None, +) -> None: + """Carry the provider this layer resolved onto the points. + + The hook reads ``custom_llm_provider`` from the request kwargs, which never hold the one + resolved here, and resolving the model name alone reads a Foundry deployment of an OpenAI + model (``azure_ai/gpt-6-astra``) as Azure OpenAI, which left it on the Anthropic dialect. + """ + points: Final = configured_injection_points(kwargs.get("cache_control_injection_points")) + if not points: + return + kwargs["cache_control_injection_points"] = AnthropicCacheControlHook._stamped_with_dialect( + points, + model, + custom_llm_provider, + kwargs.get("api_base") or kwargs.get("base_url"), + kwargs.get("prompt_cache_options"), + ) + + @contextmanager def _prompt_management_sees_a_provisional_message_list( kwargs: dict[str, object], # mutable-ok: the signal is read and popped out of the caller's own kwargs @@ -659,6 +700,7 @@ async def aresponses( _api_base_kwarg(kwargs), ), ): + _stamp_injection_points_with_dialect(kwargs, model, custom_llm_provider) ( model, merged_input, @@ -829,6 +871,7 @@ def _apply_prompt_management_to_responses_call( _api_base_kwarg(kwargs), ), ): + _stamp_injection_points_with_dialect(kwargs, model, custom_llm_provider) ( model, merged_input, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index f34606ce127..32fd15de9e7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -58344,6 +58344,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58387,6 +58388,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58419,6 +58421,7 @@ "supports_function_calling": true, "supports_minimal_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58495,6 +58498,7 @@ "supports_max_reasoning_effort": true, "supports_minimal_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58882,6 +58886,7 @@ "supports_minimal_reasoning_effort": false, "supports_none_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58920,6 +58925,7 @@ "supports_minimal_reasoning_effort": false, "supports_none_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -58958,6 +58964,7 @@ "supports_minimal_reasoning_effort": false, "supports_none_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, @@ -80349,6 +80356,7 @@ "supports_minimal_reasoning_effort": false, "supports_none_reasoning_effort": false, "supports_prompt_caching": true, + "supports_prompt_cache_breakpoint": true, "supports_reasoning": true, "supports_response_schema": true, "supports_tool_choice": true, diff --git a/tests/integration/providers/_mantle_gpt_prompt_cache_support.py b/tests/integration/providers/_mantle_gpt_prompt_cache_support.py new file mode 100644 index 00000000000..8f497fea74f --- /dev/null +++ b/tests/integration/providers/_mantle_gpt_prompt_cache_support.py @@ -0,0 +1,959 @@ +import base64 +import binascii +import json +import math +import os +import uuid +from collections.abc import Callable, Iterable, Mapping, Sequence +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal, Protocol +from urllib.parse import urlsplit + +import anthropic +import httpx +import openai +from integration._support.client import Gateway, Scenario, eventually +from integration._support.database import read_rows +from integration._support.responses_vendor import answer, error, newest_marker, sse +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_if_encrypted_with + +Endpoint = Literal["chat", "responses", "messages"] + + +JSON: Final = TypeAdapter(dict[str, JsonValue]) + + +ITEMS: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +SIGNING_KEY: Final = os.environ.get("LITELLM_SALT_KEY", "sk-integration-salt") + + +TOKEN: Final = "synthetic-mantle-bearer" + + +GPT: Final = "bedrock_mantle/openai.gpt-5.6-sol" + + +GPT_REGION: Final = "bedrock_mantle/us-east-1/openai.gpt-5.6-sol" + +GPT_BARE: Final = "openai.gpt-5.6-sol" + + +GPT_UNFLAGGED_ROW: Final = "bedrock_mantle/openai.gpt-5.4" + + +GPT_FLAGGED_ROW: Final = "bedrock_mantle/openai.gpt-6-luna" + + +GPT_ODD_STRING_FLAG: Final = "bedrock_mantle/openai.gpt-5.5" + + +GPT_ODD_INT_FLAG: Final = "bedrock_mantle/openai.gpt-daybreak-blue-5.6-sol" +ODD_FLAGS: Final[tuple[tuple[str, str, JsonValue], ...]] = ( + ("string-true", GPT_ODD_STRING_FLAG, "true"), + ("int-one", GPT_ODD_INT_FLAG, 1), + ("int-zero", GPT_ODD_INT_FLAG, 0), + ("5kb-string", GPT_ODD_STRING_FLAG, "x" * 5120), +) + + +CLAUDE: Final = "bedrock_mantle/anthropic.claude-haiku-4-5" + + +AZURE: Final = "azure/gpt-5.6" + + +THIRD_PARTY: Final = "openai/gpt-5.6" + + +SYSTEM: Final = "Reply with the signature the user gives you." + + +SYSTEM_POINT: Final[list[JsonValue]] = [{"location": "message", "role": "system"}] + + +EXPLICIT: Final[dict[str, JsonValue]] = {"mode": "explicit"} + + +IMPLICIT: Final[dict[str, JsonValue]] = {"mode": "implicit"} + + +EPHEMERAL: Final[dict[str, JsonValue]] = {"type": "ephemeral"} + + +NO_CACHE: Final[dict[str, JsonValue]] = {"no-cache": True} + + +MAX_TOKENS: Final = 64 + + +INPUT_TOKENS: Final = 2730 + + +CACHED_TOKENS: Final = 1024 + + +WRITTEN_TOKENS: Final = 1700 + + +UNCACHED_TOKENS: Final = INPUT_TOKENS - CACHED_TOKENS - WRITTEN_TOKENS + + +OUTPUT_TOKENS: Final = 7 + + +USAGE: Final[dict[str, JsonValue]] = { + "input_tokens": INPUT_TOKENS, + "output_tokens": OUTPUT_TOKENS, + "total_tokens": INPUT_TOKENS + OUTPUT_TOKENS, + "input_tokens_details": {"cached_tokens": CACHED_TOKENS, "cache_write_tokens": WRITTEN_TOKENS}, + "output_tokens_details": {"reasoning_tokens": 0}, +} + + +ANTHROPIC_USAGE: Final[dict[str, JsonValue]] = { + "input_tokens": UNCACHED_TOKENS, + "output_tokens": OUTPUT_TOKENS, + "cache_read_input_tokens": CACHED_TOKENS, + "cache_creation_input_tokens": WRITTEN_TOKENS, +} + + +CHAT_USAGE: Final[dict[str, JsonValue]] = { + "prompt_tokens": INPUT_TOKENS, + "completion_tokens": OUTPUT_TOKENS, + "total_tokens": INPUT_TOKENS + OUTPUT_TOKENS, + "prompt_tokens_details": {"cached_tokens": CACHED_TOKENS}, +} + + +COST_MAP: Final = JSON.validate_json(Path("model_prices_and_context_window.json").read_bytes()) + + +RESPONSES_PATH: Final = "/openai/v1/responses" + + +ANTHROPIC_PATH: Final = "/anthropic/v1/messages" + + +BURST: Final = 30 + + +ENDPOINTS: Final[tuple[Endpoint, ...]] = ("chat", "responses", "messages") + + +def cost_rate(model: str, field: str) -> float: + value: Final = JSON.validate_python(COST_MAP[model])[field] + assert isinstance(value, int | float), (model, field, value) + return float(value) + + +def expected_spend(model: str) -> float: + return ( + UNCACHED_TOKENS * cost_rate(model, "input_cost_per_token") + + CACHED_TOKENS * cost_rate(model, "cache_read_input_token_cost") + + WRITTEN_TOKENS * cost_rate(model, "cache_creation_input_token_cost") + + OUTPUT_TOKENS * cost_rate(model, "output_cost_per_token") + ) + + +def fresh_marker() -> str: + return uuid.uuid4().hex + + +def prompt_text(marker: str) -> str: + return f"Return the signature marker-{marker}." + + +def valid_options(options: JsonValue) -> bool: + if options is None: + return True + if not isinstance(options, dict) or not set(options) <= {"mode", "ttl"}: + return False + return options.get("mode", "implicit") in ("implicit", "explicit") + + +def message_item(identity: str, text: str, status: str) -> dict[str, JsonValue]: + return { + "id": f"msg_{identity}", + "type": "message", + "role": "assistant", + "status": status, + "content": [{"type": "output_text", "text": text, "annotations": []}] if status == "completed" else [], + } + + +def responses_object(identity: str, model: str, text: str) -> dict[str, JsonValue]: + return { + "id": identity, + "object": "response", + "created_at": 1, + "status": "completed", + "model": model, + "output": [message_item(identity, text, "completed")], + "usage": USAGE, + } + + +def responses_stream(identity: str, model: str, text: str) -> tuple[bytes, ...]: + response: Final = responses_object(identity, model, text) + return ( + sse({"type": "response.created", "sequence_number": 0, "response": {**response, "status": "in_progress"}}), + sse( + { + "type": "response.output_item.added", + "sequence_number": 1, + "output_index": 0, + "item": message_item(identity, text, "in_progress"), + } + ), + sse( + { + "type": "response.output_text.delta", + "sequence_number": 2, + "item_id": f"msg_{identity}", + "output_index": 0, + "content_index": 0, + "delta": text, + } + ), + sse( + { + "type": "response.output_item.done", + "sequence_number": 3, + "output_index": 0, + "item": message_item(identity, text, "completed"), + } + ), + sse({"type": "response.completed", "sequence_number": 4, "response": response}), + ) + + +def anthropic_message(identity: str, model: str, text: str) -> dict[str, JsonValue]: + return { + "id": identity, + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": text}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": ANTHROPIC_USAGE, + } + + +def anthropic_stream(identity: str, model: str, text: str) -> tuple[bytes, ...]: + started: Final = {**anthropic_message(identity, model, text), "content": [], "stop_reason": None} + return ( + sse({"type": "message_start", "message": {**started, "usage": {**ANTHROPIC_USAGE, "output_tokens": 0}}}), + sse({"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + sse({"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": text}}), + sse({"type": "content_block_stop", "index": 0}), + sse( + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": OUTPUT_TOKENS}, + } + ), + sse({"type": "message_stop"}), + ) + + +def chat_completion(identity: str, model: str, text: str) -> dict[str, JsonValue]: + return { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": model, + "choices": [{"index": 0, "message": {"role": "assistant", "content": text}, "finish_reason": "stop"}], + "usage": CHAT_USAGE, + } + + +def issued_id(prefix: str, marker: str | None) -> str: + return f"{prefix}{marker or 'unmarked'}-{uuid.uuid4().hex[:8]}" + + +def mantle_peer(*, pause: float = 0) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert request.method == "POST", request.method + assert request.headers.get("authorization") == f"Bearer {TOKEN}", request.headers + body: Final = JSON.validate_json(request.body) + model: Final = str(body["model"]) + marker: Final = newest_marker(request.body.decode()) + text: Final = answer(marker) + streaming: Final = body.get("stream") is True + path: Final = urlsplit(request.target).path + if path == ANTHROPIC_PATH: + identity: Final = issued_id("msg_", marker) + if streaming: + return Reply( + content_type="text/event-stream", + chunks=anthropic_stream(identity, model, text), + pause_between_chunks=pause, + ) + return Reply(body=json.dumps(anthropic_message(identity, model, text)).encode()) + assert path == RESPONSES_PATH, request.target + if not valid_options(body.get("prompt_cache_options")): + return error(400, "Invalid prompt_cache_options", "invalid_prompt_cache_options") + response_id: Final = issued_id("resp_", marker) + if streaming: + return Reply( + content_type="text/event-stream", + chunks=responses_stream(response_id, model, text), + pause_between_chunks=pause, + ) + return Reply(body=json.dumps(responses_object(response_id, model, text)).encode()) + + return respond + + +def failing_peer(status: int, message: str, code: str) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert urlsplit(request.target).path == RESPONSES_PATH, request.target + return error(status, message, code) + + return respond + + +def openai_shaped_peer() -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + body: Final = JSON.validate_json(request.body) + model: Final = str(body["model"]) + marker: Final = newest_marker(request.body.decode()) + text: Final = answer(marker) + path: Final = urlsplit(request.target).path + if path.endswith("/chat/completions"): + return Reply(body=json.dumps(chat_completion(issued_id("chatcmpl-", marker), model, text)).encode()) + assert path.endswith("/responses"), request.target + return Reply(body=json.dumps(responses_object(issued_id("resp_", marker), model, text)).encode()) + + return respond + + +def system_item(*, marked: bool, endpoint: Endpoint) -> dict[str, JsonValue]: + part: Final[dict[str, JsonValue]] = {"type": "input_text", "text": SYSTEM} + content: Final[list[JsonValue]] = [{**part, "prompt_cache_breakpoint": EXPLICIT} if marked else part] + if endpoint == "responses": + return {"role": "system", "content": content} + return {"type": "message", "role": "system", "content": content} + + +def user_item(text: str, endpoint: Endpoint) -> dict[str, JsonValue]: + if endpoint == "responses": + return {"role": "user", "content": text} + return {"type": "message", "role": "user", "content": [{"type": "input_text", "text": text}]} + + +def expected_wire( + model: str, + prompt: str, + *, + endpoint: Endpoint, + marked: bool, + options: JsonValue | None = IMPLICIT, + **extra: JsonValue, +) -> dict[str, JsonValue]: + body: Final[dict[str, JsonValue]] = { + "model": model.rsplit("/", 1)[-1], + "input": [system_item(marked=marked, endpoint=endpoint), user_item(prompt, endpoint)], + "max_output_tokens": MAX_TOKENS, + **extra, + } + return body if options is None else {**body, "prompt_cache_options": options} + + +def body_of(request: Request) -> dict[str, JsonValue]: + return JSON.validate_json(request.body) + + +def without_stream(body: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + return {key: value for key, value in body.items() if key != "stream"} + + +def only_received(wire: Wire) -> Request: + (request,) = wire.drain() + return request + + +def assert_wire(request: Request, expected: Mapping[str, JsonValue], *, streaming: bool) -> None: + body: Final = body_of(request) + assert urlsplit(request.target).path == RESPONSES_PATH, request.target + assert without_stream(body) == expected, json.dumps(body, sort_keys=True) + assert (body.get("stream") is True) is streaming, body.get("stream") + + +def breakpoint_count(body: Mapping[str, JsonValue]) -> int: + parts: Final = ( + part + for item in ITEMS.validate_python(body["input"]) + for part in (item["content"] if isinstance(item["content"], list) else ()) + ) # comprehension-ok: nested input items + return sum(1 for part in parts if isinstance(part, dict) and "prompt_cache_breakpoint" in part) + + +def endpoint_path(endpoint: Endpoint) -> str: + match endpoint: + case "chat": + return "/v1/chat/completions" + case "responses": + return "/v1/responses" + case "messages": + return "/v1/messages" + + +def request_body( + endpoint: Endpoint, model: str, prompt: str, *, stream: bool = False, **extra: JsonValue +) -> dict[str, JsonValue]: + match endpoint: + case "chat": + return { + "model": model, + "messages": [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], + "max_tokens": MAX_TOKENS, + "stream": stream, + **({"stream_options": {"include_usage": True}} if stream else {}), + **extra, + } + case "responses": + return { + "model": model, + "input": [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], + "max_output_tokens": MAX_TOKENS, + "stream": stream, + **extra, + } + case "messages": + return { + "model": model, + "system": SYSTEM, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": MAX_TOKENS, + "stream": stream, + **extra, + } + + +@dataclass(frozen=True, slots=True) +class Outcome: + status: int + call_id: str + response_id: str + text: str + usage: dict[str, JsonValue] + headers: Mapping[str, str] + raw: str + + +def sse_payloads(lines: Iterable[str]) -> tuple[dict[str, JsonValue], ...]: + return tuple(JSON.validate_json(line[6:]) for line in lines if line.startswith("data: ") and line != "data: [DONE]") + + +def unwrapped(identity: str) -> str | None: + try: + decoded: Final = base64.b64decode(identity.removeprefix("resp_"), validate=True).decode() + except (binascii.Error, UnicodeDecodeError): + return None + return decoded.rsplit("response_id:", 1)[1] if decoded.startswith("litellm:") else None + + +def upstream_id_of(identity: str) -> str: + managed: Final = decrypt_if_encrypted_with(identity.removeprefix("resp_"), SIGNING_KEY) + wrapped: Final = identity if managed is None else managed.rsplit("response_id:", 1)[1].split(";", 1)[0] + inner: Final = unwrapped(wrapped) + return wrapped if inner is None else inner + + +def row_key(request_id: str) -> str: + return upstream_id_of(request_id.split("_cache_hit", 1)[0]) + + +def caller_sees_upstream_id(endpoint: Endpoint, *, stream: bool) -> bool: + return (endpoint, stream) != ("messages", True) + + +def usage_of(payload: Mapping[str, JsonValue]) -> dict[str, JsonValue]: + usage: Final = payload.get("usage") + return JSON.validate_python(usage) if isinstance(usage, dict) else {} + + +def chat_text(payload: Mapping[str, JsonValue]) -> str: + (choice,) = ITEMS.validate_python(payload["choices"]) + return str(JSON.validate_python(choice["message"])["content"]) + + +def chat_stream_fields(payloads: Sequence[Mapping[str, JsonValue]]) -> tuple[str, str, dict[str, JsonValue]]: + (identity,) = {str(chunk["id"]) for chunk in payloads if "id" in chunk} + choices: Final = tuple(ITEMS.validate_python(chunk["choices"]) for chunk in payloads if chunk.get("choices")) + deltas: Final = tuple(JSON.validate_python(choice[0]["delta"]) for choice in choices if choice) + text: Final = "".join(str(delta["content"]) for delta in deltas if isinstance(delta.get("content"), str)) + usages: Final = tuple(usage_of(chunk) for chunk in payloads if isinstance(chunk.get("usage"), dict)) + return identity, text, usages[-1] if usages else {} + + +def responses_text(payload: Mapping[str, JsonValue]) -> str: + items: Final = ITEMS.validate_python(payload["output"]) + parts: Final = tuple(ITEMS.validate_python(item["content"]) for item in items if item.get("type") == "message") + return "".join(str(part["text"]) for part in parts[0] if part.get("type") == "output_text") + + +def completed_response(events: Iterable[Mapping[str, JsonValue]]) -> dict[str, JsonValue]: + (completed,) = tuple(event for event in events if event.get("type") == "response.completed") + return JSON.validate_python(completed["response"]) + + +def messages_text(payload: Mapping[str, JsonValue]) -> str: + return "".join(str(block["text"]) for block in ITEMS.validate_python(payload["content"]) if "text" in block) + + +def messages_stream_fields(payloads: Sequence[Mapping[str, JsonValue]]) -> tuple[str, str, dict[str, JsonValue]]: + (started,) = tuple(payload for payload in payloads if payload.get("type") == "message_start") + message: Final = JSON.validate_python(started["message"]) + deltas: Final = tuple(JSON.validate_python(payload["delta"]) for payload in payloads if "delta" in payload) + text: Final = "".join(str(delta["text"]) for delta in deltas if delta.get("type") == "text_delta") + final_usages: Final = tuple(usage_of(payload) for payload in payloads if payload.get("type") == "message_delta") + return str(message["id"]), text, {**usage_of(message), **(final_usages[-1] if final_usages else {})} + + +def parse_outcome( + endpoint: Endpoint, *, stream: bool, status: int, headers: Mapping[str, str], lines: tuple[str, ...] +) -> Outcome: + call_id: Final = headers.get("x-litellm-call-id", "") + raw: Final = "\n".join(lines) + if status != 200: + return Outcome(status, call_id, "", "", {}, headers, raw) + payloads: Final = sse_payloads(lines) if stream else (JSON.validate_json(raw),) + match endpoint, stream: + case "chat", True: + identity, text, usage = chat_stream_fields(payloads) + return Outcome(status, call_id, upstream_id_of(identity), text, usage, headers, raw) + case "chat", False: + return Outcome( + status, + call_id, + upstream_id_of(str(payloads[0]["id"])), + chat_text(payloads[0]), + usage_of(payloads[0]), + headers, + raw, + ) + case "responses", True: + completed: Final = completed_response(payloads) + return Outcome( + status, + call_id, + upstream_id_of(str(completed["id"])), + responses_text(completed), + usage_of(completed), + headers, + raw, + ) + case "responses", False: + return Outcome( + status, + call_id, + upstream_id_of(str(payloads[0]["id"])), + responses_text(payloads[0]), + usage_of(payloads[0]), + headers, + raw, + ) + case "messages", True: + identity, text, usage = messages_stream_fields(payloads) + return Outcome(status, call_id, upstream_id_of(identity), text, usage, headers, raw) + case "messages", False: + return Outcome( + status, + call_id, + upstream_id_of(str(payloads[0]["id"])), + messages_text(payloads[0]), + usage_of(payloads[0]), + headers, + raw, + ) + raise AssertionError((endpoint, stream)) + + +def send(gateway: Gateway, endpoint: Endpoint, body: Mapping[str, JsonValue], *, key: str | None = None) -> Outcome: + stream: Final = body.get("stream") is True + headers: Final = {"Authorization": f"Bearer {gateway.key if key is None else key}"} + with gateway.client.stream("POST", endpoint_path(endpoint), json=body, headers=headers, timeout=60) as response: + lines: Final = tuple(line for line in response.iter_lines() if line) + return parse_outcome(endpoint, stream=stream, status=response.status_code, headers=response.headers, lines=lines) + + +def send_raw(gateway: Gateway, endpoint: Endpoint, content: str, *, key: str | None = None) -> Outcome: + headers: Final = { + "Authorization": f"Bearer {gateway.key if key is None else key}", + "Content-Type": "application/json", + } + response: Final = gateway.client.post( + endpoint_path(endpoint), content=content.encode(), headers=headers, timeout=60 + ) + lines: Final = tuple(line for line in response.text.splitlines() if line) + return parse_outcome(endpoint, stream=False, status=response.status_code, headers=response.headers, lines=lines) + + +def matching_rows(name: str, markers: frozenset[str]) -> tuple[dict[str, JsonValue], ...]: + rows: Final = read_rows( + 'SELECT request_id, status, spend, prompt_tokens, completion_tokens, cache_hit FROM "LiteLLM_SpendLogs" ' + "WHERE model_group = %s", + (name,), + ) + return tuple(row for row in rows if any(marker in row_key(str(row["request_id"])) for marker in markers)) + + +def spend_rows( + name: str, markers: frozenset[str], *, expected: int, seconds: float = 90 +) -> tuple[dict[str, JsonValue], ...]: + return eventually(lambda: matching_rows(name, markers), lambda found: len(found) >= expected, seconds=seconds) + + +def success_row(name: str, *needles: str) -> dict[str, JsonValue]: + (row,) = tuple(row for row in spend_rows(name, frozenset(needles), expected=1) if row["cache_hit"] != "True") + assert row["status"] == "success", row + return row + + +def failure_row(call_id: str) -> dict[str, JsonValue]: + assert call_id, "No call id to look the failure row up by" + rows: Final = eventually( + lambda: read_rows('SELECT request_id, status FROM "LiteLLM_SpendLogs" WHERE request_id = %s', (call_id,)), + lambda found: len(found) >= 1, + seconds=90, + ) + (row,) = tuple(rows) + assert row["status"] == "failure", row + return row + + +def assert_priced_row(row: Mapping[str, JsonValue], model: str) -> None: + assert row["prompt_tokens"] == INPUT_TOKENS, row + assert row["completion_tokens"] == OUTPUT_TOKENS, row + assert math.isclose(float(str(row["spend"])), expected_spend(model), rel_tol=1e-9), (row, expected_spend(model)) + + +def assert_usage(endpoint: Endpoint, usage: Mapping[str, JsonValue]) -> None: + match endpoint: + case "chat": + assert usage["prompt_tokens"] == INPUT_TOKENS, usage + assert usage["completion_tokens"] == OUTPUT_TOKENS, usage + details: Final = JSON.validate_python(usage["prompt_tokens_details"]) + assert details["cached_tokens"] == CACHED_TOKENS, usage + assert details["cache_write_tokens"] == WRITTEN_TOKENS, usage + assert details["cache_creation_tokens"] == WRITTEN_TOKENS, usage + case "responses": + assert usage["input_tokens"] == INPUT_TOKENS, usage + assert usage["output_tokens"] == OUTPUT_TOKENS, usage + input_details: Final = JSON.validate_python(usage["input_tokens_details"]) + assert input_details["cached_tokens"] == CACHED_TOKENS, usage + assert input_details["cache_write_tokens"] == WRITTEN_TOKENS, usage + case "messages": + assert usage["input_tokens"] == UNCACHED_TOKENS, usage + assert usage["output_tokens"] == OUTPUT_TOKENS, usage + assert usage["cache_read_input_tokens"] == CACHED_TOKENS, usage + assert usage["cache_creation_input_tokens"] == WRITTEN_TOKENS, usage + + +def assert_answered(outcome: Outcome, marker: str) -> None: + assert outcome.status == 200, (outcome.status, outcome.raw) + assert outcome.text == answer(marker), outcome.raw + + +def deployment( + scenario: Scenario, + wire: Wire, + model: str, + *, + model_info: Mapping[str, JsonValue] | None = None, + points: JsonValue = SYSTEM_POINT, + **extra: JsonValue, +) -> str: + return scenario.model( + model_info=model_info, + model=model, + api_base=wire.url, + api_key=TOKEN, + aws_region_name="us-east-1", + cache_control_injection_points=points, + **extra, + ) + + +def settled(gateway: Gateway, name: str, wire: Wire, *, accepted: frozenset[int] = frozenset({200})) -> None: + eventually( + lambda: tuple( + gateway.request( + "POST", "/v1/chat/completions", request_body("chat", name, prompt_text(fresh_marker()), cache=NO_CACHE) + ).status_code + for _ in range(12) + ), + lambda codes: all(code in accepted for code in codes), + seconds=90, + ) + wire.drain() + + +def mantle_deployment( + gateway: Gateway, + scenario: Scenario, + wire: Wire, + model: str = GPT, + *, + model_info: Mapping[str, JsonValue] | None = None, + **extra: JsonValue, +) -> str: + name: Final = deployment(scenario, wire, model, model_info=model_info, **extra) + settled(gateway, name, wire) + return name + + +def observe( + gateway: Gateway, wire: Wire, endpoint: Endpoint, body: Mapping[str, JsonValue], *, key: str | None = None +) -> tuple[Outcome, Request]: + wire.drain() + outcome: Final = send(gateway, endpoint, body, key=key) + return outcome, only_received(wire) + + +def marked_cell(gateway: Gateway, endpoint: Endpoint, *, stream: bool) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + outcome, received = observe( + gateway, wire, endpoint, request_body(endpoint, name, prompt_text(marker), stream=stream) + ) + assert_answered(outcome, marker) + assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream) + assert_usage(endpoint, outcome.usage) + row: Final = success_row(name, marker) + assert_priced_row(row, GPT) + if caller_sees_upstream_id(endpoint, stream=stream): + assert row_key(str(row["request_id"])) == outcome.response_id, (row, outcome.response_id) + if not stream: + assert math.isclose(float(outcome.headers["x-litellm-response-cost"]), expected_spend(GPT), rel_tol=1e-9), ( + outcome.headers + ) + + +def openai_client(gateway: Gateway) -> openai.OpenAI: + return openai.OpenAI( + base_url=f"{gateway.client.base_url}/v1", + api_key=gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False, timeout=60), + ) + + +def async_openai_client(gateway: Gateway) -> openai.AsyncOpenAI: + return openai.AsyncOpenAI( + base_url=f"{gateway.client.base_url}/v1", + api_key=gateway.key, + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False, timeout=60), + ) + + +def anthropic_client(gateway: Gateway) -> anthropic.Anthropic: + return anthropic.Anthropic( + base_url=str(gateway.client.base_url), + api_key=gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False, timeout=60), + ) + + +def async_anthropic_client(gateway: Gateway) -> anthropic.AsyncAnthropic: + return anthropic.AsyncAnthropic( + base_url=str(gateway.client.base_url), + api_key=gateway.key, + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False, timeout=60), + ) + + +class Dumpable(Protocol): + def model_dump(self, *, exclude_none: bool = ...) -> Mapping[str, object]: ... + + +def dump_model(value: Dumpable) -> dict[str, JsonValue]: + return JSON.validate_python(value.model_dump(exclude_none=True)) + + +def sdk_outcome(endpoint: Endpoint, payloads: Sequence[Mapping[str, JsonValue]], *, stream: bool) -> Outcome: + match endpoint, stream: + case "chat", True: + identity, text, usage = chat_stream_fields(payloads) + return Outcome(200, "", upstream_id_of(identity), text, usage, {}, "") + case "chat", False: + return Outcome( + 200, "", upstream_id_of(str(payloads[0]["id"])), chat_text(payloads[0]), usage_of(payloads[0]), {}, "" + ) + case "responses", _: + completed: Final = completed_response(payloads) if stream else dict(payloads[0]) + return Outcome( + 200, + "", + upstream_id_of(str(completed["id"])), + responses_text(completed), + usage_of(completed), + {}, + "", + ) + case "messages", True: + identity, text, usage = messages_stream_fields(payloads) + return Outcome(200, "", upstream_id_of(identity), text, usage, {}, "") + case "messages", False: + return Outcome( + 200, + "", + upstream_id_of(str(payloads[0]["id"])), + messages_text(payloads[0]), + usage_of(payloads[0]), + {}, + "", + ) + raise AssertionError((endpoint, stream)) + + +def sdk_call(gateway: Gateway, endpoint: Endpoint, name: str, prompt: str, *, stream: bool) -> Outcome: + match endpoint, stream: + case "chat", False: + completion: Final = openai_client(gateway).chat.completions.create( + model=name, + messages=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], + max_tokens=MAX_TOKENS, + ) + return sdk_outcome(endpoint, (dump_model(completion),), stream=False) + case "chat", True: + chunks: Final = openai_client(gateway).chat.completions.create( + model=name, + messages=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], + max_tokens=MAX_TOKENS, + stream=True, + stream_options={"include_usage": True}, + ) + return sdk_outcome(endpoint, tuple(dump_model(chunk) for chunk in chunks), stream=True) + case "responses", False: + response: Final = openai_client(gateway).responses.create( + model=name, + input=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], + max_output_tokens=MAX_TOKENS, + ) + return sdk_outcome(endpoint, (dump_model(response),), stream=False) + case "responses", True: + events: Final = openai_client(gateway).responses.create( + model=name, + input=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], + max_output_tokens=MAX_TOKENS, + stream=True, + ) + return sdk_outcome(endpoint, tuple(dump_model(event) for event in events), stream=True) + case "messages", False: + message: Final = anthropic_client(gateway).messages.create( + model=name, system=SYSTEM, messages=[{"role": "user", "content": prompt}], max_tokens=MAX_TOKENS + ) + return sdk_outcome(endpoint, (dump_model(message),), stream=False) + case "messages", True: + with anthropic_client(gateway).messages.stream( + model=name, system=SYSTEM, messages=[{"role": "user", "content": prompt}], max_tokens=MAX_TOKENS + ) as events_stream: + raw_events: Final = tuple(dump_model(event) for event in events_stream) + return sdk_outcome(endpoint, raw_events, stream=True) + raise AssertionError((endpoint, stream)) + + +async def async_sdk_call(gateway: Gateway, endpoint: Endpoint, name: str, prompt: str) -> Outcome: + match endpoint: + case "chat": + completion: Final = await async_openai_client(gateway).chat.completions.create( + model=name, + messages=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], + max_tokens=MAX_TOKENS, + ) + return sdk_outcome(endpoint, (dump_model(completion),), stream=False) + case "responses": + response: Final = await async_openai_client(gateway).responses.create( + model=name, + input=[{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], + max_output_tokens=MAX_TOKENS, + ) + return sdk_outcome(endpoint, (dump_model(response),), stream=False) + case "messages": + message: Final = await async_anthropic_client(gateway).messages.create( + model=name, system=SYSTEM, messages=[{"role": "user", "content": prompt}], max_tokens=MAX_TOKENS + ) + return sdk_outcome(endpoint, (dump_model(message),), stream=False) + + +def assert_marked_sdk_cell( + wire: Wire, endpoint: Endpoint, name: str, marker: str, outcome: Outcome, *, stream: bool +) -> None: + assert_answered(outcome, marker) + assert_wire( + only_received(wire), expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=stream + ) + assert_usage(endpoint, outcome.usage) + assert_priced_row(success_row(name, marker), GPT) + + +HOSTILE_OPTIONS: Final[dict[str, JsonValue]] = { + "int": 5, + "list": [{"mode": "explicit"}], + "empty-string": "", + "5kb-string": "x" * 5120, +} + + +MALFORMED_POINTS: Final[dict[str, JsonValue]] = { + "null": None, + "string": "system", + "int": 5, + "dict": {"location": "message", "role": "system"}, + "string-list": ["system"], + "no-location": [{"role": "system"}], +} +MIXED_POINTS: Final[list[JsonValue]] = ["system", *SYSTEM_POINT, 3] + + +Step = tuple[Endpoint, bool, str] + + +def plan_burst(size: int) -> tuple[Step, ...]: + return tuple((ENDPOINTS[index % 3], index % 3 == 0, fresh_marker()) for index in range(size)) + + +def burst(gateway: Gateway, name: str, size: int) -> tuple[tuple[str, Outcome], ...]: + def call(step: Step) -> tuple[str, Outcome]: + endpoint, stream, marker = step + return marker, send(gateway, endpoint, request_body(endpoint, name, prompt_text(marker), stream=stream)) + + with ThreadPoolExecutor(max_workers=10) as pool: + return tuple(pool.map(call, plan_burst(size))) + + +def assert_marked(request: Request, *, marked: bool) -> None: + body: Final = body_of(request) + assert breakpoint_count(body) == (1 if marked else 0), request.body + assert ("prompt_cache_options" in body) is marked, request.body + + +def assert_burst_landed(wire: Wire, name: str, outcomes: Sequence[tuple[str, Outcome]], *, marked: bool) -> None: + for marker, outcome in outcomes: + assert_answered(outcome, marker) + identities: Final = frozenset(outcome.response_id for _, outcome in outcomes) + assert len(identities) == len(outcomes), identities + received: Final = wire.drain() + assert len(received) == len(outcomes), (len(received), len(outcomes)) + for request in received: + assert_marked(request, marked=marked) + markers: Final = frozenset(marker for marker, _ in outcomes) + rows: Final = spend_rows(name, markers, expected=len(outcomes), seconds=120) + assert sorted(row_key(str(row["request_id"])) for row in rows) == sorted(identities), rows diff --git a/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py b/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py new file mode 100644 index 00000000000..2af41c2b683 --- /dev/null +++ b/tests/integration/providers/test_bedrock_mantle_gpt_prompt_cache_breakpoint_wire.py @@ -0,0 +1,531 @@ +import asyncio +import json +from typing import Final +from urllib.parse import urlsplit + +import pytest +from integration._support.client import Gateway, eventually +from integration._support.wire import Request, wire_server +from integration.providers._mantle_gpt_prompt_cache_support import ( + ANTHROPIC_PATH, + AZURE, + BURST, + CLAUDE, + ENDPOINTS, + EPHEMERAL, + EXPLICIT, + GPT, + GPT_FLAGGED_ROW, + GPT_BARE, + GPT_REGION, + GPT_UNFLAGGED_ROW, + HOSTILE_OPTIONS, + IMPLICIT, + ITEMS, + MALFORMED_POINTS, + MAX_TOKENS, + MIXED_POINTS, + NO_CACHE, + ODD_FLAGS, + SYSTEM, + SYSTEM_POINT, + THIRD_PARTY, + Endpoint, + Outcome, + assert_answered, + assert_burst_landed, + assert_marked_sdk_cell, + assert_priced_row, + assert_wire, + async_sdk_call, + body_of, + breakpoint_count, + burst, + deployment, + expected_wire, + failing_peer, + failure_row, + fresh_marker, + mantle_deployment, + mantle_peer, + marked_cell, + observe, + only_received, + openai_shaped_peer, + prompt_text, + request_body, + row_key, + sdk_call, + send, + send_raw, + settled, + spend_rows, + success_row, + system_item, +) +from pydantic import JsonValue + +pytestmark = pytest.mark.timeout(600) + + +@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"]) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_a_configured_system_point_reaches_mantle_as_a_breakpoint_over_httpx( + gateway: Gateway, endpoint: Endpoint, stream: bool +) -> None: + marked_cell(gateway, endpoint, stream=stream) + + +@pytest.mark.parametrize("stream", [False, True], ids=["json", "stream"]) +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_a_the_sync_sdks_see_the_breakpoint_and_the_mapped_cache_usage( + gateway: Gateway, endpoint: Endpoint, stream: bool +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + outcome: Final = sdk_call(gateway, endpoint, name, prompt_text(marker), stream=stream) + assert_marked_sdk_cell(wire, endpoint, name, marker, outcome, stream=stream) + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_a_the_async_sdks_see_the_breakpoint_and_the_mapped_cache_usage(gateway: Gateway, endpoint: Endpoint) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + outcome: Final = asyncio.run(async_sdk_call(gateway, endpoint, name, prompt_text(marker))) + assert_marked_sdk_cell(wire, endpoint, name, marker, outcome, stream=False) + + +@pytest.mark.parametrize("endpoint", ["responses", "chat"]) +def test_b1_b2_a_pinned_explicit_mode_reaches_the_wire_with_the_breakpoint( + gateway: Gateway, endpoint: Endpoint +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire, prompt_cache_options=EXPLICIT) + outcome, received = observe(gateway, wire, endpoint, request_body(endpoint, name, prompt_text(marker))) + assert_answered(outcome, marker) + assert_wire( + received, + expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True, options=EXPLICIT), + streaming=False, + ) + assert_priced_row(success_row(name, marker), GPT) + + +@pytest.mark.parametrize("endpoint", ["responses", "chat"]) +def test_b3_b4_a_region_prefixed_deployment_reads_the_region_free_row(gateway: Gateway, endpoint: Endpoint) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire, GPT_REGION) + outcome, received = observe(gateway, wire, endpoint, request_body(endpoint, name, prompt_text(marker))) + assert_answered(outcome, marker) + assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=False) + assert_priced_row(success_row(name, marker), GPT) + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_b15_to_b17_a_bare_deployment_name_with_its_provider_reads_the_provider_keyed_row( + gateway: Gateway, endpoint: Endpoint +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire, GPT_BARE, custom_llm_provider="bedrock_mantle") + outcome, received = observe(gateway, wire, endpoint, request_body(endpoint, name, prompt_text(marker))) + assert_answered(outcome, marker) + assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=False) + assert_priced_row(success_row(name, marker), GPT) + + +def assert_anthropic_marked_wire(received: Request, marker: str) -> None: + body: Final = body_of(received) + assert urlsplit(received.target).path == ANTHROPIC_PATH, received.target + assert body["system"] == [{"type": "text", "text": SYSTEM, "cache_control": EPHEMERAL}], received.body + assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": prompt_text(marker)}]}], ( + received.body + ) + assert "prompt_cache_options" not in body, received.body + assert "prompt_cache_breakpoint" not in received.body.decode(), received.body + + +@pytest.mark.parametrize("endpoint", ["messages", "chat"]) +def test_b5_b6_claude_on_mantle_keeps_the_anthropic_dialect(gateway: Gateway, endpoint: Endpoint) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire, CLAUDE) + outcome, received = observe(gateway, wire, endpoint, request_body(endpoint, name, prompt_text(marker))) + assert_answered(outcome, marker) + assert_anthropic_marked_wire(received, marker) + success_row(name, marker, outcome.response_id) + + +def test_b7_a_deployment_flag_true_opts_an_unflagged_mantle_row_in(gateway: Gateway) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment( + gateway, scenario, wire, GPT_UNFLAGGED_ROW, model_info={"supports_prompt_cache_breakpoint": True} + ) + outcome, received = observe(gateway, wire, "responses", request_body("responses", name, prompt_text(marker))) + assert_answered(outcome, marker) + assert_wire( + received, + expected_wire(GPT_UNFLAGGED_ROW, prompt_text(marker), endpoint="responses", marked=True), + streaming=False, + ) + success_row(name, marker) + + +def test_b8_a_deployment_flag_false_opts_a_flagged_mantle_row_out(gateway: Gateway) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment( + gateway, scenario, wire, GPT_FLAGGED_ROW, model_info={"supports_prompt_cache_breakpoint": False} + ) + outcome, received = observe(gateway, wire, "responses", request_body("responses", name, prompt_text(marker))) + assert_answered(outcome, marker) + body: Final = body_of(received) + assert breakpoint_count(body) == 0, received.body + assert "prompt_cache_options" not in body, received.body + assert "cache_control" not in received.body.decode(), received.body + success_row(name, marker) + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +def test_b13_azure_openai_stays_ineligible_for_the_breakpoint_dialect(gateway: Gateway, endpoint: Endpoint) -> None: + marker: Final = fresh_marker() + with wire_server(openai_shaped_peer()) as wire, gateway.scenario() as scenario: + name: Final = scenario.model( + model=AZURE, + api_base=wire.url, + api_key="synthetic-azure-key", + api_version="2025-04-01-preview", + cache_control_injection_points=SYSTEM_POINT, + ) + settled(gateway, name, wire) + outcome, received = observe(gateway, wire, endpoint, request_body(endpoint, name, prompt_text(marker))) + assert_answered(outcome, marker) + assert urlsplit(received.target).path.startswith("/openai/"), received.target + assert "prompt_cache_breakpoint" not in received.body.decode(), received.body + assert "prompt_cache_options" not in body_of(received), received.body + assert SYSTEM in received.body.decode(), received.body + success_row(name, marker) + + +def test_b14_an_openai_entry_on_a_third_party_host_keeps_the_anthropic_dialect(gateway: Gateway) -> None: + marker: Final = fresh_marker() + with wire_server(openai_shaped_peer()) as wire, gateway.scenario() as scenario: + name: Final = scenario.model( + model=THIRD_PARTY, + api_base=wire.url, + api_key="synthetic-third-party-key", + cache_control_injection_points=SYSTEM_POINT, + ) + settled(gateway, name, wire) + outcome, received = observe(gateway, wire, "chat", request_body("chat", name, prompt_text(marker))) + assert_answered(outcome, marker) + body: Final = body_of(received) + assert urlsplit(received.target).path == "/chat/completions", received.target + assert body["messages"] == [ + {"role": "system", "content": SYSTEM, "cache_control": EPHEMERAL}, + {"role": "user", "content": prompt_text(marker)}, + ], received.body + assert "prompt_cache_options" not in body, received.body + success_row(name, marker) + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_c_a_response_cache_hit_serves_the_marked_request_again(gateway: Gateway, endpoint: Endpoint) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + body: Final = request_body(endpoint, name, prompt_text(marker)) + first, received = observe(gateway, wire, endpoint, body) + assert_answered(first, marker) + assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=False) + sends: Final[list[Outcome]] = [] + + def resend() -> Outcome: + served: Final = send(gateway, endpoint, body) + sends.append(served) + return served + + hit: Final = eventually( + resend, lambda served: served.status == 200 and served.response_id == first.response_id, seconds=30 + ) + assert_answered(hit, marker) + misses: Final = wire.drain() + assert len(misses) == len(sends) - 1, (len(misses), len(sends)) + for miss in misses: + assert_wire(miss, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=False) + rows: Final = spend_rows(name, frozenset({marker}), expected=len(sends) + 1, seconds=70) + assert len(rows) == len(sends) + 1, rows + (hit_row,) = tuple(row for row in rows if row["cache_hit"] == "True") + assert row_key(str(hit_row["request_id"])) == first.response_id, (hit_row, first.response_id) + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +@pytest.mark.parametrize("shape", sorted(HOSTILE_OPTIONS)) +def test_d1_to_d4_hostile_prompt_cache_options_reach_the_wire_verbatim_and_the_providers_400_reaches_the_caller( + gateway: Gateway, endpoint: Endpoint, shape: str +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + hostile: Final = HOSTILE_OPTIONS[shape] + outcome, received = observe( + gateway, wire, endpoint, request_body(endpoint, name, prompt_text(marker), prompt_cache_options=hostile) + ) + assert outcome.status == 400, (outcome.status, outcome.raw) + assert "Invalid prompt_cache_options" in outcome.raw, outcome.raw + assert_wire( + received, + expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True, options=hostile), + streaming=False, + ) + failure_row(outcome.call_id) + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +def test_d5_a_duplicated_prompt_cache_options_key_resolves_to_the_last_value( + gateway: Gateway, endpoint: Endpoint +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + encoded: Final = json.dumps(request_body(endpoint, name, prompt_text(marker), prompt_cache_options=IMPLICIT)) + duplicated: Final = encoded[:-1] + ', "prompt_cache_options": {"mode": "explicit"}}' + wire.drain() + outcome: Final = send_raw(gateway, endpoint, duplicated) + assert_answered(outcome, marker) + assert_wire( + only_received(wire), + expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True, options=EXPLICIT), + streaming=False, + ) + success_row(name, marker) + + +@pytest.mark.parametrize("endpoint", ["chat", "responses"]) +def test_d6_a_null_prompt_cache_options_is_treated_as_unset(gateway: Gateway, endpoint: Endpoint) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + outcome, received = observe( + gateway, wire, endpoint, request_body(endpoint, name, prompt_text(marker), prompt_cache_options=None) + ) + assert_answered(outcome, marker) + assert_wire(received, expected_wire(GPT, prompt_text(marker), endpoint=endpoint, marked=True), streaming=False) + success_row(name, marker) + + +def test_d7_an_unauthenticated_hostile_request_never_reaches_the_wire(gateway: Gateway) -> None: + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + wire.drain() + outcome: Final = send( + gateway, + "responses", + request_body("responses", name, prompt_text(fresh_marker()), prompt_cache_options=5), + key="sk-not-a-key", + ) + assert outcome.status == 401, (outcome.status, outcome.raw) + assert wire.drain() == (), "the upstream saw an unauthenticated request" + + +@pytest.mark.parametrize("shape", sorted(MALFORMED_POINTS)) +def test_d8_to_d10_malformed_injection_points_never_crash_the_request(gateway: Gateway, shape: str) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = deployment(scenario, wire, GPT, points=MALFORMED_POINTS[shape]) + settled(gateway, name, wire) + outcome, received = observe(gateway, wire, "responses", request_body("responses", name, prompt_text(marker))) + assert_answered(outcome, marker) + body: Final = body_of(received) + assert breakpoint_count(body) == 0, received.body + assert "prompt_cache_options" not in body, received.body + success_row(name, marker) + + +@pytest.mark.parametrize( + ("status", "endpoint"), + [(400, "chat"), (400, "responses"), (401, "responses")], + ids=["400-chat", "400-responses", "401-responses"], +) +def test_d11_d12_a_provider_error_on_the_marked_request_reaches_the_caller_after_one_attempt( + gateway: Gateway, status: int, endpoint: Endpoint +) -> None: + marker: Final = fresh_marker() + message: Final = f"scripted provider failure {marker}" + with wire_server(failing_peer(status, message, "scripted_failure")) as wire, gateway.scenario() as scenario: + name: Final = deployment(scenario, wire, GPT) + settled(gateway, name, wire, accepted=frozenset({status})) + outcome: Final = send(gateway, endpoint, request_body(endpoint, name, prompt_text(marker))) + assert outcome.status == status, (outcome.status, outcome.raw) + assert message in outcome.raw, outcome.raw + attempts: Final = wire.drain() + assert len(attempts) == 1, [attempt.target for attempt in attempts] + assert breakpoint_count(body_of(attempts[0])) == 1, attempts[0].body + failure_row(outcome.call_id) + + +@pytest.mark.parametrize(("label", "model", "flag"), ODD_FLAGS, ids=[label for label, _, _ in ODD_FLAGS]) +def test_d13_an_odd_typed_deployment_flag_never_opts_a_mantle_row_in( + gateway: Gateway, label: str, model: str, flag: JsonValue +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment( + gateway, scenario, wire, model, model_info={"supports_prompt_cache_breakpoint": flag} + ) + outcome, received = observe(gateway, wire, "responses", request_body("responses", name, prompt_text(marker))) + assert_answered(outcome, marker) + body: Final = body_of(received) + assert breakpoint_count(body) == 0, (label, received.body) + assert "prompt_cache_options" not in body, (label, received.body) + success_row(name, marker) + + +@pytest.mark.parametrize("endpoint", ENDPOINTS) +def test_d14_a_point_beside_junk_entries_still_marks_the_anthropic_dialect( + gateway: Gateway, endpoint: Endpoint +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire, CLAUDE, points=MIXED_POINTS) + outcome, received = observe(gateway, wire, endpoint, request_body(endpoint, name, prompt_text(marker))) + assert_answered(outcome, marker) + assert_anthropic_marked_wire(received, marker) + success_row(name, marker, outcome.response_id) + + +def test_e1_client_breakpoint_prompt_cache_key_and_explicit_mode_pass_through_with_no_second_breakpoint( + gateway: Gateway, +) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + body: Final[dict[str, JsonValue]] = { + "model": name, + "input": [system_item(marked=True, endpoint="responses"), {"role": "user", "content": prompt_text(marker)}], + "max_output_tokens": MAX_TOKENS, + "prompt_cache_key": f"key-{marker}", + "prompt_cache_options": EXPLICIT, + } + outcome, received = observe(gateway, wire, "responses", body) + assert_answered(outcome, marker) + assert_wire( + received, + expected_wire( + GPT, + prompt_text(marker), + endpoint="responses", + marked=True, + options=EXPLICIT, + prompt_cache_key=f"key-{marker}", + ), + streaming=False, + ) + success_row(name, marker) + + +def test_e2_four_client_breakpoints_leave_no_slot_for_the_configured_point(gateway: Gateway) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + turns: Final = tuple(f"Earlier turn {index} marker-{marker}." for index in range(3)) + body: Final[dict[str, JsonValue]] = { + "model": name, + "input": [ + {"role": "system", "content": SYSTEM}, + *( + { + "role": "user", + "content": [{"type": "input_text", "text": turn, "prompt_cache_breakpoint": EXPLICIT}], + } + for turn in turns + ), + { + "role": "user", + "content": [ + {"type": "input_text", "text": prompt_text(marker), "prompt_cache_breakpoint": EXPLICIT} + ], + }, + ], + "max_output_tokens": MAX_TOKENS, + } + outcome, received = observe(gateway, wire, "responses", body) + assert_answered(outcome, marker) + wire_body: Final = body_of(received) + items: Final = ITEMS.validate_python(wire_body["input"]) + assert breakpoint_count(wire_body) == 4, received.body + assert items[0]["role"] == "system" and "prompt_cache_breakpoint" not in json.dumps(items[0]), items[0] + assert "prompt_cache_options" not in wire_body, received.body + success_row(name, marker) + + +def test_e3_a_client_implicit_mode_on_a_pinned_explicit_deployment_wins_on_the_wire(gateway: Gateway) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire, prompt_cache_options=EXPLICIT) + outcome, received = observe( + gateway, + wire, + "responses", + request_body("responses", name, prompt_text(marker), prompt_cache_options=IMPLICIT), + ) + assert_answered(outcome, marker) + assert_wire( + received, + expected_wire(GPT, prompt_text(marker), endpoint="responses", marked=True, options=IMPLICIT), + streaming=False, + ) + success_row(name, marker) + + +def test_e4_an_empty_prompt_cache_options_object_is_kept_as_the_clients_value(gateway: Gateway) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + outcome, received = observe( + gateway, wire, "responses", request_body("responses", name, prompt_text(marker), prompt_cache_options={}) + ) + assert_answered(outcome, marker) + assert_wire( + received, + expected_wire(GPT, prompt_text(marker), endpoint="responses", marked=True, options={}), + streaming=False, + ) + success_row(name, marker) + + +def test_e5_the_same_request_twice_writes_two_rows_and_two_marked_upstream_requests(gateway: Gateway) -> None: + marker: Final = fresh_marker() + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + body: Final = request_body("chat", name, prompt_text(marker), cache=NO_CACHE) + first, first_received = observe(gateway, wire, "chat", body) + second, second_received = observe(gateway, wire, "chat", body) + assert_answered(first, marker) + assert_answered(second, marker) + assert first.response_id != second.response_id, (first.response_id, second.response_id) + for received in (first_received, second_received): + assert_wire( + received, expected_wire(GPT, prompt_text(marker), endpoint="chat", marked=True), streaming=False + ) + rows: Final = spend_rows(name, frozenset({marker}), expected=2) + assert {row_key(str(row["request_id"])) for row in rows} == {first.response_id, second.response_id}, rows + + +def test_f1_a_mixed_burst_marks_every_request_and_lands_every_id_once(gateway: Gateway) -> None: + with wire_server(mantle_peer()) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + wire.drain() + assert_burst_landed(wire, name, burst(gateway, name, BURST), marked=True) + + +def test_f4_a_slow_mantle_stream_during_a_burst_completes_with_every_id_landing_once(gateway: Gateway) -> None: + with wire_server(mantle_peer(pause=0.3)) as wire, gateway.scenario() as scenario: + name: Final = mantle_deployment(gateway, scenario, wire) + wire.drain() + assert_burst_landed(wire, name, burst(gateway, name, 12), marked=True) diff --git a/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py b/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py new file mode 100644 index 00000000000..cc7ab06da90 --- /dev/null +++ b/tests/integration/providers/test_openai_dialect_prompt_cache_breakpoint_owned_proxy.py @@ -0,0 +1,385 @@ +import asyncio +import re +import signal +import threading +from collections.abc import Callable, Iterator +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final +from urllib.parse import urlsplit + +import httpx +import psutil +import pytest +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from integration._support.responses_vendor import newest_marker +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.providers._cache_control_marks_support import owned_config +from integration.providers._mantle_gpt_prompt_cache_support import ( + EXPLICIT, + GPT, + IMPLICIT, + SYSTEM, + SYSTEM_POINT, + TOKEN, + Outcome, + assert_answered, + assert_burst_landed, + assert_wire, + body_of, + breakpoint_count, + expected_wire, + fresh_marker, + mantle_deployment, + mantle_peer, + observe, + openai_shaped_peer, + parse_outcome, + plan_burst, + prompt_text, + request_body, + row_key, + send, + settled, + spend_rows, + success_row, +) +from pydantic import JsonValue + +_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +_FOUNDRY_BASE: Final = "http://prompt-cache-breakpoint-audit.services.ai.azure.com" +_FOUNDRY_HOST: Final = urlsplit(_FOUNDRY_BASE).netloc +_FOUNDRY_MODEL: Final = "gpt-6-astra" +_FOUNDRY_KEY: Final = "synthetic-foundry-key" +_FOUNDRY_CHAT_PATH: Final = "/models/chat/completions" +_CELL_TIMEOUT: Final = int(2 * graceful_stop_seconds() + 120) +_RESTART_TIMEOUT: Final = int(4 * graceful_stop_seconds() + 240) +_HELD_BURST: Final = 20 +_SPREAD_BURST: Final = 12 +_SPREAD_ATTEMPTS: Final = 20 +_MANTLE_NAME: Final = "mantle-gpt-cache-owned" + + +def _peer(release: threading.Event, held: SimpleQueue[str]) -> Callable[[Request], Reply]: + mantle: Final = mantle_peer() + foundry: Final = openai_shaped_peer() + + def respond(request: Request) -> Reply: + if urlsplit(request.target).netloc == _FOUNDRY_HOST: + return foundry(request) + marker: Final = newest_marker(request.body.decode()) + assert marker is not None, request.body + held.put(marker) + assert release.wait(timeout=120), "The held burst was never released" + return mantle(request) + + return respond + + +@dataclass(frozen=True, slots=True) +class _Rig: + gateway: Gateway + wire: Wire + owned: OwnedProxy + release: threading.Event + held: SimpleQueue[str] + + +def _overrides(wire: Wire) -> MappingProxyType[str, str]: + return MappingProxyType({"HTTP_PROXY": wire.url, "NO_PROXY": "127.0.0.1,localhost", "AIOHTTP_TRUST_ENV": "True"}) + + +def _started_workers(log: Path) -> tuple[int, ...]: + return tuple(int(match.group(1)) for match in _STARTED_WORKER.finditer(log.read_text())) + + +def _live_workers(log: Path) -> tuple[int, ...]: + return tuple(pid for pid in _started_workers(log) if psutil.pid_exists(pid)) + + +def _open_peer_connections(pid: int, peer_url: str) -> int: + port: Final = urlsplit(peer_url).port + return sum( + 1 + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.status == psutil.CONN_ESTABLISHED and connection.raddr and connection.raddr.port == port + ) + + +def _drain(queue: SimpleQueue[str]) -> None: + while not queue.empty(): + queue.get_nowait() + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[_Rig]: + directory: Final = tmp_path_factory.mktemp("openai-dialect-prompt-cache-breakpoint") + release: Final = threading.Event() + release.set() + held: Final[SimpleQueue[str]] = SimpleQueue() + with gateway_from_environment() as environment, wire_server(_peer(release, held)) as wire: + with owned_proxy_process(environment, directory, _overrides(wire), workers=2) as owned: + eventually(lambda: len(_started_workers(owned.log)), lambda count: count == 2, seconds=60) + wire.drain() + yield _Rig(owned.gateway, wire, owned, release, held) + + +def _foundry_deployment(rig: _Rig, scenario: Scenario, *, bare: bool) -> str: + name: Final = ( + scenario.model( + model=_FOUNDRY_MODEL, + custom_llm_provider="azure_ai", + api_base=_FOUNDRY_BASE, + api_key=_FOUNDRY_KEY, + cache_control_injection_points=SYSTEM_POINT, + ) + if bare + else scenario.model( + model=f"azure_ai/{_FOUNDRY_MODEL}", + api_base=_FOUNDRY_BASE, + api_key=_FOUNDRY_KEY, + cache_control_injection_points=SYSTEM_POINT, + ) + ) + settled(rig.gateway, name, rig.wire) + return name + + +def _assert_foundry_chat_wire(received: Request, prompt: str, *, options: JsonValue | None) -> None: + body: Final = body_of(received) + assert urlsplit(received.target).netloc == _FOUNDRY_HOST, received.target + assert urlsplit(received.target).path == _FOUNDRY_CHAT_PATH, received.target + assert "prompt_cache_breakpoint" not in received.body.decode(), received.body + assert body["model"] == _FOUNDRY_MODEL, received.body + assert body["messages"] == [{"role": "system", "content": SYSTEM}, {"role": "user", "content": prompt}], ( + received.body + ) + assert body.get("prompt_cache_options") == options, received.body + + +@pytest.mark.timeout(_CELL_TIMEOUT) +@pytest.mark.parametrize("bare", [False, True], ids=["prefixed", "bare-provider-field"]) +def test_b9_b12_foundry_gpt6_responses_reach_the_openai_dialect_on_the_foundry_host(rig: _Rig, bare: bool) -> None: + marker: Final = fresh_marker() + with rig.gateway.scenario() as scenario: + name: Final = _foundry_deployment(rig, scenario, bare=bare) + outcome, received = observe( + rig.gateway, rig.wire, "responses", request_body("responses", name, prompt_text(marker)) + ) + assert_answered(outcome, marker) + assert urlsplit(received.target).netloc == _FOUNDRY_HOST, received.target + assert_wire( + received, + expected_wire(_FOUNDRY_MODEL, prompt_text(marker), endpoint="responses", marked=True), + streaming=False, + ) + success_row(name, marker) + + +@pytest.mark.timeout(_CELL_TIMEOUT) +def test_b10_foundry_gpt6_messages_bridge_to_chat_without_a_breakpoint(rig: _Rig) -> None: + marker: Final = fresh_marker() + with rig.gateway.scenario() as scenario: + name: Final = _foundry_deployment(rig, scenario, bare=False) + outcome, received = observe( + rig.gateway, rig.wire, "messages", request_body("messages", name, prompt_text(marker)) + ) + assert_answered(outcome, marker) + _assert_foundry_chat_wire(received, prompt_text(marker), options=IMPLICIT) + success_row(name, marker) + + +@pytest.mark.timeout(_CELL_TIMEOUT) +def test_b11_foundry_gpt6_chat_stays_on_the_plain_chat_wire(rig: _Rig) -> None: + marker: Final = fresh_marker() + with rig.gateway.scenario() as scenario: + name: Final = _foundry_deployment(rig, scenario, bare=False) + outcome, received = observe(rig.gateway, rig.wire, "chat", request_body("chat", name, prompt_text(marker))) + assert_answered(outcome, marker) + _assert_foundry_chat_wire(received, prompt_text(marker), options=IMPLICIT) + success_row(name, marker) + + +def _spread_burst(rig: _Rig, name: str, workers: tuple[int, ...]) -> dict[int, int]: + plan: Final = plan_burst(_SPREAD_BURST) + _drain(rig.held) + rig.wire.drain() + rig.release.clear() + with ThreadPoolExecutor(max_workers=_SPREAD_BURST) as pool: + futures: Final = tuple( + pool.submit(send, rig.gateway, endpoint, request_body(endpoint, name, prompt_text(marker), stream=stream)) + for endpoint, stream, marker in plan + ) + eventually(rig.held.qsize, lambda size: size == _SPREAD_BURST, seconds=60) + held_by: Final = {pid: _open_peer_connections(pid, rig.wire.url) for pid in workers} + rig.release.set() + outcomes: Final = tuple(future.result() for future in futures) + assert sum(held_by.values()) == _SPREAD_BURST, held_by + assert_burst_landed( + rig.wire, name, tuple((marker, outcome) for (_, _, marker), outcome in zip(plan, outcomes)), marked=True + ) + return held_by + + +@pytest.mark.timeout(_CELL_TIMEOUT) +def test_e6_every_worker_of_a_two_worker_proxy_marks_the_mantle_request(rig: _Rig) -> None: + with rig.gateway.scenario() as scenario: + name: Final = mantle_deployment(rig.gateway, scenario, rig.wire) + workers: Final = _live_workers(rig.owned.log) + assert len(workers) == 2, workers + for _ in range(_SPREAD_ATTEMPTS): + if all(count > 0 for count in _spread_burst(rig, name, workers).values()): + break + else: + raise AssertionError("One worker never took a marked request") + + +def _mantle_config(wire: Wire, directory: Path, **litellm_params: JsonValue) -> Path: + return owned_config( + directory, + [ + { + "model_name": _MANTLE_NAME, + "litellm_params": { + "model": GPT, + "api_base": wire.url, + "api_key": TOKEN, + "aws_region_name": "us-east-1", + "cache_control_injection_points": SYSTEM_POINT, + **litellm_params, + }, + } + ], + ) + + +async def _one(client: httpx.AsyncClient, key: str, body: dict[str, JsonValue]) -> Outcome: + response: Final = await client.post("/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {key}"}) + lines: Final = tuple(line for line in response.text.splitlines() if line) + return parse_outcome("chat", stream=False, status=response.status_code, headers=response.headers, lines=lines) + + +async def _held_burst(url: str, key: str, markers: tuple[str, ...]) -> tuple[tuple[str, Outcome], ...]: + async with httpx.AsyncClient(base_url=url, timeout=180, trust_env=False) as client: + results: Final = await asyncio.gather( + *(_one(client, key, request_body("chat", _MANTLE_NAME, prompt_text(marker))) for marker in markers), + return_exceptions=True, + ) + return tuple((marker, result) for marker, result in zip(markers, results) if isinstance(result, Outcome)) + + +def _assert_served_and_landed(wire: Wire, served: tuple[tuple[str, Outcome], ...], *, expected_received: int) -> None: + for marker, outcome in served: + assert_answered(outcome, marker) + received: Final = wire.drain() + assert len(received) == expected_received, (len(received), expected_received) + for request in received: + assert breakpoint_count(body_of(request)) == 1, request.body + identities: Final = frozenset(outcome.response_id for _, outcome in served) + assert len(identities) == len(served), identities + rows: Final = spend_rows(_MANTLE_NAME, frozenset(marker for marker, _ in served), expected=len(served), seconds=120) + assert sorted(row_key(str(row["request_id"])) for row in rows) == sorted(identities), rows + + +@pytest.mark.timeout(_CELL_TIMEOUT) +async def test_f2_a_worker_killed_mid_burst_leaves_the_survivor_marking_requests( + gateway: Gateway, tmp_path: Path +) -> None: + release: Final = threading.Event() + held: Final[SimpleQueue[str]] = SimpleQueue() + markers: Final = tuple(fresh_marker() for _ in range(_HELD_BURST)) + with wire_server(_peer(release, held)) as wire: + config: Final = _mantle_config(wire, tmp_path) + with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned: + candidate: Final = owned.gateway + url: Final = str(candidate.client.base_url).rstrip("/") + workers: Final = eventually(lambda: _live_workers(owned.log), lambda pids: len(pids) == 2, seconds=60) + burst: Final = asyncio.create_task(_held_burst(url, candidate.key, markers)) + await asyncio.to_thread(eventually, held.qsize, lambda size: size == _HELD_BURST, 60) + held_by: Final = MappingProxyType({pid: _open_peer_connections(pid, wire.url) for pid in workers}) + assert sum(held_by.values()) == _HELD_BURST, held_by + victim_pid, survivor_pid = sorted(workers, key=held_by.__getitem__) + victim: Final = psutil.Process(victim_pid) + victim.suspend() + victim.send_signal(signal.SIGKILL) + release.set() + served: Final = await burst + assert len(served) == held_by[survivor_pid], (held_by, len(served)) + follow_up: Final = fresh_marker() + (answered,) = await _held_burst(url, candidate.key, (follow_up,)) + _assert_served_and_landed(wire, (*served, answered), expected_received=_HELD_BURST + 1) + eventually(lambda: len(_started_workers(owned.log)), lambda count: count == 3, seconds=90) + + +def _model_id(name: str) -> str: + (row,) = read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_name=%s', (name,)) + return str(row["model_id"]) + + +@pytest.mark.timeout(_RESTART_TIMEOUT) +async def test_f3_a_graceful_restart_mid_burst_drains_the_held_requests_and_keeps_the_stored_explicit_mode( + gateway: Gateway, tmp_path: Path +) -> None: + release: Final = threading.Event() + held: Final[SimpleQueue[str]] = SimpleQueue() + markers: Final = tuple(fresh_marker() for _ in range(_HELD_BURST)) + stored: Final = f"mantle-gpt-explicit-stored-{fresh_marker()}" + with wire_server(_peer(release, held)) as wire: + config: Final = _mantle_config(wire, tmp_path) + with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as owned: + candidate: Final = owned.gateway + url: Final = str(candidate.client.base_url).rstrip("/") + eventually(lambda: len(_live_workers(owned.log)), lambda count: count == 2, seconds=60) + candidate.post( + "/model/new", + { + "model_name": stored, + "litellm_params": { + "model": GPT, + "api_base": wire.url, + "api_key": TOKEN, + "aws_region_name": "us-east-1", + "cache_control_injection_points": SYSTEM_POINT, + "prompt_cache_options": EXPLICIT, + }, + }, + ) + release.set() + settled(candidate, stored, wire) + before, before_received = observe( + candidate, wire, "responses", request_body("responses", stored, prompt_text(markers[0])) + ) + assert_answered(before, markers[0]) + assert_wire( + before_received, + expected_wire(GPT, prompt_text(markers[0]), endpoint="responses", marked=True, options=EXPLICIT), + streaming=False, + ) + success_row(stored, markers[0]) + _drain(held) + release.clear() + burst: Final = asyncio.create_task(_held_burst(url, candidate.key, markers)) + await asyncio.to_thread(eventually, held.qsize, lambda size: size == _HELD_BURST, 60) + owned.process.terminate() + release.set() + served: Final = await burst + assert len(served) == _HELD_BURST, len(served) + _assert_served_and_landed(wire, served, expected_received=_HELD_BURST) + with owned_proxy_process(gateway, tmp_path, {}, config=config, workers=2) as restarted: + settled(restarted.gateway, stored, wire) + after, after_received = observe( + restarted.gateway, wire, "responses", request_body("responses", stored, prompt_text(markers[1])) + ) + assert_answered(after, markers[1]) + assert_wire( + after_received, + expected_wire(GPT, prompt_text(markers[1]), endpoint="responses", marked=True, options=EXPLICIT), + streaming=False, + ) + success_row(stored, markers[1]) + restarted.gateway.post("/model/delete", {"id": _model_id(stored)}) diff --git a/tests/unit/integrations/test_anthropic_cache_control_hook.py b/tests/unit/integrations/test_anthropic_cache_control_hook.py index 8392cef8a7b..d96f00cc2b1 100644 --- a/tests/unit/integrations/test_anthropic_cache_control_hook.py +++ b/tests/unit/integrations/test_anthropic_cache_control_hook.py @@ -14,6 +14,7 @@ from pydantic import BaseModel, ConfigDict import litellm from litellm.integrations.anthropic_cache_control_hook import ( AnthropicCacheControlHook, + configured_injection_points, supports_openai_prompt_cache_breakpoint, ) from litellm.litellm_core_utils.prompt_templates.factory import ( @@ -3738,6 +3739,14 @@ class TestPromptCacheBreakpointCapability: ) assert supports_openai_prompt_cache_breakpoint("gpt-5.6") is False + @pytest.mark.parametrize("model", ["gpt-4.1", "gpt-5.6"]) + @pytest.mark.parametrize("flag", ["true", 1, "false", 0]) + def test_listed_model_with_an_odd_typed_flag_is_not_eligible(self, monkeypatch, model, flag): + monkeypatch.setitem( + litellm.model_cost, model, {**litellm.model_cost[model], "supports_prompt_cache_breakpoint": flag} + ) + assert supports_openai_prompt_cache_breakpoint(model) is False + def test_published_map_without_the_flag_still_injects_on_gpt_5_6(self, monkeypatch): unflagged = {k: v for k, v in litellm.model_cost["gpt-5.6"].items() if k != "supports_prompt_cache_breakpoint"} @@ -3769,6 +3778,277 @@ class TestPromptCacheBreakpointCapability: assert model not in litellm.model_cost assert supports_openai_prompt_cache_breakpoint(model) is expected + def test_a_null_prompt_cache_options_takes_the_implicit_default_on_both_paths(self): + points = [{"location": "message", "role": "system"}] + + _, _, chat_params = AnthropicCacheControlHook().get_chat_completion_prompt( + model="openai/gpt-5.6", + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + non_default_params={"cache_control_injection_points": copy.deepcopy(points), "prompt_cache_options": None}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert chat_params["prompt_cache_options"] == {"mode": "implicit"} + + kwargs = {"cache_control_injection_points": copy.deepcopy(points), "prompt_cache_options": None} + AnthropicCacheControlHook.maybe_inject_cache_control( + [{"role": "user", "content": "hi"}], "sys", kwargs, model="gpt-5.6", custom_llm_provider="openai" + ) + assert kwargs["prompt_cache_options"] == {"mode": "implicit"} + + +class TestHostedOpenAIDialectFlag: + """#38666: an OpenAI-shaped model served by another provider can opt in through its own + model-map entry, instead of being excluded by the openai-only provider check.""" + + MANTLE_MODEL = "bedrock_mantle/openai.gpt-5.6-sol" + + def _register(self, monkeypatch, key, provider, flag=True, **extra): + entry = {"litellm_provider": provider, "mode": "chat", **extra} + if flag is not None: + entry["supports_prompt_cache_breakpoint"] = flag + monkeypatch.setitem(litellm.model_cost, key, entry) + + def test_flagged_non_openai_deployment_is_eligible(self, monkeypatch): + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + is True + ) + + def test_bedrock_api_base_does_not_veto_the_explicit_flag(self, monkeypatch): + """The api_base check exists to sniff for api.openai.com, which a Bedrock host never is.""" + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + self.MANTLE_MODEL, + "bedrock_mantle", + api_base="https://bedrock-runtime.us-east-1.amazonaws.com", + ) + is True + ) + + def test_flag_set_false_keeps_the_deployment_ineligible(self, monkeypatch): + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=False) + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + is False + ) + + @pytest.mark.parametrize("flag", ["true", 1, "false", 0]) + def test_an_odd_typed_flag_keeps_the_deployment_ineligible(self, monkeypatch, flag): + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=flag) + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + is False + ) + + def test_unflagged_non_openai_deployment_stays_ineligible(self, monkeypatch): + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=None) + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + is False + ) + + def test_entry_provider_must_match_the_request_provider(self, monkeypatch): + """A flagged entry does not license a different provider serving the same model string.""" + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "azure") is False + ) + + def test_openai_entries_still_go_through_the_api_base_check(self, monkeypatch): + """gpt-5.6 is flagged and openai-provided, so it must not bypass the host gate.""" + assert litellm.model_cost["gpt-5.6"]["supports_prompt_cache_breakpoint"] is True + assert litellm.model_cost["gpt-5.6"]["litellm_provider"] == "openai" + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint( + "gpt-5.6", "openai", api_base="https://some-compatible-host.example.com" + ) + is False + ) + + def test_azure_hosted_gpt_5_6_remains_ineligible(self): + """Regression guard: the openai entry's flag must not leak to another provider.""" + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-5.6", "azure") is False + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("azure/gpt-5.6", None) is False + + REGIONAL_MODEL = "bedrock_mantle/us-east-1/openai.gpt-5.6-sol" + + def test_region_prefixed_deployment_reads_its_region_free_entry(self, monkeypatch): + """``bedrock_mantle//`` is a documented routing form the map keys without the region.""" + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle") + is True + ) + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, None) is True + + def test_region_prefixed_deployment_honors_a_flag_set_false(self, monkeypatch): + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle", flag=False) + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.REGIONAL_MODEL, "bedrock_mantle") + is False + ) + + def test_region_prefixed_entry_outranks_the_region_free_one(self, monkeypatch): + """A row keyed with the region states that deployment's own dialect; GovCloud rows carry no flag.""" + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") + gov_model = "bedrock_mantle/us-gov-west-1/openai.gpt-5.6-sol" + self._register(monkeypatch, gov_model, "bedrock_mantle", flag=None) + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(gov_model, "bedrock_mantle") is False + + def test_region_free_entry_does_not_license_another_provider(self, monkeypatch): + """The candidate keys are built for the request's provider, so a flagged Mantle row stays Mantle's.""" + self._register(monkeypatch, self.MANTLE_MODEL, "bedrock_mantle") + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("us-east-1/openai.gpt-5.6-sol", "azure") + is False + ) + + def test_unmapped_deployment_name_stays_ineligible(self): + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("azure/my-deployment", None) is False + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("my-deployment", "azure") is False + + def test_bare_name_colliding_with_an_openai_row_reads_its_own_provider_entry(self, monkeypatch): + """The Responses layer hands the hook a bare deployment name plus its provider. The openai row keyed by + that bare name neither answers for the deployment nor stops the lookup of the provider's own entry.""" + self._register(monkeypatch, "gpt-collide", "openai") + self._register(monkeypatch, "azure_ai/gpt-collide", "azure_ai") + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is True + + def test_bare_name_colliding_with_an_openai_row_stays_ineligible_without_its_own_flag(self, monkeypatch): + self._register(monkeypatch, "gpt-collide", "openai") + self._register(monkeypatch, "azure_ai/gpt-collide", "azure_ai", flag=None) + assert AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint("gpt-collide", "azure_ai") is False + + +class TestBedrockMantleGptShipsTheOpenAIDialect: + """The shipped cost map flags Bedrock Mantle's GPT-5.6 and newer OpenAI rows, so a configured injection point + on one of them reaches the wire as prompt_cache_breakpoint instead of an Anthropic cache_control the Mantle + bridge strips (verified live against bedrock-mantle.us-east-1 on 2026-10-07: cache_write_tokens then + cached_tokens on the repeat call).""" + + MANTLE_MODEL = "bedrock_mantle/openai.gpt-5.6-sol" + + @pytest.fixture(autouse=True) + def _bundled_model_map(self, monkeypatch): + bundled = os.path.join(os.path.dirname(litellm.__file__), "model_prices_and_context_window_backup.json") + with open(bundled) as handle: + monkeypatch.setattr(litellm, "model_cost", json.load(handle)) + litellm.utils.cached_get_model_info_helper.cache_clear() + yield + litellm.utils.cached_get_model_info_helper.cache_clear() + + def test_shipped_entry_makes_the_deployment_eligible(self): + assert supports_openai_prompt_cache_breakpoint(self.MANTLE_MODEL) is True + assert ( + AnthropicCacheControlHook._targets_openai_prompt_cache_breakpoint(self.MANTLE_MODEL, "bedrock_mantle") + is True + ) + + def test_every_flagged_mantle_row_is_an_openai_gpt_5_6_or_newer_model(self): + flagged = { + key for key, entry in litellm.model_cost.items() + if key.startswith("bedrock_mantle/") and entry.get("supports_prompt_cache_breakpoint") is True + } + assert self.MANTLE_MODEL in flagged + for key in flagged: + bare = key.rsplit("/", 1)[-1].removeprefix("openai.") + assert supports_openai_prompt_cache_breakpoint(bare) is True, key + + def test_seeding_stamps_the_openai_dialect_for_a_configured_point(self): + non_default_params = {"cache_control_injection_points": [{"location": "message", "role": "system"}]} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=non_default_params, + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + model=self.MANTLE_MODEL, + custom_llm_provider=None, + ) + assert non_default_params["cache_control_injection_points"][0]["_litellm_openai_dialect"] is True + + def test_configured_point_emits_the_openai_marker_and_default_options(self): + _, messages, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model=self.MANTLE_MODEL, + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + non_default_params={"cache_control_injection_points": [{"location": "message", "role": "system"}]}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert messages[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + assert params["prompt_cache_options"] == {"mode": "implicit"} + assert AnthropicCacheControlHook.count_request_cache_breakpoints(messages) == 1 + + def test_region_prefixed_deployment_emits_the_openai_marker(self): + """The region-prefixed routing form documented for Mantle lands on the same shipped row.""" + _, messages, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model="bedrock_mantle/us-east-1/openai.gpt-5.6-sol", + messages=[{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}], + non_default_params={"cache_control_injection_points": [{"location": "message", "role": "system"}]}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert messages[0]["content"] == [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + assert params["prompt_cache_options"] == {"mode": "implicit"} + + BARE_MODEL = "openai.gpt-5.6-sol" + POINTS = [{"location": "message", "role": "system"}] + MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + MARKED_SYSTEM = [{"type": "text", "text": "sys", "prompt_cache_breakpoint": {"mode": "explicit"}}] + + def test_a_bare_deployment_name_with_its_provider_is_stamped_on_the_openai_dialect(self): + """A deployment written as ``model: openai.gpt-5.6-sol`` plus ``custom_llm_provider: bedrock_mantle`` + has no row of its own and no openai row of the same name, so only the provider-keyed row can + answer; the stamp must read it the way the dialect resolution does.""" + stamped = AnthropicCacheControlHook._stamped_with_dialect( + copy.deepcopy(self.POINTS), self.BARE_MODEL, "bedrock_mantle", None, None + ) + assert stamped[0]["_litellm_openai_dialect"] is True + + def test_a_bare_deployment_name_without_its_provider_keeps_its_points_and_costs_no_lookup(self): + points = copy.deepcopy(self.POINTS) + with patch.object(AnthropicCacheControlHook, "_resolve_provider") as resolve: + assert AnthropicCacheControlHook._stamped_with_dialect(points, self.BARE_MODEL, None, None, None) is points + resolve.assert_not_called() + + def test_the_chat_seed_carries_the_resolved_provider_for_a_bare_deployment_name(self): + params: dict = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model=self.BARE_MODEL, + custom_llm_provider="bedrock_mantle", + ) + _, messages, out = AnthropicCacheControlHook().get_chat_completion_prompt( + model=self.BARE_MODEL, + messages=copy.deepcopy(self.MESSAGES), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert messages[0]["content"] == self.MARKED_SYSTEM + assert out["prompt_cache_options"] == {"mode": "implicit"} + + def test_the_responses_stamp_carries_the_resolved_provider_for_a_bare_deployment_name(self): + from litellm.responses.main import _stamp_injection_points_with_dialect + + kwargs: dict = {"cache_control_injection_points": copy.deepcopy(self.POINTS)} + _stamp_injection_points_with_dialect(kwargs, self.BARE_MODEL, "bedrock_mantle") + _, messages, out = AnthropicCacheControlHook().get_chat_completion_prompt( + model=self.BARE_MODEL, + messages=copy.deepcopy(self.MESSAGES), + non_default_params=kwargs, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert messages[0]["content"] == self.MARKED_SYSTEM + assert out["prompt_cache_options"] == {"mode": "implicit"} + class TestRecordGatewayInjection: """The injection marker spend accounting gates prompt-caching savings on.""" @@ -3901,3 +4181,83 @@ class TestRecordGatewayInjection: custom_llm_provider="anthropic", ) assert self.KEY not in kwargs["litellm_metadata"] + + +class TestMalformedInjectionPointsAreIgnored: + """A ``cache_control_injection_points`` value that is not a list of points (a string, an int, a bare + dict, a list of strings) raised inside the hook and turned every request to that deployment into a 500. + Every entry point now reads it as no configured points, the way ``null`` already read.""" + + SHAPES = ("system", 5, {"location": "message", "role": "system"}, ["system"], None) + MIXED = ["system", {"location": "message", "role": "system"}, 3] + MESSAGES = [{"role": "system", "content": "sys"}, {"role": "user", "content": "hi"}] + + @pytest.mark.parametrize("value", SHAPES) + def test_reads_as_no_points(self, value): + assert configured_injection_points(value) == () + + def test_keeps_the_point_entries_of_a_mixed_list(self): + assert configured_injection_points(self.MIXED) == ({"location": "message", "role": "system"},) + + def test_the_point_beside_junk_entries_survives_the_chat_seed_on_an_unstamped_deployment(self): + params: dict = {"cache_control_injection_points": copy.deepcopy(self.MIXED)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + _, processed, _ = AnthropicCacheControlHook().get_chat_completion_prompt( + model="claude-sonnet-4-5", + messages=copy.deepcopy(self.MESSAGES), + non_default_params=params, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert processed[0] == {"role": "system", "content": "sys", "cache_control": {"type": "ephemeral"}} + + @pytest.mark.parametrize("value", SHAPES) + def test_chat_prompt_hook_leaves_the_request_untouched(self, value): + _, processed, params = AnthropicCacheControlHook().get_chat_completion_prompt( + model="openai/gpt-5.6", + messages=copy.deepcopy(self.MESSAGES), + non_default_params={"cache_control_injection_points": copy.deepcopy(value)}, + prompt_id=None, + prompt_variables=None, + dynamic_callback_params={}, + ) + assert processed == self.MESSAGES + assert "cache_control_injection_points" not in params + assert "prompt_cache_options" not in params + + @pytest.mark.parametrize("value", SHAPES) + def test_chat_seeding_falls_through_to_the_defaults(self, monkeypatch, value): + monkeypatch.setattr(litellm, "enable_anthropic_prompt_caching", True) + params: dict = {"cache_control_injection_points": copy.deepcopy(value)} + AnthropicCacheControlHook.maybe_seed_default_injection_points( + non_default_params=params, + messages=copy.deepcopy(self.MESSAGES), + model="claude-sonnet-4-5", + custom_llm_provider="anthropic", + ) + seeded = params["cache_control_injection_points"] + assert seeded and all(isinstance(point, dict) and "location" in point for point in seeded) + + @pytest.mark.parametrize("value", SHAPES) + def test_messages_path_leaves_the_request_untouched(self, value): + kwargs: dict = {"cache_control_injection_points": copy.deepcopy(value)} + messages, system = AnthropicCacheControlHook.maybe_inject_cache_control( + [{"role": "user", "content": "hi"}], "sys", kwargs, model="gpt-5.6", custom_llm_provider="openai" + ) + assert (messages, system) == ([{"role": "user", "content": "hi"}], "sys") + assert "cache_control_injection_points" not in kwargs + assert "prompt_cache_options" not in kwargs + + @pytest.mark.parametrize("value", SHAPES) + def test_responses_dialect_stamp_leaves_the_request_untouched(self, value): + from litellm.responses.main import _stamp_injection_points_with_dialect + + kwargs: dict = {"cache_control_injection_points": copy.deepcopy(value)} + _stamp_injection_points_with_dialect(kwargs, "gpt-5.6", "openai") + assert kwargs == {"cache_control_injection_points": value} diff --git a/tests/unit/responses/test_responses_api_request_body.py b/tests/unit/responses/test_responses_api_request_body.py index 3ef3cd8854a..8c8b347be82 100644 --- a/tests/unit/responses/test_responses_api_request_body.py +++ b/tests/unit/responses/test_responses_api_request_body.py @@ -827,6 +827,62 @@ async def test_injection_points_still_reach_a_native_responses_provider(): assert "cache_control_injection_points" not in body +@pytest.mark.asyncio +async def test_injection_points_reach_a_provider_prefixed_native_responses_model(): + """Bedrock Mantle resolves its Responses config from the price map by bare model name, + so predicting the bridge with the ``bedrock_mantle/``-prefixed name read as bridged and + deferred the system point to a chat-completions pass this native path never runs.""" + injected_client = AsyncHTTPHandler() + mock_post = AsyncMock( + return_value=MockResponse(_minimal_responses_api_payload("resp_mantle", "openai.gpt-5.6-sol"), 200) + ) + injected_client.post = mock_post + + await litellm.aresponses( + model="bedrock_mantle/openai.gpt-5.6-sol", + api_key="fake-bearer-token", + aws_region_name="us-east-1", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + client=injected_client, + ) + + assert mock_post.call_args.kwargs["url"].endswith("/openai/v1/responses") + body = _sent_body(mock_post) + assert body["input"][0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"} + assert body["prompt_cache_options"] == {"mode": "implicit"} + assert "cache_control_injection_points" not in body + + +@pytest.mark.asyncio +async def test_injection_points_reach_a_foundry_deployment_of_an_openai_model(monkeypatch): + """The router hands this layer the provider it resolved with the deployment's api_base, but the + hook reads the provider from the request kwargs, which never carry it, and resolving + ``azure_ai/gpt-6-astra`` by name alone reads the ambient ``AZURE_AI_API_BASE``, so next to an + Azure OpenAI one the Foundry deployment got Anthropic marks the Responses transform then stripped.""" + monkeypatch.setenv("AZURE_AI_API_BASE", "https://other-deployment.openai.azure.com") + injected_client = AsyncHTTPHandler() + mock_post = AsyncMock( + return_value=MockResponse(_minimal_responses_api_payload("resp_foundry", "gpt-6-astra"), 200) + ) + injected_client.post = mock_post + + await litellm.aresponses( + model="azure_ai/gpt-6-astra", + custom_llm_provider="azure_ai", + api_key="fake-api-key", + api_base="https://foundry.services.ai.azure.com", + input=copy.deepcopy(_INJECTION_POINT_INPUT), + cache_control_injection_points=copy.deepcopy(_SYSTEM_INJECTION_POINT), + client=injected_client, + ) + + body = _sent_body(mock_post) + assert body["input"][0]["content"][0]["prompt_cache_breakpoint"] == {"mode": "explicit"} + assert body["prompt_cache_options"] == {"mode": "implicit"} + assert "cache_control_injection_points" not in body + + async def _bridged_body(mock_post, *, points, input, instructions="You are a documentation assistant."): injected_client = AsyncHTTPHandler() injected_client.post = mock_post From 04d97abffbb3fb7454823edff3ff155b08871503 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:37:55 -0700 Subject: [PATCH 29/65] test: make 77 legacy live tests offline in litellm_utils, router_unit and responses dirs (#45298) * test: make 77 legacy live tests offline in litellm_utils, router_unit and responses dirs * test: restore lost coverage in litellm_utils offline replacements Bound live health-check tasks to the concurrency limit so eager task creation behind a semaphore fails, route the Langfuse trace-id check through completion -> Logging.get_trace_id with all four metadata combinations, run the callback dedup checks through acompletion, pass the dynamic key via litellm_params, cover the env-inferred default model list, and add the per-provider audio transcription config lookup * test: restore router coverage lost in the offline move Add the missing non-stream LIT-3058 header/count-once test, pin the UTC minute on every usage-counter test, assert router-level client reuse for transcription, and replace the private selector-attribute checks with routing outcomes per strategy. Assert outbound bodies for speech, rerank, image, assistants and moderation, add timeouts to event waits, and drop the router_unit_tests husk files that no longer hold tests * test: restore sync stream, router sync stream, error event and field-type coverage for migrated responses tests Add the sync streaming logging and sync Router.responses streaming cases the offline replacements dropped, port the legacy per-event and response field-type validation, assert raw headers, the float created_at conversion, MCP tool headers, the search_context_size input, and add an offline replacement for the in-stream context-window error event. Remove helpers left dead in the legacy file. * test: run numpydoc-backed unit tests in GHA and drop a test that pinned a crash * test: restore sequence number, item id and content part checks in the responses stream validator * test: match router logging events to the test's own deployment so late events from other tests are ignored * test: restore the s3 cold storage history test to its original assertions --------- Co-authored-by: yuneng --- .github/workflows/_test-unit-base.yml | 4 +- .../test_gemini_thinking_replay_wire.py | 91 +++ .../base_token_counter_test.py | 67 -- .../litellm_utils_tests/test_health_check.py | 326 -------- .../test_secret_manager.py | 113 --- tests/litellm_utils_tests/test_utils.py | 657 ---------------- .../base_responses_api.py | 108 --- .../test_anthropic_responses_api.py | 25 +- .../test_azure_responses_api.py | 1 - .../test_google_ai_studio_responses_api.py | 231 +----- .../test_openai_responses_api.py | 743 +----------------- tests/router_unit_tests/gettysburg.wav | Bin 775192 -> 0 bytes .../test_router_endpoints.py | 261 ------ .../test_router_helper_utils.py | 526 ------------- tests/unit/integrations/test_langfuse.py | 52 ++ .../test_health_check_helpers.py | 191 ++++- .../llms/anthropic/test_count_tokens_oauth.py | 83 ++ .../responses/test_openai_responses_http.py | 531 +++++++++++++ .../test_openai_responses_transformation.py | 127 +++ .../proxy/test_health_check_max_tokens.py | 166 ++++ .../test_google_ai_studio_responses_wire.py | 158 ++++ .../responses/test_responses_api_lifecycle.py | 109 +++ .../test_google_secret_manager.py | 83 ++ .../test_secret_managers_main.py | 8 + .../test_router_provider_endpoints.py | 732 +++++++++++++++++ tests/unit/test_utils.py | 410 +++++++++- 26 files changed, 2752 insertions(+), 3051 deletions(-) delete mode 100644 tests/litellm_utils_tests/test_health_check.py delete mode 100644 tests/router_unit_tests/gettysburg.wav delete mode 100644 tests/router_unit_tests/test_router_endpoints.py delete mode 100644 tests/router_unit_tests/test_router_helper_utils.py create mode 100644 tests/unit/llms/openai/responses/test_openai_responses_http.py create mode 100644 tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_wire.py create mode 100644 tests/unit/responses/test_responses_api_lifecycle.py create mode 100644 tests/unit/secret_managers/test_google_secret_manager.py create mode 100644 tests/unit/test_router/test_router_provider_endpoints.py diff --git a/.github/workflows/_test-unit-base.yml b/.github/workflows/_test-unit-base.yml index ebb925c8b68..e5847e5fbea 100644 --- a/.github/workflows/_test-unit-base.yml +++ b/.github/workflows/_test-unit-base.yml @@ -143,9 +143,9 @@ jobs: run: | diff -u model_prices_and_context_window.json litellm/model_prices_and_context_window_backup.json if [ -z "$RUST_BRIDGE_ARTIFACT" ]; then - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils else - .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --no-install-project + .github/scripts/uv_sync_with_retries.sh --frozen --group ci --group proxy-dev --extra google --extra proxy --extra semantic-router --extra saml --extra caching --extra extra_proxy --extra proxy-runtime --extra utils --no-install-project uv pip install --no-deps --python .venv/bin/python rust-bridge-dist/*.whl cp rust-bridge-dist/litellm/rust_bridge/_native.abi3.so litellm/rust_bridge/_native.abi3.so uv run --no-sync python -c "import importlib.metadata; import litellm.rust_bridge._native; print(importlib.metadata.version('litellm'))" diff --git a/tests/integration/providers/test_gemini_thinking_replay_wire.py b/tests/integration/providers/test_gemini_thinking_replay_wire.py index 055aa2fee4e..c0fbe0c4fa3 100644 --- a/tests/integration/providers/test_gemini_thinking_replay_wire.py +++ b/tests/integration/providers/test_gemini_thinking_replay_wire.py @@ -6,17 +6,20 @@ import uuid from collections.abc import Iterator, Mapping, Sequence from collections.abc import Set as AbstractSet from dataclasses import dataclass +from pathlib import Path from typing import Final, Literal, TypeAlias import anthropic import httpx import openai import pytest +import yaml from _pytest.mark.structures import ParameterSet from cryptography.hazmat.primitives import serialization from cryptography.hazmat.primitives.asymmetric import rsa from integration._support.client import Gateway, Scenario, eventually from integration._support.database import read_rows +from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server from openai.types.responses import ResponseCompletedEvent from pydantic import JsonValue, TypeAdapter @@ -463,6 +466,26 @@ def _spend_row(*request_ids: str) -> Mapping[str, JsonValue]: return rows[0] +def _spend_proxy_server_request(request_id: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + 'SELECT proxy_server_request FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (request_id,), + ), + lambda found: len(found) == 1, + seconds=70, + ) + return _JSON_OBJECT.validate_python(rows[0]["proxy_server_request"]) + + +def _config_storing_prompts(directory: Path) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["general_settings"]["store_prompts_in_spend_logs"] = True + path: Final = directory / "store-prompts.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + def _assert_caller_id(endpoint: Endpoint, stream: bool, caller_id: str, response_id: str) -> None: if endpoint == "messages" and stream: assert caller_id.startswith("msg_"), caller_id @@ -654,6 +677,74 @@ def test_gemini_own_tool_call_signature_is_still_replayed_on_the_function_call_p assert _spend_row(rounds[1])["status"] == "success" +def test_responses_previous_response_id_replays_gemini_session_history( + gateway: Gateway, tmp_path: Path +) -> None: + first_prompt: Final = f"gemini-session-first-{uuid.uuid4().hex}" + second_prompt: Final = f"gemini-session-second-{uuid.uuid4().hex}" + first_answer: Final = f"gemini-answer-first-{uuid.uuid4().hex}" + second_answer: Final = f"gemini-answer-second-{uuid.uuid4().hex}" + first_provider_id: Final = f"gemini-response-first-{uuid.uuid4().hex}" + second_provider_id: Final = f"gemini-response-second-{uuid.uuid4().hex}" + calls: Final = itertools.count() + + def respond(request: Request) -> Reply: + if next(calls) == 0: + return _reply(first_provider_id, stream=False, parts=({"text": first_answer},)) + return _reply(second_provider_id, stream=False, parts=({"text": second_answer},)) + + with ( + wire_server(respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_config_storing_prompts(tmp_path), workers=1) as prompt_gateway, + prompt_gateway.scenario() as scenario, + ): + model: Final = _register(prompt_gateway, scenario, "gemini", wire) + first_response: Final = prompt_gateway.request( + "POST", + "/v1/responses", + {"model": model, "input": first_prompt, **_CACHE_BUST}, + ) + assert first_response.status_code == 200, first_response.text + first_body: Final = _JSON_OBJECT.validate_json(first_response.content) + first_response_id: Final = str(first_body["id"]) + assert _output_text(first_body) == first_answer + first_spend_id: Final = _upstream_response_id("responses", first_response_id) + assert _spend_row(first_spend_id)["status"] == "success" + first_proxy_request: Final = _spend_proxy_server_request(first_spend_id) + assert first_proxy_request["input"] == first_prompt, first_proxy_request + + second_response: Final = prompt_gateway.request( + "POST", + "/v1/responses", + { + "model": model, + "input": second_prompt, + "previous_response_id": first_response_id, + **_CACHE_BUST, + }, + ) + assert second_response.status_code == 200, second_response.text + second_body: Final = _JSON_OBJECT.validate_json(second_response.content) + second_response_id: Final = str(second_body["id"]) + assert _output_text(second_body) == second_answer + assert _spend_row(_upstream_response_id("responses", second_response_id))["status"] == "success" + + requests: Final = wire.drain() + assert [(request.method, request.target) for request in requests] == [ + ("POST", _target("gemini", False)), + ("POST", _target("gemini", False)), + ] + first_provider_body: Final = _JSON_OBJECT.validate_json(requests[0].body) + second_provider_body: Final = _JSON_OBJECT.validate_json(requests[1].body) + assert first_provider_body == _expected_body("responses", _user(first_prompt)) + assert second_provider_body == _expected_body( + "responses", + _user(first_prompt), + _model_turn({"text": first_answer}), + _user(second_prompt), + ) + + _FIVE_KB: Final = "s" * 5000 _ASSISTANT_SHAPES: Final = ( pytest.param({"reasoning_content": _REASONING, "thinking_blocks": 5}, _REPLAYED_TURN, id="blocks-int"), diff --git a/tests/litellm_utils_tests/base_token_counter_test.py b/tests/litellm_utils_tests/base_token_counter_test.py index ddce27522c2..f4c90118a03 100644 --- a/tests/litellm_utils_tests/base_token_counter_test.py +++ b/tests/litellm_utils_tests/base_token_counter_test.py @@ -17,7 +17,6 @@ import pytest from litellm.llms.base_llm.base_utils import BaseTokenCounter -from litellm.types.utils import TokenCountResponse class BaseTokenCounterTest(ABC): @@ -71,69 +70,3 @@ class BaseTokenCounterTest(ABC): ): pytest.skip(f"Missing or invalid credentials: {e}") raise - - @pytest.mark.asyncio - async def test_count_tokens_basic(self): - """ - Test basic token counting functionality. - - Verifies that: - - Token counter returns a TokenCountResponse - - total_tokens is greater than 0 - - tokenizer_type is set - - No error occurred - """ - token_counter = self.get_token_counter() - model = self.get_test_model() - messages = self.get_test_messages() - deployment = self.get_deployment_config() - - result = await token_counter.count_tokens( - model_to_use=model, - messages=messages, - contents=None, - deployment=deployment, - request_model=model, - ) - - print(f"Token count result: {result}") - - assert result is not None, "Token counter should return a result" - assert isinstance( - result, TokenCountResponse - ), "Result should be TokenCountResponse" - assert ( - result.total_tokens > 0 - ), f"Token count should be > 0, got {result.total_tokens}" - assert result.tokenizer_type is not None, "tokenizer_type should be set" - assert ( - result.error is not True - ), f"Token counting should not error: {result.error_message}" - - def test_should_use_token_counting_api(self): - """ - Test that should_use_token_counting_api returns True for the correct provider. - - Verifies that the token counter correctly identifies when it should be used - based on the custom_llm_provider. - """ - token_counter = self.get_token_counter() - provider = self.get_custom_llm_provider() - - result = token_counter.should_use_token_counting_api( - custom_llm_provider=provider - ) - - assert ( - result is True - ), f"should_use_token_counting_api should return True for {provider}" - - # Also verify it returns False for other providers - other_provider = "some_other_provider_that_doesnt_exist" - result_other = token_counter.should_use_token_counting_api( - custom_llm_provider=other_provider - ) - - assert ( - result_other is False - ), f"should_use_token_counting_api should return False for {other_provider}" diff --git a/tests/litellm_utils_tests/test_health_check.py b/tests/litellm_utils_tests/test_health_check.py deleted file mode 100644 index 9a79db89164..00000000000 --- a/tests/litellm_utils_tests/test_health_check.py +++ /dev/null @@ -1,326 +0,0 @@ -#### What this tests #### -# This tests if ahealth_check() actually works - -import asyncio -import os -from unittest.mock import AsyncMock, patch - -import pytest - -import litellm - - -@pytest.mark.asyncio -async def test_azure_health_check(): - response = await litellm.ahealth_check( - model_params={ - "model": "azure/gpt-4.1-mini", - "messages": [{"role": "user", "content": "Hey, how's it going?"}], - "api_key": os.getenv("AZURE_AI_API_KEY"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - "api_version": os.getenv("AZURE_AI_API_VERSION"), - } - ) - print(f"response: {response}") - - assert "x-ratelimit-remaining-tokens" in response - return response - - -# asyncio.run(test_azure_health_check()) - - - - -@pytest.mark.asyncio -async def test_azure_embedding_health_check(): - response = await litellm.ahealth_check( - model_params={ - "model": "azure/text-embedding-ada-002", - "api_key": os.getenv("AZURE_AI_API_KEY"), - "api_base": os.getenv("AZURE_AI_API_BASE"), - "api_version": os.getenv("AZURE_AI_API_VERSION"), - }, - input=["test for litellm"], - mode="embedding", - ) - print(f"response: {response}") - - assert "x-ratelimit-remaining-tokens" in response - return response - - -@pytest.mark.asyncio -async def test_openai_img_gen_health_check(): - response = await litellm.ahealth_check( - model_params={ - "model": "gpt-image-1", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - mode="image_generation", - prompt="cute baby sea otter", - ) - print(f"response: {response}") - - assert isinstance(response, dict) and "error" not in response - return response - - -# asyncio.run(test_openai_img_gen_health_check()) - - - - - - -# asyncio.run(test_sagemaker_embedding_health_check()) - - -@pytest.mark.asyncio -async def test_groq_health_check(): - """ - This should not fail - - ensure that provider wildcard model passes health check - """ - litellm.set_verbose = True - response = await litellm.ahealth_check( - model_params={ - "api_key": os.environ.get("GROQ_API_KEY"), - "model": "groq/*", - "messages": [{"role": "user", "content": "What's 1 + 1?"}], - }, - mode=None, - prompt="What's 1 + 1?", - input=["test from litellm"], - ) - print(f"response: {response}") - assert response == {} - - return response - - -@pytest.mark.asyncio -async def test_cohere_rerank_health_check(): - response = await litellm.ahealth_check( - model_params={ - "model": "cohere/rerank-english-v3.0", - "api_key": os.getenv("COHERE_API_KEY"), - }, - mode="rerank", - prompt="Hey, how's it going", - ) - - assert "error" not in response - - print(response) - - -@pytest.mark.asyncio -async def test_audio_speech_health_check(): - response = await litellm.ahealth_check( - model_params={ - "model": "openai/tts-1", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - mode="audio_speech", - prompt="Hey", - ) - - assert "error" not in response - - print(response) - - -@pytest.mark.asyncio -async def test_audio_speech_health_check_with_another_voice(): - response = await litellm.ahealth_check( - model_params={ - "model": "openai/tts-1", - "api_key": os.getenv("OPENAI_API_KEY"), - "health_check_voice": "en-US-JennyNeural", - }, - mode="audio_speech", - prompt="Hey", - ) - - assert "error" not in response - - print(response) - - -@pytest.mark.asyncio -async def test_audio_transcription_health_check(): - litellm.set_verbose = True - response = await litellm.ahealth_check( - model_params={ - "model": "openai/whisper-1", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - mode="audio_transcription", - ) - - print(f"response: {response}") - - assert "error" not in response - - print(response) - - - - - - - - - - -@pytest.mark.asyncio -async def test_health_check_bad_model(): - import time - - from litellm.proxy.health_check import _perform_health_check - - model_list = [ - { - "model_name": "openai-gpt-4o", - "litellm_params": { - "api_key": "sk-9876", - "api_base": "https://exampleopenaiendpoint-production.up.railway.app", - "model": "openai/my-fake-openai-endpoint", - "mock_timeout": True, - "timeout": 60, - }, - "model_info": { - "id": "ca27ca2eeea2f9e38bb274ead831948a26621a3738d06f1797253f0e6c4278c0", - "db_model": False, - "health_check_timeout": 1, - }, - }, - ] - details = None - healthy_endpoints, unhealthy_endpoints, _ = await _perform_health_check( - model_list, details - ) - print(f"healthy_endpoints: {healthy_endpoints}") - print(f"unhealthy_endpoints: {unhealthy_endpoints}") - - # Track which model is actually used in the health check - health_check_calls = [] - - async def mock_health_check(litellm_params, **kwargs): - health_check_calls.append(litellm_params["model"]) - await asyncio.sleep(10) - return {"status": "healthy"} - - with patch( - "litellm.ahealth_check", side_effect=mock_health_check - ) as mock_health_check: - start_time = time.time() - healthy_endpoints, unhealthy_endpoints, _ = await _perform_health_check( - model_list - ) - end_time = time.time() - print("health check calls: ", health_check_calls) - assert len(healthy_endpoints) == 0 - assert len(unhealthy_endpoints) == 1 - assert ( - end_time - start_time < 2 - ), "Health check took longer than health_check_timeout" - - -@pytest.mark.asyncio -async def test_health_check_respects_concurrency_limit(): - from litellm.proxy.health_check import _perform_health_check - - model_list = [ - {"litellm_params": {"model": f"openai/gpt-4o-mini-{i}", "api_key": "fake-key"}} - for i in range(6) - ] - - active = 0 - max_active = 0 - - async def mock_health_check(litellm_params, **kwargs): - nonlocal active, max_active - active += 1 - max_active = max(max_active, active) - await asyncio.sleep(0.05) - active -= 1 - return {"status": "healthy"} - - with patch("litellm.ahealth_check", side_effect=mock_health_check): - await _perform_health_check(model_list, max_concurrency=2) - - assert max_active <= 2 - - -@pytest.mark.asyncio -async def test_health_check_creates_only_bounded_initial_tasks(): - from litellm.proxy.health_check import _perform_health_check - - model_list = [ - {"litellm_params": {"model": f"openai/gpt-4o-mini-{i}", "api_key": "fake-key"}} - for i in range(10) - ] - release_event = asyncio.Event() - create_task_call_count = 0 - real_create_task = asyncio.create_task - - async def mock_health_check(litellm_params, **kwargs): - await release_event.wait() - return {"status": "healthy"} - - def tracked_create_task(coro): - nonlocal create_task_call_count - create_task_call_count += 1 - return real_create_task(coro) - - with ( - patch("litellm.ahealth_check", side_effect=mock_health_check), - patch( - "litellm.proxy.health_check.asyncio.create_task", - side_effect=tracked_create_task, - ), - ): - perform_task = real_create_task( - _perform_health_check(model_list, max_concurrency=2) - ) - await asyncio.sleep(0.05) - assert create_task_call_count == 2 - release_event.set() - await perform_task - - -@pytest.mark.asyncio -async def test_timeout_does_not_cancel_other_health_checks(): - from litellm.proxy.health_check import _perform_health_check - - model_list = [ - { - "litellm_params": {"model": "openai/slow-model", "api_key": "fake-key"}, - "model_info": {"health_check_timeout": 0.05}, - }, - { - "litellm_params": {"model": "openai/fast-model", "api_key": "fake-key"}, - "model_info": {"health_check_timeout": 1}, - }, - ] - - async def mock_health_check(litellm_params, **kwargs): - if litellm_params["model"] == "openai/slow-model": - await asyncio.sleep(0.2) - return {"status": "healthy"} - await asyncio.sleep(0.01) - return {"status": "healthy"} - - with patch("litellm.ahealth_check", side_effect=mock_health_check): - healthy_endpoints, unhealthy_endpoints, _ = await _perform_health_check( - model_list, max_concurrency=1 - ) - - healthy_models = {endpoint["model"] for endpoint in healthy_endpoints} - unhealthy_models = {endpoint["model"] for endpoint in unhealthy_endpoints} - - assert "openai/fast-model" in healthy_models - assert "openai/slow-model" in unhealthy_models diff --git a/tests/litellm_utils_tests/test_secret_manager.py b/tests/litellm_utils_tests/test_secret_manager.py index 7cc5c72faa7..6d088b0c0e2 100644 --- a/tests/litellm_utils_tests/test_secret_manager.py +++ b/tests/litellm_utils_tests/test_secret_manager.py @@ -1,4 +1,3 @@ -import base64 import hashlib import json import os @@ -7,7 +6,6 @@ from dotenv import load_dotenv load_dotenv() import tempfile -from unittest.mock import MagicMock, patch from typing import Final import pytest @@ -93,8 +91,6 @@ def test_oidc_google(): ) print(f"secret_val: {redact_oidc_signature(secret_val)}") - - @pytest.mark.skipif( os.environ.get("ACTIONS_ID_TOKEN_REQUEST_TOKEN") is None, reason="Cannot run without being in GitHub Actions", @@ -105,112 +101,3 @@ def test_oidc_github(): ) print(f"secret_val: {redact_oidc_signature(secret_val)}") - - -@pytest.mark.skipif( - os.environ.get("CIRCLE_OIDC_TOKEN") is None, - reason="Cannot run without being in CircleCI Runner", -) -def test_oidc_circleci(): - secret_val = get_secret("oidc/circleci/") - - print(f"secret_val: {redact_oidc_signature(secret_val)}") - - -@pytest.mark.skipif( - os.environ.get("CIRCLE_OIDC_TOKEN_V2") is None, - reason="Cannot run without being in CircleCI Runner", -) -def test_oidc_circleci_v2(): - secret_val = get_secret( - "oidc/circleci_v2/https://bedrock-runtime.us-east-1.amazonaws.com/model/amazon.titan-text-express-v1/invoke" - ) - - print(f"secret_val: {redact_oidc_signature(secret_val)}") - - - - - - - - - - - - -def test_google_secret_manager(): - """ - Test that we can get a secret from Google Secret Manager - """ - os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "litellm-ci-cd" - - from litellm.secret_managers.google_secret_manager import GoogleSecretManager - - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "payload": { - "data": base64.b64encode(b"anything").decode("utf-8"), - } - } - - with ( - patch("litellm.proxy.proxy_server.premium_user", True), - patch.object( - GoogleSecretManager, - "sync_construct_request_headers", - return_value={"Authorization": "Bearer mock_token"}, - ), - ): - secret_manager = GoogleSecretManager() - secret_manager.sync_httpx_client = MagicMock() - secret_manager.sync_httpx_client.get.return_value = mock_response - - secret_val = secret_manager.get_secret_from_google_secret_manager( - secret_name="OPENAI_API_KEY" - ) - print("secret_val: {}".format(secret_val)) - - assert ( - secret_val == "anything" - ), "did not get expected secret value. expect 'anything', got '{}'".format( - secret_val - ) - - secret_manager.sync_httpx_client.get.assert_called_once() - call_url = secret_manager.sync_httpx_client.get.call_args[1]["url"] - assert "projects/litellm-ci-cd/secrets/OPENAI_API_KEY" in call_url - - -def test_google_secret_manager_read_in_memory(): - """ - Test that Google Secret manager returns in memory value when it exists - """ - from litellm.secret_managers.google_secret_manager import GoogleSecretManager - - os.environ["GOOGLE_SECRET_MANAGER_PROJECT_ID"] = "litellm-ci-cd" - - with ( - patch("litellm.proxy.proxy_server.premium_user", True), - patch.object( - GoogleSecretManager, - "sync_construct_request_headers", - return_value={"Authorization": "Bearer mock_token"}, - ), - ): - secret_manager = GoogleSecretManager() - secret_manager.cache.cache_dict["UNIQUE_KEY"] = None - secret_manager.cache.cache_dict["UNIQUE_KEY_2"] = "lite-llm" - - secret_val = secret_manager.get_secret_from_google_secret_manager( - secret_name="UNIQUE_KEY" - ) - print("secret_val: {}".format(secret_val)) - assert secret_val is None - - secret_val = secret_manager.get_secret_from_google_secret_manager( - secret_name="UNIQUE_KEY_2" - ) - print("secret_val: {}".format(secret_val)) - assert secret_val == "lite-llm" diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py index de29496a79b..5ec7b96dcfa 100644 --- a/tests/litellm_utils_tests/test_utils.py +++ b/tests/litellm_utils_tests/test_utils.py @@ -1,7 +1,5 @@ import copy import logging -import time -from datetime import datetime from unittest import mock from dotenv import load_dotenv @@ -15,22 +13,14 @@ import pytest import litellm from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, headers -from litellm.litellm_core_utils.duration_parser import duration_in_seconds -from litellm.litellm_core_utils.duration_parser import ( - get_last_day_of_month, - _extract_from_regex, -) from litellm.utils import ( - check_valid_key, get_llm_provider, get_supported_openai_params, get_token_count, - get_valid_models, trim_messages, validate_environment, ) from litellm.llms.openai_like.json_loader import JSONProviderRegistry -from unittest.mock import AsyncMock, MagicMock, patch # Assuming your trim_messages, shorten_message_to_fit_limit, and get_token_count functions are all in a module named 'message_utils' @@ -47,676 +37,29 @@ def reset_mock_cache(): # test_basic_trimming() - - # test_basic_trimming_no_max_tokens_specified() - - # test_multiple_messages_trimming() - - # test_multiple_messages_no_trimming() - - # test_large_trimming() - - - - - - - - - - - - - - - - -@pytest.mark.parametrize("custom_llm_provider", ["anthropic", "xai"]) -def test_get_valid_models_with_custom_llm_provider(custom_llm_provider): - from litellm.utils import ProviderConfigManager - from litellm.types.utils import LlmProviders - - provider_config = ProviderConfigManager.get_provider_model_info( - model=None, - provider=LlmProviders(custom_llm_provider), - ) - assert provider_config is not None - valid_models = get_valid_models( - check_provider_endpoint=True, custom_llm_provider=custom_llm_provider - ) - print(valid_models) - assert len(valid_models) > 0 - assert set(provider_config.get_models()) == set(valid_models) - - # test_get_valid_models() -def test_bad_key(): - key = "bad-key" - response = check_valid_key(model="gpt-5-mini", api_key=key) - print(response, key) - assert response == False - - -def test_good_key(): - key = os.environ["OPENAI_API_KEY"] - response = check_valid_key(model="gpt-5-mini", api_key=key) - assert response == True - - # test validate environment - - - - - - - - - - - - -def test_function_to_dict(): - print("testing function to dict for get current weather") - - def get_current_weather(location: str, unit: str): - """Get the current weather in a given location - - Parameters - ---------- - location : str - The city and state, e.g. San Francisco, CA - unit : {'celsius', 'fahrenheit'} - Temperature unit - - Returns - ------- - str - a sentence indicating the weather - """ - if location == "Boston, MA": - return "The weather is 12F" - - function_json = litellm.utils.function_to_dict(get_current_weather) - print(function_json) - - expected_output = { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "description": "Temperature unit", - "enum": "['fahrenheit', 'celsius']", - }, - }, - "required": ["location", "unit"], - }, - } - print(expected_output) - - assert function_json["name"] == expected_output["name"] - assert function_json["description"] == expected_output["description"] - assert function_json["parameters"]["type"] == expected_output["parameters"]["type"] - assert ( - function_json["parameters"]["properties"]["location"] - == expected_output["parameters"]["properties"]["location"] - ) - - # the enum can change it can be - which is why we don't assert on unit - # {'type': 'string', 'description': 'Temperature unit', 'enum': "['fahrenheit', 'celsius']"} - # {'type': 'string', 'description': 'Temperature unit', 'enum': "['celsius', 'fahrenheit']"} - - assert ( - function_json["parameters"]["required"] - == expected_output["parameters"]["required"] - ) - - print("passed") - - -# test_function_to_dict() - - - - - - - - - - - - - - -def test_duration_in_seconds(): - """ - Test if duration int is correctly calculated for different str - """ - import time - - now = time.time() - current_time = datetime.fromtimestamp(now) - - if current_time.month == 12: - target_year = current_time.year + 1 - target_month = 1 - else: - target_year = current_time.year - target_month = current_time.month + 1 - - # Determine the day to set for next month - target_day = current_time.day - last_day_of_target_month = get_last_day_of_month(target_year, target_month) - - if target_day > last_day_of_target_month: - target_day = last_day_of_target_month - - next_month = datetime( - year=target_year, - month=target_month, - day=target_day, - hour=current_time.hour, - minute=current_time.minute, - second=current_time.second, - microsecond=current_time.microsecond, - ) - - # Calculate the duration until the first day of the next month - duration_until_next_month = next_month - current_time - expected_duration = int(duration_until_next_month.total_seconds()) - - value = duration_in_seconds(duration="1mo") - - assert value - expected_duration < 2 - - - - - - -@pytest.mark.parametrize("langfuse_trace_id", [None, "my-unique-trace-id"]) -@pytest.mark.parametrize( - "langfuse_existing_trace_id", [None, "my-unique-existing-trace-id"] -) -def test_logging_trace_id(langfuse_trace_id, langfuse_existing_trace_id): - """ - - Unit test for `_get_trace_id` function in Logging obj - """ - from litellm.integrations.langfuse.langfuse_sdk import resolve_trace_id - from litellm.litellm_core_utils.litellm_logging import Logging - - litellm.success_callback = ["langfuse"] - litellm_call_id = "my-unique-call-id" - litellm_logging_obj = Logging( - model="gpt-5-mini", - messages=[{"role": "user", "content": "hi"}], - stream=False, - call_type="acompletion", - litellm_call_id=litellm_call_id, - start_time=datetime.now(), - function_id="1234", - ) - - metadata = {} - - if langfuse_trace_id is not None: - metadata["trace_id"] = langfuse_trace_id - if langfuse_existing_trace_id is not None: - metadata["existing_trace_id"] = langfuse_existing_trace_id - - litellm.completion( - model="gpt-5-mini", - messages=[{"role": "user", "content": "Hey how's it going?"}], - mock_response="Hey!", - litellm_logging_obj=litellm_logging_obj, - metadata=metadata, - ) - - time.sleep(3) - assert litellm_logging_obj.get_trace_id(service_name="langfuse") is not None - - # langfuse addresses a trace by a 32-hex id, so the id litellm reports back is the - # resolved form of whichever source won; that is what the alerting deep link needs - if langfuse_existing_trace_id is not None: - expected_source = langfuse_existing_trace_id - elif langfuse_trace_id is not None: - expected_source = langfuse_trace_id - else: - expected_source = litellm_logging_obj.litellm_trace_id - - assert litellm_logging_obj.get_trace_id(service_name="langfuse") == resolve_trace_id( - expected_source - ) - - - - - - - - -def test_is_base64_encoded(): - import base64 - - import requests - - litellm.set_verbose = True - url = "https://dummyimage.com/100/100/fff&text=Test+image" - response = requests.get(url) - file_data = response.content - - encoded_file = base64.b64encode(file_data).decode("utf-8") - base64_image = f"data:image/png;base64,{encoded_file}" - - from litellm.utils import is_base64_encoded - - assert is_base64_encoded(s=base64_image) is True - - - - - - - - - - - - - - - - - - - - - - -def test_is_prompt_caching_enabled_return_default_image_dimensions(): - """ - Assert that `is_prompt_caching_valid_prompt` counts tokens with use_default_image_token_count=True - when processing messages containing images - - IMPORTANT: Ensures Get token counter does not make a GET request to the image url - """ - mock_token_counter = MagicMock(return_value=False) - with patch( - "litellm.utils.get_messages_reach_token_count", - return_value=mock_token_counter, - ): - litellm.utils.is_prompt_caching_valid_prompt( - messages=[ - { - "role": "user", - "content": [ - {"type": "text", "text": "What is in this image?"}, - { - "type": "image_url", - "image_url": { - "url": "https://www.gstatic.com/webp/gallery/1.webp", - "detail": "high", - }, - }, - ], - } - ], - tools=None, - custom_llm_provider="openai", - model="gpt-4o-mini", - ) - - # Assert token_counter was called with use_default_image_token_count=True - args_to_mock_token_counter = mock_token_counter.call_args[1] - print("args_to_mock", args_to_mock_token_counter) - assert args_to_mock_token_counter["use_default_image_token_count"] is True - - - - - - - - -def test_get_valid_models_fireworks_ai(monkeypatch): - from litellm.utils import get_valid_models - import litellm - - litellm.turn_on_debug() - - monkeypatch.setenv("FIREWORKS_API_KEY", "sk-9876") - monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", "1234") - monkeypatch.setattr(litellm, "provider_list", ["fireworks_ai"]) - - mock_response_data = { - "models": [ - { - "name": "accounts/fireworks/models/llama-3.1-8b-instruct", - "displayName": "", - "description": "", - "createTime": "2023-11-07T05:31:56Z", - "createdBy": "", - "state": "STATE_UNSPECIFIED", - "status": {"code": "OK", "message": ""}, - "kind": "KIND_UNSPECIFIED", - "githubUrl": "", - "huggingFaceUrl": "", - "baseModelDetails": { - "worldSize": 123, - "checkpointFormat": "CHECKPOINT_FORMAT_UNSPECIFIED", - "parameterCount": "", - "moe": True, - "tunable": True, - }, - "peftDetails": { - "baseModel": "", - "r": 123, - "targetModules": [""], - }, - "teftDetails": {}, - "public": True, - "conversationConfig": { - "style": "", - "system": "", - "template": "", - }, - "contextLength": 123, - "supportsImageInput": True, - "supportsTools": True, - "importedFrom": "", - "fineTuningJob": "", - "defaultDraftModel": "", - "defaultDraftTokenCount": 123, - "precisions": ["PRECISION_UNSPECIFIED"], - "deployedModelRefs": [ - { - "name": "", - "deployment": "", - "state": "STATE_UNSPECIFIED", - "default": True, - "public": True, - } - ], - "cluster": "", - "deprecationDate": {"year": 123, "month": 123, "day": 123}, - } - ], - "nextPageToken": "", - "totalSize": 123, - } - - # Create a mock response object - mock_response = MagicMock() - mock_response.status_code = 200 - mock_response.json.return_value = mock_response_data - - with patch.object( - litellm.module_level_client, "get", return_value=mock_response - ) as mock_post: - valid_models = get_valid_models(check_provider_endpoint=True) - print("valid_models", valid_models) - mock_post.assert_called_once() - assert ( - "fireworks_ai/accounts/fireworks/models/llama-3.1-8b-instruct" - in valid_models - ) - - -def test_get_valid_models_default(monkeypatch): - """ - Ensure that the default models is used when error retrieving from model api. - - Prevent regression for existing usage. - """ - from litellm.utils import get_valid_models - - monkeypatch.setenv("FIREWORKS_API_KEY", "sk-9876") - valid_models = get_valid_models() - assert len(valid_models) > 0 - - - - - - -def test_add_custom_logger_callback_to_specific_event(monkeypatch): - from litellm.utils import add_custom_logger_callback_to_specific_event - - monkeypatch.setattr(litellm, "success_callback", []) - monkeypatch.setattr(litellm, "failure_callback", []) - - add_custom_logger_callback_to_specific_event("langfuse", "success") - - assert len(litellm.success_callback) == 1 - assert len(litellm.failure_callback) == 0 - - - - - - -@pytest.mark.asyncio -async def test_add_custom_logger_callback_to_specific_event_with_duplicates( - monkeypatch, -): - """ - Test that when a callback exists in both success_callback and _async_success_callback, - it's not added again - """ - from litellm.integrations.langfuse.langfuse_prompt_management import ( - LangfusePromptManagement, - ) - - # Reset all callback lists - monkeypatch.setattr(litellm, "callbacks", []) - monkeypatch.setattr(litellm, "_async_success_callback", []) - monkeypatch.setattr(litellm, "_async_failure_callback", []) - monkeypatch.setattr(litellm, "success_callback", []) - monkeypatch.setattr(litellm, "failure_callback", []) - - # Add logger to both success_callback and _async_success_callback - langfuse_logger = LangfusePromptManagement() - litellm.success_callback.append(langfuse_logger) - litellm._async_success_callback.append(langfuse_logger) - - # Get initial lengths - initial_success_callback_len = len(litellm.success_callback) - initial_async_success_callback_len = len(litellm._async_success_callback) - - # Make a completion call - await litellm.acompletion( - model="gpt-5-mini", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response="Testing duplicate callbacks", - ) - - # Assert no new callbacks were added - assert len(litellm.success_callback) == initial_success_callback_len - assert len(litellm._async_success_callback) == initial_async_success_callback_len - - -@pytest.mark.asyncio -async def test_add_custom_logger_callback_to_specific_event_with_duplicates_success_callback( - monkeypatch, -): - """ - Test that when a callback exists in both success_callback and _async_success_callback, - it's not added again - """ - from litellm.integrations.langfuse.langfuse_prompt_management import ( - LangfusePromptManagement, - ) - - # Reset all callback lists - monkeypatch.setattr(litellm, "callbacks", []) - monkeypatch.setattr(litellm, "_async_success_callback", []) - monkeypatch.setattr(litellm, "_async_failure_callback", []) - monkeypatch.setattr(litellm, "success_callback", []) - monkeypatch.setattr(litellm, "failure_callback", []) - - # Add logger to both success_callback and _async_success_callback - langfuse_logger = LangfusePromptManagement() - litellm.success_callback.append(langfuse_logger) - - # Get initial lengths - initial_success_callback_len = len(litellm.success_callback) - initial_async_success_callback_len = len(litellm._async_success_callback) - - # Make a completion call - await litellm.acompletion( - model="gpt-5-mini", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response="Testing duplicate callbacks", - ) - - # Assert no new callbacks were added - assert len(litellm.success_callback) == initial_success_callback_len - assert len(litellm._async_success_callback) == initial_async_success_callback_len - - -@pytest.mark.asyncio -async def test_add_custom_logger_callback_to_specific_event_with_duplicates_callbacks( - monkeypatch, -): - """ - Test that when a callback exists in both success_callback and _async_success_callback, - it's not added again - """ - from litellm.integrations.langfuse.langfuse_prompt_management import ( - LangfusePromptManagement, - ) - - # Reset all callback lists - monkeypatch.setattr(litellm, "callbacks", []) - monkeypatch.setattr(litellm, "_async_success_callback", []) - monkeypatch.setattr(litellm, "_async_failure_callback", []) - monkeypatch.setattr(litellm, "success_callback", []) - monkeypatch.setattr(litellm, "failure_callback", []) - - # Add logger to both success_callback and _async_success_callback - langfuse_logger = LangfusePromptManagement() - litellm.callbacks.append(langfuse_logger) - - # Make a completion call - await litellm.acompletion( - model="gpt-5-mini", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response="Testing duplicate callbacks", - ) - - # Assert no new callbacks were added - initial_callbacks_len = len(litellm.callbacks) - initial_async_success_callback_len = len(litellm._async_success_callback) - initial_success_callback_len = len(litellm.success_callback) - print( - f"Num callbacks before: litellm.callbacks: {len(litellm.callbacks)}, litellm._async_success_callback: {len(litellm._async_success_callback)}, litellm.success_callback: {len(litellm.success_callback)}" - ) - - for _ in range(10): - await litellm.acompletion( - model="gpt-5-mini", - messages=[{"role": "user", "content": "Hello, world!"}], - mock_response="Testing duplicate callbacks", - ) - - assert len(litellm.callbacks) == initial_callbacks_len - assert len(litellm._async_success_callback) == initial_async_success_callback_len - assert len(litellm.success_callback) == initial_success_callback_len - - print( - f"Num callbacks after 10 mock calls: litellm.callbacks: {len(litellm.callbacks)}, litellm._async_success_callback: {len(litellm._async_success_callback)}, litellm.success_callback: {len(litellm.success_callback)}" - ) - - - - - - - - - - from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.utils import get_applied_guardrails from unittest.mock import Mock - - - - - - - - - - -def test_get_provider_audio_transcription_config(): - from litellm.utils import ProviderConfigManager - from litellm.types.utils import LlmProviders - - for provider in LlmProviders: - config = ProviderConfigManager.get_provider_audio_transcription_config( - model="whisper-1", provider=provider - ) - - - - - - - - -def test_get_valid_models_from_dynamic_api_key(): - """ - Test that get_valid_models returns the correct models for a given provider - """ - from litellm.utils import get_valid_models - from litellm.types.router import CredentialLiteLLMParams - - creds = CredentialLiteLLMParams(api_key="123") - - valid_models = get_valid_models( - custom_llm_provider="anthropic", - litellm_params=creds, - check_provider_endpoint=True, - ) - assert len(valid_models) == 0 - - creds = CredentialLiteLLMParams(api_key=os.getenv("ANTHROPIC_API_KEY")) - valid_models = get_valid_models( - custom_llm_provider="anthropic", - litellm_params=creds, - check_provider_endpoint=True, - ) - assert len(valid_models) > 0 - assert "anthropic/claude-sonnet-4-6" in valid_models - - def test_get_whitelisted_models(): """ Snapshot of all bedrock models as of 12/24/2024. diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py index 8f3867a9201..984ac425e74 100644 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ b/tests/llm_responses_api_testing/base_responses_api.py @@ -347,31 +347,6 @@ class BaseResponsesAPITest(ABC): else: raise ValueError("response is not a ResponsesAPIResponse") - @pytest.mark.asyncio - async def test_multiturn_responses_api(self): - litellm.turn_on_debug() - litellm.set_verbose = True - try: - base_completion_call_args = self.get_base_completion_call_args() - response_1 = await litellm.aresponses( - input="Basic ping", max_output_tokens=20, **base_completion_call_args - ) - - # follow up with a second request - response_1_id = response_1.id - response_2 = await litellm.aresponses( - input="Basic ping", - max_output_tokens=20, - previous_response_id=response_1_id, - **base_completion_call_args, - ) - - # assert the response is not None - assert response_1 is not None - assert response_2 is not None - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") - @pytest.mark.asyncio async def test_responses_api_with_tool_calls(self): """Test that calls the Responses API with tool calls including function call and output""" @@ -447,72 +422,6 @@ class BaseResponsesAPITest(ABC): else: assert len(response["output"]) > 0 - def test_openai_responses_api_dict_input_filtering(self): - """ - Test that regular dict inputs with status fields are properly filtered - to replicate exclude_unset=True behavior for non-Pydantic objects. - """ - from litellm.llms.openai.responses.transformation import ( - OpenAIResponsesAPIConfig, - ) - - # Test input with regular dict objects (like from JSON) - test_input = [ - {"role": "user", "content": "test"}, - { - "id": "rs_123", - "summary": [{"text": "test", "type": "summary_text"}], - "type": "reasoning", - "content": None, # Should be filtered out - "encrypted_content": None, # Should be filtered out - "status": None, # Should be filtered out - }, - { - "arguments": "{}", - "call_id": "call_123", - "name": "get_today", - "type": "function_call", - "id": "fc_123", - "status": "completed", # Should be preserved (not a default field) - }, - ] - - config = OpenAIResponsesAPIConfig() - validated_input = config._validate_input_param(test_input) - - # Verify the results - assert len(validated_input) == 3 - - # Check reasoning item (index 1) - reasoning_item = validated_input[1] - assert reasoning_item["type"] == "reasoning" - assert ( - "status" not in reasoning_item - ), "status field should be filtered out from reasoning item" - assert ( - "content" not in reasoning_item - ), "content field should be filtered out from reasoning item" - assert ( - "encrypted_content" not in reasoning_item - ), "encrypted_content field should be filtered out from reasoning item" - # Note: ID auto-generation was disabled, so reasoning items may not have IDs - # Only check for ID if it was present in the original input - if "id" in reasoning_item: - assert reasoning_item["id"] == "rs_123", "ID should be preserved if present" - assert "summary" in reasoning_item, "summary field should be preserved" - - # Check function call item (index 2) - function_call_item = validated_input[2] - assert function_call_item["type"] == "function_call" - assert ( - "status" in function_call_item - ), "status field should be preserved in function call item" - assert ( - function_call_item["status"] == "completed" - ), "status value should be preserved" - - print("✅ OpenAI Responses API dict input filtering test passed") - @pytest.mark.parametrize("sync_mode", [False, True]) @pytest.mark.flaky(retries=3, delay=2) @pytest.mark.asyncio @@ -565,23 +474,6 @@ class BaseResponsesAPITest(ABC): else: raise e - @pytest.mark.parametrize("sync_mode", [False, True]) - @pytest.mark.asyncio - async def test_cancel_responses_invalid_response_id(self, sync_mode): - """Test cancel_responses with invalid response ID should raise appropriate error""" - base_completion_call_args = self.get_base_completion_call_args() - - if sync_mode: - with pytest.raises(openai.APIError): - litellm.cancel_responses( - response_id="invalid_response_id_12345", **base_completion_call_args - ) - else: - with pytest.raises(openai.APIError): - await litellm.acancel_responses( - response_id="invalid_response_id_12345", **base_completion_call_args - ) - @pytest.mark.asyncio async def test_responses_api_context_management_server_side_compaction(self): """ diff --git a/tests/llm_responses_api_testing/test_anthropic_responses_api.py b/tests/llm_responses_api_testing/test_anthropic_responses_api.py index 89751d7fdd0..15d1a81bc68 100644 --- a/tests/llm_responses_api_testing/test_anthropic_responses_api.py +++ b/tests/llm_responses_api_testing/test_anthropic_responses_api.py @@ -1,35 +1,20 @@ -import pytest - import litellm from base_responses_api import BaseResponsesAPITest from openai.types.responses.function_tool import FunctionTool class TestAnthropicResponsesAPITest(BaseResponsesAPITest): + test_basic_openai_responses_delete_endpoint = None + test_basic_openai_responses_streaming_delete_endpoint = None + test_basic_openai_responses_get_endpoint = None + test_basic_openai_responses_cancel_endpoint = None + def get_base_completion_call_args(self): # litellm.turn_on_debug() return { "model": "anthropic/claude-sonnet-4-5", } - async def test_basic_openai_responses_delete_endpoint(self, sync_mode=False): - pytest.skip("DELETE responses is not supported for anthropic") - - async def test_basic_openai_responses_streaming_delete_endpoint( - self, sync_mode=False - ): - pytest.skip("DELETE responses is not supported for anthropic") - - async def test_basic_openai_responses_get_endpoint(self, sync_mode=False): - pytest.skip("GET responses is not supported for anthropic") - - async def test_basic_openai_responses_cancel_endpoint(self, sync_mode=False): - pytest.skip("CANCEL responses is not supported for anthropic") - - async def test_cancel_responses_invalid_response_id(self, sync_mode=False): - pytest.skip("CANCEL responses is not supported for anthropic") - - def test_multiturn_tool_calls(): # Test streaming response with tools for Anthropic litellm.turn_on_debug() diff --git a/tests/llm_responses_api_testing/test_azure_responses_api.py b/tests/llm_responses_api_testing/test_azure_responses_api.py index e2b1de33182..3007b315859 100644 --- a/tests/llm_responses_api_testing/test_azure_responses_api.py +++ b/tests/llm_responses_api_testing/test_azure_responses_api.py @@ -6,7 +6,6 @@ from base_responses_api import BaseResponsesAPITest class TestAzureResponsesAPITest(BaseResponsesAPITest): - test_multiturn_responses_api = None test_responses_api_with_tool_calls = None def get_base_completion_call_args(self): diff --git a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py index 068de771e2a..01bfbc3be35 100644 --- a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py +++ b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py @@ -1,233 +1,12 @@ -import os -import pytest - -import litellm -import json from base_responses_api import BaseResponsesAPITest -@pytest.mark.asyncio -async def test_basic_google_ai_studio_responses_api_with_tools(): - litellm.turn_on_debug() - litellm.set_verbose = True - request_model = "gemini/gemini-2.5-flash" - response = await litellm.aresponses( - model=request_model, - input="what is the latest version of supabase python package and when was it released?", - tools=[{"type": "web_search_preview", "search_context_size": "low"}], - ) - print("litellm response=", json.dumps(response, indent=4, default=str)) - - -@pytest.mark.asyncio -async def test_gemini_3_responses_api_with_thought_signatures(): - """ - Test that Gemini 3 Responses API preserves thought signatures in function calls. - This test verifies that provider_specific_fields with thought_signature are correctly - preserved when using the Responses API with Gemini 3. - """ - if not os.getenv("GEMINI_API_KEY"): - pytest.skip("GEMINI_API_KEY not set") - - litellm.set_verbose = False - request_model = "gemini/gemini-3.1-pro-preview" - - tools = [ - { - "type": "function", - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. Mumbai, India", - }, - "units": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "Units the temperature will be returned in.", - }, - }, - "required": ["location", "units"], - "additionalProperties": False, - }, - "strict": True, - } - ] - - # Step 1: Initial request with tools - response = await litellm.aresponses( - model=request_model, - input="What is the weather in Mumbai?", - tools=tools, - reasoning_effort="low", - ) - - # Validate response structure - from litellm.types.llms.openai import ResponsesAPIResponse - - assert isinstance( - response, ResponsesAPIResponse - ), "Response should be a ResponsesAPIResponse" - assert ( - hasattr(response, "output") or "output" in response - ), "Response should have 'output' field" - assert isinstance(response.output, list), "Output should be a list" - - # Find function call in output - function_call_item = None - for item in response.output: - # Convert to dict if it's a Pydantic model for easier access - if hasattr(item, "model_dump"): - item_dict = item.model_dump() - elif hasattr(item, "__dict__"): - item_dict = dict(item) if not isinstance(item, dict) else item - else: - item_dict = item if isinstance(item, dict) else {} - - if isinstance(item_dict, dict) and item_dict.get("type") == "function_call": - function_call_item = item_dict - break - - # Verify function call exists - assert ( - function_call_item is not None - ), "Response should contain a function_call item" - assert ( - function_call_item.get("name") == "get_weather" - ), "Function call should be for get_weather" - - # Verify thought signature is present in provider_specific_fields - provider_specific_fields = function_call_item.get("provider_specific_fields") - assert ( - provider_specific_fields is not None - ), "Function call should have provider_specific_fields" - assert ( - "thought_signature" in provider_specific_fields - ), "provider_specific_fields should contain thought_signature" - assert isinstance( - provider_specific_fields["thought_signature"], str - ), "thought_signature should be a string" - assert ( - len(provider_specific_fields["thought_signature"]) > 0 - ), "thought_signature should not be empty" - - print( - f"✅ Thought signature preserved: {provider_specific_fields['thought_signature'][:50]}..." - ) - - -@pytest.mark.asyncio -async def test_gemini_3_responses_api_streaming_with_thought_signatures(): - """ - Test that Gemini 3 Responses API preserves thought signatures in streaming mode. - This test verifies that provider_specific_fields with thought_signature are correctly - preserved when using streaming Responses API with Gemini 3. - """ - if not os.getenv("GEMINI_API_KEY"): - pytest.skip("GEMINI_API_KEY not set") - - litellm.set_verbose = False - request_model = "gemini/gemini-3.1-pro-preview" - - tools = [ - { - "type": "function", - "name": "get_weather", - "description": "Get the current weather for a location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. Mumbai, India", - }, - "units": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - "description": "Units the temperature will be returned in.", - }, - }, - "required": ["location", "units"], - "additionalProperties": False, - }, - "strict": True, - } - ] - - # Step 1: Streaming request with tools - response_stream = await litellm.aresponses( - model=request_model, - input="What is the weather in Mumbai?", - tools=tools, - stream=True, - reasoning_effort="low", - ) - - # Collect all chunks - chunks = [] - completed_response = None - - async for chunk in response_stream: - chunks.append(chunk) - # Check if this is the completed response event - if hasattr(chunk, "type") and chunk.type == "response.completed": - completed_response = chunk.response - elif isinstance(chunk, dict) and chunk.get("type") == "response.completed": - completed_response = chunk.get("response") - - # Verify we got chunks - assert len(chunks) > 0, "Should receive at least one chunk" - - # If we have a completed response, check for thought signatures - if completed_response: - output = completed_response.get("output", []) - function_call_item = None - for item in output: - if isinstance(item, dict) and item.get("type") == "function_call": - function_call_item = item - break - - if function_call_item: - provider_specific_fields = function_call_item.get( - "provider_specific_fields" - ) - if provider_specific_fields: - thought_signature = provider_specific_fields.get("thought_signature") - if thought_signature: - assert isinstance( - thought_signature, str - ), "thought_signature should be a string" - assert ( - len(thought_signature) > 0 - ), "thought_signature should not be empty" - print( - f"✅ Streaming thought signature preserved: {thought_signature[:50]}..." - ) - - print(f"✅ Collected {len(chunks)} streaming chunks") - - class TestGoogleAIStudioResponsesAPITest(BaseResponsesAPITest): + test_basic_openai_responses_delete_endpoint = None + test_basic_openai_responses_streaming_delete_endpoint = None + test_basic_openai_responses_get_endpoint = None + test_basic_openai_responses_cancel_endpoint = None + def get_base_completion_call_args(self): # litellm.turn_on_debug() return {"model": "gemini/gemini-2.5-flash-lite"} - - async def test_basic_openai_responses_delete_endpoint(self, sync_mode=False): - pytest.skip("DELETE responses is not supported for Google AI Studio") - - async def test_basic_openai_responses_streaming_delete_endpoint( - self, sync_mode=False - ): - pytest.skip("DELETE responses is not supported for Google AI Studio") - - async def test_basic_openai_responses_get_endpoint(self, sync_mode=False): - pytest.skip("GET responses is not supported for Google AI Studio") - - async def test_basic_openai_responses_cancel_endpoint(self, sync_mode=False): - pytest.skip("CANCEL responses is not supported for Google AI Studio") - - async def test_cancel_responses_invalid_response_id(self, sync_mode=False): - pytest.skip("CANCEL responses is not supported for Google AI Studio") diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py index 2705b91ee3a..bbdc7d3ad24 100644 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ b/tests/llm_responses_api_testing/test_openai_responses_api.py @@ -1,20 +1,9 @@ -import asyncio import json -import os -import time -from typing import Optional, cast import pytest -from base_responses_api import BaseResponsesAPITest, validate_responses_api_response +from base_responses_api import BaseResponsesAPITest import litellm -from litellm.integrations.custom_logger import CustomLogger -from litellm.types.llms.openai import ( - ResponseAPIUsage, - ResponseCompletedEvent, - ResponsesAPIResponse, -) -from litellm.types.utils import StandardLoggingPayload class TestOpenAIResponsesAPITest(BaseResponsesAPITest): @@ -29,736 +18,6 @@ class TestOpenAIResponsesAPITest(BaseResponsesAPITest): return "openai/gpt-5.2" -class TestCustomLogger(CustomLogger): - def __init__( - self, - ): - self.standard_logging_object: Optional[StandardLoggingPayload] = None - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - print("in async_log_success_event") - print("kwargs=", json.dumps(kwargs, indent=4, default=str)) - self.standard_logging_object = kwargs["standard_logging_object"] - pass - - -def validate_standard_logging_payload( - slp: StandardLoggingPayload, response: ResponsesAPIResponse, request_model: str -): - """ - Validate that a StandardLoggingPayload object matches the expected response - - Args: - slp (StandardLoggingPayload): The standard logging payload object to validate - response (dict): The litellm response to compare against - request_model (str): The model name that was requested - """ - # Validate payload exists - assert slp is not None, "Standard logging payload should not be None" - - # Validate token counts - print( - "VALIDATING STANDARD LOGGING PAYLOAD. response=", - json.dumps(response, indent=4, default=str), - ) - print("FIELDS IN SLP=", json.dumps(slp, indent=4, default=str)) - print("SLP PROMPT TOKENS=", slp["prompt_tokens"]) - print("RESPONSE PROMPT TOKENS=", response["usage"]["input_tokens"]) - assert ( - slp["prompt_tokens"] == response["usage"]["input_tokens"] - ), "Prompt tokens mismatch" - assert ( - slp["completion_tokens"] == response["usage"]["output_tokens"] - ), "Completion tokens mismatch" - assert ( - slp["total_tokens"] - == response["usage"]["input_tokens"] + response["usage"]["output_tokens"] - ), "Total tokens mismatch" - - # Validate spend and response metadata - assert slp["response_cost"] > 0, "Response cost should be greater than 0" - assert slp["id"] == response["id"], "Response ID mismatch" - assert slp["model"] == request_model, "Model name mismatch" - - # Validate messages - assert slp["messages"] == [{"content": "hi", "role": "user"}], "Messages mismatch" - - # Validate complete response structure - validate_responses_match(slp["response"], response) - - -@pytest.mark.asyncio -def test_basic_openai_responses_api_streaming_with_logging(): - litellm.turn_on_debug() - litellm.set_verbose = True - test_custom_logger = TestCustomLogger() - litellm.callbacks = [test_custom_logger] - request_model = "gpt-5.5" - response = litellm.responses( - model=request_model, - input="hi", - stream=True, - ) - final_response: Optional[ResponseCompletedEvent] = None - for event in response: - if event.type == "response.completed": - final_response = event - print("litellm response=", json.dumps(event, indent=4, default=str)) - - print("sleeping for 2 seconds...") - time.sleep(2) - print( - "standard logging payload=", - json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), - ) - - assert final_response is not None - assert test_custom_logger.standard_logging_object is not None - - validate_standard_logging_payload( - slp=test_custom_logger.standard_logging_object, - response=final_response.response, - request_model=request_model, - ) - - -def validate_responses_match(slp_response, litellm_response): - """Validate that the standard logging payload OpenAI response matches the litellm response""" - # Validate core fields - assert slp_response["id"] == litellm_response["id"], "ID mismatch" - assert slp_response["model"] == litellm_response["model"], "Model mismatch" - assert ( - slp_response["created_at"] == litellm_response["created_at"] - ), "Created at mismatch" - - # Validate usage - assert ( - slp_response["usage"]["prompt_tokens"] - == litellm_response["usage"]["input_tokens"] - ), "Input tokens mismatch" - assert ( - slp_response["usage"]["completion_tokens"] - == litellm_response["usage"]["output_tokens"] - ), "Output tokens mismatch" - assert ( - slp_response["usage"]["total_tokens"] - == litellm_response["usage"]["total_tokens"] - ), "Total tokens mismatch" - - # Validate output/messages - assert len(slp_response["output"]) == len( - litellm_response["output"] - ), "Output length mismatch" - for slp_msg, litellm_msg in zip(slp_response["output"], litellm_response["output"]): - assert slp_msg["role"] == litellm_msg.role, "Message role mismatch" - # Access the content's text field for the litellm response - litellm_content = litellm_msg.content[0].text if litellm_msg.content else "" - assert ( - slp_msg["content"][0]["text"] == litellm_content - ), f"Message content mismatch. Expected {litellm_content}, Got {slp_msg['content']}" - assert slp_msg["status"] == litellm_msg.status, "Message status mismatch" - - -@pytest.mark.asyncio -async def test_basic_openai_responses_api_non_streaming_with_logging(): - litellm.turn_on_debug() - litellm.set_verbose = True - test_custom_logger = TestCustomLogger() - litellm.callbacks = [test_custom_logger] - request_model = "gpt-5.5" - response = await litellm.aresponses( - model=request_model, - input="hi", - ) - - print("litellm response=", json.dumps(response, indent=4, default=str)) - print("response hidden params=", response._hidden_params) - - print("sleeping for 2 seconds...") - await asyncio.sleep(5) - print( - "standard logging payload=", - json.dumps(test_custom_logger.standard_logging_object, indent=4, default=str), - ) - print("response usage=", response.usage) - - assert response is not None - assert test_custom_logger.standard_logging_object is not None - - validate_standard_logging_payload( - test_custom_logger.standard_logging_object, response, request_model - ) - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_openai_responses_api_returns_headers(sync_mode): - """ - Test that OpenAI responses API returns OpenAI headers in _hidden_params. - This ensures the proxy can forward these headers to clients. - - Related issue: LiteLLM responses API should return OpenAI headers like chat completions does - """ - litellm.turn_on_debug() - litellm.set_verbose = True - - if sync_mode: - response = litellm.responses( - model="gpt-5.5", - input="Say hello", - max_output_tokens=20, - ) - else: - response = await litellm.aresponses( - model="gpt-5.5", - input="Say hello", - max_output_tokens=20, - ) - - # Verify response is valid - assert response is not None - assert isinstance(response, ResponsesAPIResponse) - - # Verify _hidden_params exists - assert hasattr( - response, "_hidden_params" - ), "Response should have _hidden_params attribute" - assert response._hidden_params is not None, "_hidden_params should not be None" - - # Verify additional_headers exists in _hidden_params - assert ( - "additional_headers" in response._hidden_params - ), "_hidden_params should contain 'additional_headers' key" - - additional_headers = response._hidden_params["additional_headers"] - assert isinstance( - additional_headers, dict - ), "additional_headers should be a dictionary" - assert len(additional_headers) > 0, "additional_headers should not be empty" - - # Check for expected OpenAI rate limit headers - # These can be either direct (x-ratelimit-*) or prefixed (llm_provider-x-ratelimit-*) - rate_limit_headers = [ - "x-ratelimit-remaining-tokens", - "x-ratelimit-limit-tokens", - "x-ratelimit-remaining-requests", - "x-ratelimit-limit-requests", - ] - - found_headers = [] - for header_name in rate_limit_headers: - if header_name in additional_headers: - found_headers.append(header_name) - elif f"llm_provider-{header_name}" in additional_headers: - found_headers.append(f"llm_provider-{header_name}") - - assert ( - len(found_headers) > 0 - ), f"Should find at least one OpenAI rate limit header. Headers found: {list(additional_headers.keys())}" - - # Verify headers key also exists (raw headers) - assert ( - "headers" in response._hidden_params - ), "_hidden_params should contain 'headers' key with raw response headers" - - print( - f"✓ Successfully validated OpenAI headers in {'sync' if sync_mode else 'async'} mode" - ) - print(f" Found {len(additional_headers)} headers total") - print(f" Rate limit headers found: {found_headers}") - - -def validate_stream_event(event): - """ - Validate that a streaming event from litellm.responses() or litellm.aresponses() - with stream=True conforms to the expected structure based on its event type. - - Args: - event: The streaming event object to validate - - Raises: - AssertionError: If the event doesn't match the expected structure for its type - """ - # Common validation for all event types - assert hasattr(event, "type"), "Event should have a 'type' attribute" - - # Type-specific validation - if event.type == "response.created" or event.type == "response.in_progress": - assert hasattr( - event, "response" - ), f"{event.type} event should have a 'response' attribute" - validate_responses_api_response(event.response, final_chunk=False) - - elif event.type == "response.completed": - assert hasattr( - event, "response" - ), "response.completed event should have a 'response' attribute" - validate_responses_api_response(event.response, final_chunk=True) - # Usage is guaranteed only on the completed event - assert ( - "usage" in event.response - ), "response.completed event should have usage information" - print("Usage in event.response=", event.response["usage"]) - assert isinstance(event.response["usage"], ResponseAPIUsage) - elif event.type == "response.failed" or event.type == "response.incomplete": - assert hasattr( - event, "response" - ), f"{event.type} event should have a 'response' attribute" - - elif ( - event.type == "response.output_item.added" - or event.type == "response.output_item.done" - ): - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "item" - ), f"{event.type} event should have an 'item' attribute" - - elif ( - event.type == "response.content_part.added" - or event.type == "response.content_part.done" - ): - assert hasattr( - event, "item_id" - ), f"{event.type} event should have an 'item_id' attribute" - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "content_index" - ), f"{event.type} event should have a 'content_index' attribute" - assert hasattr( - event, "part" - ), f"{event.type} event should have a 'part' attribute" - - elif event.type == "response.output_text.delta": - assert hasattr( - event, "item_id" - ), f"{event.type} event should have an 'item_id' attribute" - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "content_index" - ), f"{event.type} event should have a 'content_index' attribute" - assert hasattr( - event, "delta" - ), f"{event.type} event should have a 'delta' attribute" - - elif event.type == "response.output_text.annotation.added": - assert hasattr( - event, "item_id" - ), f"{event.type} event should have an 'item_id' attribute" - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "content_index" - ), f"{event.type} event should have a 'content_index' attribute" - assert hasattr( - event, "annotation_index" - ), f"{event.type} event should have an 'annotation_index' attribute" - assert hasattr( - event, "annotation" - ), f"{event.type} event should have an 'annotation' attribute" - - elif event.type == "response.output_text.done": - assert hasattr( - event, "item_id" - ), f"{event.type} event should have an 'item_id' attribute" - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "content_index" - ), f"{event.type} event should have a 'content_index' attribute" - assert hasattr( - event, "text" - ), f"{event.type} event should have a 'text' attribute" - - elif event.type == "response.refusal.delta": - assert hasattr( - event, "item_id" - ), f"{event.type} event should have an 'item_id' attribute" - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "content_index" - ), f"{event.type} event should have a 'content_index' attribute" - assert hasattr( - event, "delta" - ), f"{event.type} event should have a 'delta' attribute" - - elif event.type == "response.refusal.done": - assert hasattr( - event, "item_id" - ), f"{event.type} event should have an 'item_id' attribute" - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "content_index" - ), f"{event.type} event should have a 'content_index' attribute" - assert hasattr( - event, "refusal" - ), f"{event.type} event should have a 'refusal' attribute" - - elif event.type == "response.function_call_arguments.delta": - assert hasattr( - event, "item_id" - ), f"{event.type} event should have an 'item_id' attribute" - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "delta" - ), f"{event.type} event should have a 'delta' attribute" - - elif event.type == "response.function_call_arguments.done": - assert hasattr( - event, "item_id" - ), f"{event.type} event should have an 'item_id' attribute" - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "arguments" - ), f"{event.type} event should have an 'arguments' attribute" - - elif event.type in [ - "response.file_search_call.in_progress", - "response.file_search_call.searching", - "response.file_search_call.completed", - "response.web_search_call.in_progress", - "response.web_search_call.searching", - "response.web_search_call.completed", - ]: - assert hasattr( - event, "output_index" - ), f"{event.type} event should have an 'output_index' attribute" - assert hasattr( - event, "item_id" - ), f"{event.type} event should have an 'item_id' attribute" - - elif event.type == "error": - assert hasattr( - event, "message" - ), "Error event should have a 'message' attribute" - return True # Return True if validation passes - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_openai_responses_api_streaming_validation(sync_mode): - """Test that validates each streaming event from the responses API""" - litellm.turn_on_debug() - - event_types_seen = set() - - if sync_mode: - response = litellm.responses( - model="gpt-5.5", - input="Tell me about artificial intelligence in 3 sentences.", - stream=True, - ) - for event in response: - print(f"Validating event type: {event.type}") - validate_stream_event(event) - event_types_seen.add(event.type) - else: - response = await litellm.aresponses( - model="gpt-5.5", - input="Tell me about artificial intelligence in 3 sentences.", - stream=True, - ) - async for event in response: - print(f"Validating event type: {event.type}") - validate_stream_event(event) - event_types_seen.add(event.type) - - # At minimum, we should see these core event types - required_events = {"response.created", "response.completed"} - - missing_events = required_events - event_types_seen - assert not missing_events, f"Missing required event types: {missing_events}" - - print(f"Successfully validated all event types: {event_types_seen}") - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_openai_responses_litellm_router(sync_mode): - """ - Test the OpenAI responses API with LiteLLM Router in both sync and async modes - """ - litellm.turn_on_debug() - router = litellm.Router( - model_list=[ - { - "model_name": "gpt4o-special-alias", - "litellm_params": { - "model": "gpt-5.5", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - } - ] - ) - - # Call the handler - if sync_mode: - response = router.responses( - model="gpt4o-special-alias", - input="Hello, can you tell me a short joke?", - max_output_tokens=100, - ) - print("SYNC MODE RESPONSE=", response) - else: - response = await router.aresponses( - model="gpt4o-special-alias", - input="Hello, can you tell me a short joke?", - max_output_tokens=100, - ) - - print( - f"Router {'sync' if sync_mode else 'async'} response=", - json.dumps(response, indent=4, default=str), - ) - - # Use the helper function to validate the response - validate_responses_api_response(response, final_chunk=True) - - return response - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -async def test_openai_responses_litellm_router_streaming(sync_mode): - """ - Test the OpenAI responses API with streaming through LiteLLM Router - """ - litellm.turn_on_debug() - router = litellm.Router( - model_list=[ - { - "model_name": "gpt4o-special-alias", - "litellm_params": { - "model": "gpt-5.5", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - } - ] - ) - - event_types_seen = set() - - if sync_mode: - response = router.responses( - model="gpt4o-special-alias", - input="Tell me about artificial intelligence in 2 sentences.", - stream=True, - ) - for event in response: - print(f"Validating event type: {event.type}") - validate_stream_event(event) - event_types_seen.add(event.type) - else: - response = await router.aresponses( - model="gpt4o-special-alias", - input="Tell me about artificial intelligence in 2 sentences.", - stream=True, - ) - async for event in response: - print(f"Validating event type: {event.type}") - validate_stream_event(event) - event_types_seen.add(event.type) - - # At minimum, we should see these core event types - required_events = {"response.created", "response.completed"} - - missing_events = required_events - event_types_seen - assert not missing_events, f"Missing required event types: {missing_events}" - - print(f"Successfully validated all event types: {event_types_seen}") - - -def test_mcp_tools_with_responses_api(): - litellm.turn_on_debug() - MCP_TOOLS = [ - { - "type": "mcp", - "server_label": "zapier", - "server_url": "https://mcp.zapier.com/api/mcp/mcp", - "headers": { - "Authorization": f"Bearer {os.getenv('ZAPIER_CI_CD_MCP_TOKEN')}" - }, - } - ] - MODEL = "openai/gpt-4.1" - USER_QUERY = "how does tiktoken work?" - ######################################################### - # Step 1: OpenAI will use MCP LIST, and return a list of MCP calls for our approval - try: - response = litellm.responses(model=MODEL, tools=MCP_TOOLS, input=USER_QUERY) - print(response) - - response = cast(ResponsesAPIResponse, response) - - mcp_approval_id: Optional[str] = None - for output in response.output: - if output.type == "mcp_approval_request": - mcp_approval_id = output.id - break - - # Step 2: Send followup with approval for the MCP call - if mcp_approval_id: - response_with_mcp_call = litellm.responses( - model=MODEL, - tools=MCP_TOOLS, - input=[ - { - "type": "mcp_approval_response", - "approve": True, - "approval_request_id": mcp_approval_id, - } - ], - previous_response_id=response.id, - ) - print(response_with_mcp_call) - except litellm.APIError as e: - if ( - "424" in str(e) - or "Failed Dependency" in str(e) - or "external_connector_error" in str(e) - ): - pytest.skip(f"Skipping test due to external MCP server error: {e}") - else: - raise e - except litellm.InternalServerError as e: - if "500" in str(e) or "server_error" in str(e): - pytest.skip( - f"Skipping test due to OpenAI server error (likely MCP server unavailable): {e}" - ) - else: - raise e - - -@pytest.mark.asyncio -async def test_openai_responses_api_field_types(): - """Test that specific fields in the response have the correct types""" - litellm.turn_on_debug() - litellm.set_verbose = True - - # Test with store=True - response = await litellm.aresponses( - model="gpt-5.5", - input="hi", - ) - - # Verify created_at is an integer - assert isinstance(response.created_at, int), "created_at should be an integer" - - # Verify store field is present and matches input - assert hasattr(response, "store"), "store field should be present" - assert response.store is True, "store field should match input value" - - # Test without store parameter - response_without_store = await litellm.aresponses(model="gpt-5.5", input="hi") - - # Verify created_at is still an integer - assert isinstance( - response_without_store.created_at, int - ), "created_at should be an integer" - - # Verify store field is present but None when not specified - assert hasattr(response_without_store, "store"), "store field should be present" - - -@pytest.mark.asyncio -async def test_openai_responses_api_token_limit_error(): - """ - Relevant issue: https://github.com/BerriAI/litellm/issues/15785 - - Parsing the in-stream ErrorEvent must not raise - "pydantic_core._pydantic_core.ValidationError: 3 validation errors for ErrorEvent". - The iterator routes the event through litellm.exception_type, so it surfaces as - the typed 400 client error the non-streaming path raises (litellm.BadRequestError) - carrying the provider's message. invalid_request_error is a non-retriable client - error, so there is no MidStreamFallbackError wrapping. - """ - litellm.turn_on_debug() - - # Generate text with >400k tokens to trigger token limit error - oversized_text = "This is a test sentence. " * 50000 # ~400k tokens - - response = await litellm.aresponses( - model="gpt-5-mini", input=oversized_text, stream=True - ) - - async def _drain(): - async for event in response: - print(event) - - with pytest.raises(litellm.BadRequestError) as exc_info: - await _drain() - - assert exc_info.value.status_code == 400 - assert "exceeds the context window" in str(exc_info.value) - - -async def test_openai_streaming_logging(): - """Test that OpenAI Responses API streaming logging is working correctly.""" - litellm.turn_on_debug() - from litellm.integrations.custom_logger import CustomLogger - from litellm.types.utils import Usage - - class TestCustomLogger(CustomLogger): - validate_usage = False - - def __init__(self): - self.standard_logging_object: Optional[StandardLoggingPayload] = None - - async def async_log_success_event( - self, kwargs, response_obj, start_time, end_time - ): - print(f"response_obj: {response_obj.usage}") - assert isinstance( - response_obj.usage, (Usage, dict) - ), f"Expected response_obj.usage to be of type Usage or dict, but got {type(response_obj.usage)}" - # Verify it has the chat completion format fields - if isinstance(response_obj.usage, dict): - assert ( - "prompt_tokens" in response_obj.usage - ), "Usage dict should have prompt_tokens" - assert ( - "completion_tokens" in response_obj.usage - ), "Usage dict should have completion_tokens" - print("\n\nVALIDATED USAGE\n\n") - self.validate_usage = True - - tcl = TestCustomLogger() - litellm.callbacks = [tcl] - request_model = "gpt-5-mini" - response = await litellm.aresponses( - model=request_model, - input="What is the capital of France?", - stream=True, - ) - print("response=", json.dumps(response, indent=4, default=str)) - - async for event in response: - if event.type == "response.completed": - final_response = event - print("litellm response=", json.dumps(event, indent=4, default=str)) - - await asyncio.sleep(2) - assert tcl.validate_usage, "Usage should be validated" - - - - @pytest.mark.asyncio @pytest.mark.parametrize("sync_mode", [True, False]) async def test_openai_compact_responses_api(sync_mode): diff --git a/tests/router_unit_tests/gettysburg.wav b/tests/router_unit_tests/gettysburg.wav deleted file mode 100644 index 9690f521e8443f3b0db97b1467a9d1fe85d964c7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 775192 zcmXV(1$5{N^sqjR_kY6Kre@hcR@x zyA&x_mw0lM-1~n&;lKMzliZwp&Lf{k&U4OTH;x%KD($dKnKW$T$Ooo9U6Q6KiixY$ zL`4~~NKwp6nsVR0Pv2XA$fXQa+VRb)7=~T(8xg~)W|- zuXLyPVLW>?v=%~#--sG5m7YphekoOAh7Wo!8dXL!RCb|1uTe*f44y89lWcm+V!T>o zxAB*;#MohMH1--t;Ui|K+)q*3GfKJATj>l%ne>0b*l8>?w$T2N;WaMNcO!itHnwx^ zrSG^QoMpnX1D=zW5}s}e-^IwT9WqT)QkAw!4`fh8k15dHk-iHQ8*S5-s8M5VGuG4l zQRogK#m)3v&lm@d4aN_~Tw@-iZ$#3kk-MLUibPKuXVF<2RHh>13yiiC9_G^TdE*rF zT3~!;d}}O6o@?Q26~CT=o=9iX8wga;5jg<}HZZh0j8T)qSX|5X?r2)E^ zAitBK;R54rr>AOcEFa6t3^Q)9Qu&Al%>}oXY29?G5ipx-1Prqj5Qv(;r@zNI17QsIECwNLh$0Z=m z&G<52E!b{?gM1`V!e6^m&r`>Ex(*(WBk5w0)eVgFW6Y}Wn$+d!E^Mw6&Y91?`AAy9UAm9f{ihcfY-hF_j+CtBaboL z;$`-u#f$Ke0nZnaVjaANY0->lZpN}ZLRl%kpg*|kj5b=qK_gmN2OUibpC{3xksgy1 zG7{@(PtUzU{}7()gBA71i%4XY`?=up0QmpK_#4iu;cX-D%Z%B^EUwMm31bN@hz3pY z;)mKwII6^kni9Gf53?GYk3nBH5*6%(k?^4eE}|eO37ZKauOKw3Xvsj|Vv%m88IC0G z#z9I6oMbVI54oL0T3fM&W%!ZLj5m!>jNg%46@BG`jCg{}BS<$HEOq8N(N+uOkW3Gy z*!wkH=0nvgA1y`cvW8j-{qy5iPKzY$TWozk}hg z7kClulq(b9cN~0-P##bo!M6>koyN!&=;a{#-Ht|_^e1`3QsXP*HRCzsDdP*{A1II* zxu4(nGwvy#F*BwWK74#Xh#t>DuiR;lyH`*#sc#CX2zZ+Ti<%#Yf z!>)LcsuRr>)2jwR)ogYD9+;1?86}F7b*b$UYez3}KYkP~H>oG#npa2xcve zF&Ko51#2U?JAnH`!PcY7lf1447j1Z6@(jtW#A3}@MLPYpfI>gxo`Hf&Xxl*Slo)=L z*t>$*D&FNhpR3@fCDQDW@O;@wA&XJH{CXMrDIl@~bcsJul;ilDI&fCalV{NUaWr*_ zJ9P=)I{>_2pNL-9;bFTX=?LRVK2e2?8tLyc9M^%{2%hQ$ayiE96j)e}hG!U4jOoV9 zWJzo2I}L3%f}FjyiK0WvuKeKTD5yGvou?s}+=ND5c!n!7iyo!!($n+41T!qX| zpuZ+K5Zj0$olV9Z<4-hN2S>-jZar8s6Ayz4YYaiFWd43oD_(sQI6n_ZN07-e@DStj z@T+9qLZ@WS9$FkmTFG!#&gcsEk_s9#5*dPlx2wj^C1>h`&f22w40KkChVp6K38Xv* zG9Kc(7i*n{hbl~zuQAd%7*C?TF4z%_}PvR+C@x(;z?B*NXl|A|X3C+Q$foX;qJ3&4v3yw4VVFHhygbPCfj7f)*E`qk$35BqWlK zv_@cW65WPDb89@64H`S~v;j5*M>4B2qb2bOI-b>!hQ;$n@VsfzT?F;L(VmZXK6CCOpiAmwY&$xR7ul%k`K$V9Ru$rVD3 zy$#BKA^-jvUoN(~nYqXTbnYPA-^edNCt~?r<5O(+BXac*iKuhX#3jZS&rr#0B^>?) zEGK@2uplWNgb41O$V9xV(9jL}MzF6MvZypP)DBD*Lsx`n>|{Pb9OPVv zp3872QMd=s$V|h67HhG-zG$nI?DL8a6);|oFDR$)TgmY5Ad1~hJnoL}{2<~0`jTw4 z777%l2K|d~s)P33a6O+W6~x<@C9XV1iO{b`%P7=eMkZ^Bw!bDM@h?wQk^Sa_lLGYK z4a<8Je=-~l7GZn6K|)Wga{^ND1xGCs_$Y(Fd~8mv&kepD=w71Z8ss#WzNObac%Gk) z?|AKDY>7=WtKNpTzcyY+^4}0EWfpr7X@7^5KZfo_-1&v5IS)+<`kV1u;x$r1dG zw+{?9qn%thX@$ltWVEM{(=ojF2Ch?}F9Ypd0LQJ6P9_|Lk++OmhHP@UBQtUp8?u1y zDv&k@4)&r610*znr~q^JF+{2HP}T+6#<>$jYjOUjBG-{nJAwOYP-jCs5@{q4FUGoM zuGN$09!2}D$SpULlWs|PcO5G{g+Dn%KavG?gqo}AX&U*qp`{5 z)N`Q1jGiSvq(XI&RzafxORyuKG^lQehaG}Mq;hi+-AIlbWt4xQX%AAG4|;a+n*v=@ zCy*JjM7E3AhuD`?Bg(*R9T?q+FOd1mUyOec&F$mf6_1;a>~B{dM0;I%K8T#W$WQ!_ z&L^3bOI+N6mfi5s4NZ;&SKXl{3Ay&-a}OjUd777zE8+eaa_x#G^}rfK*sD|vGU!`H zx0P7TzqF8i>k?9Ji%tuny$s3rp|@+W{;?pa2rY>WMv$?L164z~R|Z-{lj4_5e3C4v z8H7l@_t2M(eq2PL8svT+?t=-9tMnxG!94KMmiX2Yi*WI5fGG40`S9PopGGEIk;!@b zbD@EL*x0jFEFMy(z(sq;jf21#zvaNGfmVHJv?DEt62)Jj<#WnCXsZqSv?k<|g#^1Y zVm?$!)^Hh`i}+RmQYEJ^OYkGe6#X1UKOY;9lS?mzo;A$qPBC5=at4XylIuz>pb0vK z#=~g;0DrH{GU{n5H9IfpT91~Opk2Xs7@6loK{ov#<~j;eDv4Q#;dMVFtOMco3E!Ck z-FfI!stx{xJgq!83SCSlG&?Z-3hgStadwFY~Yip>_@7sAad-uEFr4PPs>9GMLjqT7Ag*de5Il$X#av%o^A zkQ`WMrya?Td(nO<7#@r^uEPU%g^Lon>H|``V!>1Jj1OWzEy1A)SxXJK2Y&Yk>}L>? z=z^w8v4TF_y^^!Zd_ywFLtOj7htyb|MCX&}Q9QIMF`tq-n0N-!u3?BQs%T$L)?5nC z3$dp#PtRelE)lzmHc~Ot@P(Tb7@13TbsmyA4C2HSix+lbCo+dFBWg$n+<+bxFmok3 zONCPAR~~d(iSKjZODY*rVt)k`%d9(vXAblEBzGi(&*GlsjMeb89(wkGcoRLu;6(D^ zBOvP>J+(v2Z(*_HK|p85>x_*Akp3nlAEoU%p4|t(k|VZ3N7vG>C(@pXoxj8!`+-E> z>W1eG?#q>jU#@}Ex!Ax_uqd^3nSsWTR|7duFSOhj+ytRTGGiCCdqBIN5xr2}2&H1% z$9ZqSXJ*1jCe|aB`TvmHQD_qGrIN6p81*M|y$Gc;%W1$C|HxWo-r10_mUJX~9u1zOH>rMm7~wK<+m1YCW+{~% zE12qz%p?{yz(X7fm290+Ek+)-cw}IM0 zNVz>!cH%1Im(kSE|4(@WElr|TDJX~&d&FPwfulW)Q4f8`_;JbsT=TVyD01*(A3#!CN-= zG87ER3}14^Uc3*4=0dbs3Yx~iQ6F&B0gQA`#O%qS>rU*h4NsR6CB`M@D8tF;q$1KE zTg}3!_vDvOJl_v%EQVS$mBIaJ%>Dl1(`@q4`S>D<r;#Oo@&B=WWg4=H%O zj?m>ySWq2Ss^X(%r6&pdOoPfnw3MjkN_a*$8kRa&C3>zV#>vW!P$Ql zWvaod)X)FG@0{Ru39TMVP*h2)16Y$Uk!7}rr(Jlbg-B#4?G7WsgGfPE_JUwY^3+1G zHW*8h`j^Icsprd_uz^oma4mD*9CRcZh{UL!SmkMW$we9x6|PjgrTTjmT24Zr$Uy2T zCgjndzD3GuR4T+KgfdyFlNntay}CfHWR`AZZiUw*^xXk`lqO^;s}K2L@LIBs$FYv< z$$`gUg{^p6>blo}PVq3Epx@2iuDlN?f?tJ3WtFgyUj?PI-c&-nt3a?+C)`+V6SPZa zC7DW1LK0F>aYB*gqSug<-vyQ*U~cvhJaj?|(!(I0EYr54 ztoT#OysuQEJ$RxaWGLg{tsMIr2wh2_FNrG)>e|DP3$CO_+?uNu@+yJm0%GY9xNnDS zCC3>Gmg8t#VvJOUPD1M?yh0s+Wo@UKdwap9R4|UfWs5|#s6bM6Xi+@2_*Idk)HXs1 zOOSbRB=NiCr@OFc@u?U2yNmuMf^EVUU!a2eEwYjPSdb`{B&h%&L5fmq7j0A{3q66r zGU$|9xkPB`JHV$5aIkfBJz+{WLURiaWc;oFFpd_-U^G6Tw9Q3W~^)|ZI8sI zdeNQxrC@O+5pEKZ=uRx=;Y56!Kt$_Lq88HHK>Fzv69^X{UjiQ*gYNYXhw+6H#1NrNuM; z2T~)nIKybt;y{9ybZ{VR3%MYuiT(m0ssJuzRwDVk11@ETf1Z~Y$sb13i=cG_v1JoT zl{#oD63OM&0)352;B6%Gki4}ok}K!F)ZxUM)4-ruh?#HM=r;>FD#{tKa27pAsB#xz z;U}TZhef#2#*NtbaAa)*r~T>ic6{FiG*k|C3U`mf*}uqVJA8){HX%8sM6xVsl3gNF z_p{%dL~{_3&fKhb;*pJi8vWVCyb>>N0%2amGHp%b))!0Jlm{TI@cUZlbxK3-Pm zyW%shL8k+tst9Sx+&hyIi;$ws;a%8U4t&de)5*vOu_5t9g?!T(L)QCxgSTY9r*O$C zfR7du3uU*7pA0(}s*AbAN+n9mjQ#@J)36v7)Ja{foU1=5y*5!%+$<}q5y?C3C_<2X{cASdzZZlodA#6qZU zfu1h&TQgY7N2cjWDayzS?ZRX<>39gKK#KL`fU+(;QwyS3qPM@n<99@jT}Vgvq157g zj`MQBwM6ntD7oSzHe&k~P$67OO(Klcr3UF`1eqP0KuQTU!5LWY9n1{J^LHeYYX>6Q zGG{<5LRKtG*sFYm?vKDZWZ?bA0B$w%gEXu)M zd(hpU_Tn$r;n8JHq=H_W;Nb{k96++75!o#u*}bfk$m)3s8WnH9361Pd%yzHr3EIH7 zQ;a3~;<*HZWc@-`c?7pDuvw`ETmzQ-fM2QGOLP;tc4Iy}nJUB-yuo$Idj#L6;R(hf z`OeVV0b95Uy^Z2s_BBW>Z$#QM|928u&V!U=+)=R9c4%}cb|Etk7aDh=cRT2kj4PR! zR9Pes=z;I*2vvf5D|hPX&xJl^9pM5xTZwM=;lHh5SXM(y;GvlJ0;uT?599D%!|-@z z(0e^qh5PtcLQhh!YDODj#z;Zp8vRS{q5|o)XCBZG%49w%Rna7V*YQ14*AjHpA~{)| z7fi_RGL=_0FIl;hNF+5a1I_x;>}7PLb5GWe>bNJX$L*1>RMv+liGb2 zfPS=)Tti1YySb$9UI{XUfAIn-Ai03wz1WjP(;c8rR!)XOl{x)cKBe)LtY2J4CUI!( zh6Rs=cUhfoq{Sg<{)1WkPFhF?=tb+|-9_`VLYM+Zgffk*ni2BQt5m(aLy3vl<%4?3 zd1Ws14_3Mi{+442)d|06N9GE>gs?N&g(!8L(a=2{>LqiN`Hha}YsMl^^30XFdO8%R z@_R178{o!@4&wBYhCZ^8jO?qB)s6=AbsBW6Cq{oo-m?P^uIwd{ekBj<1hRUA*ER_| zl{s1g-|kG*E{9@EvU6=XG_^^1GuctH0qx9$0}nXW8LSxIy@CcSDnNMrvHx%6E zVFQvu$(&XEj;zXxcWh3eK5AUC=OR4M7$i3V84XT&wTZ~875X({;WCGms?j3`*AYO2MY&H&PLhwI3%4GO(v@@FeT|UD0kb zm&C;ykRo->ZP=RRLbBf>4-LxtLt9$3Wi(k`%Of+8wRBmpE5Q1CCn9Kn^qh(RlIoP~ zkcdO8=u+kxRZx8p+4vdtG9Ei|(S%?H>XPMM9#mXn3cbV~tHDp1n_^(U!AeF?^_@85pREZap-LkT3b|sS7 z=s{-5GBcLF7ZSyidC3}w2MSIzrtF%KSlb(fbZ5MLK1)_m1jf&xfpgfG>`xZi=%B3y z7LcQq4R{OR0RcfvPc3I1ri#Lc8EZR@r4wxdlB*b|%&SK}2)e`P2>Rw1IP# z-`r3yvvpY;m%6^hfxocKBT!fY9WoDG2i=Dfeklx35{G3aTK2$-J;?gqSw2a1Om+iF z9`_I2$S$%CP%JZ~WJYSH$0{r^1?@@o!3&SlM)rapVr0Ra)JbLEtW=0g(b;J5AakV_ z=w$*CV-UKL+W&ZRyGP0BWT&2Jy$mE)fv=-v#>??6t&v85_-o{zL}CLve_5}rx$B6Dn6os|6b5d7Aoi*)2H zvl7{_C7F5Z1MxFixuZef9Msp-nj)hji4LCI$D^(%j`>^KYHjnn8~_L)oUt%D=kxs?Y+J;?_o zuMj)#23DIG{}lL<8Q3KxFEeJTXo(+{H8#m(CGN!;TXyb=^@Z{3DR@iS6E8FE_UK3U zL?1!2;tQneC)rUxddfg63fx9{vWnjG;D0)e)%%$*y==_i6y~vVXi9lt-XNc1G^z`z9>yUuYl`Pf5*Rc2USG?g3qpCUS^6edXg0^sbfg}CK>rwBOkGX7PObK1YNSDWHA1DIJ2B5xt_yM zkH9xwhrD~x!a;Ac7p@F6%8uW9bRuhUveHt)6{Sb9Op&crJaRyh)Wl?Gxnuzc;9BNC zAuzi!vFAp-)i(T+?2=rJZ7s$Re4ogMzGFtb8sAleHaaqTA(9dwC+n56I#!EZ&ho1R zYGk*;RSByV-!HY(uJAtq-el*k83_945kzD=yDtmC`* z^F}m$h_R%KC3_HM7AB}T08KK3mXj4Vc(zSsOY?YsCsL4AJu4%KFVI27l|3vH%~~?% zm0fhQ(@)N6oTTqdykzAfKu@ysqZ1mIQ$D@%k^@0z3fvSSaq$dtPDgxFlwMQOr|i!A zk9_e@D3Cqf9*{5dOb-<9Vg#9Ud62hMtYl}sjr>FQqaKB_CS)g(QPwTI*hg1JEMp9@ zO*uei`R(McRHkJ|jO^Z&Y+0hPoR`>>*kkoA z`Q02OF4^%Gxc{BF{2TmMkTr-6SQS~#sX-IRxT^6p#f)!5Vsf@f=6^~ezrV;6RiL#D z8e3pTvL9Tkr*0&u6Yt95unfCtOrZM`lFEXo9BfExDC-$tss zehrjLMPF73WJTl%h?R^+BWAT>jC`=rgFNXvqJgYsiRYIZl$`CdGP+c=WQ|DXH9}V> zo|4t17GO$ZNl%^~nD7CIu>+|Y=Yb3vK`dWZvSn^OgtaBvOEw+*y^0>n(OUzwZ%6k@ z&>*oT7pbIR7hSLh*$pQ3vx{ipd*gkiya@WGh9$dkrAqQ0=YwV^B)^RBC$R;o{kA}= z0G`amzsWg2iN&%LVkDTl zo!t3uo@xWy`h&8*(3(y)a2K+bUCcZ15mJpf40?9Mk18AAZ+4?1BY*!)20{C~)QfjwiI}W~Qa1}#~ti*|5m0jsqp&zM>>;Z{K8Epr> zirrN5k|@;{K03gcfn-zQs1*|JPd++;o;zy5Fg!@5G#h<(#L8r)F$+qDaLL|v1x-l>rHsE<*3#s>p^q7~oIT5cBiXqtl~F&Q zB8R9Lgz~jm%>v|ckiMmI{*&>H@e0|3tkM39R{w&g@5poC#RvY0Wr^R*K&Gk1D zfpV#GO5IFmrah6f_%dvcsr>q$XeDGyo9U_4FXHo<8?-$LzzoTy*nH1 z+=;$q4Y4nhXqAwJ#4I@-BbnI&kgx?$62te&PJk3fk-frlqC>oa)W+PzY1zRpb7t8O zE4w#jPAhAUvVT(ih^!JzrT-+Fl~otnPboEX*{kn?x`AN$VYKC@5-z)^gLt$Hj3FmT zr0RYg`OBVWsdSv?*8rEakXo^vBa>=@mMVN#70?iv`a2}wzv(7<+b;d~G|=-Bx}qihmw7C={zaEP(~21k-_ubq4{R!R2|?%DwMj> z8wq#7BLwj<5<_KtiCG@{m0blv_?6Y>KFr0s@%&h%U(PR5N`VMaiXtl_?DrZ1#O#=lZ<@<&C4zzS;^*cGuDaMjZc3)Sa`4sS9iyg_%{gY@z_P)j#H5m!Yiu9FAh7CJu znaEYD(UHs_W$#&M^2wRXU&?pv&$|jN$huSpt)wC>`^n`b-4SFYH9FY^nh(tuEJXGO z%BtmgV(C2m*5~M2&OU#M|9ZxlXpAr(W$k?)-eE88WHm}^V37n)yhI`cjy%vS5npN` zvZtg8y5rcP;6+yZ&B#O6o24?U(N@+52PNtbvM$jHtH?rg50M2wgf^twa0_|Q8=&`5 zIF)+Vf5@mkZRC7vv2u*~Ifa^mL`j)x$my6yv@Cj(dcW*hk0KRWWfpH&gf#N$X9q~$ z0$q!Uvxmaze+Kl@;a_j36I(ZJqqf2z=fy`F0q z@{l^19~tE;GB?`7ICADkvg+T#ocK!}+{*rw@3G{g+>;eN$>TdCRXO#0ir!`IT+Wq> z2h@@C7VO{=Wf8t*B$>qRa5DhAwUZl4g;h?%jX_dfh#~*uZ1?Bvuvo?K8onfiR%DJO zdoqXdLvZUn`jC}(#>&+7#7_ke8K4bJ^e3etf zze4v`a-7cSdmDOPj5oS+BE|}3Ikb>yzk|Mi;dPE&b0lrsgBjUvE?Gb!8j_s{f)j}x z6>uoEf^^0?hSgrcZe*2cCm5eYFG1rG_5I=16AXROH9`9g3lJ8NENw(teWX~Obd$|5-N5a}b>*fe5C zz3~!JZVS)vqF&e@ZS}+NNTuaxP`VX-^+jjzLGwy*{ug?z$70SJ;z7H?@9peuzYSSG z3D$G*OYM<(5!}eiq^y)KV_eydDM;x<^s*3d0!B|{_LB0GvYx*!K&X>7KAA)Gz`w{G zWjs`f-;#=#%;`^pj3XdcPFeN9PD*HBk0#TQY7b($tUteG^v5g7iE7#VScb20U_Y|| zH5rr~Hb&4(4-nlJ-}D1K?8kySfiWu>ROn?Z$gwC(627Cx7>RZ!^W-VKh!xGGV}Wuy zPG-nmxi9|QtUumc?)Fs?&7GLLP86n7&biRH3-?IH_tVL!*w(h5dgf=+iESEDB#6jXrXpYipw z54tniURPxw?*?ANj07bv|3)O+h|Z;^^(HcX0I&NTXuSoBrt(BvWj$OE<<7OptC8ro z7rG6iS6j}I_eKLAbhsRC%IxM5=z9&Cql_SDF#0Frs-4_Qc8|!3+?Lo*9NGAglwI$45#Xwp(Ik8L8Xf9rv<`p$Il7RO&`TJ_V3ZnsPA1d@k=80?agn$zr_sBi zM;#>Z0fqnJrH+FQ*>SatXXQk63|{X+yEi8+Y&r@0IsI)G2f zu*M*dd}Sk)>_=WQkCPpAAJNZR{KPw0&2Ess6Wq@MFHtmbh;tNPM*9F;+d-=pjJOQy z8lgu{ryQq#ET}IBS2tq8#n^u?n6fByjj??b}w$L}+YG9w>c|7wiGhm3`ng~+4=>y#6jr?B1j*z2$K zEvvjXaMqR>=_U%v3C^eaEeE~CXkVVVmrdnR*0KHo;olK?9>P+(p~c(D+P1OcmL8nz@e77-rHWHBa9Sb_>MLd(}CN&c%~J8^!9{TkkcHp3t6J=aikGsR22l(K#xRG9k~a{+4d2uD$&M8 z`rnNO%3LKA4EWG?6HhKfk6(iG?f9*8AfXe~N#&tCK2G+_O-3F=@NhwVcrFMl1sQwk z^DUlvABtpOQ5C)JK@zgJP1Y@BKZ=~V@*ow-(&QZEK(y5Xs$5Vl=Z)oGf02FjgFtQx zesDfH;+@3zkNJ09W)PR|#8TyC>KSykpG-_nUxdg5Qt5p?6d!{F57?G}bzwTlXp5%0 zgDuIHWFN$}U`5Wp}Qr?hOOCXOP=2B7oGa<=og)U}UoLGhT28Bl8bPVWINxUL2t3$>25@GVe+1TQUC!%Ae9d}U55|K^GO8%Rg6?^gI; z`4>vE8KVFmWvAG9XuKWG$iLt+2Ahz*VsdtEFqTrynE!(5GQRb|$H*MD7ApJUyX4<@ zC`-%)Wq)sHa4*&EZOp4)$78$?hu<3aqn#hnAbX4^fSx5vT&Y*q(ARe4 z|1t7^9PP;&wIM{`1)%3MG9NiXypGo|#3tEQ-aveBN7YQU_9j?=2y6&4r4HH_tV*Ru zP6^2#Z8?z<;VD@uJx)*WpvNB)J0D)fk{&m1G1?lr#wgJ9CBLo!k&h-U;0s2a&*)3= zKdbOoKVTL27`I}H_hWlKj4^y3$EV@OwZ=94@+px>PJeC3Z^?P*$?UKgYbeHHeU|>U z{=WW+{=I&mK2{&3zogIC59wB;7}UQB3Z&j=#x`ZYsQkMht(naX#K*{*qQpa~1190K zN5kn0$Y>UsX^j$A+@=nu2TgNK)ut}yDdyScdUKN6UY(@QSC6Q*s>70F>1=6hQ7k*v z1?p?+L^V@gYQEWAWtwT4Y$`R`Ob3y*i_Cet(Zx8XAJGr$JN5tcYQ0Gh>no$hl@X_9@NA#9PXQQQ&Wq9<$AO9o%WBkkb()jlHxp=9bs+)DUo~O6buhAdW z-_qaK7wY?YdJXU2>+f(qt-r=EqxF7Vz4cOkj6O=As<+l7@lEkJ;#1<+#@oct#r}w` zjctf6i5-fajpfDfi4V|+lP#|{rKrO!cU$M$${iNx%dVBli&93WHm3G;zvh0>{aLB%``B(Ed=Cx*%dZ)TYZBWmu z`_w;T3{MuI)VROW<>P==^Ns` z<7;BB*gMfy(UaN-+6`K^))cX8Ewu-F3V?@mo3j*9s==4)L+#P)LH6YHK;~auj)}tEq7X`TE<#hTQ;jB)s^N; zrf#N($kbwbx&C1M>)4)XM)U#gLS$oPe`H4_5E-bwr!{DwMemE~_BsrhB|dhG4m4hJLW;= zMWzP4`w}CFJ(R>J#Ck@DX%9!{h2x=Rp>INSLjKS-VP{0uUXH#VA8f2s_L!S3pV+%N zmnOG$pG^BD<80>e?ElN@lshT+o1CfH=~jXMNc+Rc&iNM26c%KM?yadYhIJ35L{A*WfFGOa71iH~MezzvK7$KM#x# zb_sO|_l``{c1Kg=9rbZ&@_o}VbG7+yb*{Qr{TBOMY;JGvZW^KVH*$4KJ-5SRGgzSRJ@3SRDE#yiWTtHd;?tdYijgyE=+oX{ooSm1nHZd?9;m?yGtG@+RfI zkUKsnFZ;{PR_Q%b*EoCF+ge^Xh4t~V!bo0luJ3}UjpwoEil+CQCO5^KMtf|&GyW05 z1ED7)FGkPCEy^)dKeex=uQkVZ%=VytqCH@nVQa8Xw=T0BRx9xhSDTiQcNQ^Q{e$TA z7B%xB$}#0ZliB>1`M9}*ItD*8Ms1-UGT&mJZ?c+(D~}kD>BHkQVyUrZ(RR_dwIuDA z$XAhXBQ3Pu+Vj!Um^EIk_cKN-x0{|a->FWud}MvvcEOfw&#||*JMD*UZ!>wf|p;6(hBh$4dQFr_n z{deO_WxT1$G~HZh?ybJ9-lSeK_b|6NbyVu~XXBO8?%KNW&`{6d(}DN>y3gfbO&=+t zZ^Msi8L`QFk*T-kR{Ij?3n^#ZDH&t3R^&XJcSHV7`PTelc_VUFRyz|EVA1 zSNaa)pptL?Mh#nBw(Yh%?3?V#j+-5m9TvwF`$x7-)+Ltv)VS$UMb~G>%VMjwHzK{m zjlrbg4S{n=x!CXW-{s%$e=<-KObw-h=mpxS*u3~fof9;sY;(lCS%@;!zL0@2M;DJD6z!NMBKOC8>O^n%%j^^vE(;eMh&!n7kU!QR# zYky8R>2kx$`U`bWu?6_S zhFhAZdiwf?2X=(kX});6X{j2r9-TE4 z=|yFtp~tt!c1KI1mH62^qD!OZ*w+bP`-8Dj`N1^Eyv6*mYN(G{Ub76ae1nJSZvND? zjQZ+bx-H&5HZ^)#yItED=@q#U?jP9@c~-kC+9{S6Z==6%SWMTOUs0d3%&@*>yT@K( zud~mxkFYn`?z6pXonYByzRpx{%*N}d$3D{>kzYd}1s4Yb{)hdIzP&z=?=DjGg@Gr6 zkA!{=H%BH!e~kSW|5E?L_zu5uJ@I>zIz#QP{$h@rTAIwt7JWwinplgdC-Pr-PH127 zR8S2a2_;2bTEFP3SdwwtwB54PKGpeGvXMG7-I-OJU79;LcTMi<+(&ai&)J^UBO{*r zxNDf>PHVaO1AU?PYp{=RX7l%rO2gQ?ZndA*Y^^D*yHtOsab@!<@8f|_!bP#~j9KO$ z)_3fOlhowul&jr_yD;s9`)>E2scTb)B#(4vIW()?GRizhIjMJuFN{8`?Tb7WxjpiF zWOan7q(!5h+j#}pB!iz%nkJ?684O~8(ST>GrR9@Dl^xb z?^8d)Gk;DV`-1s((=?^X*b#p_c2l&gRu`@gwF{+&riP}4pO4&u#vSo*jnB-lTkmtk zos&{Fxj)XhJ9|Lx?Ri`9#~boq$$KexX7;L#A@1K?cQ~%KZZ!QD{~_{L!0COY>9dB~ zy0qHO)g!CdRX5grP&cumRns`n1iw9;6yt=Ic@{o?z4N-18Ezx(-Skh>Z%YrS<)&>( zjU{h%KIrIT>tg9@9-+8(SL{`7ROILIgz#12r^CNu^F`Wy(NjcZxAzwmG1cvs{@?VIS~*!B7$<2uvV z<~^#9<>k0-wY`nw2FGBBZhzeVz3pMEN4?Kni4FZ7KNAf`Muz7Hp9?(af5#W~zU#fu zyVg72XZUvd4+qrH6Je|NezZO|R)51dphQfI&8^kP)Ti)Vi_9lXKPVH75yXgCv|6i* z6h+!a-ij#NH`)+vX>9yYqsF|#I?r*BYhKD(_d^*IvhT@VpZ7`rhWt7Cx98uP*D=SQ z`LKJYtJeOvrGx2W?2d4je?s#!4Snkd*I27wxODzv-laP(2dk=T^t$^R*LXe&9FLgv zM@%1CmfOS5lGM#HZ5B}|c(l^aJ$5ZMFH5YoG^9=Hq_*(d%4zvlK4$s!6#!B^_ z#w62k=AW5eyiNWYRtKuDnqM^Cq?8*A&uQ? zT&GihNc%H$LC*TToPy&8jux&KX9{NK=jZOoJm7xU^{>5^<$!Tks|d94bZzKfyR+(< zOZxdQD@!XcRBpfU`=#?$ZEF4XdzwcC#%cfQ)6IppbxH4~*wgo9uFu||b5-sixjS+n z$sL%ZXFiy|I<-skpGo=l!Isvh>-CywxmFY26IvIn2)r0r7nm7*J@is|T%?!wVKgWH zkM1>=DXHWfM@_ewpE2KIKF(}$tTLba&3rx1eB#cSKbjM}G4_1yz1Zs5$apH5#AKzj z$!R`su2qAUEM=JdZ9)0DyHpDh2_N|Oe;8k3)K_sH0p^=D3f?u5M8^Q`$#=RcCyHfKQQc=w|& z)v?yHNeM>VhOYLy8Z+w#Rd2b}^TN%QFP&R=Zrb^)FE(A;TQ#tDTf-=CWAN?h-9{Hx zv(0f1N*$5@b>@)li8&K;GxA2~Ey#7|L^DsMA9Wv0`O$S-(!aJO%Xg+#Mw0$@Y;ZI~ zD~fy?ekJ^2_+73E;m5*9!f!<$);wBXG&8Dc1<|Qsv1e>#?2B0M_`djfGK>=by~zS) znCVvYD76LobxZ4h>zlT=_D}7niDA3!OYQgDf3baUy~k3jMoa;!(FOX=u^+Vj$f?kp z;D>?9{(F6ky%W7xd0TrG?`}`7_q;dYYZ15*m>+y1^m6!Yq+4`qY(l(H->N@N&HgT` z3zw1H|55MwlDS=l{)}EoB{8Vpzc z@A2HdIZAeYdUpV=y~zUOJ}NHuT5?E zqA86doAbl}#+EA^)pFba98;a|Bwv$ygZmHnSMFQgL)|X-yQ$ZtyydD$dfV}p-DkVk z_Ko#B%L;XZTB17DQ|3R-!_4)j-%Ml40oEF~>X&0%qsz6=BV)q#!8-#c|5#tMcZv5s z??2wBec$;z2ECzGk=;>mypPgo*08kCowp>PO>wxZ++U=9npWoyrv8yKD)~?64-U7j zjrx(&MUO@6BZosT2Hy6$y`kpz%^xbK`iHhy+oYApcEsN?s!b==nbu}|qch*V zF(a03y~zV=Y$Z|BxHaobDdz46B3mv5}O(vF1nX`&2)C@l<8& z$~EUMpBrC!_4%@k1FBZkH8#EKbAC2|=O{1H` z&5a(HudTmbpjYtv(Cgv(k-xP6i)P2ZiVcb{i~HgiMi3(qpxdUhVKi0;D6Ou?S0kT&s*tP?dj`X=&J~f43E`pu@Zf^ zvO`^ED^9xXEN7IRsTJ;iX&<}0rY=an%cUiaaa?UXq&iLO^pmk0qZ=c`!v6#(1%C8@ z;a}#j_Q!d32<#2)4ZarI9dhDTOw8v_Mb<>FjU1=;vMzigyf{2Md^|Ke^iF7I=!?+U z(E8xm;GIDw^lK=G45KZ1+(*$z^~I*IEWg?JI4`AqoOU3yTi%iu*5Y0z^GZ**(pnj< z9&V+UOe)OIYs|EzEp;8RJ#KnGdQEVI=i`QTwZW?HmrNI?obP*K<;69Zw^zSWdt?2v z#%a1GOf|w!`& zTQ$CJM&nn_KYRZScp^o5FEX@b^k#Xc*3WEz zJ2G6)Cx4vWCHaQrmTWnTrwm9AJ8rNJH*YYe#6Qujp<%vjn>RI%XdKvB+GuUOrP0xR zo$r=V&*)uxw&}c@ZT~;#y(yv8PVPYJU#U}4!zoS4PS>-{8?LfM&HGFXmH!#n>GR^J z7gEn< zp2>f|Qh!o2)Yna|l^3b-|7N5c*Xm2+<+@EdU|wns z+D{~%b@fl(lUALvBGa4EB7I|Oi{$%~N^AjBJZ26r@Z~m-Z78XmSd&t1t?pOzN1eSX z=sgj9L))xBr+#Z6?z)&V$=yBepR~gC2h;veeI@xw(o4)~A5r_5h8hp*-^5FF)}EEc z%oJZ#+gU5DIn0T=Qd^jys%Fg?9luFy9op~z#oNY{);yr8u`#1*Y;#v{p1&lhhO;zJ zbWeP{k#G9MG|==q`!ycWZ;9=~lWz3In*VIvTwhtcrDko-m$fg{Kiss)Hzzzuf5x)j z`E1&=+421Qi|3X0Z+)W8i8i}iH@Et)WNOjnya}1bDP{I-^9S*tA}0eMd%HBZY5K2e zOY@(ekG%i*t_=(i)kFrx2E?buZ;gLI9sXu&9>bZ-PKne;evU4QN2w4jwm2QHx*YD~ z>Az>r%1X|@CVOAjpv>FT9!rke?^74+<=Q&|fAh)u?KP#<8>{ZBo?G)#-GIgyJskqB z$eCESvY1SFm;H^Toz5p*FS@>RPEDHVXtvF<4z`r2gQ%YLHkYbNmgSb)tQofdY#-Xo z9NirU>?dt2tw$|Css~IIX`CnmNM4iF}w9Ck!J$Up3J7t8x}TXHGS25 zmG=|>xu7HRbacA@xYEm1Xg+7YPW?!|Uj5cQ%2cYnPvyLWa#XRH3oHw4FD4Zw|Cahz z+W7Rx)1B#IcWLT_t|5*(%R?roK3nS%+UwiXJhyRd!=d_|hN^~{O)EVQ_-BNsX`a|F z{eGp=)R7v;cHD|k;Zh75ewGOj>ZVg(mCC_eeyWcv{Qe%G0 zG)^fsI_fELSL~zZeH-=^R1!|!X)$Lixt^$y0^Al>mb&u_4$InR{ocTo;^fXN6<1{X6R!a%qQBst=F-rcH``7q{4@Mr0*?mU zhac4LiEmZju{`JKn!L^J%p9H5A^)ZppB1hv^tLE0cqlg`tH#~QHQBbv^hx}BttoV4 zaCpGy?-r;aZhaUk4z~@*LXL1jX8=RBqfCZymcP+9i5>^pEJhF+KKCyoMRfr>xGk)Ze4B zzld7%q)2(VHuwuv-{l|ce-65b1;>P541W_Dt2>g6pbxjUi3*}MZwQ`%d>MbmZg*@O_gXld|^z<7T*-xTi|&!?U@JS#o< z-jm(~RB#pr9t`G$UJ6|ZJs%#XZH=WUG0Wkk&Z)gKUd$;e*xzDY(fdWi3&$0_k~cE< z&+JxNwv18kHwfe&hm zejD2!f5n()ikhFdd~Ka+d&2H?jBw1e&$acjzM}3TYQLni@UqKhj_+& zy1T01ng4zHu#il8N^aeI?m4H@)10XfQtfHO(pP3~%QhVO-RHdz{e<@C4)D3f2P5J&2KW7s=PFz@`H3eCE7TZaEhhT?zu%=k`I? zb>@3=b?H3L-7xMRJ3))-E7UaQw|BBWpZ+CfL zQ8h{bjoM~755|EsRt`4truM)pZg@J&$a_(!An+CaaAs&E{giWAGJcSZmDAkO!${)RaBiTXuN z&}}S=Yt2^|&q_DsmZqAfB)PMMjxgEDhUt^lAb$~MCc1)YUMH}_UEVBotX8-xJ5T3q z%}&qKGXKe}mbE3PlRMOZo$=xX^8|YxR6}K=KgSG+^~bR2X3=G$r$zl2*&paxYse;| zY#Ys*6eQd>#?VXJU0{lrfsN-Vhm}{#6kusby~90~+;v=WF5Xqub=1|-J=C+&J6sv0 zw$R(LXzm-iBy^VU$@fgh&2g5ZmPpG@^8~YM8fz+GVsf~tw5g4$2e6MB@)PNjI78^j z`?wWGFT~UweH^g1q55JyTOUfp8M0BZftrK)+z-6&Qt*cBfew^~&8JL%ra#s_7`LhP zE3L}TF$X(|o_%L}Ko8PSY3sG-T4~hQGqvwpeY|Rn-cFmT&i9W|%KJF)a(6@51gG2a z(Xrk6%5}*zN$H?*Mm63i7B=Ow6tLy84-dK+{5m8%v`@Gx@_gjQs6V2oM^%gr4*xS` zlFenBEnea3FiBsn-tzbI-}B!=uW_L|KwS*RL8gDM|Gx6KvfEeD7lL(K#24mU>#M4K zR_3Tq?E>9pG$UTVp14$!C zM>X9}i_*4q1ARmH(uH(5ZHCV_r%z}#pj>l}v0O*;kXTUD)FS_Zr=AB&f(u}p-UXkg zAUMH6+$jh!hhD^QPouj473hg zX&mAflEHimUr{J0)D^0M!*EOZODrP2liJDMWve_w8X_hL)%k+tA8^p7a%S?F9OFZf zmD<5f_`#Jz7F|NOYFpI(=;Op9=WoP9!SJg9mRktv4F=Ib{s~_i-TW><+p6%rNI$N* zQGlTfpeN}M^qLssS5#(~X&$^zVa@M9t=#ZE^S<@8@m$0C-p$?Bz1980eH7@#XeC~K zs;M*&ObN4vbJ7-BlP8!Oo7bBwS$bJJ+Dh6V+MoQdZk>#d)dlNr^DEgdZ4~Slz2>T0y8-Wc%2CaFmLGWz`%^Q5P7j ze-IPzK;PuGI7(V7H2{V<3Eikz^4`E6XGg&_s?EN$5nz)J<~BiZH4ZGeV@7Vn3HH+k z)VgQ!>o|VhWtm{X95JeLN4UIXA;zZ`KbKGBoAENgiR2~wuzDUERgLZJ6n1A<{l0cU zTd5t>ZX(kbz#-^}k-qQuDsjpRpfGiOqkO9Gm9huXeTH^Le@m|eIjqAyFX-f1o$ibF?qoOH?F(YF)G- z;G{8Hw%Q3J_m@^ptAQ@?aqXS9MSlaP(>P?_g)qbUu~%?2Co#|uD~t6xSx6ExgvsJi zse(L89wC>MGo*vkaH*)YL_99!@KeEs3g(xP=A<+UC!yf^1d-R^C4L9bj~L;2t_=1S zth9z;=dA~S^NZ0I{KaQnHg^vi{)XTPJ_qNo9k_XZ77v!E)7Su1{uZgmALE+>UAQT{ z##2n-1%4kHKy2g_cLC~>DVT?;7=>G4$bDe1SUGU!W`PIT7Nhr-ePBP)H4WTFvIGoE zFY1!^;0ta6XSE!D(sbiHc!BR|F}hR#spZqZX_K_KY7O+_Bz2rR3@k0D+E06=*P>g{ zy~ten#1=l8A1UkwH)^~%0IY$*h%9&c(_kNU;qQ|A#7=fWpE@41nI9dN+2H!+ zHp&{+j9Q3ZMZog28gG~k4r^=Fz~4y)ej0y?KLh4cJH8nI3A->KKcCbi1<6P7cUxnO z=Hd31qhN)auu?8?AGv~L5Gh9|Qs`K=8tXTb5wHn4_KJqFFtB$6Y}OF2HJFPYthIXF5ioDtk&4LN!;J}SDho1_ zxwhmsc}Yf+2jHoWCSAx%(jTMw9qj3NzC76iKI(30QKph^d{e$BX-e+nZ$tRo{2hK4 zey>Z0a_g|FoZ!*x#yW7i@^h1n517j}#%r*g=8;O|92nb^Nh*0q{^6_hrTNL2pO@fj z<|A{7#E<982)_k0_$h>}!VH%o2MLtT(DENP9$;5aVy)>~y$XHJ@^MRy6KoV4fV2N6 zTVgEZWURn@Mqw~HZ?iY_0)0TcvlE8dkc?dHBt3&z_rPd^d0Ysd_Y>|V>xI*>D+R9# zYq~CC@qXQ@FJqsK(j?%-9t)o6JklRyGm$0JfYb9FZVB0dHBkU-cruvyeK`llvn1!? zIv9D`H1>$zhQ_EK-hCnV7OIY0K*$;+CLrM?cS$d-hL*-sHVEvRwftIP1y>8R2bWPH ziLWP=;4iU)VEK;a8wfXy+F+-aBvC>$lACnl`ms*NE6%~c7TfU82~V8H5K@J2A%^lV zjCo`#d5%}0*kM-VPs~$4-YQ%+#WC+gmSMmcak-r$Js7pCnu6) zs6ceRI2lC_8u7*yHWkX2`d~qxGX@cXPh#)cYw9t&2^WAjhJ(4=kXxjsv*l2vd@)ud zZ~u(b=_y-kSXpr{oD8QU!Hw?#UGG%d!uY|@<|~R@j4JelF^iq%(#chl%FWU<>3+_~ z+8D2}dW#^UcHmc$wJZ#%M6B^i*e|{^X6vi3IUV_7H z;^rC6`1$-sWHh6|*p1^y3B`B^nGUYT5i*hYf$?YO^C31A6)TH-ga_m&YSAQagScP( z!Y||sVFeZCuJKKvP3g~e)7i#j&d*H&=L>$eESlUELJ`%P8Y1mX?sDz`lK zQwV2`xfHdSzTc=Xb{6mZlhopDHW^KZ0lDpBP{GapMLuxF_#J$_FiOdCV;|DnT>w zgeA}w#(J?8ICKG@0zn!{3UVcwVw@N68lCtRMBy~BckdfpSUvFvcKrt92W=$OVpX|K z#t4-c6mA^$w5^ZEgo`&h4a!SQq{Smy6t?1@!iOHeIa$#Ln%3Q>B+~p-;H~ z=py|jw~O30(pW{#O^4D&L>Dj@1`j>U3z8{q6-xVm&?AVzmxMZeabqg_%CYug*_r!!HyF4m!Ykq0&8KM6g^T49k6v03p)L8B?$M89h5bot2tT+h^rLY~&(MoV#RNi6>W#qZjo{`7 z81i7c&lENChhBi@H)^0JIAYAyhhyg+7v>3%!P120fpp{UkTYQ4wbzU5e;AFxz8=O^ zph-$Gx#dY{nld(b;=S1B! zg>THo8Oz8VBbu}$*XVP-p7EV$#H07r8W=N(pLT&|>|?f*`HY#QjS<0;^$V;qDMDr% zpJ_bn!@o5)(9Prw*2Y;DtUI}VLIo%f9@2~YXMQ0M_5{C?jnJ*^pg3Jh)^h9P#Z|&m zHbLK^&li77d$x^BtLnET5dl&q{-4*?y{Ol4{_baZen$#ie8yz^Yeu>>^Io2c5vYbBMuD) zyL|`U!e)}?{Bffktp$Gm5iYj?7Y4&mCxKzTn5#mrq9&+<69I~SewI*%31XGH zb6fQ0%z^Q98PmvK)G~1LVEc?>d>N#o&V;7lBOh&T)oa=8Kp*^{$;HE!k z>DnOftGJK%8L3*L5zd1TD7HiowK*xvHga|Om&O{}g}W+jArIL@eT!k`=W?@HA-bGf zM+8=e&EU3kYmNDYu@p@+>IgHrW$0|pN8Y)fOW`Jh89z;Ei*8FFg35^wA=UYEP-sjc zS6F%Wk<0e4jIG=|B4a!s87XWKs=$p#US8)8fGL_n-*d+hn(B7mM`Aqh)7PL87 zwhu`H-_fwpg3x{>le#2`>uwaMZ4ouA@Nl;>BH0MKf#1cC3ut;-9qlr=R4yj;W};Su z&E^{sD>_-D*bM%=@vq*S>%_NU1%bHiKS!Zgc@EGEFbeT|{2+ zA8W`e^AZoI7sJHg=by46+y;IhJxlk2z3*Zc<1elO`DrAPApQ}JH_AcJa+ezbwbN_V zx+ddKsL!TA544@mMbsV)wN5~@naU>6D)gWcDx{JYEK5&f#ZXs2C$nj=(FEM&{>X4Y zKxft;6~kFJ3mMUK#Di>JrIqmRgHZd8f$n-USBe~g?r8xOJJ*mg*5!_(@^nIHx73hm zyk3(~ei3J3`L$#EDbin<&9!8~s3$(s_ClP%v%Ks*Nhiax(=&zlMn$MxgGdWLS{T5M zp^=;kJeO%)Jq8U3w_8}vUBIrLk7{&`@tox~o&jec&#tg-IK3~UdSF~n$!(Tktb*F?9?p6hb-)ngF{=;V#5ky>1{sxr6EKFG_P9JG zhVMp}aSmFMBnY{=!cbPsCe66FY%B8Wg;34iU^Tc|;B7U*>W*YSoD@%tG03jB8B@WN z97bN!Oxljr;UbNjhK_zq%m95oytF@s$d7W$rwlV|f!fURPQ5iGP7xj!28u+SQd%TxcnC%9*jASr( z3Q++20=aD=@&h@~b!5v^Q4_x67C^f?g*=0ftT{Ibd45^_t#Od=ME0Q1TbOT#Y~mTJ zfW_npayJq6csYIqDzKqYZ&Hk1cRayiob>H^!j)q$jq?0I$dOI_8qDruM6c6qJSvHk z`ZsFhbMfa9yV|pQS}86}MCGloyUW%ol^iwnl4~jCI3JT*qCdljs|k zi*F+~#ahqJwip-5YM^3+RJXQE=qRd)Kt*X&I+64eRvNMF7VxP{Yz@V25&V5_68lR3 zG``}MTeBW?B-fTV89!J(K9ujx-JWR^VjA1*p$wr9iAuCxZaGhcNOp-%8 zvsL68lo5x(VI0owAU>`Zcw4neb#6LsXe{Q}lOwUaMF?M z%QxY#>z%Z#TsnUYdfN`{lKPiXRA|LVV^1v7_p!&^ZN3u!%s8TtQV-EpatV1h$J2Oq zn(pN_afJ}Z-uPGO=Sg8OQ%e&QeWxd2@9q*!rfS?qU!tFJ6Qm(hs?io0TeiNHZ4_I{ z3)uqYt-gL_&sZADIr%Y>$eORc9xvF3bz$;UURrPS&AOU{c<_6|n<9<8hJR_sgq z0-3-a4p6v#CLe^;M4-?gv0p+BafnfrO3+FLqrWf+2=h^3bYlbOIcaV*0?O0^EYWBD z43f(JW!1P15>I-L+>GzC&n6mjns^t1bbbhR)xp_0!_R^zNY z2o_*5?lkF2<{I;j8$dgF(vcLzxwDE4Cr@ZPEdng6FOZ&EtU0U1%_9$yFZD;4`V8WQ zM07;-!p1?=3Z=M#I2(&XY1Eor2ZrNEz74l@zB@sf7qx9H)F4c$9>2y^RE8IPkPr#u;`SnwSjY z4!k$uNe_U6MFI&a0ljYz=$~_pQbvSf18P_rSws&gFYd4v+*tCNtB^z)U0 zkqtr(RvEh4Z9wE+!?B?PwD*aqv3F zM;O`63-^lA!0t*KU)g!+X$u2!>I)~4IoRJjxG0Q4B&mrkpc;3VYm6v(5w8W566)7m z+;2E+xLFz09wUHwRfKLB9&)4ysOMm3qe`BVBX9G>z2yJqG zIBdZ60^?pD?m6{tg4pw-d z{DJwrgE4-Bxoid;NH==n-6!MylmhZ|5RnGHIml+-V&xnLuFfGNDvTcE609p^qNo$P z8wH_Ae~2~t1IW^IWLCf6sK8srHVb;6MLZ)zA_oRsU!6G0UwHY z82Q@RMR8m=JWplJ>RdS7yur$jz%xYQ-7disU&ZcthJC&So+HTQjDFxzkB3*vCFH{i z;H0~->Vmljc%J7*Ka9k5%)%MWbTU@*T)flI*yUy5co2iVQxjeu8Sq3Yi`B9UD<|Mz z@-Id+2_7?VFlu$t#~F`^@CM!)`SE@0fyVM&;C6bKu@b0sKg9AE*tH(xA<*3yh*66$ za*J?_R5@-cYQMY47!&boe`0RC;`xW-Up27;Cqd!g2wbZc5{sD`K9v;cFP8Ukmt~WMPy8-Uf%^ z-w{AeuOKD{Zo(c9hk?M2!#+IeOYGamh$BVexe#zI@ngl0!Fw)%wcj7JLs6wvL5_bJ zuT>4_#0m5|0=KvS!ur?&)Z4%s2;8WegP+(4&JPQ)_IF?meq-bk@Qwm^_h#ewI?z~j z#s0~McYFfg3&qjj$&EFC0PkutM!W=&(G5V410IStI29bkQ|!evI`Q)wp#zeR?=FSs z*nu@G{qK9PV%(cyKJ#L=1e~Lm|_&wnI))2^SHLQSu&r0Baijvq>PvOxLSa)NvPP@Y8E^v#{XEVz#_+3kf*n1=hTPh`kiEQU&oU z5x;lF{A|Kp7RDLiM{e-{zS$peS_a&B=A(mg74Ky(*4za={aXBa5c$F+oDZk5Zbsw% zIWQN2e)I*5R^S%UKXBp(?z0HE%)8*-k%sS+@Scuh?Ljw@vL>R zrvkTD_k~|ZKdhOm*i9|5(x>D3YGIXc#=B^aJ-!rsZY=gnU|j`VXM(WaH({Qf@DzHC z6C>b2ejRHoaN~O5jyWG9?jo$pgLpS8oPpEL#0@Hb-T zOE`wjM9dG|>0Sc6W+Z+}z>6~AlsFt8(TI+T*qdW82Yrznb%FQCJN(3r*y(#Q7Ewk4 zoaYlTH&?J`1|zmL!Rl^?XhN}%hH_gG{R6lAn(^$zklO@2EAQi3KVWqi!Waa6W-4I> z3ZwEmi2nXnjNMQC?;iYwL`0Xf7|p;f@4$*MXBRP|74bbDM8^|&nt-=Q1{~FD;;Ht) zx8o+x)M3c4{=@7jMwkDswjk)PpJ2wy;WduJTWSvG&VvkOHKJq}#EZ@NTfpteg;>!B z<9i8_yE8mqj^Q&o7@-+J$o|E+oy7OO!}<@b z9s98kM2y&NtfJjmV@|A;^?1c&h}BQ=zGh%01)SjnyEAZaXhn>ff()r9R%r{&^*y+e zn9#p2_&+XmUX0*7?0b>hfiZuG{hJ#|syt%%KEF*}bi8V`}vMDmUJM4&)pktzKH zZ-Nha*k1JSH8 zI*=Z&A*qTiHHq7UGkpPK_i9AkR+xn~*k_yZ>a}p9uEP_&#N0mR9^jM++;|;|wY>-b zKN8L*xs58wYX{+9>v7V&!fp@T6wnK~KqH)vpDOMaY8>v zMsgl;x-z;%uUIsEA#Q_lISHz*VJsP+`GH>NV|0QSqi4Dkb5)XcMxXc#D}?iJDKd)P zTvd_`w}5H1AyHdgn@ zpLekqI2Vf`J86i$e-h_NAB<~jR7hpeNzDK!Wg=Vyy0KMEL2vUqolEP{_xc6B2R@c* zGqg5Zbub(Tt2fkX>MFIm_DNj_{ohhL-~A`gwxEg7OKIvX^}bp}leKngPyZHZ z4__)({j2>a{Y7wvTYbGdR1i<$V=w_2x}PsCP8TnV9#N1gO2wsvVgg(y5(T(w2-oh`Te@Neu z`A{%c4Bl_|SWj6ln`fC`N{hjijy2NYNVHBLuXR*CNZY=F?bZMq_CI|dZv$TyCBMIe z{}WW>qroGeqW+_m)Ys|_(GhrRbR&Iv#-A3hh$eZ7>2LE9^Kf&#IRyIe@1|jMgZN@^bCy9hJkpp^_v%*s$1%5dN#h=1lAxQWaoU8$FYI>Dw_)obZ_^3i_}CBVm@KVM}?!R&ExX9{KI;H{Fz z=M~>clg&eIDM9-~7KUbqwhGG%s{>B(xzG>6nfB(k`<9aChtj`1>UO<__K&Liqx=hf z)xAlc6i+x9*cFs!$~0vxE}&?m-_*D3ozP$Q0SB!Ntf&#X)$Nf%>?aq{X;=A4!U9p0 zyO{2o7MWU`{xLN$M_IaA4wyTch#V&M5|f1${6)?WHXYRm=>KSa!1fuZv{zOsW&HcW zch2%hf+Js0v+1Yx!tk%UNJ|6tpMj5II3ad`Pc#&{NZ>Bwz)h~5xcBIUj)l8aHrz?N zV(#W(AD@9^#&Ob$SjarEO`3qozC)j?*VGS!cXdKN1tqc*-WD=Ou@##JOZ_%{7om|D zBgyhdd7G)Ixv{xATpiZI3ueB2TG}Iy1dpx;X$lmjC2K_c=$Et+S_iF>R#7VeM(G62 z4u-;gJ%&!k%xs}CVCxJsMb3egPT?RNQq7(V83lqIr%_{PwY!9#49O-BSDI&Fv545gv3ho`aYtz(rVx6|i5Rc803KjP#e(85VVEGoMP>|guIhXVz9e6bpTj@JevB6C!A)s7oUCra zE9^R=<3xTE92+(pE7?{0LeJE!@IB(dv_0bA>JL{{wThlYR{+H;&#!{Z%vQO8xw6G( zO|)*euC~^-jV67+#z%E1*ul&6 z>WGs+s77!CC@%{r#~YGQh;G0>`rZh%ED{g1s?D zd#(;vfB5J4Tln|+>#M^x2e<$(V-YzhRF*4SrrJk^oDDk@(Jg9z^v;+Gu@hr=#pH?3 z7x^Hpcu2afpxFifMH_vzU-#~E*LH^H49nV{nVz{f>tyzsoHWM{m)W~VG1P2*9Q$hQ zB}qajI1X%(8^~1hNu}jDFk-KZk>YtFM`$fJga@8eXds%UDEXHxnXbv>#XNa^Gx$~^!L~1&{^OqE8v{h z0rOTejWG2!nGumo%1h9x?Y9n}P3{~I2 zN8~uT3+IeK2`AK&PMfw__XG){(cupwN<{UE?h_|w8eS_tVbeTM& z8&#)ok|)-+HD_E_yG&}c zSl^$_N6oS3wx%(1wCs|8OUI>HX``4d+N2{Ak)6^^sg%UT6mVlL;$hz4I$;Le>&MhJ z$`J1&w+XIhV;tLao@f7+eIoncoJ!7rU6nlP-V~*nwwmHD5wch4Bfb2eCTz5Io>j7r zxBP(eu(zo?)SBCb-TYbd2Kjp)aMSuCw>yLWYI(RMoJW1u8M*8U)R9Y2$CgLMg^ZWA zN2f0w>%It1$CcOv9avFd!fU~wcq)7miCoYWW4>$dWvOVnZ(e9FWL{(1E$5Ru2|K~| z(AXYYRo|!f_Rm*Blqy^kKRKZ8DN1LA3_dQ05|r@ue6-`X6#3q8g9aewfk zVlKIjd5^VeP-aLeIP;Z>3X8rNT|OpPOq=M(k?X?WhggF|>tnffHGiI{C=pcQjwp2an z|E6?T!j$K}cRs}ztMr5_KMo#D_x(-Po0x^!@VXj{T<^GkR(}VLVJ>7a)seSZQB@V8 zzj5YI)92~O^o{x$y|MlUvEd0ca~st#?9=c1L;8wYfm60Y#G7E+Z86zb1gD3(!WTsD zh_Xai#K)JYgeXtM8^lpB6f4!Fd!#NqqptU5xz9lv)G^zVRXH;v^IRs$zLay$dD4Bx z>r<+NuL3lIY!r$}hvYPqZeC_tW*KWKY$;(WV|i&FYrbO2V>%)CkqgVeq{Y~K@$iP@ zrTcJHYznuV2k5a>;1o8J-qY%+n-v|t1s`0;otGTd9H}`+bGGJqavnK+&dzS)P4|VV zCG;3(=B|^$f-GH?4?=8{R{MI*?N|uS(9f##4X`I+ukoc!SL+&to-vLLmR_JLhrw?^QYo_ghHg5>n`-had$~yQaTvfh9@v#aO zf!|+8-K&<=@GkXL`c^o^Jwg>+6`Alt^c<(LjJYIK3>&JE2{W&HQUx zF3d2}wa>~}Z!@>waW^|Ct5BvZqj2Vn%$%&PIUk)1JVlh9>J2>+`9&Lkv^Y@yW_oLG zU|DE6ZaIn_@t?UIyp49lNv1v2=&j|~(qyR()Dh3{u|TpRBe)MpMtfQ40p!cRh4`IZ`LnPV;5UK5HRc zyls#5q2-qOjA^`FO8N!fXMc3$#~KURMRG?9= zW_UB}W=jsc>xgHJQc=4_vyIo}Utycn#gu67Znb zJ!DC~BF&S=N#ms^5+}_OvxExJ&9ot2aMe%hG1>-yCFPWtc)GdXJ3i)2${C(>Bqu#* z8T9^z-MYu`3s+5gQP!WE$xjoD$o)*i%^fV6mWI|&*4)+=mf7a^rYQNLxE$`9r@6jx zG&qe~e>ukHwss0GEi<6@O#+T^hXuNgH-RY4A$w8fzXz`50>9`q*nQK;6fzLp#3o?> zcPI0pocIY|dl7yO?}U1(v(Qgy4zD?e?0P(O8q>&qplkz-|DY6ULsuiK9gb@JG0v;0 z>JYWH+8z3_6tyz&lku3Zf|#kDh>Qyn8<*nrn+soDD^%|Fw3ljs_}N}TMIG$FqV!R! zE3K7Ah`f80g3zIbXo_A7?lt+vc$u0c>o|K@NJO|h@@sT#?5)^R@Rf|pRVlV+^yY}e zp>KlXY)+FPHZ!)W|9LOCwmDv8J2UfTJWt!7)-U}+M!~Ga>qu*VtB%N7$9&MV3EoWyOp&Ix@@A=}^tW_b+5l&RN5W&?32jOR z?m8<>Cum>&Ta^*MqTVF8#pQF1aSV6daWr%ObnbJVb+`1kf_qFI-9?`npGlfc1Ue6CCWGJxJ zDL^E9pk}NMY$p}`j%R!Yuz|M1^XaZ&7ylIJ!E(yaiUw;F4Pgy_ziq+ei_L? z7iJZD(_qfSpZF#say?p-iqu5oXaQ)A$^sdf2o+vCT8PHr^F?S0L_n@fB zAH_DNg_g^0^ z>>Zh5n73)ElYejgeKGZIdR$hyoH*xTcd~DgR@x}aX9{0fuwoqG& z^^0|^b&92`xuz*cu4_7IN;Ex_4@x(HlMaU$Rp!&dIq)c;0&#Df`u;biHwkv<6oz9Q*B*`KYCo)nln`8DyT1*Cp~- z_epnsED+#+a%MAjL4x@-C@M7w|B*Tx4hLnuMY zK<^f0w19e}By@E*z@FL2S^>e_0$imtF#e+OV#D~9 z^OS2on{S%;ou{B@x;w_rxf9%{;q4gim7(Ij2{uw0k|id9C!)1n2XoiV zdphr|yzlbxd9KA*$+bG_kFb6A&*opEgDa_D_GUPKW^PNLk#-uFl$w7R{yil1c>2Dq z3XcBp7(M7?>P|54){A$gIdW&y8~BJ_wKcVSYXjtb7bZ@VgJiCQ0EqoKw}?h4;ad?|G{g<2k4>)M+I?Fm{JM1rzy z@2sONi%s>U7yM1`Hgo80v~!5%Hs2KQQuq(2`#vZa{BP6{y&ZkR3WNO~iG4c|KAFRy z@azah;9g*#li@P34O*vLa8)0H85}|n(iEtpj-zULi1TfZwj6m)UA!vwP<(7LkE;?S7B@J@L2q_sX=+na6X2-O1i@ zN+UH>U(DIW6LK~4LvvG9v}bH(f^r8p2>umxEXW?bH8@wu_28vJMeJ$Tftb0CmPY1C zxuXD17Tc;XSF0#TJ^ft=9our!vP)$Don_BzmenjfC8xS;zUPSVnLk+H3+{3kA-6P3 zJ_fX=qGg|DjU|sIpZOTPx@@8iK3#pFSGWTO1UPiG7QF%wlmwmAALDEr2{dB>Ja@M$ zui>4w$amitr__g+!EwJ?d#yibnOqTJH+8gy+b|0SL{}+ z)#Majai3|FHcTn!UG4q?r_<}MQ1?i8lxL84gp#B_*Dtd++-*3zOc94jUr|RakyLSs z*c%nZO=t-x0NE)3#?c!1+TVr#^{JkyH-eJ%12V^a{+7yGpWmC;Yxk(GP}gku{55e_ zbk=vScUoQPt~7V7w;yn_QEE4R4;(i4i8IWV>?h%edNA^Qw3KUA{DVC8^Igct@+Ril z9zQS+lt?_60^p#M`V@CT%K_=V^5|%dqGZq=QY>go|nGMD#r?utHLj77@Q}@TWVNO zTTfbxTdP_snOm5q%L}B&;x7I>mk8uy32qS>0`*mnu4}zCURwmOWnD?}F>e>Rdxd*{ zc}{pHd2V^mdIu
    azW$PQ0zq_Vd^6>V|E#OA=+OF)g5;m#7dm1dlCUr<|Cw}TT< z_b1+*pEr&}i@jvO&Ecirwn#v#?ftud&Y%gz|&evYB+A5slJ#fgrLU-<~1?Ozl&*cU)7{l7r zgY9cg?EEe9uy88b4v_=3<}0n_4!E}29XVmN2K1!Pt!q6=e?M2F$z z!$>m;YxV}6e#-Fv*Q{a_)_xLeUW1ot(Y!U&bv@55!X4*8I{pAZQJw7fI--?zkeLtR zfZObP8?tLB=Q0^Oy2knk@D!alW`(b#u|FC^Nejtm^n+?23-_^0B3S=$YG5;=$E^uH zg|Qi}irAfGuxInTR*m}?&f*vU3St)$(JE{5&K_v19_ZP=;M5lrYx;|wq0y%}3wCn^ zv@i;pb{$KxDqj9p;^@J`MdU~#H}M%sn*l2%o^wtELgOl4XWc&bf-Hkct{-7v&@sSGz_)TQ%7vyYiYW-~RN($3B z61Y!;mCWL_-*C#&{Cg$Qq|@B-YU;B8@cpYui&yN>nxPYqHpm4xA7fqhq1l`K{e-)j zfwVi%s{WGOjwdVhfxMkH2crTJ&mYL*HrVhj;rO9gNA)tm7K_$|M;^c z(&RbSse_T?``{XtKck`ApKwiQG;$0a{tLPoi^luM3deAA^N8zNNY1XT_ADHKAGuNi zJ#rEmY4w)Ll3LGS6cmXU4N(^gDa4!e@ZN&l=}$CMM`BgUXqL>}a8778mRMdg_i#P+ zcC8sViBQ@Us%1ODga1KoUC_SPtfs0w$C^Ww4<5)17N7>xWd zzK1oH;QXV+&Qw7Tm7XKZW`KH525*r@XeQnldrLV$loSyXjK_4!DGG|E0NBgy(EdLr zVCEE#&?%(|m4kh#aM(<2#+o8Ng*v)+=&2oMKH_;J;jIW_hUxLE?~^eILDOdd{rHvV zEJP=67RoSJw=dl~<}igcpSg*6p;bp)MffHb!xLG@Jw8RwZomfX0Bw$EubuHlmw~gW ziH6)yw6i=BzQ+F+OeN8Nyv-B;&C{R|asMkY*0%2h!U zPv-vepoM;+aeSzLS(93STZG+Ob9+A`+d|>WUvScIe#YvJaa4E?g5{|A5-U6-*~B4W zC(|RpC$sXE?DG)opJg5gv%QCScbt(1G~;MEXd`bdj$B<0zN8*|@^ z!5b~1X4WYl;FObu=S%`9A|8dxpP?0hQCE=(yZ->>#5mY*#~M6x?<^V@e;dd1&HN| zNRTkT-js}&g-K4q&pC-s)6i7Esi=*&CK2)bmR?OWYH zF|2JOalpSwN>GGoX7uB3rHLejSmr*0jcCWbt{P z-;MpnV3&`>TI$b!m-A;JvZ^k>Gn=2z!HtSw!I~h+dUJ~P&;{0nxvNOArD%vBSdUH6 zRrH9XV^SO%A{vc+61#FXXB|bQeKYVmEr@yth`o_mpLpjO?4ysuWP0s4CK4Wr-!%eT z_YStRj^+Jcs0I$O9XGxLx$z78G#{C;GxU?3#k2P#Z#S~8$=qXC&Py;eBW=gPQ@zj( zRgkvZxVehR#31ydH9z<_JKlw^t50ordQRjIcG4@JYR%*-i@)@n>~sh0k$zYjcaXf# zxF0)~(@kz}6JC``w7)%i-~_bQADXxgE&E^v#-nW`k;jYCB-TX1*X);$*u3K&8f-q6 z$_lK8K0MuWq#m;Ww){>zK6UuwYiLd8Ou5q^2`2KsWt+eRH&b(pv2Wh$1@33$zgm6QU{{Gel16=o1r6;=4>4pQ|qbG$mB=Sm_a3uC1^ zuyogQ>!-1`OCs?~!?(IH73}pd-mw^Z>yAxWADWKkj&5)YH?Ur_>SDkM|ki8bHUTG`-#w)H8Zjv?=OYzx&{ByvJJ{oEz}z>icxp8=j_uGTxv=|UiNjd^YQLia zm&1>V!g(T}E2x@IVH&^%GWFHzWad;}$!S3Fl%{gPlxoTEsqQ*X+-WUYn-|O%zee}J z9iSl}(v$BYnmRkU-b67knZaY=lBaTim!Vv%^Th!8eQ9)DxIk9k-U_5>Ju2_YP;XU` zKXXy#cZB>{9&(rg2#~Iy_XxvKb?$ zl^!#9d^_?bsI6B zBHZUqDkC&7nE8lP8)Qkp(|OS9Eb$4X@hEOH1l{Wrs*;Inj?Gmb9rqZoCjvRYnESX% z)#PBR-Evq}qIBs@09H#Ifk}-f z-w~|uWzNcTdV+>hd0$?Q0DqWGJFGQk3d2w#pEO1}V~_Qj<@?cZzW*!#3;w+CZMsXPM7FrVF8o#EzTb6rdAinI+$ zufzMp`K~n~?id>W3=(QKpIf=TdSG+U!z100I>FTb*g4;!oXAC^67p#eZ|H=DqEbWF zh)!Y`__#^c!W}Vzs*ZE;dKa?FtMU9^z^%RTf`5Tl%x0XX%BCAtLz#4)`tu^xCiZ3B z^RUfKy*xDm`{#XA(B7ZXb)O44_!lQaVM9I ze@1t7(A`u|UnFXh2an=4cKHCT>RD81ddVKu)9+IQc|$u17Bw5wIC`nMnSC-Ctjahh zoQ(yY6t0zqdM40Ox0&MaIO}xyF9-+;Y!LW8AbUVI|GU1GoY@_pl}}PIophg4J9`1d zOL{sHXQDD@x}F0+B-yMk#!4UL61GzIi}q9YZ1yR(5lTn7s#Jm;n`XSG7IOf-Ub|`c zw9ZuOCW9_Y#`DgI4*vulZe_*g@vq0C6H0JHSA~37730Y?PZG0=XR%8}`goQhI`WCC zlMsB9Y)HOw=+-CLP|L8C@6$ct8zIRPRiJldl^#h|&s`j@g+P=VJ zFNBvj1YMj!&Z3|sG23MVSmH@i0{U?`l5-wCZ2dtkM}r!wjCDT;D_k?exc}qWTNSaC z(oi)aT{?nL$!PqkMV zwXwB`*IAZiEF89welG_2n|N?yzu~a8;J}BnpLtmDG34~?Ayre@#RO=3BGNpA8D)F| zWjTs&4C}a!RYdi3jSp3>&H@YHp?i2eKUtjw=ZyOb?l^^<|L}hmgr7>zSf$q zyalP+SOl7@vmU3*oJa%gs)IsLQDS0N3Gk)4ls?L0>MA)O8U zwHjzsYbx3oBR!T>6gsUNXBh`QSTk5hVgJ`4D>9dE1WV*NIY~~d{F0B#-Q}I++{%gr z=+ZUKSWNHy6540g1UWSpMAcNJ_#Qevk3*7dK+;yko33b_fxFX^8#xNPY6lWA5ai4S zpDSCT_6X$3G|rN4RJX>`+BH5BU<7Z z7_GkA45pbJ0D0D4J7P4Ky4t7uCIs9KYL=#7(9pmu0Z{>E1Ah6Xbyl^VAp6r^pRKvn zAKrstk;^l4r;`?FG(t8B*wdHATe509=veAg*C&r-j_tSnj~xAY>5kYzsAy!=x_j5s zN&U5Vj`{->?g_0Wy2Xc_zlOgwn|cy!7Rv!F*Hu`N3$YcSa0aXJK*pg_Y8dUfvl7&I z&(Lx*U9Ka0$%PcVPv#*b^=mV+{;Ly9C;%Gt0eYu2S^WumZ?uX_zi!N-!`V49m9?mI zGw5xQUhXANq+%+BilF<{&OeZ@QAv|tiUqNk4V29@eC*GunyxlyIRYPN8FxDX3$#1l zj%J<}+R&eOCUtaenUAnW{>l8WXH;ILm3v8-$uZqAZ&52$M&GY31FyeKi$Y>np|@-f zrWJI?v;UtF#7Z-o2eA;IF<)&va-kG;kaxJls#x*C<}+g|nqt20r%%ydsk`}*sO)Wm zW*%mg!v6e?7iINM9f=qG2S20+)>tw&lbh&AZaNcmV!f@9fZws3WTTP(oW4#4s3M(2 z=WeI^fGJW-)bm;&vzFZ5(cbTAz~#Vs0kix9eaAXiJFhxBJ0l!!WrdVed=Bqa5L58# z^Dt|x6cs?zSZ5mJD!A&)ppFX4V{G{y;~X>W@0CLG2Wr?G6A3zKZiSy~dq%l`y0dy` zsBu~!s>z<{PC8XZVQu6yR#C-#mT3+L^!tXJ{_vUc>=)uCTT!|yMjZxVRSD^L-(ukbcu*Ho|}uli*H0+IT%1n9*_Au78Y< zXo;ctQy1|EpF!)H%?b4B`a?}XEIqS5R0P)1s!->9K`Q`ebC;eQ4y}Sud4-*}5hqZU znT2@SZqCQD@EfC@ni}b;*eOZe>^gCSBqVuv{7X04qWt6zPH-+oIgQd-+`hc)GkkuL zN|IVkal6WNl%ncr)uA^P1}T>q9Un)9z&F4BzAb&X`4;u7;3xRTJHl)gVzaTGC- zo`$3+(F?4F+E=}%{!*QAVQ&3~@ju~>6vwok%C?$H9eF98AG%RddQt32ZhSXb?AugB zbYntK1^O}!p&xi45_b`@D~!tEF~m-)W6$0e7fOokl9o!6Gz^UCRVOA2zcc8)rK4%Ri}E9RW_BfevYh zcPk)a^75vFVkta;P1Isu7LPy)O~F2A6(Z5Q9f+5%q8IoDu-}1jN{qowD!A}CmWeeH ze4u#`&od`9QWd_w#H@<@biL0;kD-I~5^sP-JQ=&QFuvz;vMFDM>EIY1LAx2n1y~r* zur&taQw5q=sGGJd>IQi5ZtRQLD?Zm#JfRR!@e$zSt#0R6sTZ4JBRvgw0BqNoz29=a^R<~ zs$OqRZ-}=(a$|y8Piv?TH2!B!rCw$RQwTEK1}jVDI64XpqvPfR@_SQ^VtP(y!n9Q@ zsiUY{>O?n&Um$`rqiL_|&F~HP5Cz^NG^A5V4AV$@Qy-B>-hdxlTwF`m<~%<4NzNmW z(H5^Sj(!%Wk!^lh(?{{0XA+^CO-kg93&McYcC1i$p3c$OFWtCs;^+5b*(5B!Nu`)@H(f6 zBAFHrYXBPN0I`B@Si%e8lofa@SBy-=Te=WkzsARXqGlSa-orj4h)~6{{*Oe!=EHw| z@rv#e2N{T^TN@b%h88`So0vvlWW*Vw7Z0%|#`D!yeAVig{n_|s_!2F<1=`jh?6fsi zIlK6lNbN~v>1JXZeYyK)=!ybV(U+$RFG`v%bpTcNkiH0i*heT4v>mLZ0y3pJ9b}5} zw*RBf^QfEUS* zr4hu(f1F%-o|YgEr`EU_^KbXygWP5E(>b)|R4k*m-Rn|gwVdt(f2sL7B$Y!K zE@1_QK|Ho4Vp|y7rww!8XY%a**qVpfeM!TO|IwL>hq2Uf1%kSdCz?=}Sl<=AEgk+W zL!|uyv9yVBy%WA44ChaQ>o3FeInnP;*~ekBekHMZ4r80$zz*n2yf!cVF#x_FfaG65 zbi4%Gu{`|nn)8{6?i0DMR>V#k<2{71zHap9tVNFFFOkZ{y!9=+9!rGyG0{ms&)-#{E}ERm*@ z*ySy$f9;7RSO@OjMukBs`sqg!3%Nlw;ye8P9UD*K+%teJdxq7%3)yyt=vg9hg40-i zHTW0{5}dOv)kY)C#gJmw~bUoSNy(2)M%Z$*Vl z_z}b&KT{<;mnwx_^v!G}f1=;vPipDYGZ)NDy?jaOF1fMN%s&W$_v?^3`NFg7a|>0t zofe!}MdI{bh(8Ks1+pU%+C#}LIJvDv6Ef4|L&ZueN!+O+69?AfNd-gg?clgDXud1) z*I43XR)q2|^6(iv?gkBHCueY&pJ@T-RAPUT>@Av1`)cylzR&x<#PNSwxF|kQuO(v3P>~IKi`8nqYaMf!4%`Hemn!GqND}J928nQ)9K~h`9L@ zRS&?|UqBSFCU#5&GeV0|DUb;Lejx&3b>oRfj&_Dl%MzDp$Xzt&Cfai!p?uWCZVJPW z9#3w~>L$Dxi>ohD^H4ZP;mO(QV0{Lw_5-J$j8s#J13%;*mUAy9k%RxCSH7Z~dT=(c z=(bS}`#6-IL2aSJPQEcu%k!~@fzv#c)dpRh_7!E^73TCc>< zUW>Mig{L2*|BAwmR($C_oZFlnc4_j=78b!yp59~!3CQ;SaA_@mvMLggIlP_6N<8nTx2S(jE;r}?&ZF|vZnjc^eg^tKwX*el3H-%o4Ktx z`j+}&FQr9WJV)Ec6G0lmbF$&xN?Vj|B&_@xVd`JvDKIRJM>hX?>MmCE|R^y z50`J^#v8MbES$qPWZ8FUDu67s)tfpPdd^M`?lIiEi8XD8a-y-Mk8!GRp_{avkP{F3 zEfjyAon7MzpUG244*t#U+o0n_GNex4nH!{m z!Je!R+D+JV9q45|IHk?p?l1P~#jB`}#nBzVy)*hMfZXUgboz1D_nS;$F0emWpzL$( z!j2vvNX~5#TCOy7Vr7{!u%~~laubxe7zr|!e?_y-<8Z_!{)|hd<_Uaf3^MTud)>i0 zucXR11uw;_%~J7gbg1-oq|tutx@*UjYi&%rVK;GP{wkkLdJ{sY%i0@|=>>@8(MTWx)o*K`-DkDCPstml)8J?NX>8&Dfe3E-HL0#5EdJSZC(~xoV z@Oqk|Zz^#@e)xWSptW_-++tRF9+~=uo%+HxtvK(#SoxK)5K3?kpO6~2S#?Hk-{PWD z$efz|Y4s*a&wi`%lT~^$+GA<-W^ZLdJW70KMZ;xA7Jucwk8{qa@H7MA z!Ry?`dhYZY=bOYk?A+OJEOcx27Ov?hloyM%x2P?KpFYDGTC07_YAuQTj+1ym4ErV$ zI3v-JVo-^SY_alsQE<&{VIfXwMDsiVNOOM9Rj%pZoZ&Wc;TOWYk)7 zv&YdA)!6-1-kZ!CR;7AeWypEeBes^8{~h4>PQkxFkQt@1ajkh2=gG~4B7-h+HhH14 z*JcgLhde(lY2u{=RqnT4CKaIP-&-9PZ^G%-GOr}N2D?@(Wu7g&xY{TGH5eX zYE`E%@81bs+mN6w!Q`CeG)+!sIy@7_`#Tdq`v%8%A@X(#&72j_dnlMo=E*@vGs)^% z@3rDIJ%lXi;r7I3YO?;9@ctQ)*9zJBG(tJ?C-^LTf&$!dLq>Sf!dzRN8U!s5C4w2r z`?}D{CK}H2F&Cnj&yoMyjrW~~-?g%LRxNdYyo+4Cr!wfMRy?;Cw#0rc$Ry$w8N~8P zfsxSR6zJwSC)W&o%~!H!i;-1DupoAjC4E6p&{o(GUyM#r&Mass!RUjWXq<|;_{+ZA zvg?XakW;+C=}trUMzh1^skn*KP`(Ov)&NNwfvuc}h*xe-tRXZJ359fq(vPy*chGM& zc%%iZoQw<+gL%`F}tgFW>P#ROd#svkqT)HwdID{_kBUDRRp&hj-Kqtd1Qwhb79#u zrF-T#xO)`#*mC4f0+Ek-XvbcxMZ^ky_`Vi*o9T$eme4K{H#<^2Af90Xjx4$_MPuQ%JWDz%e5nXW^?GNYzVlyaLVV zM!uirX`Uc`ngV^X5#$rsKaONrjt7(mS@oFTu0x7R$dmlgN<8x21%ICmSte)6ixuMJ z>wvWr%5Q}q-3-mt9sgq}xN3^b?7#^$@;rr*Tl4wrEO7Haau-9O;7BZDv9oLvIQ|RE zMr@iMjTQ4JbwC-pV;?YF4XUlrXA79>4{h&5QpNDTL_CR!K<5h@;0QY5BNR0U>2n5Y z@C5oyK>FW?3+F*0@o1}8JWFQIZ}9YK(W$Mdzdgfq+(qyIgD%O&J32ErP;7@9h7=43 zd&1-1Ok^lEKGtQfw}Rh}$10=C4obd4ja(hxVZhyKz++*mw;$necEx@^NERi`x)OIndx?G-i_h)3f+}~+A0Cj2EgDM_pS&{_T@U8;H@T5Nj2c#0$Z{JK1f$^A}SGvzz_Yv^Khir z2ejF5q--6i?@uW5FKmpmz`H22u^!l}4Gz{q-Sd%=!+4%Gd_4*rjE0xfp|L;(Ri|6g zTMwbl!+71o1093S+6H?ojuX`5N$5?6(ypU}l41Ehgo|waFChy*aB50@fZ;TB#bUT+ zGfz7g`f0&;pLs?DZ8aOY&=f0pFgoZNS8-yK)a98)Z_i*TqgSXdEgXH)gEOuK&JVD? zS8&>m$iYcK{#U5}V?18g0^lg%-5-yj5;RZ%P6_9kyYNH}d8R0yx+eH85AGK7>__<6 zf{YdO2)5I$J_cwHM)UqgzvjY+Peyd?9CeZFh>%!_EB&IbYYEb@Auw%@+!NE=DgfCF zSRrE5%rIzk5ByzHO6o;MyPT4FY$S z(JqyNZYLyKK0K%5R072k`S%mS{ltXN>v&Kr$*Y{D*2_*5;Rn&tN6d6J11Y#;7Pzts z(b-a5MQmuP#FYfE_Qm$j$wymgFC~$vEa=qx#FAV@1D#|;4-!9b%^ZnhWC>NG5b5#M z<9Lbcql(1fuLZI}qjM~stgWSSa%1Hrd&&0o9IdtGQPh8|J(OB1>QS{1$`BIs|gJx67kGLw#80V^bVQK$J|Aw z-e4P%pWjSBI*%v+2f4rc=!a!cd^>6+WMUJm0v)IjS`AjF0K-4&tt`d1%;m&4(qP%u z1P|$fcn-L=9h@R&c+P-ZyF>Gfp_P98JeEAk5i;{bsT!WolOLfHVjfwtKY5iQ4iwL% z!M6NP7n|=9Dg@hO9nwvmP;v zu8BV%xwuSx6lP*&V`^jS1*X!8)K{t^_h1jqI;OX7SJta5*ftYwUuxfKA7anXmh29; z47OuhoYqfW!Cbhk%>KO36v?{O*%|Otd~i7Pb0!i=FVAGVy7ag2FlUhCeL_~c7;)Y7 z%uy^yY_}58=OuV-z0qGA;oYOm@vKUP^<1(~>zN)fl2~+6vL!#z7OSvy9`Kx%sl`fU zHbw$)84&0Ur2k-I?i(iNo?`m#PyZ)s1HSrG@r?6<`E+8WR|6LKpaoFALCC)Hoa-=gk3(>7 zHMr{`a|kz+`!<=wl?-}mNX>Xn_-_bR2}xL95)&hn6Z04XrR^ZDa2|g#0nc>{x+ZY$1y4;^9!uKCbe&C#{D@K1tN`q!W* zb^uZ6hSd4=V?Jp&vW~@xwtgXEvKK8e9jjv!+F=lo8h|fS15J<)-}D7?coDIRKhZb6 z;g!dDdpWTS(_?KH<*RsX+)CJg1CTvK`Rm@mu{A$Qg;gHL)5T+_w1*m6BK3=7rHlB| zE2!lY9=-4@-}3(tXzvjA#yuiAYtVe-d5+D<{N704(ddS8;G;Ov-zR|^(8V#N2Q$?0 z1YS}DcN`2|t1d*Z?8S}{eX5I~mZf~2 ziFBF*-i5yX8)?y(|4-t%6vCSqaRnVs?&qn)h_n4fW)4S3lqS<=5^?uZ`}G4F3?tGo z1bSl@z9j?jokA~moV6rXCB>v?QV*uhRb_VUQ>JkCl<%|8{e`@mU1II!Hq2zb%@&G( zrL|03-XJxjLi81SJQ?>b4c!KL0vGaaCfvH0IR8FsAEwiVeT=V`@%3Evnh)8ZlIpte zp$W**=(E4DmD0dXsewdeq{0ELBp;E5ti-2tK^aZ3U+zH_VZ^`!9ti#>EvE`P7g(6fwl}CQoL!y?(lKl%PZbLp_LNDyb(m4WTMgCAmHmpQD zR7NlKC1272jnj_#(Cg^7-pp3wK12qyAW3vG&0n$h=3yD6L>FCwW0!H&hD7=7!1ORx zxMj(QBxA2|BGpM{$*fPH?&t)%cME)Z4xC@6PWUFe_5~bJ7r*Hbq__?3aE-q{%sbwr zy%aL}1&J~ifkPT$6)(er5cTT0vBk<`M~x>AzXok^1YG$YR&8u?fq>9wf^d8wYV10yjpJUnUWx9V2{`v^2txF6~O#QGK<66P27e#DDS9v zsm^os5A31RIKgbs43-b{cub}aQ1=f6yIbHM5#K0*T-!qwVjK2e4*a<(_&75-c>=t% z4%}<_T66H$o}n3nfgB-SSe(xlv6=?LT`RC6GeB=?==WIy4n_fyMCiW++Uf~Vxd#1L zK$DjOQ<3mQ3>N(@IQ9{oY+{cd;_4z+cpTdx6ZV~mfvkg`7!HCp=%E6o9#xYA=&;Cw z90j4E-jDmX54*CVsO5T1kr+Pz7oz zf!)&|Eg?2m=7ZPbky>?tp4gr|0G+!Fi@6=S@D)&d4*Z-foGA$_a3|Ti@W54SAtnD_ zHog5|1J`CWYfqvssjQ3SIqC?_s!dTAGi$KCmch}InRL6ICzwaq(e};O-TuN>Q_H0^ zry~C!AVLHh?R61*v`}yh^QqbxRrEeoR=qNZV|@%^!_Xe{ga20`UvLBU+2@$ox|~SJ z8en}592E!iE%04yLX~&Xgav@lbM#|=G{fsa2fVhdSauWXwmKRRJ5zTe%f$ZpD`4jV zHexhU^0e0D$n8q_PFq{uc;&bo5^c#{A;o4vM|=eSnF+T7B+e8X0CmAq9?`KCE!B}yoZM7B9q5sFFfX2 zI!`hmS>F~Za}aEN;kjV?f;vEJuesMDv_v_+JB*hs z_6*+yMsu+=TRC7K`7Gt>i%l`x^*AG1I3(O7n!`G@0#6Ez>h2VHD=PV9yrlk5a97%Bnn_vadQYmt*5&~X)rQaQdX^SQaC*PcvuSam| z8tD21RN^I3|1y)B(3w<^7C_zz?X!>?DGgiu8`Ep5sC#V>?XB%oHBJ4d%-8;QoC@0= zz9np~V~Q-{NG<~+8= zL1rrS1ac{Zov6jO1PUSPocR6cz*vRgG~}GvlTi*&trRsPso{iWfwJg&=Bp6Ju~3>1TA=|2@u{N(4f;n z=*?qLRDL*lFTThCuHKipjg6e`Og0LfpnA3?9W7m$A-|WJg3MU^rRnH(v3=n4x=>OzK zSJ-<~@$}f^Mm>x{MOYMG(@6ZApZ;)aloV=Jeg5S5fxUU(YNTQYtk%3})#liqeYoCT z?1S%tlAPld)clL*-i*e%8nSl$$fDWEx!‎B@%(x7he3Lf!S<|1CFR{A70S5Zio zLHMy1=y%DAxA;9U)N)Entv=EQ+a7A?)OJj*-e{ZUcqmtk;nSa{+So%~&p5g}60k~2v%TpORjgI0{Lh3; z*g+M5J@^F+uwdW^VU9vX9rECtMDWD>1Fd+{E#&VPVpV-4Ryh+3EhoN~1$kT+EAI|k zZ7JE#9ef^$MLPsf~;ppcq;A1#Z1A1N1=0Ax| zW`)P+vq$12F@+xReOBsr@^FtbY=?M9EJ!8y+6@gb6&vsmDmn&Y6_iEGDnz>PV1e() z{=W?kj{(*#;Jl{T=lM8GUt&SGkji(V6`|AC(}^__?f)3mNB2r~=ESeZcgn#0$G_qGWk{~o_-H4v%z zGZfJUui!LZ$Z^io3%-9FxP_IK20OPY+VmIFZ#dj#G6!`IGPW=r{}WvuMYZSmzyv1R z%#f}q%e00ztIf&GE!x3l!jNxr6lsQ;-Q!#33%xZLtG;){FhJl6sJQ=)Q{R zhpupZTI`*6*#7I`b5WDljjRb19MQNZsCj!yT(vSjUu7))1n^LwXkHw4*JpI^B_g*n z_WoO--HvLpPUymo_|Kc+ftAR554n@&L6)s}S@mToc9kTW%IZ-)qO`fR?9{m;c*~@^6Ez=+G@(_N_NMwCkB1%=z5n==NQGU7# zi8+tojDa&t1>bPrDrlA8ey=|*eLDy7HIl$VJAV2mP}ovbDz7|KcWNEAnQE*$QQfRn zw6Ad#aXxU=bL_B3*n8OyvRC3gbNXjt>1UCOT8E(fMLvB7867|IpA1wpZ%4L_qfaP@ ze~Ld1)O(xQgA|yDH8aY8lzG`1i9tLK6ogY^sRz6OZ2Zu|O+3NAROwz1RKmJ>K@`~! zM(1PW%tkxELbH{?POAa$?t~6Zy8iAHk6DJ7kO!DBXzyO~Oj~g;kkTrvdC5}b{9p?UR!2T%gsTstd#z7GY*ngj~kni)f3qx_yOxR_& zkwR0jhL#YWGLhH$z=4U!PYsC0w04rKD5I5l_ElSy`N{%yl9t2P)mG7#)%IL#uU%8q zs&ke0N^#{Q`*r#=^FJ+plefS^LG}zN!Lj&}w!j1I$lX+Md;E3KStHOXkKp)ANSqsV zb^Jn`HQ^aIK-I(XU^amNt$aTN8$rXD8HOEp1nj>?b0!g45xe(9O;lwfGJWC00x47EZo%e!uz#C4|6M5I4)UP{ai$zp$N%JxVdw@69)&>^Fp&<`r%0tS z(1MUU@9-s-gMS5ScoC`loVedZm%{9g3x|5K5CGmsDocR1`oCi^qm^-Y#|=$G|ssSo?64_X+WqpoY)xJ*$3B5;3;b{ zZ~D0PB|B#NNLl5(avNngouUhw%Un`vL09h^;)2=KXUbsZyj)s-&t`x(QW|O?AG2BM zHgTd;*lS{|+G?=28+gpe!pV$I7d@yg_&ScR+$HpRbS1{{2n*yCwc#`TYv6&(KxqdQ zx)_cb%#)WV8{LiAM{1zf9xRIZ*#cy_i`oE@zqJ#W6W9xb8@oUqTZsA`fEH#V2O9Il zm9Ud)LlZU76Y;$2!n6I5+<(Ehefg+K^fUuiBGKs6Fya6b^iT^4vkDrxMpWq?_%CW5 zk4?CM=?o*W2wPzZw8A&ch)pQ!`UWFqMD1Kg&hi*O90)WEqnXoT?FZQla2$;F!TPKX z)epiqV$vjfvMHPri={1N;U(jh3G1sie%S)F!V+xpf!JXg$X|%fip`MOC$aqQVCkJf znodJr{|CjBns`fQB-|;hK@a|PI>0hg<@}2-?s3e7{~M0)$Mo|P%1b4nY*1<| zigJ-1^1Yc>9!0!wkn|Fmyanbhz=;bhzfSNea;zZF=fS!xMYJoDXz)5bjrDXpT;pRI z6RQ{am-4fl#KH3uhwgzTGL`!81zFe+p<DdYC?;o6XL}SsVIw+0$G(@X zUy!5a}|iBL}_UfTfl_$>7D3LtwH55tH4j|Pt({(QR{qZl&DZ3dMfr<#$Vn&uXMhf)%+Pol^g{`#ycLx!^Uz=SiJrO8 zY|%{S$%)s~oet_ROr6Tc)|eALpqbqa9xdp5X)d0#W$SMbSVFv1lsc ztBQ!}6s-0qaBn`&niu(6hUfVcj7;Q?Bfv{DvTc*F*I$tPw-TGVi5)(H=jnjd8-YDH z6bz2T8tBM-TOpwfA=kfC4{`{|PT>0E@YW~snu`Y9#^*6W`2bKBuNxs>*+ncb3LeP; zq`GpwE#Utk+GQ;#m%5 zRBvQcKh7`=${b1@s0~zH6k2YGbS#YT(vB0($9q}q0xR*qY_R%aWQx& z97|Tr2~w?>LbV@T(3>0iiyeGk!@KujTaO3k!lFu#OwGfGm>U`keHACJoFb%opF{n7 z@UvEcvzgcwtB9@4$NHQCbmyaMW}~6j;~}r(ce~)P9pqPzK-Eim=O!fBF>rDPd(VRO zD$E`7K~YV>+(fdgo3X&oV2ORVq=K(YP)Q%)>-$u@{f5I|l8ZgaQ%(jF&7kWH(E1yo zb{lQDoog@Rz6*fp2z2sX&Nh~lts&;SnRAIcjD4ZAEr%XPQx`Olcdj8`DrN$@sKLn$ z-DLp#f6x*6kci#^qJuAxv%j(PuJQLXc!pJ6X*Za;&FAgpX^--pSCGapc+YG8f5LaW zfXo_xlYrH-7q5H|+;WP)xC<;q9k1}u#NS@v`~ARTIoIq5C5!-uVg}+ z_kkTdB{Q!mxZev+hoJ2Vv!EDE4kb0A}qA!|PKJR*Bk*V2rr ze+SDCEZ9g<5eFSkAf|bnmm6#T6ZxKnM3(aL>WFn4g*1E!_rD8dB!iWj*u=}gAo4li z;lDyWe^V&AJsNfq_%90Nt02J|Qh!qlI(-bKPXgy%$>XP|?=%l_xLQ^(7TOP{3N#=- zSJ-+R9b6e0h)uldk-cNW};9m%tx98urB3_El<^m_>ToJ9PkEV6$EuLaoFi+IloB9FZ} z>%ZjUQ)3mTg*xY;f1Gq$Wh2s53R`I-yjY0rbt&s7{Fh$H=<`sls3)I=Bs@l(zcPN} zDy)Ii_{EQb*-_#LZO|a$XvgJ9nOsQO9?yC5h0~9%yktPtga7yAn#%@r->?n2je+jkKMBth;NGkPgq0txM1o zVZ>NP=lf=!Y$#V2mCq00lpNR-$ACc-?8+5kO`I_=a_=LSsze;L6}l}q7HcK& zoS*M|1M3aI^$OB=1(^TNby9=zlE~^VNY*0gvsg}?A9@oxb~}eT z8&8m%&!0opcvm^qb!7T2PPH0dToaO9$Do3rRA)N5`!Bfn1F!F)Ji48q1UcbX^o7VJ z7T`|dz%T$L74GmHIU&6C**xz$t~Uvo9|%dnc~HzMUWd5eXPzS&=V%V~pCT$HUScw= zs6-eD7tTj6kAge9bAof=?IIH4BvL<2v+NW&sTZ0w4t*2ND;yu>9kZYQ zL)Shde#j;cthDN2qdXd@Bv|Uf_oeV!yYrQ(7XBMqnZT1Zrj~9Gm^bP2y-(&VoAlNC z!U-J6)*jp~P(9OVs&^Nw^2lg+jlHd6HqAs2Co{ zL{2>f&Rc`Ok_FtXf;R@kFEgq}VmYMEg^eWkkBa*0^n4u$ z?n`3(SBEk(LT$g0=MRZHi8}i5kfe3uH=W0`JBJJw)NzU{@8g;`@w8L(H>I&m>O-rg zxK26j!?s{-Fqo4a1r=XWd zz;As>OJ3w8KZsi<5y^SZ>ps%{Ysk;ZOXWi*PF@sQR|2Y;fhD(;j}^!P;k*0;wG4yy zN8{yofZ{u%$D4DKEb~x}M(%?DwVJbCr_yMoU#~n3d8a}kxE+9D@ zoQbJ*lhEgDIHjQ3m%!&KpC9w_lk>gdCHkP)6oVW*f}YF&{Df<0G*$2E+3OLl$iMP(0=yC;G)n{zIBBgPINjk(F@I0_1qrP`!reQx#LA zxAOmZY_AP`y`5)yf-HN_-=;=>MKj?EeM8b1>hFn%OgaFM2_aLZzG0v;DwBJX8>e9J#?s zt^?Ec*cG+N&ZWTai-8Niq7yTO=vMUN*Wk>X!8e zaPpk1T?VE{f$up^cNq>9HpByXNObO8M;2TPc?b$NOEKU&0hlgE&%242|3IP~N8=yj zv|D(Jsn}LC_^*q&NVv@`F=`ysyS0WL@zY={>8KBu}_=i2fzv#Z22Q^&AUOj=_ zJPP#1*0A}Ux;?t67~EO`2@%IhQ{gjR0y@H4*vsn-TK^yH&*{hjQPDGx&-3w7=7e;Q zurR;zdIy~dJp9KQY|zUyq``ftKtw+dqRl@;wHD%@3R*Fd=Q#uK9_IfiK;R1RI0H-` z@Gd*2?#5Zy1IvlL#uG&E2 zh(6ziz|=Jd2oPIss`frlv07`yHI)GM`sK6LXOgqVMPg^m7y_ zFXLG^AwfiyhNz+W2(20Lh3M@SlY+}3zq%mHgof|MnfoKdg+;g`1mQktklxU}@c&xy zsskkz!}?1KLFpHKoDUd(H&P|Wa_5rB<1fIY7 z`}@e~gPe9CPct0qILtGMF3mk)PRuv|23`#82rrP02E*mypDO%c3SCeW3nm7-%n$B6 z!)MEZ?Ov$k5GU^pRpjD#!tRNODuh*E3GLAr|E?~SEV>hAUSGhu$r(jN>p#8<0{`bo zu=m{iEU?=F+(cjLYT$W;_dG)az67qb;X%<6x0BzE10VZ&9RTw4IQL^9Vv?&W!X4WK zE79#&4t^KD-XC~+nK^GmeqILuQ&?do`MwONYzq9F^L)wRoK*O0^^q4vd5U8EzBGT8 zoTo|7ncdK(s7C${)qaCoUxQV#;atQp#CuXfPpQF&L|*rBNTaTT24-8t*&7RRFwb|8^StAFuc5|F zoS%FfP!}~?xkGV|cy#wbXk-i&UWT8=03{LK@ql{|^l3+Ki0(w8+rrQiU!l@RSlptc zZ!|a-{_UiY{8)iD+r!!ShV+Ec6t_5+m{WWh*|vwzVlIDrB%ll0O%1JyUQ8XT5_0_` zSKSLeZGpZM@qrwn_q_!EU(l>p_(b%~Ch?ciSkVIdO`)O={NDq4SQ4Gv0SWUbFcWcu zilHc51u!R`vQS9B(=31on=@qRzXm3Pd%gn6XM7yv+`FM^As61mtyb=y5!%YZokX8( zC!~FDAn+Sl`@pQ=AQ8P6TMZ5Y^;N+CE>G|rNZ-O9SqX=nNAjEm`X0{g0qY(iP_%mEed}$kPu{+oO=SO62bXA)7(Sfj3+$64{a$2)uy)zCg=` zp_j6puLrRHo6}W5V)jBZihOJ{{_le&GnLl>o-ZdfBWV5=*t`tq3S5uFBN&E$UJi6b z7w#nF^LF&oN$|cA{kjA?n$G8iy!(H>P#N$OQ=(JKaOxJotvPpZ&Gkjrq!M(M0j%m^ zMRdZeaJZPFa}*dIe`28&4JraL%Fgj=l&^X6)iVZutxN2SWN-s{o4lTGW6uavL3`+pZ+}uAu)KnRI zF2<<@rv88)it@~5`HkqLjzyk`y%ZX_5j&XN)bADI@9RLDO|Ss|2CvzHcMoW0CK3N7 zc+kzcN?xv;3F;KOQ`G2xf#w{=vdrA)W?W>Wd@#YkYx4YuDU=jBXEi1%BhjDBD&(? zT_3P}T;Mwj+BNvghdj{3B3Y7x~QjECDAYN-guGXeebP(fvML?)i%J`g?yC!Pf#SNZNX-|gf3pYTLF zt`!N44X8kPnNB2nHn=p1PWXV%_{mR7b01NQoSpZDhdeSdv*0b~7ZZ}&g2%s!!S$f> zW*XRB1r4nRN+0=KlV^GZHdX%4PKU)4AeTrzdJ`ySIKItjPCX*zMQ*{56g{M(zDdkj zC;){uMN*3$5el^Q44JhN%Gm%kw!(3nfyhP9kcbCTh!a-fd52>!G=RRv+yy%@7nASG z^4l_8ry5eZDpwTNXj$+sxH%nHcJTb7LP8><@*MaI3qwYF{o?E@(2GQdr{aH^cW33g zKhar*;jflZOkF6aHfI&`KRNf4m`ig6+FlpJm#}u`LG|m=U9X`FKQ@K1Dt;pU1*9c- zRwK^#0F5SM>!o z>7W7;$M}G(N(GcEgy<@kCoK#=3dvImIA#Yv9%SlH@FOIvnE8E&Cpv>3eFUw&XUM5bcUyYsm8o$(sfI2rIlW=ZJYLTqgXM zB(Nkrt=mxZ6L6LYXI$Y~#JuDK$N;hNKtzZhLHEb`$ton;OyslJ__iMJK*XlEgQd-2 zXAkeW!kHg&rJKl*8=PEp)TH8>1@3=By+Rhe=lrLkxXZk*@$LY5(c$1i&{Yz8(XgE-I{=@|48<3h zgPjY&>mpZv1}%yhOTmz(F&wy;gyV!Al^2Z0fl;AdM3rt?zD`6wJ%UP3!|m6BxA3kv z0JklC{~US|@cG71S7L9U0!JR=!r9P{<>8Tdo-;Gwi8!|CK#t}4#NMGC{G=&xstTum z1nOsjsjyN`bLJ&Tlpc5q!;#tI)K`(&8uXrp_hx}Is_~ZtkQzOC4M6ud1o9owYAum| zHK3NXJV#C7FQ8kUCy9Z}M8@zoXL|^*iAYxhw6F=;^$*eBJVfRTkb|s-92>|_#&d!* zP@9e)P!vwX!&^YefD}=LTK>Qn56W-VvC`nA&5%%IqsOUWU;Sjtpbp8ym z@f3_nSXo(-LoK0_sl>>ZW2dY`-;V(XS+OL<{1*#4U&KVRLlGL+_wv)r@YN)=abN0H z2Y@%Rf8!ESmE%au<7D%5k^OOFql;Nxvx!jjfwsPLu4kO%9p|~i-9LbnL|_;LK1bkh zpTZR_Z~+6Czh`>;&t{zPyMFKb}5mNhh&%BY-7*I252=26iwk?Hfr$bV&` z4kQhAPxq*Eq7#amh}!;A%s-q@mBcK*xBAbTPJgFBHS0I!yKSrEyW^BSoo#?N#rD`< z#Ie%;MLVg~k#`Y&FCq1}=CRBQ?4+Kep!r$9<}2cx>XQwt|8?LYJ6i5pep%1U4b@zl zSA9XHPCn+;ZIBnrubAxnCsF!6;PfMgdCkCHe-<{4?lYe0Df9^anqH2r^WMNgc%?Jl zZ1b#Tq#e>hsiE|R?us3BhHA{D4NxQWf*Gvg)+oz+>Qt9d({qg(o>lx0>H2+Zj`vrg zvpfzvumG`t3zcp?kg!wfswhX*$RBbf)4m5v!>KemAr+B^TaUt%%)J5H8}PbBFMcod z!8{`RF`+KlN?a#5xuvDRU37!q0{X>*kEoQ{OihIms2!r;&G3{2KYSn;*p#ebLnfxS zAV((CNtj6W!9eP3d{iaQ@XzL~`>1?hOKoy1e+z1GE>H`bf?9hERq>|32)*L1sne<$ z*ygY7Uu7OQB8}sEL4B2Pw=bD)_p$lo`*lDNGH}A z9?>i6V)VzT72$;(gQ%PQq?OX@DaR~^X)#jhIenGAhdoU_k>1L>=D!%IM%4#3xxvEL zX!O=HS`Z;~LQIG!l zPv&s5I8#h#`m+YNTjo&3zLV*$8>FN16y+xK+ULk8rOeVi>r=G;1|l6TiAT>2bn}-m zHyH1jdA-c9&;hoKs`HEf>w&XO5^88QtwW_nRQNoj=X0htOj;#KH*{vk zUTu2e3b50oAy|4%70WZ`6Z{8OA_9Hu|;|(`%+-M&@+?0P4jv&{Lk%+-0;jiWmir z8b&>%xba*sqVMpH_Vr*seuA%zK2)EhXE%Ns2h8PxtI`u&{_szcEu#8HEOkzE6blPP zG>SeQy)CkA*b4h@+a6oMHs6*_ZEmd`=uDT=dfy{&TW@-=->d68%$oirbG?~pwhRoi z#7n8=Bh)muw$_(ME334NwoSII+BNwCd~}TXbRA34;C=rgwsf>NCoiTz^*{JEC=pPY`V0LCh>p6NvEz&UgU*$R#bw}iCa&5T=b3}hLZT%3_*544RKOS69 zb@l^#{SMKQSBj4t&}JrTXfq&JzR->N1Y05=O_N}0M|a8v>m+c|$~q3L|FAx#FEOk2 z0#;^2VqF=7*#j;8*Uh%(e?||((D&){_2v2keX+hoKcQ!3mq?s>-R#B$QU%N2!gh@~ z=^r-m3}A}oFDR}NId&g;>s4r*RF;2&N2uYv08V0g&8fwxm*fT33c;TKTV@cbmS7{wBXgC7TXhomotq#r;9L_8MBIjoWN^ekQqM4(sBVvpV>7HAF5LlxW)Pq!Prj4#G?o^U8sAZ8~w#8#__-W!0#EQQV= zgKoTlb~}fL&1;!O1^y=>-=4|hCCpf6T%R)jG3FRojN0ZUxM>{yem;L0s)}C)+TlM= zp)RBVdE1WGLQ-jYqOw)(t1Z>qY2&n_wlen8%sqZ!KToGmd0S`O3foiLY}*g5n6^jF zpw41r%xE?Rbdw%h^I5B~t!orMTrDcyHU?h#)6hpc-5kSahD&f!GWr-ze`o3#%46|l zz@kY)lJ5jEk(~d(P*ub?vO6=OvO3s}$*CAF5Il|coJ}WLChAj~QGq{-UK}g)8j3Pq zH)yRX4VC&cm$*9As>luCp*hk{I#GUMIaCY==pr8v9-bqC^3iv^)*Qx;gt_KlW}NxX z*l27ABI(VMOdrWa6=F&tRuOGr_|pemNUUkprXKb`W_QwZdX&1NAwA}KbAXu}h-Nn% zn*EWqtC>-L!#@a#SCnX9FZ}LFd^E!&TnLU;vaJc&4e9739M0^>dC*;7WaJ{MH$DcU zsdX#OJKJGBfDHKQGmk_sQScq7aa4DEF0t4DD~B#cMIz z4>d{6uC>(4YWvh*%3sP5X0iNI?kdBTXyvqgocY90Ph|eIr>Vnp?7N%z4(aTKO5_3 zCwaFur~_|B*ZdGT@CY4si>TYN zS=ZsK-X}`Ykm+t`s9*O}|NWK|oMIQ%aAam$ro&u9W0W!zjcdjY;~>~tXv{Q58bkR1 znDO5DZ4@ylnTO4Ay2Gn6brVQH5^Tt z9o#fy{!MxHi#k!eqm{I6wxzM}wbyr?cDNj$9Ge_%9d^eCdrJFCTPZr|8fo*@@5&#_ zGG?c2k=nECWCC-9IuHY$f`vN-tN9=5g>wX7QrX>>>i$&hK{yI_viLu;*<`TU73!P7 z#*7V2%ZWt>Hp7ZJNUG$uuVm`+x=%gAQsaMg5 zBg~p+fUO%H45zVGZ?5OobLoHRQ*@s`&PXz9anes_a(^Wz2<-MJ`YZA*TY>s^p#KU! z@daM;)jEE<5dD7$yojx6`*_C@IN%+%Vn3;L?nk|U8tAYwm~aI|y+{N66w%o~37@Md z^&mQX3$9wLf`vq>8hb{*$_ zEHc;78Gp+B!sHSw+okHkH^Y(2{rnT5+xgVBFQnS}D0p~9S9Kz?Bn=f@hXUED#yO45 z*NDhWVG2?Sb}lGVJX~2+O2+1#E=&;bBV|O2&ZN6=82!r`sC(Ir&r+Mez`n+p8jXZ0 zLj`d^YO4GDqy6X6e{D>kG2Zxt9RyePqxwO83)W9HJ(Hea|5Kl-f6yx#OAH%SbHt2* zj_#qA3PIOB>8RKfaFSJwfDW~GVRPqSM24H^r#i0GvNsm z;wYA(2l+UZTIIS(?;=zb^s=UrHcHjy6LNAk0+nFtbb{8}CfmE&N3tz$k$tiKxc#2p zMgRB?zAnL_gjcrRwm!CWwrg4?ZMEuChAMXDk$hGjLl4P$sS;3_N?*kn^jmt%80cXt zU4dumkBVnz(Ia+&WH28Xi_whvjV!!M7;#2kBfXKsXlAT15{+`^zsP9?pQt5r`#Q5L zND@^yjRfj3(w@Hdp0BGfrl|(IkA3Nl?*tcseUxkb{vr z1<>TTn3w(-xU2Mq8_-lb?7$k};6A;yHiR`LFd!-GxeX%vP z&$J)2r*h19Bst1Dn>m|1OE}$*v+OZw&nBBc9oZar?Dg$SY;U#Z+AnpzT3LOi^iX!n zS0xv6UC`iJVigy8(h7ldc&nNGAJ}<38LQ%{vCilY>YmyZh`fl`}6C(rzYm?C!6 zY+-&irW%Et@9oBE$~hB4e$-aTUhLS zUl*8B4NTf3nl$#hNbA1Oh$~LH&S~J-sni`Xj?KFqN;RV z>LM3X(yOVo6IvtN30qqGFnf^R$@0!^bk81kZgBqX%;)^*xaYXzSm)^M@F9^d+fvwO zXib@&qN{9&QpzfQ*#JiM?`c)@D$#_0JNg&f6qj%1!ih<7MlQnLs4G60PwG8 zWCZqK^l+mv8|S7Pv)L6m%9wBLXUj}9w)ro!H60cg*>9E`4JWn|4<%;hCMMT{>J$r8 z%KM`O7IDXe%sHzmZDD_7q7=p)m;Ulfws0~)O;(f-@&#mVWj46|knTzA&>+9y)nuG; z3GtVr%w0_u9EtzDiH&eykWLqjGR9?eM3V24Z-}qFFT2n0ec`?3ed0aueeKQdYX%+; zf|v8YPrkDHHr>?ILr;6O6WdoE&j?=rF_z+=DELWV%~W*s*TDLI?Qcxy z$2I!3^I>~zAd;AZ`0rkHT4Ato%9;*)ZG+^I8p*Td8}f7ZquH>tb1J2jib{4x$Lbg* z*9z?!*e{J|{+cG8fG-s5D&npQMDq-+w&nzzSNh}p33z1tk&vhK?s|2-upXn|@Xh!2 z^u_yf_)_^CzDQp>zV74e=Iibo{-ub$M-+g{rDJ2ERD~>UaVUBH%Gmf#26ZSK4#^CVgiGqWs8PS3omy})z;yLOJrqm(XcK&__vd5!|= zPi*RU%4VeqTYU1%t>hna6D6gRQNCeaObqlQam%)3p$g&?uV5qSBB(y6{|TN!b~DCI zZx+RWn*%-jp^IYrHeUf>k~arYiv7MvzRAG9g?FW=lV_ACn|GV{uy>_5);rP@aG!9i zp33}mfv37>k2{|`gS(99lCP%!Z_91#2Fq6eLo|E2z%iwDSjwonk;$De)V9hlZDiQ) zsOV%p`QQydBqRDtVtJ!PS3GeNadD+;g$LmqXb#qprNN_}8 zrzsn)^!IuN^RWL;pe@n8VsxT+#A@!39np{Y@H67Ud&o?DV?OdrOEz|J)Mn$jm@oH| zC|7U!9hPSXYm#NDbqLcWCAR#gz!u9xWKg5C?3lEKct|O{y04N4nZF5%{~rHV#}2%T z99Dx~dL$Ex&22HuViQ!sU%qLsG&4gZ3yBNeAvX8R=kYbsPwIE{F8WZt1S2nj-%nnBC16^3cnT6D(Y9%y2x8$SsdH!H=VsBE=RjsCa*1UIwo%mW^C)rYojIO7tPBag7~)8>!u1&vesI?0{*^?34>s&^@(?jWv0! zmtIeH=r=oYM{@UNp(#JY7Xf?YbiT?9{R^B>)L?neDa@^H! zs#`S0e%D^Y;c!+CZxD4YIw}fzt&Wuj%av@eoiSmh>@$?wQi3!VC|8#K!Fp_6Iulqx zheC#6UvsAM&8!n#WvLW2jIzG>-rC0fz*oGQSAlX&oG5_iz6Cz=U>Am2Mo7Ex!{`BK z6HWhMA@&@(iDvhtdwexfzT9k7m6#1qj@0_i`W^ePG1-domP*9hdRY6giElMD{s%tp z1mL!sn4XEBC0Y6dewD~PZG9%Sw*Kv}uE**{{NJU<_GV!h!$vtrIr=#YvA?;cZIRa1HpzJ^ zvRSe{(c{Atwe|9DIZ7L3pKTwdHBg$#vSLy1$*U~I{d0_=CXq%aqCYc}=}mpPjf?)C z!3BX>)9btIi!*F^?1TNfdC7k>_}W@Qs%6>aFJs2|?*zLs5Acq(oSu>O)>CwjH;`hi z4S>*8DXr2@VQ-Fgr=>2D>6Y>Zsf)ERnZ~o0>()ACG>#KZ5Sv>ftX?MC%nD2kma`ht zLV1T|v-S;6CnAv^Pv{bGziH`$W}eE#{7LwV1M$xP#1_jMyh*ogCp7gnEQO-}H$-Xb z6CNWJH*y*W(?_>R`?~OMq?GY=(`-LrVjI?dlzT3KmO^f=KtVptB z5z`zuG#isrGio{2+{!a~mePrRu1VIafeA(w+eANOiJdSEZ(+|q&t;#@yk$N&#^{%Q zVR~IX==Sde*YgT2x-GNNNp5l1(-srNL^BlD4rc@3!<=?#2G}zES~eFR1V_k$^VV7R1~K zS{DpT}eC+hEH)sh{K+!(gR5$`DH z9OO*2uheeiF@`DkPDzyc66nU62v#MkLWq*?{q`^V#{95R+0e;N@`;8}gV-im#K_k1UP zH+&y_E1^i2r=cf@r>7^acbE5$cc1q!Z)@))Z!&LnPkVQ)d#q=WK0R>CdO|vBRhZ=e zBKXTXO&P6~v6pbx4KEYfA?j6R;fUE`eZ$s=w~o9Xl^8WQ@?7}CuwP+0!{3H&cOJ9% zw0+SAXzA4*(vzT=LNGb_ANG8^z!amP&*fR^`NK2bozZ>FUC$e%Kh#a%1CP(O&z0TN z-B-*w$#&~F_kDzP|5%67^N+SnA-^RmRHs;P$*HyD zwr}=v&W7QcB6~#*i<%g@J>o>fiAXj2Y4m_-Uu4&arQy%RD@1$>pC6Xs=wyGy&evG& zpnTo3p4hXEdaH54*g#=(8rEq(-(2rD&sX;&cXv+(?>cXocf7l>tEg+6`;9k2Z_4)I zhI(23IQgsg#v=9}U-usgHlzo8IQrN@-rGhz_^JF^zAS%|dy_p1Q_m{hm6l3^GD20+GTWGA)XS=>Emjq^m$FaxV@aR4u4bCAN~cyVnbuiE*1rq|6!H31L9wNE zP~cxSjt?Yvdx$(;T{8hoTQk^diS7Euw+H{=tGAZ-v*&g-zQn&TSbs_V+;+Uy$T>942q-wJdms&qEMEClOtIZ27s7TcCMl7%agVlbZ{svS8b@>S%z$b^Wo;rGIBhP4m7>A0sgkT+q|%%R`2f|bs? zzy@Sx8soSxjW@yF-`&P@$y?jk#5=|vOxl{Xz*WKXulIxZk@u~4n6ES*S!<-~J)hP1 z&zwwW>c7lcI^cgFcxf4nmslQ8cAlJ4NK$s+mSn=gQnj${uC0e{jFzB|Q8%kk)fehE zb)Ra}st|LpuVzqMlC7Hp4*p?>c|B`$EP^mH6(^YAn4frQYce|d0$KcZ*_7MTR70L( z9kZFSMX#;PcmQ>LvA#UMqCPuxaLW_z8Q@Oi4!Z8U7P-p19InSnTapeY?M%9o^dRYO z(ut%AN#BzGcAxZCFvbRx*k*%W&MYoJD8C5W1Sy+y+@nOTD%thK-ITw3q%g9CK+1A)#ZhuSiXOj6w zFYAl;hU1TS^WFAl@w9iXP3q>#>{;f`=)3A|>Ye4C@5;4RB!qGU6$kRC|WTmULZ3nQ- zrM6T{D4se}xAN79L zf1AIFUVig854;HGm3k`qw4Jsn$5rR+@ZOP^B0ERsk6asBIyznQv@uJPhbP+@RWRyQ z)QRXv(TAdnM;46O7M?!*va_r`sGOFXN)_eY%3b+^wP-Mdzaji`-`J$5@j1Mbce;0h zx07dr%bR#9F|R9^r?)qqH_4qI4b;!G#G4hb+wVQ&t8aWXr;$^VP2vOoC}i$Msw?U< zTc)m5UyeZ*=7qN-)VFFMZL-!{%c7;%W@+_oM{OHz>22LK6;Ju7_Dox$eOJ4xG3qB~ za}1XMlol`rTWtM~wYDG=m_+T#zf9C<9e9av+RMBhiraV4*ZHbaEi%-%(O1e>!s~UH zb*FNdbJuj+-CbR;puz-T_dRh$VztB?iQ5yyk|L8565|qg{LY(L!!_Ew)L20*ueASf zv$lU>Fj{J$q|nCMaybTuEr~c8xh*nzx zWZ$C7MrMu}9o{@F(f&YPEN7Q1Gi(01`cmF%+3eSemwY!aWA_g7{o|Fr7d&M>CEUN0 z{z&?fG~1outFk#em1jTLPU~*uneQFvOQnw?*3zGR-g07i3bFYn0lG8sV_s7S(vrMl zQ%P0Et3@>v3F%ZTsRz`G+7iuS8(_<1Uv3}4Wc!0!O6{}SPy4O;$Q`s(3n_WziZ?U=3II35LCadaIberDZH^w^!%`(>+-dxWUEEh6ZN&R)2aj7aGS4>?vmNQgwA7Ur z$nzAhVp9()HI+o!C!bPGwXW?SduGRPdzk&Dc1ta)PE+e?-?b_>U8_kvr-U*}&Ln3; zw~r=n)kkV;HG&(MgLa#lH|?;Jlt3weelx;ItxqRPmnh+%?aJXQ>?-1F=b8n_mvwDSDw7nSv^lA*Yq>kxn~&O!_kjvjg*FQ& zQ1x@k(n0BF-x?MlF%ZkLR(QSeb`eU{8R+qQ@;Axjlh;nxBl=v_$f)vBFYxn|!uN%Z zaCUTLvX9gflv46cX(D;p3({Ka+F&ADJcVzdceA&audx1^Xh?s(kZ+ZzihHBW>2Bs( z>mBWTPpor>Z=`RVPxmz@x}2X_c(hsAY-)}(Uzv@Gzz)O5PsZGe^?30om{{;1z6?7; zr8;t9<)<=C-K$wSf`{~bY45K z=qnK!c}tANW_?7~$VXPQdtf6m(2QuZ+G4*q?UYPdsocnTv0F4Dqx*Ms_pGUoF67IZHB5 z-io;673&!(tFlS`sD;}v*uUEk+LzcT**DoEspE)p)^j#;mT{hNv~dLOx9y2`FTXi& zo2SL9*W`3;ww@(N5!+lN@1d5B+U?+8GP7l<;>%0z>r&!8N?<8ksj{pos}#GcsuQvITgnm4GfSXKZH|;d z8fQIE4QB$q4b6x{B{Avt5k7knvoDrX6FsjUsekp2^_kxO-mjh&o_U^|K(n`ZKQYNN zzQMkqzW!u|Dq=MkB`Y_BjG~o1)j%>M37oozf3E)pr=ElFWub#&8uh|M=?~mU?L%QI zhe}g}bXo4N6jHCMKcJD->N54QT1e}!4b!S>n)X4xiM{e1i>0>ug!ya^Ws6)-PA+?; zXt|+0L@p;EC8{wDdU*@=c`Rv|<51d~+4_WPyE){mazso-l1d`r!iso0s7|x z^(x7kr#u~c-@v4%L(FC9M;`RMr809?4Kktk@I%v+U+YT_t|C#Xr{q1>QjaToTOU%# zyqWBHak~6hlimK}-$9;i1%Bfdyy%mBHIrO(Z{qSZ$xrSl|GI@rzHek-Gc#MQ0~7PE zGTpK_v*o)%Id|AO^N1MsKSZ!}qR1tUoBANVvfh%as6j@4b1{+5{pLBdFm)ZL1Gn%F zi{c;eBWpeg8@jNRQVz1_|6p1cg&BSM#ldKzC%+p%zR}$MS~y3 z56eN+WhxblgRmbL60_Rmi|{SQrYwW)mew2R9Yma?moFFc#YG+3SftB!bF9B(pb?YU zzgqf1Eor1z(iM5Ha#%fvHfdw4Y%5`_VmoQ;ZO`FIaO8HTL$;K4+_W#VuOiFV%F)MR z+T-nGY>%`*iM~{%Rwe~|h+bpiKbH*Xt0^(Lu}nRhZDBhOlSqpbKOPNb5AyrX&gKCl ztFcQTtdG(!>FL-6-}V1kIt#ccwzh%K%zdc@ zR!r;;#Kdk86*gzS|1*B>4_tO<&YbgfoMZw?=W#kS6VLBP{$Z}`0nyK< z_)D9}Hivj>k!|rI>#~v^`of4qy)yo0Z@B!#Qjd{MJF6A5Ot-wZ)L}2pE7o^p75r?4 zZB&dx51Wzkg`lM*YpV6L^^$cBxu-)!!6$0>*y(6H6<$HgVQRJKvm5&gBCzQ_FWqa9 zcn4frU3;B9iENK>u5(^>ej$5y%-P)ehU{izM`=f>~r<8A}RvjZ=??RHU{lFtva{zdL%o~?tIMhKMg!2`hN8t<7@Hl=rhzi+`FGw7B7FVE4Jab zb5@t7qGcr6%+*wFJ)|nWnb8XhKR`|?#dU`aWj~kh{KL5sy}!~vhxo>MyXJ`CFN0j+ zM|)Yv21kD9C1+XJG}kEdX}iqXcBEFfJ<|6d_}hXH``MU{Je!6Ey~oU9X4BS_M_x&U zMj>nYpJfH-bSGHSlihw~ZDkv0>%b1C@2D6Fv<7UD9yIMb`|7lY_{$@(7PgalF64-{Z?!KY3NZ-Jr7F3IMfMVoP^aPS@6Mw4q)tDx zkxKTzqVZ5G;r*B2CjZ5L^L?g!SMq7*Hzr_N;MBnX@IWdBJPl|TXkf>5@_*vD6X<{P z>EzuOJFC2RX&*nI5ni_~RLN0GzmA=I7SWee7_C|Z%Qc^8H&SIhbqWctG`O)1^&tn` z2a%&+-K(MdhEUUM=Vi`lNcC)?Mn|V6ZWI!+nWq&My>bG{N4+8S2E)i7JSIxsi0X0K z4<#L1RUz+vUyIOEIAKsu8()dPzCj(2v@eRTwW@pRaw!W8w{q= z0m0Wp+#yFo0zy}Xj>6j>p00G@1pjM()BLLXPVlN}i6nM?gveH??Sz`q)5%dYZA#i# zd!nO+vybzoQ)j-Rj#FtdX?g7t&IH#?cNVu)f|T=2HoT|>TW93aN3&z%bN!ZaR9Q#` z_({zm@6c2a@YM6{(`yiSY)M?KA60~T$*T*5UW`L0-IbA0`KWmsreL=OE`zs*Qe;Gd>0<7sC>| ze}XdK@QGkpJ~5uf9gnZHlt$Xmh%-6xt`Pm7>;_*iBygab#@_k?1yfcg6*`3 zT=ZY|A!!xTV(fh#FPvjMU6p^eMz(rhC2b>#(T3Za`|e0rFT7*K(eSXa$-#NkJxNz4 z>B%#cs99Tc#dFA<=4+S%cZ^iIrC?5YHHdj z$028k>o&80;)rnsGoR=9wadu1<;R}MZn%}{SX`4eP6*@dny^s9ujzx*Vk zFvG~$+z9P#1J6pFI*vGOzQ!)Bg<|~0z*s4UF6ncB4Q>+|$ z_de_caDi?;4a|Bg>&X_nsU-GsM`0EGaNi_f9*b;Ug!bw~XNTRav8YPKHnVG;bDHyt zvp4>7XMB?_u8ppjt`=yrw?wU+?&h4=^?(YCL{6JgsX{8GFQ68_Di+g6;;TCuYl`Ee zy|{g@{f8sUmBTYh8Ed&s?CiL$vL#Hdt!4Fo9+)Y7az;zUiqJhlSp&QS(gL5S8x~wW z_-eYPLCXW4``+>jw64->k-;c$ly&!Z_}eQu7Py|f@45Qg)BViwy>D`!lutirrgd_3 za_)8>rJ}92qo8A;^NTyt=mJG&wrrw4rK)v}t+!WBuT9okmh(hpT3Y*YMnQkuaBD6L zn%>Au3@NLzmfTV-XEg*GpE+qP7^{dAys7IN3p|grd-GH*>&IG@Wj%3Hji|w3b)ELW z(wN=UmrTPh(>^b!HQkvWAPguD|p{Mm2R2wm^1r3k}t&RFdVS=dA{D?wM5igwO?5 zfiA_D)C3P>k9^%d)N@|v>{YU|{n)1|3K`goy);{JzO48$R<$F&Z+o#DhfsUm!gxj& z%t{<#4pk=Sz||nv4(CY6W&4S=$@q9@Q@f_#`&BtLDD7|i9M?oWz1r52&vI4Wjg{Zc zQr0&%cuROl26t$OptXJ#ecSt02q+%diL7kAe`UXT?<3ZpU}`Y7={Zj&cL`^*y%zkO z8!Kdxqe$wJA2WXR{x!rt)s^CzM^|cbvZJ}^i<+by(W=|pc-QyI=u_M~hxbzN?mmxv z2K#)VqJIss!aG_R(SDKsoG)gK@zf-u^3nCyotbz@H@%OimivOMsJp*smoBw?$EcDW zz~VBDh_?Dr-fTVKg^(tYmwW}UBWfkdEU{Ds;^Iu zP0m`bj#S`TTp7XE4|f5whE=HI`kOrgqSY;0Kg(09ZN8ZGHKQ%5mJe2|*E+8SWTmHC z3qx^X)IT+{HYTIzZ#kjP<;3J`WN>fm0qDm1dNa0cW!RB$lirzAUbeb-p=CEwU7F3k0f||e?xMSnxef<9E5e?V zIw`eoS`m9Xdm;N8`zw2-W3}^`$43pdzNSWHmL*XguQlA zMgR2ye*`WH3=Ev)ALx76Hc5ML9ObNuXd*40$c$&PkFeiyd~m*Tj&sycd;2R}>f^Kn zPP?aza!mbBJ#a>~9lGM6YPW3mTH{m1x2;dS*KAuP=Ro}Jwbbh)v7ZH&o$Ls^Ukz1< z(BYcicR?q~3D?yG388S0Dxr#)&7Ma_%MQ$huS=Xb{_GF|I}GTp(F5o6D$iXt?74LZFzx>vC_JT{i>&1Kl4c;%Q@`uqSl9G znO<|AT|=VRcR1y<6BVbi)brfI2kV9nbCAe>I$~ZKj9v7D48k8ih9q7`@5?OeAh)?2 zyPr}WRo-=x{K(&qFh_l&J|7%*yqQw;(AgcS9?HI){b_H1nD2?eE*z|9(K_ zz}EpD|K`4BZBLZP)bqOa6WDk8^-a!E_DlA^T?IKs>x&R3@&I)?c_>0rICJsxJ`B{%6sVdjpEdOHsNHzQOX>ke8MQKCTbmtEbg;j zB}UkXZk7AFp=L6K+V6(M4Q64JS*fZ&Pd}sd3Jj%lt_TwDE`4k-sQqq^7g-MvIL>|FmBH23 zImofi{)$MN9sL#R_EO5*7IS=!iaJWUhugw~A7>aAaX9p?|52|owtu~T`rPvO z4qBQnG`M}x9N)>7MnLVVa>%m7I#b)EuSZ^WbQ_+6dZ@dRqhp$qmc#CFd~sd&tTL?R zP2!Bfo>#7|?&eBWTTb76e$#vo+ZtQyX~VQicyXb&MB5MewTf2Fc-k8Zr@e zJBlQfbF6pL(|bm7s3}?l%U*4aS{qL#8-2F>)lo#`zFG=Fm46VC%R|rN80`RA;Bi_N zt&`T5ZqadSL#3th9~}g~^f6aqzpv&cRuhmQeOg(hDxlbsQbIif=s=uJk*L4l~2&Z zlwzK)Q1w&yM*S4Jv#`-ZU+CHBrY6`~1i93Qj?tg)w)8vMJ+0ZNy@jVXQKnh$s`@IW zFT4J~!Ix`*{kfRx;25=ymT0MB`-qp^+4FJ` zr*2Ts?pFSy!)BV+K}*0gh`C&1O$2`mUr)t1$L@W6oXiCW7_TQ<~Fxy~*4z;O}8n zG{vLAJ5ifagdWyB=%V%1yhRaFenrG#4y(yZx6)QB+A|W18BJt*F44ze>i^%kmwL(> zYt;X&-@NX7b+V=@E`6G^)8-X0Ke$RryP))bJH2MuvU=_DPWCNC^uA!I zyj~e#*+O=iJwrWvU4>k4>Ejqcb#60q}w0MZwoVLbm3zbOtqNP$+TL~}ivP{!_)oDfvy_;vbXRq#0EbtL> zv@IQo4$m|ytIN?97_a{{Rv;%WoZ?cK>WbRz65Eby{p(5^QT}&Y4XU-4u@gaIdc9oA ze`hCtIka-$Z;e zgebZ~-+z0oxHzgLT2N8PDGuQIJ~ilv=?ut2{oH7FAXu;D;grRebf9ln->?ftq>-f8 zp`vaTdmJyPW2d^YO8-W^*9z)thM)!Q)VRB-@JYq08s*;Zw!3@Mb^Hgl;*06Z+(eIh zM`FjFb(Os(uF^?+8_V)O7;miK)Hl(6cZQDtzGP8O(E~aSS-%PG^-!V7J4{xsUE*Evr)i#AqytylCsb|pCd zT_I#!{q#6A?kwUR4e{MZVd1WK?|_pR(dBYq&jKWB<2|%OD*nmyz33r)Nnc0`9g}mg zZT?h#!k>AyZkk1_Pu<-@_L{h2RK_~}muP$*Y`M+sn$nc>=%^w$R&3KssU9aQ|jbYuJc|Fq==tV&|2GA~LJ@ ziRFB!8!{Kq2~*Q^VuL|l@NVRW?D{?x&$u-;R6FTEOJEm?hwPn_h24pU1Bn`JU+&CP8c`_cCI3Og2^6aF|H6D z4@MV+VXOB>zx6;awZcx{?QfE)%u3rO+Usz{W|QcnZm6qo;8W zUAG;GOC3{c5}&?I*Q|q@&Jk4KUL$60L&M#nD|5F>oJ=^QGwJknqQA@b3sCOXM@#TE>*dA)9>68sucWck6 zKuc#ihtC&c+^1f22IuOg14{KCusK>p&Zvy*Et@zk}hi)1pkkoRjteykZ8vOqc( zhI8s}U84Xssw?QdoPyQ8op|0E!-t-gADr?XtYuaIhiw%?hf`rS7>~FDR+T}|Q!65U z>y!whc^lMqSj*k07VNI>C(twxpu$hc zmVfbecY*WC^rx!y=)UsYf#=K9RneKu;8@O5`5!g-pU^Bf+1+6PQGJ8cYj@LYQH(CC zPn@K0m5blxu~%72xgUBuIT*>yrrAE{3j#(u7BPLyE?l&rEF zi$Z6m>|b;W{u->m0)M%&l*^(ME1@z}&It}IYgN4KdMj&Ccfs<-rB`^Dr zEYr7Q!@lAinX6QVl+(|%14bTnaJ z(TeO(J#{HGe*$0Y9Fp6cDAW$D^j7SP(3l?J8{}AX(ye$N36aDuGDX+}usP=dT!9nk zb52!O`0G#dfg1UY?Cc7!K{54_C~FH1xv^$5vBSlF))r1B%r59{9HSQ})^Uf} z@&C|0fxs|;-E^LiUyL+%lkI9xU0_{SQyL3>0$41GjBMi>wZZC$Trl5c0*l zsQZ5B3BV>eOWpMWVhly-Y#U66>O;;UNhUHh8OgI8=@|u0|1frumv0A_U7Rfv1Wr4U z30tS8Q7K-59HsRCZzGTPUcFA-G!M0+z3`lGvKzx%c%%;WaSe(00XU8$^ORYg#{M*C zsRYkLl&1uImO#DjRQ5J%guEJxrdfz((w_4WauelPZ|-_D1eqs$r=%i{5i(jngE z!J-q5c+w}y?1mC2 zN=LM7DeI7ZQ5VuVimEC+CP}gZB!5UtZxNizO z-pRhk6Um=8fl{mDcZA~+JRrKGq06HAtQ0i77-+Y{${5bO-ROLky1njneGfth%C2?w z=|vBMCVvoHuMbW#64%|1#(BZWe!Lb_M{tNPzh>%HqBrSzRl(m}f-IZCX$}@L9s96a zHUMt}dAShnUxX8BW&nZ0@X%?_6yC@Ey{M#l&#os;*~#WVxZ_W-J=dfQD?JzC#z7=P zyfK8k8Z-VSuKZ#wRW7kJ!UA^BZO5Dv$V%rY&+r2eye1s5i2Xpeady`z>Wo{WDR04R z_sG;7VGp2*WR@kSVqrhFPh?fQLM7jUSWBK50sL!n&v4}$lJzv}JIL;y5|u0fzG|?$ zU`I4TM=1LS@kd`e!fSKlZWwur`)K!haO_xY#V(AP1r4#ADwajOR^n-|Cj(0Qj$N$Q z(yuM2BT)~{r}?NQ$Oz`jL$R{oE>`?MB5LQb(RD0yhkg;80DLR z4@hQ}x%XDy;5(gHu0RicS>r?KZk9Whk;KJGSuU+XYH8;>64fU4zdpGsVc#YST%qk~h>8&|YO`+~*I-clDrLfVKlZN9w z-HkLX+z3`P+-O49cr#TcZeuXgx;LNr8@JI3HI?^h@&1a6&iZazem#_4?04m@ref1Z z8{_o^JyEHs)m8IArBmpPd4gY?4E;>OMu;TK*#+zSEuDscped;uR2Jd?)CStmm8L*7 zK>5Hqkn_-_OW1SeCf2PJ+MfVzdc!Fe{NwRR@ab4P>x}nwan><>}GcZ85~VF)mV0e_*Y~NG0%6pO&z3NMB2QT zo#g01`wLyT5~z*CYwn_FR)(P8`eRvsHU7k>=9EIl@I#iQr#E=MaY!!>pFGnC8V8h$ z&}@CMh*ydHzQOM7m{Fdd?{9PPOCG%Z(3F0lid} z{exPtL;gNw$Vf2%AFCSz=j~+1+xR{|S*0Ox=5naC9M*PKvOGK39q=L0ughL>Zlpyl z*`O2Xg+TOd7qsAMLjmHe=(IC*oxD*ZT(Az?^EtZ+w3C;brxMYhI(( z8Y&I6hiV0-DLaH(IcMp;Qb?<&uEE=RN~P2d{fu&)9M?3?Sh}JgH$2KAwXk{%8{>z* zfV~vSRTEJer^rcU-|=(yE4z>n^U%;a@Ttba*&X4k+vp)1P&|+AJqC=NM>9?}2BT|Y z=%g9xX{6`FrthX~r~m#M7FLYWiroknD;J0>ef7-M^D0N_>avn8S#HSwnb)Dan`n!p z%>Pd`Oi!f=cwCA^UaFTdb`vMOh`fJAMN}4~Zx-?33l)?i&#)4cHD_z2eK0A)&AHvw+tORsDPj83#=Pqy=+%n8$tMo+lbdi5w(^bpEV;J zyTr^x%T)nKlNHWuA+E6=+h!V zYh|2-KQ_=MxfD6IpS#B+Rf=H|*1!hp!;U|f^ruD-Wb9uGCzT>M$&l*4Wb-UgRU8&$ z9V#0J1Gl-*b0i)>6};WC#yqrPCb;mcc0o1pW-c1vz+Eh{;vs6FQjsgZB3LIvs3GskPEYs-*akLzF|Gg!qTkF)4Z|5lCenILq~&*zH}IE zM1E`Zb?%0a^3r*Jm$mv6qwGdK_(dbuvj=%6yW=mXR>GprfOBhtm1Fckzd|!MQ!f+E zD$Mz2k2%Mz4D|jDUuY;6*EDcG98OQcx5`Z3>xVvt&!!O>zYm_uq9tab2Q#y~td$O( zx8!*zA~E_Zvb*d^D545dJdF6tdZ0ZF%XOqFi7aU1sz~XTjC2zGj>is+2gmi`rm@Cm zq}fd3sqcX7LH#fGx}OE-)ga`T z0zVwYa@s}wBaJ;@hQY^2vBw{vLr$^!N5FC&^d=`xy)qhLRYsxzu0Z3J(bYwfq@CgN z)r>X)eLn)+_=5SZP-Ri%Z%+J!$yC0?>xZGk6?ijam1w%JqKPWyMRqoaM~{Q$7GzQ* zpomh$5uYGwPw6Y@gC3%u!7_|dda`@sO62`D^@`$ewAb6RCfN&k1=v21H9s0ktWOl^ zE^@mwPpyFcJ%{X6FXT?Jnp+8lW3r=Z4jZGWg5AdMzt@aP=;7V0?ixGicEd`w1B2>N zb}Dy8VhPG_d0W|$^f0ogAbWIdLOYHklhuSu*IufM>_{Z@n~I&Yi%8r8tjS4a#E$a! zNGh`)V3|LMiZ- z6;P8?W__6XNls}FA-7Ur`A%%3Gpmomj_}7*DX$bWYEu`GsAr%iY!UMOe~Lk5izo+{ z%2Oi*=<inQ7kQLr! z1dijcK2|o8x4wez<=h&gl0%iE$n()aItp6$r3!8_`qG2^oq_($X*~BdfX6Ok2M-rJ9$cjJZfbI{$lkc$&OE9nf zXy);7XJsO*7l;=(fLfx#?JcB9XDWM7@Z=bLhO34*vOXVN?qZMeT~Nt=?k$bRO2!7h z0Bl=>%TRMi>$b?=-pIGSbn+)F!>I4?2^3TG4aPY5Kqsy~1FNAQGu;PY*ChH>AC2$~ zZY=@T9b!MW75LtqOv#C}Uyxi!sq|J^*)*)Td`P-PXren*S`W);fzg!QTzhyh9xnZd zmCS>ZWUs#LXuBE6U?)EKbF_b5IO#I|1`EjXJm%Rw@L>m`)n9|b&EV+>lJy2wnv4AF zGc51U$^)WLr_hpl!S$b5g_<#u{QYcf#3S?)B*5wZtXBgMRq-y*;(^L3fO)X=w=zOA zYMu%aHF<&u+y%cO1Ue`~JkF+&v(;kk!kXUj^7vWe4!?04LP~7yHd z(ycS=U4%s)hlELlrhYN=z37BX_?5+oS<6~JfR|wG4{vbX6??7@nkE)Hd5`Dei!9bz zXHz7Yobz^*S)3;}(SX>=Q2rl+7vY1IxD%Qmfwl69_`*UoQCsLs_ElL;g?TAr9{&;* zcn-&=7=56=KiOAk2`iA3w|g=7XIRCuhx$^a*?261%B*n)dLYK!#d-?yk^S&%eK6Ju zEl?Gl>cFZaKKes6*m>Yl1l`gKd%iFC&t;sha7}L{&T^<+C?*AMz8o!74!nn9kynDU z{{!z+Sl2)Bx;LL>Bku45X)RIBc=%e*)s>U1JKz7m^sEsB=`zLk( zjrcrO-APP$l$w_aX9|&$h4``)cy%PIlgzy5Kav8EQIKURRZb!fAT zSeh%L`JH@Hi^yn3t`3;;I$@V6QwVG__nCDG;$f$wy%7{wJ;u(^t( zRSw`$%9&GDkR5x0>pEaG70iA|Q=Nm0KQnO1ojZ zG=iV{KJ_472KUk6Dv{3Lr>TxZUyeuTE`_eza7Q($Cl9)_C_8fbV6Ei9cg;yY zHa~i+1&|!f&noCRgBjb|#@Zx8t1B(k#p(rYi-}}=*5hM+q@p+mn(3*PBr}u1Cx`j{ ziN3X&MCv6gauSZ|iD$4959b##uqDKD2SeYp@!I5c>#SHr-{}Za(a8UUMi=1k42H(# zM40PHp5ms3{1P1!4#l-br$3{%d@Q4`G$FeTDEH?j9`8ixWfr_YoF@(<3e*89Jqnr9 z1DF>BuGhe!1L-Fx+ni#3bFtqRLc6V?l4E#DQ<0N(=!uD84x@-Y_5j{}vAdch3%WvG zn}~`Y1|K(&nj4@Ui71Am^@^h}(!gCUW^@VfNg*OSkWcd>O}gOYZABVYLR#HH@*PJr z-a!IAH@UwZP?!iu+~N~C<8}l%zKOqakH6%ksO8YXaXQrGsqt{lzr;k2Kp$&?QGUF3 zFXBL{WwusW zOH$L}`OK#NCYLr*{f6B&mWtI!^b!TCZ(09GG>#k3JQq5pIXk7zMuYwV73{^ryU2>h zvuZh$Y$-e@=MQ}VHZOtnM6UW1y;Xr#w1az7$w&IKp7Lau#{>U0;HW!mTn)6c5rZg5 z-se5gSx7|jIM9m)(h@0p$pn0dj=!T$HS*< zp`?-g-H;bm_55Jv@&q~mLr$=$g{}`lQdEb3D`OAkL~Eo+Bjm%L^FyA=NnKMqX`a*M~p_kiWU8GGK za}H((<&Zf!c(#J2>dnlmF`qtY#v9n5XRsSyp$R_`54*=Zm0WrrJhdozu_#%xnq*7c zl54L{6sJD>CFJ3AAL{7dP|vUuDK&)EM6$9BNM<=jTu$dZir07zJ+liL*c)mc4dhp$ zZzKcM6YH%5c#@hoIlX)W+*1tf^x(?Bz+MUDikIO6Ygu`P@Dog=T&go-sm}TVEY;U=Uu~pfEYL{> z)(zmWD6)}7$#(9gR=yeg+DxY3q>!b9WeeS#r!4)+RbQt^^C>&J<+OxQQC6D1?Tgfq zbVIwvsdP_bIZI3^7T(&!y5-ED^GK_f#8;{yZ4aCDv>zTC2eqz12hT@_$w`WGiqmxT z;5g{C783P?eg~~r5=~qONm9mW3tmUT*DJ7&CwK2L~giYw`CY?(Gge`~$b_2M1f3iJboN2CvGFeIe((r-Len!moSKjz`gvOL(n- zf;VFkJ*Qh>8tamCFh^kL$w|=-xV{kDPtHY=Q(!YAmos507KS&!Akp5Knn=zxmeb*) zp=kpiN?_(kv6bY++TF}#9=dlRGaZTaEQic#%Dk)MBjka~TQKVx@Zl6@yOF!z!C`XR zV-lmtxs$)}>ifcFtN0m-EioP4J`LWRiG6pT*BSmE$*kw%i!33ERtei68$WUu-5sRH zCFI5xFtUM{oHo1)xps)>90tbIdFD($Kgdk)@N*vi`G!O}2$jmIdp+R87-}oRu@gRE zv78~I6Nfx}%j*=>)}5-4p=3JxArFVJ_f`Qt8rak~V?;8QUWbPihJ5>Sm+A_6Rmrb*+?EgqnrUSXueCWV+CLKy$^^ z%5)q0lQa87OnEetMe3d8)SoG6udH}6a(eO&C{oV58h|XU3Vcff@rKw^9q`J^!cSS@ z!~U#JEXT#XmO)K3pxpz|*JAi|7PKR0yxxUhgaY4khKmpKN=~qO$)|G4%sXBmku=4y zu1le5vmj@RVQFQ+(H}di4EHrhYfr!nSOe|Gq32_f!Q)Lk zejaqY1H0-Pmh0b0ywDBbS528-t6iNbVlU!e01xOW{p9V{->kxyMf&Hsy9Ou>knr#Bvm!aR#nl3dG-F zU)_V7Zh?;oyzFP>PK)uXgjY0_y-HOq^9$NGdTe`OVg1D((5u+bdXn}(syfSSs&*AC z>pXRDTgjXGQ$2ME-e1r8g!6&=A4sx0Jaet-k&FWx&6#ZpY=+`UXz~JR)?Uz0T`a-s zaK!(htGmf^}tUTeit+CnnqxF0N9v{ZzdEXCxM3} zGraI56X3)#jMxP#T!2;eH?WF?Zu>!*^U!VoLvje^Mq$CUhGREFN3t(|Gv@FQ7Qr!S zRCdHW$v4Ze(!@S;z-tyXs#s3CAMQ(J0cd9YZdeB-O zxX1xzCjomoEAR($T=LHUAYCV8(S4&IMI#cOVEXz7t0)0&7G@=yBIhB?*_;vu_{ekL z62TMe|2wluW+oCy65Pk=ucrO2Zwcpr#hAr4MrJF!W<%zLCgqIXCKm$hf-5m)vlDd!E)3{;dV3@`0&$ zKv}S*V&kMI;$jdv&c>b9u;P0nE1Kdh6@rU=7^^yxr7>6)>-!rvU{5UnAy}VX8MPZ! zASb}q;iov<@ELyl2=3*KE~%dB44oIl{>sDes^F(77^(`LRz)J@06X>IloEJJInWeV ztd?x7sQ_zg43+lAPuT#@L$Qd!=dN-@Ygh`Mm)GK@b@6zOGIKh95;|vwl^^{97(Y&yj0w1NO5mnucWFEj8N)b7F00)Z&@zUh#BS`2WV4xk4tPTb$W9_bjPbMJ+ zX7PKhNp+HOUI&~P!&NWfKo{|g0ANuTZBde!Si-eT3D6v>YKUf$Q}J5xJ0tKqX4*^t zfs?aHl>JDtM{wwWjI{}vU51Aa^M5Q*y~^k3Og$KjWww!PFQF3?fr^|yn`J5}x zF~$=xy$-IPjbxKEWaonajZnc{W2}yYZACBEQz++g_m}HVKY~ ze06~4dzjo*2y0E^;4#2En(y<%KR$3z3ern{?FwX|hbqgnYf@*`)+cN_)=Ct2YV;=K~LKaS-p zC)QjfXL$*@tVGHW1mFJX8mTgndNT1*zqT=>cf%km}_lTHws8BB-$h=`!xaES>cC9z@<7E zkAzl}z_n292OxA0{+6>rx0w1#vcP+gBWYOQf`#r#(-H9Kd|*D2cSEtiS7Vb;f+h#@ zoEhM!10zdzR!*EP0^L=C%8K(zQ)ssm9M_QViy_JKLVFoapEn8lEtNwaVn;cUcDc-Y zOfP2U1B_kNDwJccHaztQ!0r@~odb1kL6!=2J%UTFBO_egof}TBi8N}5jj@m!4Fr>& zncHk^!L@M36lOXcKVb;bsY=W$2ehFvvO%=og`|y!4iqY<fZyiG&0p$u=f0kO(!5fV|o5>e@o8K`~d}gXXZ(~l zdr?+Zh_UN1!>-7U;?P7_IDWjz=Msyq4n}Q^EFNDVI@`lI&lye5VHPYOff9cE++)$* zUD4Ttp_wUYZaG191r#K;BFCW-q1sURC?i&m4`ccPgGe~Q4lkYN>f6{w$;|aVKTb5e zhg!-U$dF=Sw=jI43o5C?yAbA)o}V({el(mfar0*U$Vsvt(R&S8UlXkSuE+rCx+u#C zC0NCKV0oT-zX1+6dB4{5jK!~b%rg_2#a(8+4!T$g9|2=It?rI# z9f`(~6ME(B<}=9hE4+UIY~`d{KWbY}zJFUq;~v5bAm96g1R#3xbkKOeE0 zH{5SG&tVS*#?6>{2XHC=OHME=@#3N;73YLva>506qAdmZrYb+hfO9d^@~FohmHFHY zYxNbgNB|;olD(X_9?casq3p{1F2$!-B<(RY%@Q=_5%4<{SvMECH3(@p3@NgR-~X6& zu@wkkLe3m!Mc?5qIT88`bAJNY?L^aVgxZg=I`LIEA|0=o(!4A)&A}@>no#2JDR5*- zo-G-z^n6+wsu*C#>SjY3lhMKRiP}xVI(>$vdLKDBk=V^RQ-=Rd#<@*F#pEg zqNQ&Gg#aw%$KY2^y~)i8rP0I987qa^O@hltVEN60V+^zwAL zP&Ce3utCiVV>;lAQ~Z{5g5?zXcdYdi8v6>*FrcQKz$QJCQ7BjX7QR8Xa-MTVe&oFH zRFh&$FkgvBlmiA)&~hYnBeAkBL_nIcuC`#Q9Ix7@yr~Mb^CB@TK{s;ByHv`@nD>j` zkh9J&LmgYtn=_FntC85#u)&UC+gyToud;rD=ozTT3)~ln3ms_hcr1+z@bd$(lgujO zpszz*|GP@uf)ugw3M41-6Z!uU=^sKIXq%~FB@ZQiV7s6^u?22&g+!KKLYWF!4L5C* zFesb~cVHh3GzuX98zMV}k21seF1`_rOE%{QbI1>ER)IpJ$lkn#Bj5A;qd6NpToeqw zWjATA99S>Kyxb;b1u#c9-or;KqFSL-{(+YA z2R1tI*!~}i@fiBR=MwB|NbKW;(S%z>)6LOb8h{aXYtnV_^h+))+jkb@O}gonO^i*L+C;=m1< zMNU3{2;E!+du|}}9-JrfzgQ@~K)y8)j{(cM7$*mF%gIRHSIOaOP**0PWaa8KxH>O% zI>d}q59D2LM$mw6ebz5$B#L}UWXwF;JuMa6M>KqUI9yC~V5qZzj#WLPj`%=Uv8m!EEw#XKgSNZAwI)jA4GR zFAiMOGhYP>mkjJe`MUrdk{#}|aAgkek}8Dq%vvZT4P4Ta!Z(?@)?n7j%Jo!3Y z{+`z_W+XYH2h3aYWBGwxZf56?H7fq`CzD$O&;*kIDgnHEnchWV#tZ-^HhAg_vk|E+ z|1_Rf2o4DW^EuH9bxnE+=I0-Xoj1$AGBo2|vQyo+*RDcQ^g2Uf*ot3MLB3J73 zQ`F4ozUQjk%)kTZ#{sd&Cgn*KU55h2lDhx}I^imbu7`t}EMTcF{_y{>(Aq&0RhUa- z-buWhx=ehkCD=V*Ud%-4a8#n8*Md+XhmQC9|Lb(Vm1XC zzXZ6=&U%V7%gU^(7!VQ45#5`DSDppTM1YN^d|MZ~$ptJjLL={)|4W`H9`-@-`2acf znNJ>JO(a9zr@_%Z^QzNeMxse);fQ$nPq_FETzwiYNJN6z;ftTl`yQV?XT+mG;u?QD z!P^sf_W*aEgkIv|?s#TxVK&v!hrRge#JsA3iN?I^j+}0WJ^2T^usl*_7`DJTUR}_h z^}s?U)+u>_ToZRu(L0hRQC1kvORHES$x5FU;o`+@G5#c>zTq=)}YNUP4C& z(Z6Encjt-4kth$4782`}`i_0bowLmODZi!aNurKI-#-~SmMfnlD@8us14?gM^>@BY z;_pO8{=#=RxJsavZ@LJOVr#H8_Q79=8222X9|!gyxbuc7-2|@1;AhEW zyhI!MGP@c;p*`~z52_6m-yIm_BQk!MxgCUGcQD?6rY$MwbA&vgR5^TX=`IyNkaQYa?C9&>v z@at3Fc`nd7jEv=nta}NJUxSb1tm`rN11!ECac+$yWr;AhD{Wak;ppo4aBOeOS2 zc1E%?+k&htgujGK3h{qI{ule9t(jqI2JaUH0tFebEF(lSvN!WB#^+U-a|T8z$ve>= zViguKM~T2fl6tyWG}RZdQGi!wW+l~s8k$w?yH9ZcPv-js9EeYO3wRy_Q{uNPTpPl5 zqI2HBVb@uoFVe}MpJZ^K3U;1?aiPydM)&}9^RT9x&__KijVj0eik@O;Nghc&T8Y6(zPk|jYxKQHenGzuI6kj6fpwH08KNu+ssJ-}%a7;8&z?Zmx7P zN704_d7Aw7iy25pDKm5QK$W+F=s~QMGeApx+V|jlH-9IZGRp}szTlb<$SJYSen)d& z!gFGUSkS}ofW>k2%uaaWEGxK#jeZIo9pbJRP?Th{Zt>J0ASEj;#N2{ek?c}nU~Pz< zT%Ea=1oQRb#F{1@Wiolf%2nB!sd&!<%eO$|1?!P0NIq~WaD0gzIAhAV6+n6l{Pz@I zPho~{!2VIzasw{B&YVu0lH&uTyaj{57)kQ6Vi(6V`ZvbD$zShScSYtC1Glt*S}VbU zcE0__cVa0@HOhNa8(N`;to-Iw}NAB}|@tN98% zZy>=|F{dPCN;KT@3ORlo*`Py6eoMp$q(-U1(kdP+Y&{zvbHBcTrAix_!1de zyF`w}MhpTU!u3CqDiV3j0}jiAH4oRl0yCSLr$omjcPg3Gqu8T+S@AaTmH@@rxk~JC zvGeIhfWIUTnuZKuPgv6*EN5z{T#Q^E$k&AC^DvfFWPW5$;zb0p&TPzB;#6U*Ac}V( zz&nL8GV*x_a8VJO%E@PmVAaO_rB0?llAtO4kehY4B0+{?1ApeLs_+WzC@xbfLF0LyJG9~KnwK4((DU{TQkeb;JGaH zCz>c5xOxC=S7W8z=b0k0#P2hRj0PbQe1Pj4=u091vXWqI%e&C5a8hBO*nsHKXyQn{ zp!agXEid!^&aAF+MFgWO$l6C-zlYaxAnDIaBv(_O`4!>nEQ}Zg^uv+L5(khfeGkyK zgWZGB`Xy5aNk&?FA(mnPHYI)*$G4Yw##!k82v_arbqe@OOg;f=FV&YRNRgM&;B~Hg z0L=t~UvDg)Ptd?uDE~FHlz8VWt_@?Id6~W7H9eTD2yW{Fr~J@{)QxxJulmT8HAK@I zBMHQ=ba0paFP_tN#!2QW5_LaE6i9MlQyI?%wF*s%CwiO+&I;@`so{Es9=KrIRFVg9 z@J@0t;sc2M78~Fm>x<*B6JX_=iF1jN8oW;iI^sz^gM&{1+57PH1EA#MyCk9;S%IV* z-jV2n6&jS7t3)Hxz?cWh4})8y;I<%EmYwfEnh@FpwBvw@#CTFo%k~Gee!?tHG9MqL zo^n}E+7 z)7}&xULpb(v}779F95ZQc5RGAkqW{}KtC7jaFT&63yt+A_9NXhBBlF5Z&RQ*sj7X$ zh;xbZ9fflJ7*BXOnfZq@hrF!xE6{$+HIj7`YyLGb-H#+W3gm9^msGikhxwi7RD|n8 zkx%KFlWykB0+{s=zP%5=j+p%Mj!~qtKq5e*y~Hk&xa}`AWkGnW1^m{T=;|amtr8Fu z@6?SqSOp3yhfK+4O85i7^&^xKfjkUn2H{{d8eXc+s(qBbHh98w68L?SI}4%rO92g^*m=Z+yTGS8!F_dRCEiv6_(Npykei8Gq&}d*&2p9ofNS7{5h6H8eFrF{q{r z-w3>=cBvlJBl}>9$5k15-iWIj1A{O)R(!!XP{~O+VgovJCOTBQjWz*W$29LSw zChreJRY##Gv6J@$^9lTHz&2w~2e|PvKHn0q77rmEBj-hubVj4r0A9gBQD84UC5?bh zZ9eUZoT<$^Y(P{j9S^)%nD5#mDIIXGa7hI)QNvtG7!bF!a?t^Ap}rV&(?G_{%jXJX zNbLBQNdpoeJ`RmYHd<`+N6>@;RAo*I&ymip22gHQ?z+u22bsNad^8f|3$o=L>wCv~ ze#Z%0pzox2PGXpnIh0Bcv5GACdNLEi{tGA}3|`E^b)`*lDwVN z=sf|C5kTTQ(nD;?jNqg&>le#NJ># zyu}v%7cJ3%j*9WD_yOybY6yv!mW8&Zu12y<5@V4H)OV(>dXM>Jh6bX6Zv*C=4Go)! zB&)!z8$dfz{ImwPHB5U!_|c#B2Qr$9q!T{82p@g|3z9uif#NZ!K=KnRoF4S{~r|#SFnZ`@J^yGHTWhJPO|_%iB%^7 zr8w??%@4aaARQ0#bg3B-4W1J!4aTmM{7oEmewz6w!2gm97N1mN`|W{`^zwiCl`|GyeE%Lg6#MNXSoZ;%kAT&GKu!4SAy1Ha-eX<|(D$q0 zB;oo~+#xx3r%7E2$OFl8T;WN_`1UgYKZf$WkU=U`BkK==8ze(5RU#4#_2>B#DZYzd zutB$Vfly{(`W2dehs;W5MiSL6XUYk&fpfx5p>49Z#URB`rM%*oabO0RsfVDni#SghI5PV3N+;y-Z5s7!q z@I2JB0ef*3(n#uPZ-JvEo+h>a)u7CFjF_IaO18xpPG*lt_+MsMg`YvxtQ@3fWjR($ zJ7!%K9`HftX6D=4(7O1;;lMy7jteZLH*EtSFenkl%s`+T@aaoLs22B$|Mvnu%LP;= z=6sD=ON{(7us;tLq~72jv?}@|)ErmhvN{qb0DA3$-8>3CQ-iUcrq&3Aa$mu}lAX_t z?#z!Yj$kGqfq|@^-G`tm$?7Kplk;FqtlzJEbI#L$WGn=0L16zoQtcVD zdW76a2kg3I@eV~A6f|v(_uL`$`WX0JVs0n+Ejin}aJ0msZh_ys%uoDDA2S*i4jqd{0CQ^M^Y^_`R*vQ`wsmTW^`Z17bz=sczMm|mIS{Un3>4KM~o?FMaV7=vDi2L z(X5Y{$2-;|m2tbk%_sgkjaPV-S)F1g3D|Vej8q5689@Ir5EWbfGL-B9Mj=qgBOoFf zX^G~hXO1O+npA(q!*!DDl>B)+q+uI0Mk{1yI_UT-5?O}^B_n$gScy~|fxg0*lBc*gN;h4r@v>JovGYNWwX zrBvG001GXErc{ai0yk3GRtK5g8~QE;?i|d-pOGc*n4LL10aHJKnTG%A;;A2*gZR;M z>Z>ntSE;YP%UT?)L}dn|g~alcO#FNz?^BtzMC!hQOX)FoA;lzueuLE)M+$a^$3Am) zEKm)Ct~HaRax+7ra)TKP?j-}68A%idkM<)kKb`x^Lur>^%**-bJ@b|FqOz z?1wgff!8SJFFBJUNC2sr7XPRWIFTwd7uJJ#29l%y6W!GhI*Q@_5Tw)qR#O=Gh&+tv zc_opxB7cIp^CN2!{1s$W$^25%4mB9ex{OI^_@l&Zs*-To+Z(L zH_|FElqB(TsUAs(3`<4hh$bun7NwiD7b_ZsB__4)?V!VM)R{?7S8JqcWA2mug+w64 zL;b;Oa=>F!GcJ*E(NV#yPK9<;;oU?aBDCfOjwK#k3>b^0BhgN&zl?+OM1xqsk!0Pa z%i%4*C31I!d9MXa*TJY{1herBi9tm%`y60J^l@2ePCDnVG15tRQmO=w1D6yNTWPFU z;^zfftK=nr_j|X3R;BhRGg9^z_kQK76q7?m&lX2Q)Hc7B7_-Ft5`pqp_`4vSS(`f} zkpE(Lz2O`2W$b3OGK$Y7hLI0A)&?FGd0GqXbcugRCL4glEh{0KwT;h%0Piqhn*Lwc!L+o*%jhx z>G0O2``{IlA~R5H2v5r0QfFEgNzxP#H5%#`JLVhDO5o=zQdhcR-B9{3@biQjTmuh@ zP={n;T&y?_=&pxak1~6SC`l)>#Ei;A?~*OZ#XKT-Vj=k94^}U}i)7J1pf`%cOWt7q z8?clZu;A%55RkZ(U_+`!rC%cwotwn?H?Y#A?&>XC@E)IhS7LwK4 ziN$h+@!y#~fKY+ZrTEz5r*1||jixqdGCV53|6@+C8BOX2e*r(~7m|8`j3z83k0d^! z7v8Y+Hi;)4%v0|$n~WviFrAHNo;$AYb6q!ALvUT`U6&b0e+G`BZ*`N@Dk|AiA|D~ zlq}d6M&1qto};^^YC!hAj)u}oLuuu~b|a`Mk~@-2nt2ZfL!f*sR4tOk1rO-VL%Jb< zXIB;MY4HRlGMB=N@320pv&_!QWT)6pyc)Cinn=@u)Ui&%b6>*$LZjh4-w(<0i*-uR zi$qYRf9(U5`4FsU;>n+Qb^z;_OlKNaWH5iFXSS`8(}U3PQG6%fm-u!+xmGH!ivZK~ zjGh6m(b4sh{4IS3W#EnStmiY-B|e}p;~UVA73^C1T(~!tHAZ6nl;Fw=&}}%QiGQDq z=g5Ark?6+=c%v{}QNy&hC9Wvd$l0Jzv3mbRdsKuzr2bW6*#;Dl6KqHoc@$UFV6-Zx zwVTWIM#T26%}N>or7G~RNN}mIkm!MmmRFhOYxv{Z^*T0Hr+=#c4X%gyMR zm1u)gTqD)f(m$|v?o{xSih+N`V` z|NEGJ;uWAOF`7H@vsh^o*M1Kt2wg~ZgIG}FCB5f!sh^YBs|!4c2X&9x%tKEvMuQ&! zzp|UyROH@8o@Rq0zd}RLxatgg>_5hTY0g!ArhTSA^#q7(&`2m8AUohnO^9^Mx%gDH ztn6=L1+pT8zOuq7AlQ`o*8>ASwEmV;ZdPX$q{Dc*^)8%!QGEp-)Xq=4xILqF`hC?5>zDByP~(Gk|mmXilrgh z!EfL%m=Pqm_uCtgzG&$pkS=(s;W>i!E3u|uK;#ZoX@ydnVY^D_R9^0h1S9!)FQW)e zXM|RM?^WB1byncpPvGW-$uF|Ys^}A`g-!#$C4fdp=qfiO6bGN#8Lc6azvcMhO}JC6 z^AIFzZT@P_6D2pBgJ+e2M*CBjE3PmABOn@lQZog&(CVTe@rCGS|DNZ2in!zd|Rgx$Xwgl*ox> zwZfT6L7pwYqmW;sljC`U_;|U2v}A6?E-nkaMp6&`577hJg+=y-sKgjj(<2fx3D`<) zxKs#Au1B)(79b&(Kz>H5$}0yml1jddaMn~}VslvMRnuydn!D|+LOf>aIG3nGGLq({ zX^}|X>lG8~&$&W$t8{T*fbSlXrIGq^KkoVMyMKqGB7kr)X6D6uGC?zivB^gg_mF+7 zCATG+!oRStXP7_IkKY_2EZsb^qeKI+{Fhm&Q=1W`>P<9> zblg{GRH+}9n4;_rn2Gs_w7d`O&j9h+RJKXKR!4p&nNnpnl4Kq+#W|)8F8kJq4Y!k* zbiNB`Nbd6sx@XFOvSZD7 zxO1{;r!;3~;_IvE{D-{0!_yJ0v?ftS>AGxh%G2JgRIaNDEF=OVxmtO$SldbPQ3^aL z*!BUFInl==B@)4=WMrnG$!D>iMNs4cR(;c?-Xq{m;zTd_OCr>_q3aurD}8#R6Tk9= zROl;&(F(JUiYBbfLp51}p$GU~;wAF-4wNmKD2csDUM>?f6$!_SCHI=^oox)Qe!$T=ZT-jCT8W0o%skSHx^%Mt|QkN?BlEmgD z_9Hr7;*MgOyMW;X@bCyv@fP#TitLg&mmhJ0+kEnwwThMq0gI(sudKzJk?hQIDVpgF zT>lCPKI8w7JX5S@9T`*)sUp1%qM;>*`x%}&g8n+l^S%S`Oh8K@@EPb7N74#>h4Unv zFMe`;e5;P|tp}>wgY_w0R@-^41#^;Vybc5d_Un_h68#1F;NQ^1!63DOGgYRr73`uq)0w4>uToBr$>~%tPu-^7DOrbY2zK_yO7x4^+4;GuV*sPbacmtb=H1Njw{YgasQxG6%0% zuhd$8W$hn*BeO$!tyyO@P>2HJ;tM@S3f}-uVt<@RQ*MTat^*OVhE$#<^eMf4L99i(4sSE_ za$r$|PUM+V&n}eU%`?1^NeTQFiKh7*DEI@RhrsA3Gt-gXg`tfSjQEJxM_>{Jbx9@s z0qmXwXhqpO>OZpzeG&86%T@8rNV4e}fm2Sn!UxI;hq`)zD~Vk@fY?=N`ah(Z@N{-) z;2a#V9o;FBsn0xLqBqaY8A$Y7x?sPsIt_{x3-Ua_r3=7=y<&k^r1Iw!T6zrpyTY6q zyZbUH>07&x{3?Y`6o0EeD@hOhu5$Ge<}aR`?BNoJl$W}W%FHhsT98_BiOmH=^Ef}jcM8ZQm~{I>n2+F5e*+2AP2yh}#*9`G*D=cR#& z$MpB|@V*W(uSSfp4|vYa3ey3p2FzEwz@!#WtkjH*S`%$2{XEinQ4!vixPE=pF3bZK zrRz=lDkP61JyQu#gLu3lj39oc*jYi0E}dCff!Teei)01eP^axDtf!qSuh?L&qRMthhNV?H&TgQ8@}t$3K~IU5?hftQD)XEwd9iX zmX4DCSl&IMfCf;7*w-2OG>R2h0A{U-LP^X*s{2|HNoY?zv?DxRpYdGqx>#O!;4Z0- zlw6^BFcR83u;k!J*HZwn>dw+sc;S2G3 zr9SpBYhhsieLyB8@!c8xjdeU%_T81;6|Ql|JEW2g3FOBZR#tig3t}C#D|xgZz-c#k zy?`dgK9`(rJs=}J3K9bosVseBVhxL@wv%VPX3Q_F>nl(E#(h3W6zPf)yDy)~>uEgG z&Kh!o@AhcLVffB1fMR3QE0>I{cmt=BW=W3VK5 zrv_fm!kuyd$I@AVTTy*&eCqTqx&>5Px?AaPX=!N?6cm(BK|s2t1?iHMR6rU@r9(g( z?&aPSGqY#D-{${4=ecm>#LV7nuXxwH)?&3}>@9Vk>{EGzQ@E4I??~KcR&^n^Uv{^X zaqE2Ep&;)krBk9BGIo=wfsE~PASW+b@ylHELEbTptg0Y%GD?+kjb!&edMD)}?os+S zl0zSR=XFc%7v%~hW-i&nvcgj4OJ!yGFyfeT$YalUCsE=wH&`=?B}isKVRo$s>rs{e zWgj+)CP*D2C!u_Y=lBzP`!DuPdJ*!9?{g(`-DPF8%xXR1>lN>nozMJ`SC`_tWN!kA zKuMO_|6(zTXKa2K;oe8@yyYDvH|iq0XJX+c!cdeQl=UgHK0~rcWldxzWTFszAsOsm zzJH(l&ckmdA44KJ23M4WU2nmi%IXh=)iAjd$z-mK-&dIDN!&z6+0qZn#rsMuuQ8t{ zagsPBLNb?RXPl+@oRVj{0WBbVnry*l{mS<%u+l4#UfDfzF23p_WO)XAxt;5m*<$Gx zN;F7XsMGKK7Flm6xhs<0EHmy8SqIsHM6!-cqHSe2i8yvD8{bP!EtyQR3-D)Xf>vC4 zB_0x^5qKk6xQF!IgRJC`JiF7J)-=xK3-)jmSGkGbO1w~dcg2vZOzerwWyu;SnMGU6 z{zXJN1a@#cJH&$^M zYkvhPl3ZW|ZFG$*liB!Z?B{=6-7R+S4ktNf{zse?SqPG7lM#aDdlDLz$p4SCFJ zNbbXZcI6cJB{N(ydwiDjDy3GUuaCLo2(omD6_D7#8U8NW7cyFr_E}<0zIU&ZT~_WQ zS5jMK#tzhFcVtFK)}>4TLdM2rxtrqH%sO17jKrHEU7fHNlJ6>!lYHohLhNKE?kC7T zN(R5w3O-()_uXFR#LN+B9FLB=UoRPI$wSqiI zzfoq0Qjys`?7%(NTxu056R-GQcB_?j?2>OTtF0u`9YZen@{pZ=WtVHorjj*0Qj^H) zwDQ=;#@v^z+z)VT%gRendubmk za!+fSrEn0Ot0gD+5T49t-v2V6G#jsRC12l>V;cx^CF?k3tR?+u$&-=HmKSvK9E7!K zguX1m70Z0dU3Q=t8O}M#Pe!`GpX9ptzB_3S*FKVd9kt;v*Q5JT3%t^+FxeV%3M5jL zhy+RWIq98+mI#K-jLIxL887xZ& zaFZXo0@=wx@{Ob=lE|dY0LhGj#93$KX>8$6Wew6(cH$4BtWD^OJ*>Hu!3}&jhVODK zvWIijNk{jo4eV#iJ643CdrU3*cXZ-68+fJUvE2ia$V-v} zL)q7R2C~(LtCKS-x1cM=BT-lAyK#iOt4nTi0yykExOz*$sF)4IUVe-YlvNaxtN8?} z%E`_aL!VmI|Hv$l%tOxQJwIWcWM?2*DfpJFlsIcX^o`7LRpAwd(fX3vznPttjFd-6 z?F>4nWTCfKW;zm$MP?)HQy=bPEB?q)EZJ#xQgRtxv}z{eqx*Ozley2F^ru~=rgFZA zgR=A>vMEDfm0a{Sy~k?K=WI!=Bb7D(loB%#Y z4kA7Ao2QZAmGdhk1MeQsma)u1&OsOOj@Inl13qUQY-@#8+J(nli2Z)WnOV#U6I{FO zAv6bh{T9i}$fGiM^(FVa0#9lr(mR*WkrV0e;#IZa{Fq#&tPqpl>NCz*CJ>*0h@_-$ z&x%FOic~j6H%TdMh}^6uE|Y!7F5s2O9_G1;4p*baPVwnwI6ZPkS{ZOEIgjHZJCzwb zu!Sz5f790|GbiSK?z}3Wl7*c<#P4?z+0=M_ZOCF*p7)s!>)B-dC*^-tJ68R4E~ z7s{g9hjRroe=MudWS{(eT;C&Bei$ChK0IW}nA^+GQ>ixSitg~E>mu0C&*_CHXS2*@ z-5zo>GN47)Ap5cg@B=(A$-B9YCn_ubrA73DEB(Qz%ebIAtNIzAn*rX!4t8W99G6Y( zg%<|;SU#m8JqUZVyJykKCAoizNtb@-*U26dWm(zc#Aa{MInqy-HUB^1d!6JS_G;sgv>@shCx}hysVM-<0Z&EU?NgnjXSTvT2_7M5y$g>9ngBeu}72nPR8KuBkjR-2d+-`v`w90`6*;`dXG}u&=W-giaQfxD+(H6xvP&{&EPIddz(UHbrOhYJ z;#^2HK<36K^8H{U*D+k%2&z)QhHv@RyPjm-(a(+Pf6|D@WX_iKqJCpPuk-Iw^uN>z zlErbCQ!t9(o6OXx!yYAYj`nd5$M9J9PQFTVO1Ar3*6A6t z^BgTap9gcl&|W%nF4+q*Ll9v)g6vs)NJ(MN##`kc(^CpE)9xgvMq=Y*(4vRvTy}ze zJHXBj#|N2#M%sd{Zj4Nv<%-X-N0ON)5lR`y%KW5?oHanls>t9A?xrsBkL*6Z0;b)3 zyqJMl$4&Ia-N{_S{e04Nw9broy15e)R-V0+JC?Z_*@1EfXGKo6 ztU*`9{CstR#afXQH3sRB-9{Yr>|zk=Y45l#1L*x!ihg`@T5neR!e6H+uH{rmi&@O2 zD?mTvtb7+oceuLrb$;b!;N2SYuCkMAe$GxFx+M1J%ns)nvIAZTAgXk9V9o#k@Arx+ zj8C2S`CD^-m!6I`Md-p?40}Hkdz>4~m76(xJDJM-)G3M$+eZiJPnn-pmj0`|nG$=Q zZqa|xIr_HK79ZoX(v-QSub48vg+9B3bg!|%=;|r#{oebN_m%gB_k?$yx3>3<=cMPn zXT7JLC%b32QPMb1AM+CWI?bcaU~b<$R`5F7q%FN2w=f6zff&u)_G9*Td$ir#ZfbvO z_qUhY7j4VVC>n_`m?wNzBr;R>FUO!;ep}Y=9MUEIL&+4E-R*|qbNtE7oE-FIy@5PD zW+w0*WMVyE=jrixm|d0nwK%&aYab0xv#i(8$~la34#Q}aAF0ol9q48fG06^t>(Fnq znmUFSlt{#5p7XzskAvLKtaB^+QdVR|b8=0k=p>q%PPdm>v9-+eoyLrwip*l@#XRoK z%xAd4Y~Oz!mFIrorqatZ;=JW|TbQ`Ml|G~jUWUPJstbJYu+CBX9Cx5AuEPAJ`}F;M z&eWm(PKdqk0Sl@*-iN~68sTK6OHmf)e*8`s@A33a{@nSD$vtbBv$==P-Gk}w{R^L} z@T~So#{%TOD4h<=)4jI4wn59JPtvm)7mQ|}Up&d4F5au&g1)T2pzpQ!y7w1vJMVj5 zkM}>%N>5i$sWD-k z?99R^l5N+H7wMV$TUHbm<@mp@=q}cZ6!9@rfw!|>7r9^4DTXAr!xqfJ4$G==8GmI& zC%T;1yX-;)%_*}95<8H5_ZN7)FVSaL(D3p+%iMh`v2rc;cmO`(R;E|(=KMZj%2saX zn!iz8tgP(sn8u`l^Gt#%!ad@jxG_C+NrN=iW%m!E}j* z%qF~w*Igc;>n0j`J$j=glMR?s&K>sV-t#i;e3MwnZ11L`hR7yT?7Q{}d#}C4zF;Sa z%AyeaZwUR}yz>T31)nIEh~Jo(8e!^lMy9cMp#S`4*5C`{sTc+*M_G-UrJGD*Pc^>DqquM@n(j@KMcXL$AYbG-@ zrqM$+QMHhUMa1t}nSZtnPjWE+OmpIpimY{T;(b^ zGk<=U_(z-2nY<3lH&=j;_b#6EFMoDuhgPRH_3n5SKplV6JI4&#~a zH1FMGH$C^?FqOFuUiSbjs`ON4NBLo#$rXGr=SQ!_&K|-(NM?}ijwEwGz3B>CmeX4b zja?X@_H#V3YhV&f6^~j^9j9(pU9|`O&Z}}V_v=B<=p^Hg5%lEo4D@VbcaM4&dbW7h zdiHr9dQv^tIp<$^YIA&co>5o~-fHq4lrCuhot;5-pvpYL-qW92s-4!>v z_rqeD_*C4o=i7trj&^IihF#9CV3)E#;k$x%KD(G*$*#_CYTIq>zV<+SlKqo?(=LEc zKO`*i8IuLhF?B)5!j;EPt>I)>#G@IBw{?%xn1dNNwbXv<0_Fi+R_`-~w~6*O{kF$z zz3JaRjZWMfwN=_y?YQ<@d&Mk)IZURn%!Hhew2WE`nrtC6&aTU5>q-Tz1ey1e6+nZDWiBkhPY3;ON7I16&)(_;o&qInYabi8Jk7bn_ircD;uFy`HMqHU2b8dcOBu^eEoa-m2d6XzTvoUf$;3B3Rzto*z8DJY_r& zjip9M<1b$GL>r+cX@{9qSwX#o#~ViKn$m}VIQ{kCbDkpQ4Mi>yKwH1D|FO5(%k8h( z(Y$tg_BX%%zFiZGJkXwEFR_2K57?LOWIJXT6XV21_BY${eqy1L7*6)#cP~(hk!xdk?+Wr_pJ> zg;s}-*;V;BxAusB>#OK=oS;6y*ZUT4GV41`{eM%34q#zBvm<3WkM}sAXT=|AqyEg5 zA1R`?Pb7)*;zN;6lo3@$4|ct)XePd4=evtOVkTPa2hQ&mv65NzS4B)b6?a8J^m;SA zl0xW$Z`jqXP9`jGG5o5{L>t@K)z3jk?%;jMT5?&_qqD0m-|_V>FfB7L69zJ?1`~hF z&{O_BaF~Mpd<#$HFgEBKbLWKeiaxu;u~VBtWXj{2zQ*QdA#!<%94y7V=zs?)dr%qZ z^u>6k*O>MfSopVoFpZ|%$yuybcIUJh!_Ire8oQTW(Qd}+Zen+`zp|&>htT0! zL>t!dYfi%f@j|@k48>k=$4~ql@8mnK@;cWnyH|I?v-%6m8sH?CQ0t?a8uM>0R-qL- zYCcno`m2rTV?6-vJxT4%;A)&_alv)=}5YPxa%S|`4RIN)?vYACHq&bYza!f$jan?>VuSA}?zp=@Uut>L9@H#r;2aHzY5;9tRz-&#Kzpcb88gWmu?x+W` zWFo%sPsBLwiF9u=Y4ilUxZz!Y+>YE!Z~QEov1`R^BwpN%&&rR^zQp@nfn73!sBQ#t z(-dAKBf$?@0on5=D?2xlD{sIj$!~=-E|@GkXEdt%z9VGoAACvO~$-K z3jQK0U5ySIiHA{~NTwq@Qw@tTo!^xvn)2f}7w7*w5C8cLvXDzzf!5x@6yKd-m=Dp{#qm^!(bs-5 z@AxtI^@7-79GJ*D_haWN6MXj}qfhZTo;c;*p-d94#6ERop5aoW^=5pkt$a`PdzsUa z4ga8wD?OU?x*rfx%DlxCu^L>QkDXXe!c>PYu4SCe0p`LAqRGxaz-8&dzqN>v8)>7)Fp zn!LwoJZm{2aUyKC4OqaX#M=%(y^rm)l`n~chA2%>Bt}hOE4BR=Uik zVOJfzp)SnA&7qDa4z}H&o#k#TXEbQpLQdub$6{LeGv{Bl@(JZDq`emrVGgD#4`r6- zJurl9-05p3nm#seMQP6L_Xg@gC!;dmt)eb)SBpNJ z`SIc-x3@Zp88$VXU(_YeM?{5Z(B_SmI_^~Q9eC0iqWRv;JY^!Jn#K9ceW{*clICx2 z8Fd>ceS!9bwb1p-%xym7F2=(x9$f&>!~C< zeYCCWW4IkJYsS97tQdF5G%MqUzwTy{QET44h$C%d{xDPxaE z5;i#r$`?*i@cAchb*Gd&%ZRw2*$!`um;>BA&RTtzb5v~Cnkq9zadb#){gU0v zX{bI>cRGKFyDB_Bt)^px6D?sEy0~XWA?-49UypNETAwBQvtP@!4DL;_-0cLiH<&pz z--{;NWOb!;%zjVps^Psm!_@)GcA;aNR=eq)@!A5nn)(v2ZlN*VUQ4Vs#2Df}FdM6T zwVg@_vC6rj{H#~ApEA99tCphtZ1&R|DtX;D&K9?HeYpDi&a` z`WWB1Rqfy0`g&Ha0*yX08ke=@)&ys{zFaYh16OFB-DXaD{XccA-BdZEZqpqRwa$2M zC^zgj+WYQ*oTy2nhiYIQuc{raVtR9JjHuzXVY>BpW~>)?RL!eD6#MNY?Y6!bAE+^N zByN} ztk`0|c31RLx9bt>KWDAl+f&6JB7V`Dt5IU)wM?m4uc+dhTR`uuU9^ka1-*;(vEoLo zp|;=GSLCvEx1Fbx_N)2AF6&vTPIVgCJ2a1HsP)Q9RI7T2IqRYwm9Xcco5|V(nAJnC zV@-oH@c=7w(%PuT>qkUkd%B*a{Uj2c|J)*)S4nX5i_g?`MoITeQ&GOx^Qk?QTUIXh zd$qSRU9DnQ)OTq`Vr{gD&Qt|ENi_DKa#zPdnLUqO&6z9~Dl_!q;!kI#G6#KI&YgnA zzG1Ia{CX0R;Vh@J(pQ_N3=o$^Q0=X66JJ>+^o!bHJ16@1vVKFPnH`LZdKSkKm7QK5 zdgxmP_1}fe7dI1^+>pqQN9(h)GGHh>)K^9~tB@1#S*6q#7wrMY0=<*H#Q9YHLY=5w z5S@t+|1}=DA6Y)7k@u~z$> zF+x^b>$3VnziVHyk2(!J9wpICQ~p8HZBfqcuGvV~=i)bYtR5AMOk00wv=a}Vi{!&f zDu%v6-DcKRr}&SWPo2ivLo{`M)z#{Wm*SOPQH|OqH0HjzUtsaCVLMK6-#xX_AnA&| z-mR(E1|_icf$masfRWj#Z?7}^7^A$~&8hYf-&!lZc0pC`549tDcPA)@DTR%3&fb`2 z#Or08J>(_+a&j;!^-Sysb)QkfogRy+kG(Cd)aY8zPEVugF+8e7wYq2_cI&V40xGKq zt-NkIUq5>c9&ABFGZM{C?jmEP8?bjNz14J{s?pzFzh|-&wTCH9j8mNPeEMAFnpsRO z?)g(}0hjzmI$6_}QJNo^ZOLPpdh(w65+)B7?RLq`kcO(fGvp+3IVr1ydWY z_OVL1MSZ3{*8WC)q5k4*K&O{ge^Cdxj@`|^;kl#6Tg&ZuUwv&z^lK-lx1|0PYRHu}E~Y zS2JIGnf|3a$L=Nm(T4i=ih@>iwV8H6%)@#WP)mrF?oB3)H+5!PU6rlcf7%o@O-c2E z`;R(GtXB$Zxs_xqznJPjqR&kIJ9;&+Gx&4lw}|Sj^H%q~Fek*mP_B6nsL!K=?JJ(@ zt`MK#LEKkbsQKJ}G3p{iGt^gaGngOy_5|*lxuXy4b;dWoHFk4zt3BBNtEXJ}3%kGP zp?A5tAza44!(6LC z^`+5Si<&Y0xpvaNYVS}!^DI`^IAzV3%1rINXOr_HS`*)Vx;G-aShn(&^Obws)6vSqom&7`Ci<&n(E7xk@ipO9AbpVJpZ(QT|E|? z;EdLr>c_+q=bZMQ`@1#XXz3{y@re4Ky81#Sbgy~S`ZRdYTRNH}TvcBZ-^Cmf#as7% z>TWR~JB7T})mCZy_27bX?iA@ENU!s~uNE-x3Fbt3vgr$=?Zr;*j5n9~HkQ-a>nSQ4ngi8VezGLi&rW~s zjP_C~CR(_ks`IrWZeQygHBmogDWa37vys~#9LuLI^^8?2TVZpq{;^(7ly@e%J(PM* zQDucU=xmP8byumI^l9oH>$p=9oZ+f6#5}D`^>kOp+v}BeT6yPo9 zee15UHj3}nQAPpnvh|TQSzV}IbC2oiwHczg8Sgv~6n5uCO!r&;gfUdyuq=@Nrh0L0 zy4hB(;3;l4G+iy|ed272%~5XYT|9a1snPeGsIk^N(8*%Xbx)|@C{;oEo*H3$o>^Bv z>RaH93=dVmF>b3Fjk8LF@NF^Ods=VeZn67W+tpRZY|kodNhC$>QBqP8*vO>iJGB2D($-4eQ6#St6G<)tD)= z2(Q+~-KC9iYv9+S@w7y3mowh2s+PqUO;vMiaZZ}GR(WRhBO}nw?c}5>UDa~>K>Kge zS=pvnclU{$T1m7u)A*c=#3yntTP~uP1nr=><9tUR=T~)}Q(ru1qVZkv3wn5-3MUiK z`>JxlKID9~(9B4N$N> z*_4C)CQ;2nOghSGuU1sOc;=1N&Q2}&k&|EFsnij9$aZ|eDYKdVeTV6mpAtFWR%U58 zz%r9nkFF9GGrLNE%cNWSnz{z^S5B=$E@FaOSUoFrH=^z)Tc6W8?=DflBd7RMJ)#!T zrig3ApLdGR%b)|$m#_6r-L`CrTL}~Yh(n#&6oFihqNv!&@vdUfV>{L3bR4su+f1#dLy19cu zS<1T$m7eY{XPz=l8|jvGt`VgqX>`(dcDg%=O)HS28}D9M@3;+}H)0bA&vGZ5`<}B% z8S8F==`qNmVo4pOlyRoHr<9OpazfyJf&LA2zHwVCrL^yrChYb==Y2J|!u%;vqY-X* z=Hw)P8Yy zDNVtx2P(Uq#Y6(7wU%m4vS4xQb(PLeP7|=bnaT*Fh?Yc*J@sSi67Z58pe-+*=1P(_ z1srL=)7PzuoKJN6ie+F6)zL;%#1|kaDva*TAPz^FO-$9Vx)c6FUiG(k+1Fm=zCQwa zT*uCR3_@3$tm-pzLv!5*#0b~5TS{3$jT||FkCdOBwV*2llqU8OU~(k9T1@6Z}+ zL9k9aYsia9*3(<(7xz2xuaYpAj&S-5sri+*ZeudBeK>8uDi=W^mnq}aN$h)Lcy4FN z1<9VzP3Y=V5UDKARV^X=^ds7`8u%VuU*;d$Nb_j0vfs%T^dk$DhrF1M?wq2&CX09p z&%Cbs1E0Q~-1jF)^*k`JKk2l6PAQ=tQx>@^#awqSSjQ@Izj4Y$H(69i^R$NVqp=!I zoW^9VGdaJzbJcoUGi9RiIzNIR$m~x5uf4XrgS<^q@(@KiA^X4pzjexjrfo(-JE_ay zuWx3 z71(fJFviaAW>Bu;{C%!k7W@1IXEIUSrp{5bsxyezFMv&EP^W{M{J^Kg+yf5md!?+p z0o45qw>8=0apdj0f;{8_Mg82p?`#kS$dO(m<88v3sL20^$)z?RTe^%)>nkGi-JloC z!4@AA#a$7$nCv8yYw70R1OK=~#^+bDl6ma+oOy83j)O*p?Y?5GqbXmLp>6AAK_)+U zE{Y9!^a1fq{6Z$MG`aa-ow-cuJSirSM<_;Sum~EpubYz$LnY_6m@LNGx9zFU3MaR7 zN;Kt}Tf_+`2sXYG?#YkveNHML>3?|Qypud@j9$ho{gILCJ>_fRuODa`huz3h3Etbt75i4Ei2@ zy8gR1LvyrtdMkaJ-diuGAJO3tXnD2F%sVyUbhvPIhO@u_qFp9}j^%OwB)?XHyy#U{ zproi~-?qkDQ>^CX3~$;a?FH6M>x7kTh3wqowmsRtW(Do{M0TRj0%TfiI7{utb{?*5 z3CK|mXCYafl4NvBlC{Yxa@mutOI-a(tB&=nsaQwM5@t`crS+XS;0!_rb2=SGnw<%6 zqOoz)TiLhGBYmXh`beX@??B*0a6oWepk$z>|G58opi*#mphw`-z=ptme_j9I{$Kp# zeD%DJF;_pWcQO_lPmB}#Pue1-k&^_P(~|n1+He6v&K3JFE6u!NwzHDRiw(1CSgr9E z>sfcqAIn>)y4n*g)oN-jwO&}wt<~mNW^J>FnazA2+Z-Ej?lkM0narwY*gR{FF#DK`&C_Nf zYlKz8&SPJ<{7{kh`qf%)HKgU~>)6KFadU(<%Ip_=9{s|sXD=7&VcED& zIcvN*z)n*8dh-WA2`2cb8?&`JdP!fY(4X;r67I)miz^Vk61W;X5^5dSHLiKwM{#FD z^+HR6&4LNRHi5Rjw%+T;N~5?Zy|=Hore}pV%*`&w*nbEEAE|+2xaaNH=H%G4XpLBT zbCBh+_L&9DAu%x z1kZk|Ow#j{3vcVGv@S_7J~WTg*DvD_bMeTU`vWevh^b zze=kSsUF*DCUXKWn#s`vkztWoBv0&T^OdF9m#w_^QKY&p8UEgOH*uY4vj@DAI^<@n zX(h-ZZ&ADJ=RL2z^L)8|0dFCX&y#Fq_au3n`fB-e`AhkBcno8Q{-M#>xT~MjJLt*U z0x-iL@y#lLD^GUHIs3`D+xB`stB5tz{Ln0A&N7p%^ww~5bgVd!&M^<`dok9*tZx1o z`!n`gY)-7GIm{em{$l#97iIzLEHb^{Y-0Xyer?`0^O##>C!$xQWn=fF$D-fGrd#L4 zd$6sxv!V;ly;e^ppZDKjjku433p~fPQ`&6r>rl0%uhX4LjKp6HRSaDU^^VIJKPmph zgk8w*uW?O6a|2%gcwcGXMs)9evZ;@l1KvWN?ndohX4zPSSgN_fUS|>yQ7lVrLNr5kaimhDe} zViRJoV;Rhvu}eta9OQJK8E-vj*GF0p%~)(iY(?z9n9powCYk|rL##G?vB#`!9~PtV zXEkS^ebSCP1NEx@exXUBFa0Y#U5xXdM!}(p^U@bge>iDNLMZ-tT}<2|TPv(d))utr6Z2whRn#AC75Oim7OoeW8rctK)G1OSVuc0QRW>px zk}vvIbXPP-tPxr%2ioAP=-6l^`X<)jJQYie=8ydmtr2Y(2}PzvMn;xLGguz75hI*+ zLKFSbwjD$hwSsSL=u-UeaY?j%ENS%h`r~${%bfXP=A#*_q&pcuI4(zg-^5Jm*Jb!2 zW62EfrF#&6FsKK5_!}T8`TftlV?Fcq(%L#DE6DR^Gc_tAL!uL6|Dq}9gg;7slw9Xc zr8iBJjkhyWC#OA1%bqs>?W<%@^1kHRZ>xqsiz?=C=%{nC3b8}6pw$B$U(s6YDt6}p z`tX`@++%q=`ZEPPhkC?)5;rvTUvPNv=U~Rr7ops7hvL%Wl0yrFBLmg^G4DiAHqVEi za-Ii9dSe!tZ;JD?SZoJu9j!UuDurC;jm?kz9%$CqHaK-4TKk?3uTbjONmY=hh&$2y3G_hd9i})#t;dBX^ zW@KrQWoO2$NsB}C{jTpwAQC(hJndiQIir88-2?ZSYF9NUMz2LWMBj3X7Dm)?$<#b4 z?ca2Nz2o&iZsWRxi(@GR z^q-7z##!Cc=Ihfu$-YT}^}*|*nsM#1hlS%3;~s=^#C;uiFD_ktpZLM?;kZkoV6eUa zh(Sgr9}?M9M^4MgNI>64@Kx7!D%0DXEd0Y2UxCk-Xxy@_O5wb8l0_H)GYU z2G&RB^61b=QgoO3sncIgZ+z~l=b2{Q!K%GA_IZuKtzdlI?6@9r+2Y=W>`;xk&T*yU z9*53{I>cR$`zx+;+^gUV|5$GdOqguOJ);)B@(F#gI>=dPCCBE)x|?gPUtu}rwPr<| zgma}mOx==po2FfxqpPD=BBR1p()y+@Nga?jBRno*MBhYaN4iH|g?ohEG=DfQ+&?@o zydc~rJUQ)V>a4fdQm&>%Q@%-k7Cs+KwjY9erI`oA8Pc*vo4EJ<4U#UV-=CN#_>a+A z@8(S(H#EJOrEQM$IT~iGlxahv87dGrC@CRJQqH=$24#z-KN%OnhMx|U3Vj~x7JTA= z>`iCPS9*v}*t&}5G4#tPku_=CQ}3m=NxPhSJ0Qb4RLi zl1GQHrxi+Dnc6Y+!_-`(q`#G{-3p#a zESGdLK491#QpS@d-v)Q&}8l2QC{$~8`beA))&OR>twaf|W&&1~n zoe8uL^bV8?{OW)1JLyT)Zo4%_R{M#y6;G^QR1J4dO?!JJ^-$X8v?-~hQ*OP^{%Xz3 z1F!bJDf6~`TJiAFG;ivddye@nXU&0U4YNSO|r=;F}oBnN~l8lD|u7_O?J;h*d1@JhLyw zJ_&O#1=9bX zsX>;OnKxwcrfZ%!AiiX1Q(&BbuTS-L^<2}a1SLZLjF`+7VY6N&dusg@D|yJcgLSC_*JNJ-0HZ}ag{@LgEIq@0(pY}2DgTehu#GH1g80Z_go=z z*k;^?>vmlKMBA<80oTYZ=GfEhiSSbUTCL2svEQRJqCKPEMrTF8jOK_Qj0}(TjC>LK zDzYQ;cO)fJE?PEvBGM&Nhu=37!v351bE72whD$pMFB7_cM3QXs2tD zG$`rsbTcyK&G=D;h5X08 z89Wz3A^O<2>?zC-^3?LuanLtHcPk{Hz#y3m?79QaMM57f7RF1JHcqG4OJ==moJ9>_qph6SF^I1 znPP3C3DFOunWFn6MI#NviOF+^$D7)rcz1p#@z$z+D)0{u5=1JgJ7Ztg#&|PfKbI)`;gV2h^N1G&MIuj ztgozX<^c0cv%ERp{MXE7rL&^=XNAo}vAwZytcUrxxfK6=R4iBQgIKB9l-RXcX?%{< z*!WnsSi9I!V$UUJE3{L25WSV;fJdtv)k0dLKFTmX&3qI6-2&Bu{!so*)BqF~eLCVCG5f> zO|No-nv8>Z(R0bQe+lyDah8kP;t(;=8v72I*Ij$H-O5&phCa1t*=hX7gt7di-Olbn zCVhcD2M+Ton663o^LMyMz#d}%O07zBxZv^ZPY%&mtP=G=7MsIJly%NG;ky2>dh{#h zuF{0+r#0Ne3GG{bw%$?SO%AM%_MVw^2=VR-gCYf(JjFx}G?_j^tuC5AgVQf@#!r4YC;%ogy%RzazVsEb7CM{nDw$YW^eEgTgn*-VLGNAqnREr&PF9 zfGzBIESL{DoJ5$kmElOQ0tuZ9N4h%ebMd5&@`AMa&6xHra*FR7&h7qp_6A$$)ztgx~8t7pjvL&f6Q|;2__DNuOcmKXekPDe1x84x@&sFKa4M>fE5$lfdssQ46vF z8T^5I+c~hu-;gmK4O3WF+V52|!n*!Zy{O(+H<2B@|E{`t4XoR0NJ~!j0<|xL;n_YU zi*bwEi23057085I&LmEcD>ib4wLn!12-n_Wm$bu{1xoyx-H)>}pK6cy><{hnFcb2? zogYr6Kmzx1gsP^eWWP3n-nUlIs*Y;$=^m}WX6WnnfKkB6Z&c<{#E3Jp80A>kTcDZ` zjoYlTJk{Iv>zs`P`V76R-bT-^|ILZGq^?z`bGBxyBh)(T3+nZ{qdDcQt8y?$53{#D zU<2oHGr;{l!Q%z|-{>{H>c z*H*K$rkAO0NTVw79CIX|k3L?R1dyT*Jxi1gM;m{P|wU z?f@r(Ua&-U%^#fKMV#h5T*G7M1}uwaZUyp#F>|u1o(%l%oY;9iGlKQmXtkh?`|3E4tI1d5umA z!O{7GM;}(Q0Suac=nM0-dVE7x;m=Q6ZyOF=;BiQs zT%T(l!j&zfPUjfR{}A;(qtqqjYe%9Rm%~0n0sS9WFXDos2Pcoa_jO@NIXGH&lB+A~l0xu&Qx%5=-=@8G5{rEO=FK6EQaJQ-0 zl3lW8Kh7s;nJKL0Ao6)rc;-spRrculms;bi@T6rguqV871)6;)(zt|&?8~+gE4+qh z&xb3#f_?p#{cQ=~^pkg|WH;7eGT(KCAMhy>Fc0rw6CEx-Lkl&ds&6i@`Z}Mm-GHj3O}rCqUTQf?Qr**-^;-Z-;3%<0 zMp!&W(c?q0PP5T5BhV^Es5Z+^wVY6VoETlLOx4vgb(gvlsh_O2=NXOFj@a0$ShKP` zGO1~3=~vXIrIGjDj0{eu6HQ-wHpq_qDXhwQ>QN8#I6_rCwFBhRO?2i3WPhPE!>!{iWWl4cgD+WORjhnGSC{P!xB!xR(#uQ z@4753zYUB=oYJqAh1|=(yo*jhiV4X0 z8|3{M_jCZBP%$)56Zl~Q&P*=q7$4HvrHncdKVl}`QTAbHwjh^>IT3QjXQK(^9FKV3 zwJz&rz%Kg?{pZC}SEH_H7o7__V^KH55X!2R(Wm&}m# z?BiT?^94smuMC0(lY{*8NzU{N_o9-l6;TJ^zl~?VCa@o4l(XospOsP6&3@ywhq)Z! zPB*#aIJ0YL$-_8Iv5PTGpV)fXviqC32<$OBAuYidS|{d@hsx;bZ%I(HN_p zhYk|Mobm9AGqdXFs5#n)Z4NtMqcJDoJzU{k2C+xkTv>J9&z;2;PlQv_6`j)rEqOsX zrk>Vv>DRQyRCaIC8tGq?JL^l_(_NpauOU17L?3AE#A6?!-$4fjwPe*)t7(JK`s=8` zd@{@gOyZW=XmgE|T-IGnmG5Y$8#d90{2Zb4 z!Xu{<+VW3W@a@r4y|KWfs9x!aP3p^jR-!&@7W}LUc(5JSooX>Y`3BxvUTut45)N5D za`sE9bo__<8x&#Z|l&{pfu0 zG3dcTk&*bUrYI;(@!B3PV)lKJCPvtU@Ng%ADD<%B+57E-R!t(^?Nsniv?^N()TGE)Dgz*`%NN0$knaWi9@CrWL&$5Yu;*88XTxwp7)w(m1v zAzvroPrl;5=Dr)gV*We6+TO2Vn>bWrx6mKyPxZIjr?B=W<4b%@&1x>!>r_N8z7#XW z9PwNs9JEl|E2e##|DkkjjDQXEutL7!`p`4IOx0-Z}{}P zW)gYugXY&()LKvPlo$3mI~7FcjJ*RKrW~vlM>KRYx~0hF&!z^uuzHBv&3|EgO$Aw) ztA^E+>L4|YUw#$d)@A&WGRn``-xgTN^hmz^sj|Dl$>^wf+TY<-e8k_H<6-}T6`p{G zzQ-hqJ=prr#Dkr+CmNYvy|kXF-_tVc#dS^Jr%lnu5_MPPR6M2TKaLu*cx3@z$sfeL zA?mmH(i_AldQ)*;n63hepg6a!qgIM#!}X|1EczXq<$zt9j96VB)vf$i25Y-niCpau z;9wi06UjT|js6hX6#g%*JsHm@slTKaPJ0u+5c7(j@ENah2j96f)Pcq#-{?ToV9Q|n z(4)A;35OG>C(TKEnRqK9l%U1`6Dl5>9(oyC7&;LADDaK1ptp%?pJcfVL* zuY?m;9em@o*vXk|ZB4gQEf3c<&yL!~#asJdd!xAI(7RrVQT18~7U^_H;iM$t^=Pnn z`k;xXpn-bfu@`W!fw8<1AK|x6NADL>HsMpehtKdgXP`RO#$)Makcto23lF9%@!Bjr zvyFJ|Kl1aZ_-B2HsHQ1zm5FL??YdS&Z>`s5y-Tt7TlD%yU!$v$)%Z_OBVv4_-y{02 zqPONw`cYpjJBM5XlS!evJ4Lh?SBXSx*ypU0)?XmJ3NoC{tYr={*P2_+mF76}WB3>y zVHP!n#WLIc0sci-b1J{DZeER5jO~wBj5ds#k+bA+UxYskKTRt|mh4Vy35mLz38E-A zdL*mxzS>rAG9*n%T%6D)enH&exIX0dR)ls3 zI|MrTU-%mM29eYA>F=pGh)&AUZ|?>X{Zx9*WED*~qkZTy(#>9P4Yq!?Hj$OuL-(MW zu*o)o=oJ;Esb-%HDlkqYz?vV&nf(l$tqnfFW%%@s@xHZ@ty7bTGJ%r}7(>?vKHemZMD$V9zBsc?3%p54I(zZ|@{Z>cn4e@~le4Dt)xu z+9T}}-F7DE8}wONuHNiUK3&kaqbG4mi0&Ij)s0xF1;pJypy|Jd?RL_hNi9%!yDMMa zVc~SJTiV~#n<1xYDe{S3Fms+-tE_LW&#fHR8nY}p@M>mB^JZ*oY<+BGY$&{q%(3m! z>SRAMN3W9q?G+R?>{QY6V zeI2L~>=8O0mz3~#!rO$;6E?&r#y^Uy6u&wCczp5r__!Uxt%2eJBM|m)^{4vMygNM8 zjS1ATeMLW!`aMC9{{Jb%T14ZwG9LvFma_>HUS2#YVTeZ|@apKCT>{jEk;7MN#8%u(FcZWu}p z$t}lX&0y5rGP7D`tpL2WZ_OIyZimBADg)=TE&0RM)?F(MfAKqbZ9kbUO%v?=eoVw_ zo4d?^tOSvTPB^Xb_@nN0Wt#ede$A-wz2w{N*8{;o^T6Z4-r(KP!MF$vj6!iYLwiE6 zVP{N&WtR}wKeQS)<0XGZf12-*uNJ4Ju_w2YLvN{_#^p(HntTRn&{V>o-$_-FZ#a z*h1@$fAm~Ez&@_U%F9UPIbAf$(S=Y>fs%8IFM&cdgYPZlqkpl?*O32z(Xz|!bM|w4 z89gg9+rM!Vc32Usy}b@wx5oYj;F#O(zipikAOpo`@KRgQ(_#u0 z61m`H+2jb~$yeOB9s4yp;~P7hecI|qrfZM62<>LEtBYduV>4q>K#nKf|}h z`=R%aXP2juXFnaa4paB|7SHA%FqhNdBFE`XGXhM$C;H5^N7*6!4%WOl*0318SYC3U zbFha`tU6rjKlUcHLRPy7)~F4bb1QK0L&T2P-dTzi80n7E3pVc~wJ`kAH(E*TK_>kn zSm0XvB|Xyq0vQ~}shOqyq|FDn>cnnn+IoEE{aE`V_+{78XcIsSGGm`tQ}w-=9mwVU zj=wnuboHuuDE{JcSlocod;m25wUe3XFDJ-U9ybPZvL7^V4^`u*z$BhHHPPBL(f`AU z&u?OP{zA&TVCnLLV};zypoi!2XQnyj@h_s{iFibQa}Dg_iExy!i!dl(DjA8j_?PwB z!Di%Z>WJK81KP2u9kDKOvL;%kIg!WA<7U+Ci559&9}xdi$)q%%WTcYEsji=Abvx5`FR>l8v|sCiD#`S39y1wo?hq-VXn$7=CeKI>>fJj*FvH z?!3ci&hd=;;0c1xi8Jw1HOxh5QVg?!I_dMiA@A6pDo8evD; zupfWoPn2LM%D^d{&I4LJJ;^+?(V`C6$J$q14Q8V%zeLS{_oxoF5EkJ=A7p|=gHsmoX_AKE+JGbn973O!NqGlJPnNp37?XY6iW9st~=|kG+-%Pu!BHATRgwIxH`t z+e~#ik?6rFthEkOBzAQsW9?0xMMNb(Yu!;2(RvNAPj*3#e)uXD)uEK716Dz@HFwA^ zbwNYxmG|xoU`i{yzk@Drg0j*j|{UXD!15oc9b2RaP( z0L$-7^=}-0@KWiT{Ga}ep_Xw6U9F~&hkasdXIbPm)l2ePX-Pyr-mu)YEVZ<=477Bz z_*vSTkD2<=8!4YD$GFhA%dl0SjrG|X4^fhrNG0eXIRi^8fw9+RC4VQf{2LjWH|U#G z`Wo#~7APB(y>PZKbdJSde@DK)0=c|>M$oKlk+ z;Sd>@W`?)=kyJ%Kl6n2DaQi0yq1 zG=4{9O|Uy1D{7#tItX2hD*-*-+F1Yu`Jc{~*o0pkHypbh>&QOWb$kbfcFR8AKGr^$ zym|}!2>We&eMcGROxGj#bLAbi7&nQ5oYvb6uZ>$xBh8yEJH6_8cL!xu!sn6CM&Cfc z@_svf-9Gz$7W;Jb$?&f1y%4)%xaEL(kLf1e$_5$?NXnu5+42SULs#bT8sAU$OrmOi zKiRdR_-Q}LU7dF)(Z9-#)mH$E?-&wMB@>juie@Ugkt!DaBr5oo^7OrWP9(>Vy*ZGa z+E~pBAB%Nyi#Y8XyvUVAyu>Lud$6$^QDxW2ix$?GOB@>(sxOlO zTvbHb1Y*0BJC;sP>8=yt37WXdyM8*;ovF?T&Wq0DX!fR>_1??b%K1BgtwZl8q4j?{ zGn_KG_9CF$$AVSQ<4VF7S?KHzDmx#(>{&4Q&D~A$;NPk1s9ReuO_L|sae?k}n?Y8O!Q!fe57@xnh#c7_R<=Hr-9yEW9~r1S z1cL^$6BnN$l%9(90J; z?K)4I0@b?k?`C-D3hQ~#t;0q*q2wXE?Ms9p4{~>(&YwKN1R~lyrQ^gTE+I8@p%b>s zNpiG)g8mk0_eqBRhTCL1Ul^_#wi;#{`WreKq7AtWA*^6yek6mU_oElVZE~dD^be>< z-i=*$9N!@YuB+_1Pj)Sg9P(8vT^dqrdjah~o%P7&x`vGz>+FKX+?qA1^?QcIk&1RKt4XY-*2QSk`$(dN=p(?|sa>rB8rwGv6$qu|B>& zPH)va#e0u;l=pP6Z6C zHpk{DO}1qZm91^0d(s%$jHlc{@4ypyAY1gKLD$zdj5f?QEF@c9(Kr_r?l9AA({R&B z(|%qhOdpL~>4q9?+-8V0EYp|N=hgopBIzdQ-WYGEx@R;PpLq94wDB=l4OanZ-N4nJ zDw}PtbFSU!7FRfT^$@P z>@oJ!_FQ1!V{I>U^5#sIW2M)=G3z0x4pLc*bmz( z*aB@!Y%lEtougcPUAU-b;0iyy+1pON_J5R5$siEGv^98>R2oBs5G(I>osGnc9CCM;V>hCeAYu$EdVMi<5 zpq%GfSATu|RV%xS)nz+p*V`{!i&D7|>Kv>%JUg)a_RxQ$sOdMaJiZ(Kviv6b#rmG` z?&mee5@D%hX=^!W_A|ZIUzDaI0rwJj9iY~6J+=kr?9LjRbvdh8c8lyb+1;~eWLt7v zIa6)j91?Y8N$8y{`d(C2273BSbMOK~h(x{9zeD@iGX%-=r~=AeuuS;dQ(MHFno8#T;FSq zPdUGeeiMD3U|L>ro;xmcRY_2i5lHV~q(DciXk=YIxw^5r z*LJ@{f!~5Q1o;M)2|5^DHFvkri=j31oCsMF^dqoe(9@uTL2vyh`y`vM7;?+2$XuOu zHgL?aTR~56%JIpro^>W`an1-^P5VLHM9`I0tP%D^XK$sZXQQr!>}$v|wlOzgN6tep zulAO`4_ac)ai;C2)#g?BI5*8_Vro~|cot|y>=mw_W$i3a|JwN?uMr6juUjq;zG z;5jUTvNDvV{^X;<$5`K(+qBX2$aKs!)l}Z}&^U%p;sM6a#zw{&+;bS~JBVD&YJ8zl z>K^vodDkQ7UGS&%sogjRQhFT`uO#br>s4wXrrFYM#qC`{1Mjgb_CXH2V~les6+}_= zp+10KIEu|aAN#B=nbN+}Cp@i=#74J+iO5j9s|%Eq?j+>EF6^d!T3u9Lke7PrFGoe} zq~-MGsYYj<6>=WqP16+f-{yPftCn%zn|;gs-|=7NALbwL7wx~r|BQcQ|A~J4sZ|*3 zTgdm8_e@J!(=@{~eWL!R-Xq6JN9mSv0ShP-t)J`+a?WuKB%*Z79!(9(C)+1z|IAr| zxNH^II9Dh!4j*z7$;xeN`!DIXfQIk_>rj_C*lKwp5q4jB8TsGd^l!UHu3DT4(UHD} znV`ygscFg}tcx4iR8z399=m#@k2+I7*_;(=h1V0W6at@pL(R{3rjhearUG2}i>syL zM3)EXPttj`h5ik(gC6oM@I_1Ts=kq1UMrm=*5EDM$$bq0iB<*w^(~p(yC8oeh@So+ zX0jVOa$WZkf80xw>DJy-HWO4z99i?q ztj7XyKDWWF6{N0tI=QY;YK#`RYN9u4<5>o{C%a!0(UIvnU6x47K6>{CfLHp7PxV|W zN=8b?qHl$79!ajvhfH=J?mCCeTQFFMSS;(NaKaPD_lOL0Z9KUm_?{i0qz^q5#ktm0 zt>NuDz*l?&UvUJK<4h{=pQw$%t?Z(6poN@es%{!_$6oSzSq5Qk1K%28Tx7gwd~Up9 zd}OqkLO^NPCJvN>mTYG%WV~hg&ETe+>v6jKM(MB6-FT)HB++S(F{OYSh{3yF2XZC} z-|!9o8F_4XoO=M#ukvnR;#JLv09?X%tcI`M5byGv`x@G*7}iid^|%(B@FMykx=kRZGqcMunRMMkb0HP=&>`O0lNc2@?W+MV9b<@tAmYGUTc z@$YOPL&UlF0car~qK;8y^K%jRve2WU2tIvlx>O9}|5fqe0{FZu9`s)F0uMm0WssBi z=PZf}aB(pynr^=z$l)($_5US@e~D|hKv{8Gr4DcHJ~C!D@u0f&F)fJaT^f3H#IF}9 zp)hhid5MAv+}Yp!E6##w$|{Ld&c)et;v~q0?3xhJ8}Fq?;LT3RZ{+XPJ$Sukz^$$VOC1BI zH;J7ROa!DWk&tf01SYU@vZ{~pxorJ^9;I^I;`I^XQ z>14_6(A7(N0Y6U0KBxlTA(HMzHsZl=h_?K}er?VU9zYCwAIQxK?B)9G$Y#*8EKz|J z@O%%6RxV-3>|%$e5t%a(uc^s3_1OjDw3%68cW!_fNg%E=1>b8ssGqMyvr_QZcav@F z1lp)RR%Hirn_XBlfw-zkw4f6!G?AFkQReoO40a6j8V5S?GRCkRiO7bAq$4U@7|z>Y;-TFj->C3B#0q4Xr&+97Iw&Gs4<7kKz-y_9__50*Wj1d z!tZ6k3#S@KP5ztQV247T`)-^Lul4Xfd$5gV0n-;57FTJzlQKuplJ0!W`cd zIjs%#)__PkOAPr7uiIdL#A!_5*vr*;{qLO77Epcw2vdP$>&u)UA=l57tGGbdn{#*& ziR8KC`E4SIsxs^d9lNb4*H$GiXe3T1&cl7md01b8R)fwSe)(99P_z%3eP!>2P|+(OF;$w4F+K%KcgAfWT?}W)fH#;iZkK` zPmG5bCvo?A%uSrbD9-VG&AgTp%~+4#5$B2-kxfxhRh+xsnH@C)$?%_ssH=e7a#Cdx zMD8RLt-Ti%VpH~sI1h0h8T@8Y-mUt>FXf4oG^IQ7cJ_ccU;9I>_(L09t>bopqsa|fcc^Vr!Z`OX6J(33%GG$(`F zP{TU5=ALoPcOsvOE=q@ykZtIJ{gdobHnckfb;pnepj!^{EMMvp1V+^YQqfK9bOk7b z3HVq0kS;sWvg<%qZ-FCb@%IJfObPhBDttZy+AT%rJOf8Umn&`mh|}#&JV~5>TMj8A z&My=>r624DLDzUX3?JonjNg8twTrXzL*U+{tk(kQ@E3b-5+fhTZ(~@yjUcYJF~`NA zGw*|s$Yym9GV{@7-+M5}zI2@sCvM$@YgUmX_#JK^h_s18ZbYyzRnF~+fPV&J$@Js1 z&V1I081hKgdj}kp0mf5Dcjmh6p~5^>bP68C-`&|oZq6<9pd<644aDqn<2#)J6TA^= z^$?kOleneIr)}V_qu?(JV?AVouF-*OwPDN1$W<4yV;i*<&%xR~fbTC7)7r#S>e08k zJe_!op!@7ZKn1pPE!@}_o#e#I`Kb!5bQSm}3cXYVIoJu#>Vh=wN`*yn-G8j!R!^d| zfNXZUv=@r^=c-Ti^L?hvEybfVBdOWh1G0AXd$jma)fwkbqnjjwIq`~A!dTN;b=E$*@pc)&24=&DJ z6S;O0JERKy83#u2cWBvONx@yIGv1`4R--1gz#Q z)NnoJogZ-4YP7~EDM-bJ@PkvGa0@b&c7@0Qy2119abSf@-l>-Y~_jfkZ4!o&~5BhamsWf zBAuQ2{}4_8wP7rSu{aZu3on_+1FjY4*8I*h`|#9R%wQzH_kxa{p#5$*tTDTFJkMJR zO%GtBT!SC-qNhd?*Zh}%yMPWIgEm-+tlvz8%r4acVOR`|!WYh9I8W|j0haneGC>W{ z;uX+y)FF`-vv9tH4;{u{Q9m&n>_kmC_8HQ7K6BrQ1QX|t^+9VzlO3#${oRxO@fUlx z7i(RfU2i5Y=7WEtB4hKTzarq0GHBfz*hV?jQ+y*{e~$X7YrJlO9V`awNX&3Qvp$dY zKAm~CC95%(j7(Q{*AOJ>0=Q@@?^dO6_gnnVv&4)qAW0_ErK|zfH{q&*YNuSpZZg0x z#wuy267>mktFBbK6UDpE5JA1|%5hb9mjHWIlx*5$_fRs5bEQFs1?KNw(LU>Z z3i=%I?&ICqJB{kPJwAPWi{tH0@elN0;M>nTza@s_a`PKn>1)#4N_6L~Oig8$^MqrX z{jKext)=}v{^&IOK3k%-vvr`=YE86Naa?g8abH#s(;q#LydQ3kMJ^STmdMo%Z;h4A z7RzeO2I|Dum}i*&HLf=l<7}-Z`rq{HEFtuIEj_&4Ik<)qzcU%EoV6Hyw{_Ah0f`?V{# zYmu|KhC&+XJPcCgp_a!O=B(wc#7|G>9OrY86+K*y=n2!;X>|;7IPBN#gY6~kIkp^I zYx_V)Syw^zn>62);a$;xQowuv2*0g9q4Z&B?Y}ATeej7~XY>3P7MNEFJ(25DKws~? zrq}vLpi0+r0#YuCdMh=-{ma?eK04>quQi!lGQ0jVWnKKWH*@pP{y$e{CS-lFUU8In zXDKs1*L4GsOdiJGL!INtR+2EE{o{G&&qdR;jU`szGc!tai6jx@(JI#11|%TaE}CoSp{ zfz?cUZwob2VM@zlcz*z zvCxINPX(I-I`~Yrd^cs9>RTRpUGuKwt(q?wib!vj;;!nBeRLyfYi(zpY#ndaTNASX z%v$(s^DqDGM%E#8b6E_+ttTCEE<5+Tez|X|<)ki#o94&f8-3^ce)1k~X>Y1+9Aunp zx@cA{>0V>K2YS!&5Xf0J#Yq;mR9Ikk}ob7aXb}e-z*nYQG%^8&aE$e(%barG; zPis%xQ|i)7*{)fWt#@q09X(tnltt%%jjuS+q<1t zX-iY{B~z-YwfU6!68Kyn(_F(L`H9ZQGfnC3zTx`n8t(o<1R%q;+?ir;XB%amZtZA$ zV4v!2=-#QERX5Pfa|N=$m()Q%pdV%IWmYU9-mSb7ITH7}d9&%Bu?hH;sm5u>qV)Hv zt52qr@bW|ZvP|8-!`VR=dIXMy5kJi1m{Pv#IR`;O-ye<{izK$^G z6j!*iM^!l2qBs`GD{u@Q@b}Kje{$~JT0?^2F_qOeeIHOT-f}N#J7}Lo^^vlLUPMiu zWHamyY|E@V>ysQG>niI5+ZjhscW;kbey?}YanxBYnQ8_dm^0yn<;tKp&q+#pRA|9!7Rf^y;UAe-lMPnv7wf!sd=ZlzSaXO&CrLA za2@ptNKkQR(s`~rBOTUt^yE^D(xt2qxy&|1hlaV^yPJ~LJVHdzm&&T6ROHnmda@Ub za}V7s^U8&_>_g%gLfu zMPvR&XP;-F0^*VGZ=4;7_m3jl^uwJ++|5n|inEvKIxNnJxrIen#PbF;QaT;k%hN$H zMb_(A%U7@#RPd3@v45_Ev3vt+wU_*soM9f$)F?quj2=pFe8d;z&l_WBb>^IhD@t*q z6y=CcTp^w`8^0i3{fqd`RNXPU+!o>VBP*xe?a0cY_o2r>7nzKz`jhhUA z==aD~sr)JhLU5woSAL1b(nc3gL}n3I%2xcLTxyuo-Te`5|JAjK9$k6xJG!a`)Y*#5 zoj`AZ<5Yxask87O3gTfUVRgB&O`GCd)WC0OgH3mVs6~D0H~2grui*zi;{&1)rx!Yw*lb=%<8`1&W+2;YYu@-w_^SkVF1>k)bM5DKnCRL~VyO;j^ACRY5&S$6 z3sH!wbSHL{O+NbzC(3Mrlf9^68H%6$Rc(XKGYcy*0J}X)tB3B1{pz8+t(n-$QLMLS zlAEftIQ-o{_-h8TDoeGPYQTGgfXGdp zXfqhBmx_^UgmpxIGMHsOMq%UJtP$ko#aSnH){ay(TDt%(2}gG$lZv)i%1WANW~QX9}worww` zA-XUV2^Npekm%TVo;+b+V#b}M{dgOj@v@3woj1jbGr4NXbd+7ZR}Z^;GtZ#fB9zW^pojX*{a^#F(r^j@}UI z*@@@*8g3eiM;3(MYsk!c>CjlHIDszOWOjlxKSBbKQ z^8_~;w2F+Z8_^*V!F1!lt;7pB$<8Q3mDOf^uy52l?8k%ZhaDM-cUF^V!2@Dc4q_pL z@Q02d5sFbWA4^PS6LEtr$bn>Hd!^JnoPQSw7V!XByvBIlqOx`myw;s)O{gkjg*~a~ zsf4Bur=Rr(e2V=%aXoQ?O?Wh$hy$I1f+d(wLwto`DT#<)BvlJ7iM)s4HHBj1S0$2F zgE(*}ysB_I@$LozeM$|+Lm0+hiNiaa%l$uq1g(tx66Z!u!vmj*&)uKV_lM3cc(*i? zEuI+3N}>RB*wwS~sITMSmB-_&NCfsXF%CWUTc z0IlrvHk$tvM=WRu-sv*?z|22W=?KH(xh>yCdEj+d3fsYqc&v8oUuFjDdL z1i3bw`hd;MxHSH6BmDd(#I?JDS z7W8I*kS(pi*7c_XzBe@<4aovj!-K0qM(P%^fOK^@^KJugK4<+pvF^LrB^_Di)~Eji&)1XVnoSm87R^RY@~yF!D-O(1D@bdD&P)d)x%`@tqDDmFP}69_zN5-X?D$THb~oC#3EbB=AfkU3b_;R(9)D zB2@W3@zjY$sAlyg`d|^WsYCX%9C*HI;LIYifaAb_E(8htiDdpfBwl=@wZ zoNhy^d-05ma!36N{ak%NeGHgrAAOGefzw)7Q%$}LyyXxrX50?mi{Ol?c65eLReI4g zFpYfaVColEgAL9DV%b5Dl{)Ou{vd1q<^0hG#Ju0D_27$PXo&HkF!z)DXhbGY;M|5G ze@Y{XUJ$ikL7$2K?BfYk@SnglKF#Ts5$Y?jG*`gcmc)Y$P`7JnKp*b*BDXcw_4JTV(9MzC7~<&XGRrWCI+V$V(&kw2 zNMA4C_tet#GWnY0EhoGu`i%EJWGQFfZz^s+VSZx%Wol%ssgIF-&;ngF-Sin=zpgx| zqtkry>5IUJym1Y34R!wQ@B=T}m2NjbTnXT7s)C1Zs_ayk5JRu*Sw;<4wpvEFU3ww+ z(~qLN#3eA!L$N-d$uat)`b_H9o{}Aj(^u1frXGBe^p$w6IAwe?(X$S^^W+%ws6#b8 zhlN^;(uzg7i+1P$vaTD5fN`M7UGA}1b#15}xu*4?xCsZhAcrs$O5o|v2R7YB(YahJtFVck@#YFv_TW1 zT^Zz)qNq7N1g96LU&C4=jg8sq71{5(kyfAB6JLn^Tu1J&gi8Zh;VD#+7(hoRve%}A zd2bFXr4c<{n!q_l!IgX5K_IASgGPO+7+C8FwFx*?9XXG&?6YZb$|T}8Da5S$BhN>& zGxLK`-^0#}Axb0a%Fm!Bmy!D{1!BFFvJC`lEogL*oboa7$n)Hv-MvAGH-Q`cq-FH@ z`6OQ@X8c^rXXro&w>jQxyzZG7nbw+&mi_caoawy@yys%`eDeddgZ^|wObWFm>%co! zl55i?=)Uxy?vwh(JH^F))y2d$U!JbxjJaG)5ZbWq)aeShu?Jrk8WfL6x z6ckPuF#ko>DV}juHtVG8MB`I*1F4$b0+PsE@6Eo8#3I>CH|rsKKm8Fh+1sgG5EVL) zz?xn}lHNf>&qAwpr`Ja`HJZ_&NZ->9ZoV>#n#*xm_B%oKy5NRHaC`Ce?P$$@3dQzl z3FdnQ67vIf!e_{4R?zC^w`1LfYL@$W>Ou=+_kNTI>aTEmZzuf$F!;lWw}p{cKZ%4| zBA1poV(o~N=@c@RJGft6&qp$L4Y0_blK=Np63BKgrw-s5_IDyH(1O=^teY_Eo!TRv z%HU%+rP|>%RiYh`4_8>bx$OHrNQHO2a*!93k+zS}Y@*goyjmdhhGH*>JjDbuC^g8a zgpqgMLd-7G8JjhN$1tss~8aq)Qqu+X_tA2DiJmCl z$Zu+zv8xPZZAxQ<{f=ey0z04~ws#ru6Fubw&M@}nB(O1blhPAc`;Qu`Q*tr#x0i{% z&L!_sL`p@^z98dpfQrrDj35p>=`+;B#npoSz6Gv5M{V|F>Z)2`c}yUa*b(k3h`trI^~H5J$QbQMo`oW(#Mz`D z(dVh;a>KzxBoa&3Q^k-MX>bO6_6`=B38}jT>(EXt=P3w{_QY8KK|-v99(J;3$Fb|S zdWJ|X=(E&O-%mPCj`Nnh#MIqugVzT0YlDm)j5Av4s5ru_wD=RN0DS5G3iOI_=TMh1hf{DLoc z4#{n%GBe#BspcbZ;i6y0IQ*1I?1U)x*d#fF-ab)S45hFOVmTS%??EB(mDG#fCS(&2f#O}z* zljJz2)1zo1a-lo6_EoLsFpiwSRP5y4WF^1qs?m=j0xj`Cs*iNoM~zN2XC(ZA?)ZxR z;!9rkFjG@91riNsopG{ix0xLB)_SxtsxTX=kX@3H$3VbzOb%c-ph}HOW){4bKka zrx|~XtX>)NXdlU%iSs>&5O42}?e4^5h@w8HC6PdHcE)QmyP^+MYotUtIw^`1e6GlQ z^t0qhP7j-+u91?Br!3bk<4tw-BiRS*3|-B3i($mo|hOQ;iRe4-7T+ zwW;V^kC&%nbuaYX!F~;A#1)8jzf+Q_ph$NW!y?(O6jLWCx!oI_pNW$`a+QJ8^C^0F zUDs%06!Vq3SSLf-mjzjsBz3c=JGyu%tLTHQD-VKdG}SIIso?L8rFcm0sozL%stZ($ zY{2V0FSU@b$?u4a?50BS4%(zX8oiWshz>13@ifF4!7H%Yzakm$A!Y9%U-uvjdLtV% z6)&EfgQpb6PFSYCR+n+YYzkgMI2vX=h?g4VUpDiUhGbFlkb}9#InVWoR_uZ&5<%y* zmk!~-H$V%9Q)L>D4ZIu=sgmbCyR0#GS3aI`pOfgvq2*tJV}Iao&R^H~+W<1|9nqzO zXegDl-1jNF=#N#19z4zP-|NwzVJ<#=9By2ChSy);PZ@N+zpX%>vjD&JF+XM2R_ji!GCWDop8z|~f%Jy^Rj{Pi0Wpc@i%0uhsJqA!s|l7jgu&)vg3-_$U6X;rdy zjp5eO*j0DPEmpzGoS~r-IGYDw_JpQi3KI8TgNE9QZ*Iqre2>jA5zXHOuUDMG{0F+Y zKX%^_5L3m8A=ML0P`mUM&G!oo*aR$v9ry=N zs9xElC`kUl@wxuSiYQ1#Z31zmJdAT2{?&b|9mL7Y)948xGWj(?pWSEWH{w6k!+U7N zem?~gVFmzY14Y+ zSba3TAG;X(nbORCEHBK{Ox2AkhIxkE#vY~!v)%Nk@jg|!ML|UF)-RD~&}U89bt8P?QQ85+gZi5uttsHj`a80FRHO5tiiqGoQ6*Q)6`*mlOE6cUp-BK9{9&C_% zzPkaQQE677KKaF1GUav9z(>)gUhJq?a-PSemS}&Yem=dTet~h^Cr9X;6Q{Zf7j&Vf zH3~bFhLW5Ue2)m!dSVLgu?U}V4{@IML2TvqXv=Nr_#fop3(&V~G}MYA<60OFX+~C^ zb7b&V_Yj?13H_4kBQg=IIvTXVLwMp2F(DI@y*w5|K2EYZg{8Hbd#=KhYX>L22FEmt z(`JUDTlMtyx*P4{XagCkw)9un zLoVMRPrnHq8BGKx7!DeW*B~MaovD6r%ATIXUvsbpHZtOm==Ve-FD>D2FV^4?dGncA zVymzS_9N}~l6OuA3A2Iqk0kfo8EWdJDfDn`AbloErpPO>tY?B1&8^Qvjdef6eQHZi zbIM6|>L(g=QpG{K_;#Yga?#N~50OfVh}#w79QlZ6PGya3 ztn>?FZPUpwH(|d_g}aIn9aza;@AIx2=7ug$9`D}IQW;2t73uaLYEU_bn!ba`SrA;bX=vhVC@W~-)m7b3%Uu`3H8 zcMD;?&A=K{u{XD2)B7QnukqAcRLoQayVo13+z)-f40~)p*WAZS`-$J0LblzH?06~m za(krL6V7rn<4@kk8_=`&J0NfB!tJrdf12`_I7hxQc61HUVdL;i+akYi>sCqqiCS!- z16Xk?MU#lI{E!E8+RkWJJym`rzm*%INxy*XegL|)BlR{7^r3iOKe4q7$j9(;i-Pj} z1hytO{dv|fkCvc-cB4lc;KM#ZbH!8fRh8Z)bLpb!Aai{XD>i`sIIZw)x>EUb7~d<5 z$jlO~nR3`evFP50@Xiu;=1kTp5+u)lqPT}aZI%QzGMiY;UeFDR(j7X%ZI@orL7)s{ zc~8{m44VG4+ClF;qqiHPMLR(bx%ZVeM0~xzF*=Mr`yY*y3&I>-QSjcn=H>*)uq8B-ZvbIL(Y* zZYMI44Cj3XgIn7Zhh*x3>@LUd_d$=$W~CkY`LDUB%Tt^RR+a7wQ;>}%utkE1R=m-* zMe@xic5;EkU{RktrlkA)@1zRy!D5?7tCmD zgSyFPhn*!td>m=El}Ou5W}3tdbC_RIESMNpqcb+?UtmuTlA|fa?&*RQo~zqI{B;qM z|2#OrcgTTnbgM}vz8_9p!bMaz8{X)Me6PtaH-MXZ!;asGg?K`D72f=d7)VuSpb$yE ziWHxQ*F2a!!)I*$HN;My^147iW(8i^4J@FQ=)!&cCWD-+j4dEgvNGe(#gEY4>&YIPfxEikm&C!JH^Dju@!cG>&{uGNr8R3}j)qpfi|x?>|I(M($Z^m> zZ;3nI1mAoZJ48hCH!=T}*d?p^>jCvtN$mG{qThu<{|p0fCGvPytTH2Ub8qO@iB7aS zeAT+SMdj`sH1BIX)56g1cMWB@icxQ14$rWl{zDh7 z#ljYy<~ovV`-i;IP^4KL(Y`_C4(c(UFl5|7c6S%-jFQCjD-a=Q2$jNhSSCT5KP5EDk^w(qQ7EM__;2tI}~|5kNtTHUJ)3sE3CqMqSKjJ zoVPV=N$lmPP+&crv`uFKDA3bddb zFFJ@b@m`{h3NrTm#K13VojL>1bYtP!&iHDDxlbQYjy(4He|J1!n@1JV2JLBQ=n-p zq7gIU?bCeg0n+;&KNrZ6>}6&bpy&haCp#4>wLo>6$aLuO;5uve)L=AJXYAz3n&mte zO*N2}tjKi{U>i#_lMakD4C<96uh4||OW@h$f`SiNi-)YyCoTIDOg8B!&l2)gATvMm z?D^=EdF=be?7MDAqHcUD*6)gr z7!ogm{c@J~WTase__{YEUd-AqB`!V$&h8KYjl!dvLyq$wq)Z#K*wy)FVb-=Fd-xtB zyALlW^E7X441q^CGNatc(H!1OfWO|r0cYUB+05u2^zuOS&uChaZ}UfU{lK>0MNaMj zT)T;}--975(Q{`6rKfz&t*MSj)R);0~>`xSQ&X9de}jfLwSa6l^e4QG{lBEh10*UA(0{QrNe zoWZzr_zB@Vq1@jHH;BJ;Sf5Aio0H7*6*A-l`aXyhU#411&mfts6PM>G7}?zz+Fvz)&;3D25)#M<4fo6FY$mn zvOdFEmsZGCGpi~3u&T(bvhZSgC}M_>#fZxxe`@khNiDOH2rV<9ZwB|i%?M617SKUR z(z9p|(W%uAg`V??8SAMTTC)oG&&B8kk~)Q5_l%uj&^?5^g#5Y7XrA(&AVyq+r@e;H z9zjtRNz#?^jo{zr@XS|MD;b}s4SOLs+-D+poB}m6pg{r@;_z+unBZI!`__j@$Q8!- zg3o`lI#%wO!yZ2a&tBqr4pt~YtNM^w!M9xd0Ui|nLGtizU)C;(xm@P`&z=J8x!+mY zPH>YS>k!JnMxGE1jf4kz5Nm%B5^@{!zt3Gvy!%1p-N%eijLglB_CvR(!_Vowgca$B z#o`ZzW#mdWGZ9|P6aJqB4jI zD!-j&e{Mj_EF?cJs(kkH`5jG$e}U(PPy2zpCNTPBIO{G?6a4s;Z@z+xFM0n1yWf}j z*J7sm*m(wIi17D9;G3%OXntr_0FG?|w~HPJg;}L6zT@JXLM|k6e;-ysf*L7soTx2p z2zSPCbu)gtA+!22s~9A4V{EY&?2zC2tvY;P0*=dMRebnPRjv>|pvdtBu-X#O^@d{p zJi9F`(URRF^p}v$Ay7YzH4xJ5D04mwt&c;Smr(pBe~BK(@$lapMwbOYM6u5*bN}{8 z)GqASG3?~#yc3P=6#USg-O!Csy0J5sAy>!pp4gqG;D;<$Ok$KG>-dO!Ca}|~%uEdEb0Mux zpksuMkj^d>5$Jx*R#-AVNR@ZIp9CMs>{VewZAZ_ZhRcqzzGAkIS%W=T#Fya05I8s- z%6P+@c0T>c_X41nz-@@QuZ7=BKntPa65+$|%qkq+8G+=j$O_be9-WBqjppYMXd-GQ z+Olp97*!+a(+)}&WEP1$Z7n|GQsyRXzFUlD4_a;^x^4$Nc7r=V)7BsjTR~XjZy242 zbq<2Z^09iM%-w;s7D$yGM(M?fB&37rN*xa+Zz1bm!x=g7zmOiGjIj=DRtGLE#VkY> zl+c)FW@Fp z_^pg)lUSjKot<3|+BW2Vc{DEea5te3tMF5V=Y%uEhU}Q;P+9bNuE6uXn4O1t-sdR? z(a1ZoCib%WyIFtHomx~qt-zXGuW{gQxa<}DW#GO-ZVT(A0=dkZtY35P+yQA@mHjN} zUkXZA(j-|p*9Z;UijhP?IWfLSzR`gFQUV^##R$AuEf4bM0^GMoL*0$V<{yoZFbSVv zF7IvD?1Xh_pGVN-2z(mPZg|7YGc^ttt1Kj7Bs>_vjB`P8JMZb4V<_V&2IWdaF@K(; z=kF@W=%T!$8AnHEP!wFU*V~2c8~`v*&AJ#%+;ck zyEoL!$J~ujrzGQUh6OnU5AaWSRKVWPY-smgvELiu>P#&qckV*r#bo-_um1Zi5Ri z@jRK)sz`}szG>y@1>u=c-g6*xiZFh`=MomuNA7Y5DsF}=*W%+WXT>i=&r2Gulc4Ku z?z@LQB;>4vdu6loZhrp>bqhnIM#$qBR#%)dHRPhx#VM(H~8@jU-NXXb&dlNIc+un0ZO@h58`Y}{b3&&M3TLW7IQ zc+sc-9<)y6=}-AB33<|lJ=>L)okOzb}$dh`V}xXb8XAXmS_k*|4PA{0+YKRwr;eHR&&OfIY(_iv0`ZN*Pr zp4S?w6{)SmMJ=+i5!wie8U~+4Ks&)XHMzDu@}~mxQnhvS;i*YbVIQk-k0-kLjSw4cyp3|qB9OKC$%;65{~Mg}oN+#cCTY#xoOH|Mgf9N8!19tZWwRQW~E9P1`kX`7MUIHe^0F zM&RUgCp>0iw~G~5c4xoEP6yqbiRmjN?@Q{+I@d&WBUk|UoeUwW>SwChVzW-JV|Kou4un5 z$dk_OhdPWb2L7!HkDK6jKgLml>nduJrXG}O%bw|g#OjJRtjsP>XSTw&7QW>-DD#|A z?!*S$&8i7&J(G9SnV%iL7P4M&PC>Z;2P1mV%D-g)??Y3rhp?aCPJf%ecgb+g>(#t(*>RXXV>^cc|Uel zCY+kg{hq?rg7*}(XaFMG(B#E8mC#Ohscd=o^pl1Z$Td~##COrdmgw;P(_A|^25K~p}mNF2nyHX z@3QE6vFkpvdf&Jz4`T^{D+HE$3Kn2XW>thAVQ)u30kNY?!&kzmtiz7(gsn7ytJ*=0 zddNVlwtv5{t1|d4O&gb)nPo8k2h1iO86x^`zkxEs3wg%A71jFr;Fqe*J0E&XL`8&t z6xHZwS@Aopt&_F=!Tbf9Tn7yV<)WZeG&2?Qr84VKj#p_YC#>HPeu7wiApwM^C?W?2 zXzGuyOM#xDSV5(rQBh|99gYcu{#m@Amw6RJ&nH3?QH>>d>Jxtn31Y{d`rkT-a(6d> z+qh3jMk8!_QKxR+tAt?(~jl8sQ%gtDVJsLf|G9n#A*rThLr+5+Tn%X?~Ef zxJ8$da_p`5+~pyokm0<7Joyt(IR}MbF+w4$g)bZmb;M~gM*bC6HINVP?tq&_1xOCh z4q#?QxJsN~AhX(`+&O@m3ce`}Jxem`q8c4k=4v21`x?pg1ddE$ya|lq2^wM&)zp8H zN!$qM-bPnkWDh=J%tGGVSg%)%^bzxU2oH$d&_`(hoVlw=T!~Tn!?&H$I!#%t8az`V zLxmMxA3kgWABqYU;UP%GG-a+1VvXN0*9^ulUV*H=_(lo1O+*!3P*UQzNWR+|E-uPn zpILV!R1jw}72pmPGAGqQ{rp~3F#`{t%sG>73olm6)p>v3P2qjYhi#^ zg1^f0eo>zG7Jfd=jNe0df36AU9v^t`5cCih{w4U}E1!LaCI@(Z#x@9qzFzQWBGTkC z*5_`x;xarb>`EhA@iWx<0QZKmCyKHLA|m^WyDh_3+6PxXz|Qt!U7b)@4~N>I>0PM% z4O=k+YKU$DQP8a_G|0>D&&_B{Fy{YR4If##MCcL3E-MTz^^Ce0*A{2@m1CX}tU+O} zHtXK8pA+$5eW3hHURiLKUbE~(CBNuXAbfC<^{;_^X~`kc+W=hbHlSU*yd^ zaGRK&j=6-h-m&O|mRgLe473oj7X#6d@9ff7jP5f2^I_&Itd|VtlmR_%X*)+?wuF_P z3LlC72OFWtR(8uHo+IMKIb3-c?o^?2IP3h2_l0#O;$4DLI(Et@s3tNL!ZPz<7ZhNo zVxI|mhePi?%%B#W7X#PH#B9>|uE+%>YT8>^VdXV$D*@m1Msi0p=V)|DWvEaI>ih8% z1I7J#dIop((KLXFJA7eJc%WfnMrLFzKcLV%cGLxSO%iwi&J4ub-(qH|a6)dlDI4y| z(d4bL;zT^5AhKRqXhzn~%y`A=K?N9-@PUMf@`VvQd5*9WenHKTj6F@GyqH%L=w1hk z3tEXM1aKdfXT-C2kMnu~hX?X`F2-1r8CB=%(%LvAc7c$wA~q!QJPM-;(Vid6{t+EV za>G9&y4sNc=Vx@EdA`_96&PCu_(k+4Qdm{t0c7%YUwA14uJS+!fy(*8eLt{IuCdnQ zgqxjcL%{>jk!`EVsVvnpQjZy{(2J*d3CrdrSDr$;{6LtEPOc&yS)OtCBG(5{opB)yAYnuch>wKqd(3Z zQ`j@_7|CU3y`OhP@QzQOL6g^T?g@}J+p%oV!Yw};dp6$@mhxHbpl#6N8Q*`& zJFocp#J?iLCb+Nw8b6IE3a{iD6u8E|dIn8| zcU=Jwp*d@xmv0IkBO+M8ut0kGrn`o#o5~$}^Y=jV!Tms9 zxxfsZ1&eV5e86RD2>r<`Phm_;wA|xMsMQce$5Wk?+SB{sI$u&3@)tEg)#(ua7PMbG zPJC?(=I;TBpj0pnNkl-4vvSYiEHR$ja9tXbPs9rs6)L+Ue22RsdfGN z1G*N#D_PG<)?rq)8GAvp*#>ZdY2>jTU>sTyGdhTjF~VCBJR^eD>w=xRk|&P9I<3Ke zD#SjQS@&=3+x1X&8WGl2{P22bUC{rLY*_JN4Wi>#T*Du})8jV2NC=n|TpRh=C?7>}Yo z93ko)`|#HYBH<%hyVYQl4uNd=mwel7Xg`@Rw#w&n9xLedI{da&OFUJb_1D} zjO>^~{o5D1MP^gQVxbnyhaQ9C>vT-1O3(U#jD?LG48exp^sL_nw)i>gG6jxZ$6lRAJ+)oYgW>!HB1`OwIAq@eIK2XE zc^#?bL*0q!j#->H+$B zf2Bfu9Ei^$^x?_xeoIg6`*aES=4^`^obj%>w<(puQi_u@FR+eQ&r0dKA&fXD!w&pES$&L$bgY38-&8Aczus^^(9Pi>+Lq94CqN~D8EI=#~; zxvwg1sa%R@|LAp9J!?VtEuw}&oF?8#SB=U8c#nF#;!-u;0kyyK!d;vCg1pGyx}Gf> z>TJ$7BI%CQufwjbOHGMM>Q8QbB~-7a^(QUq zc?gG$1f{(cTw@}+em%EV9AM$Zh?I_hx02Es9b!9W()^2W}xea0d8uBu09JN`mnOy&={@B z@~=f2KcZjy52~JrVr3*E3y*MJJurO78DD)mDOr^>%35mfu7f`Rt~eNFDtkw!AIvo> z!G2I-Sb|j_210WNXxQB#_fCO^eF(2LLK3>^A|}w4Rq!oDzT_G8%AZk+9iK*%2u>Av^_g($6(oSj7tDaWb5Vy855~1Ff0sq;?Lhi%0jcDq z&SoXZ-*@18>u~ywOa6`ezed#le&%$XWBNAWk*^v4H58^xzn7t;p`0O~Ar;G@9UZJY z(aCiaeUC!)o8P zRrbme{N$mm#yzlDTbb2KDzbgS&_+-vcaq(Cos&n0Q>iqYGs}7~zL9iF>n)9^#_SNC z1g_G}@iKJ{N9ikJ<({SS!1toH7IWWIVE8Sbe?hKV)Mt!t76|%&)R3G~@^X58a}c=8 z=`LFm)NO6*k>`Q7{-u1R##q!rccI2&Ib1N0s>s3oKUtCJ2=STuKIA0GDCT?&v~eUl zQ)ER~BXRGu{^-2u?Q$30w^XTsd}pd$gMuJ8mI*;sW@(=hO#^im*TE*%2?9>5fyL)3SO|#XJxc zZh!W5VQSg~sZ8^tH``a%wkq`_8~FZw*0d{@#2u(N1%zrWPx%E#_6SlpKNa|UsUDvV zAC6%Siz#+ay)sZq9z<8DKY3T|;+b@3`iJVF2UIoYqsm|~h~lLn$JbJYaDeK9WBh*~ z6ZuN zT)z`tY#~M=I^-V4(i71X(e*Vy-o+$z*-y0C3#z4lz~y=923!f0?g_Bl-|0>VlSzS` z0x+05&bm;g4!Qk|;Px}ekBaEeG5D5}sqW*pT~qG#nNjcQ%dny0aM1ah!4R^4v2 zb%tgMi$0B!V5djY6T1^NW)bw*D@A|d0@woI;EYvpdPTggYuGYdpy(20&R<|AMJ-l7 zbj@ul4J%TE8-@;6*_|(`3fjWn6gA)#sR|6IIl--r)a!&6)cw*D&AdJ2B{i!LAf zZ8CL>JE*ak$X@&p?X!IZ zs=$kha~S^uf$}ftzKKYP8bmpV;He6PSvS1yao|rwsmm$_rCRZ<6>x?Q4fqG=S?!k3 zflogpKc%v^I+c9YS?TYbd-fPCdpcaPl09{Y9X5gbjtO{q`J^+fL~i8aEiA&L*d+x# zPvH6y=&p{)@>u$Bhp4ydezThuKZ4G?M$O$Z>P==U|52l0<ANqs`z$j?%n9P z6sjRB!{z_L@elACt6@Pc)v6J0fv46}@8Zs|%eXnqm!HMiMN?niJip zn-VAaf^Hm+rmKN1TY<`fw%pZ2f27Z7@$0Nx6LzmA>@s-Qrt1pxfXdR{u)FC)Hb?^gqeChHHPKF#L*OFuT$t$POrM@>@7)@6W z6ZK^~czz_2>)Y(6aYPgvA+rk8PdP<>OXbEc>K7|w`=_x-PO_8tQ%8S;*CVXi^~llr z?5?-eRo26T-bBsgTXdX4t=SMp{T+;(@ZpBSue({zTSQ%)xQ1=ZPM<9E^ z#4boff5pSi_t;05q1p&6?2<$eE+7-{obX0pg`zua5Iq@#-V}ZH+~{3_2+YEBdyEaKL+=`~1w_<- z5>oFJ6(9-xh<^RMu%AY=cCmam8oT2raw;C|_;_~VNU*p)m}6hf8<5#M=h!)CuuEUy z|4C%DFQM(*VOtu}mZh*7hv2KN!;agFHMdKP+etiIWT(RLN&kkcrVt13#EEyA!IDxPFaXW@Vs)NXvl@-brhbY%zMpei$j`jmWhRVYfecnB4H53qO6Ak9x} zTKqV5DJys_!~*<>&Q~pws%|``XK?jcc<>J}#A}%AZ}5YNdK!?j_nFU0#yJ_^OL!!q zX#Nt^SBPHFCTbJ{u+iQlCo_3H#Hw9~7TgKnBv7mH1Pe473+^ob)k6AZ*zg#-YMnJ! zW4SD*f@2ITyB!CTN#(gx;Ay@?@sdQ&MTN-!B6Y!BZ$frlW=&4RO9?zhAV~z`;wC%lA)g4W-gln) zzZ`);gp5bJRKzRD#73#X%wv)MWteRi9VFA>sC42?d3c3-()nFwUc-s;2zz5P_WWE{ zcQafoY`4?w<`-CbBHvS$z1sopC3-3khRUspKh49=-+`Q2OeAPBKMVM69`79B=QI@t zJ2id~vCCt`J5IxUFYuRc@X0nJ9&_Q;t#IfvWYBo_*-&B?v$VC0A(HG1O)~Ko1bV6- zaz>zcO2cOo`do#o?})*QIF-mR2sGI@{_1c6k0bg(o0lB z2)yKGM)3|=c!iN|g(F|lzw#?S*mpGGH?;XXq`Tm)%h+=#;MrYtN;-`-^O-%qA8ogl z@vld+iuV`c{Zz&(y1?GtOpNM(m0gS2IWx4_*>E)cFr>ap-Ch!PMlY!=e8TQeg1^%7 zEsLQ0MbCFHVguRK#~6u!6lHyiu!eg0wi!0UZa5%;*&SzQ=ZMf`LW5$&3k3e6IXb4L zhN?e}6uJj(=c9E;@%oEB@duiqGZ~74$Y>p-v}v&&k!KZ{ur${9DnFtho~T|E(Ue2j z_e+_{3bfuxy!k)i^ld!HeVVF5a_LSCb0ZS8#Gm zB9nKp+`ACV*v2k=jV!!FmDWpqurbhOCOTU5cMC*U-S7w;L@M_E3N%g!c1|GQHNgAB z*-fvI2}jv2ugFeJWDkYX`=Sq4(0=^a_w)_dNps=M@o2+o#BxOzOvEL>AdQ#tI;gGH zY+|X)&?J+P$u*EWmyni8tXnT6k)CG{L5J4mwGP>_5N=vZ)plmn+(lnspi|B^v_Ll2fd_A2bPJn~C6=UQ5F`0W6d*@g z%icajRpWB5Uyi5sUHu3C)UgMC!ef^>b=aS7&>N{td?j_!4>o);6f#B_A8~ff9>aE{ zVk%_0Y*}gXvXr7X>~Ygs(=O8pQ=n-H9aKi+=Z?bP-LIb@SC!5)qPfJ@sxtOl>Skq{ zJKJ@D6ZuZM3Q$-5+nSXS%67#HC=dmy2sa-#ogWA-2w!6cX$0D zxCVE32@>2B+@0X=&hEm>SnK?6^FRBLAS>PX)~#E0>eMOit5z1}kZ_);1ZTG<8Bqj^ zY+LY&cPx#V3s8;uC$HG4R;+YI-ggJf3%@W9sV{ru#*qteuUCj^)rF z{+C(UmDC+-A=GNFFt2ATD9Cu!$QFYsEn|)P;Tt*-1)z_*W-R_lIE#mV;GT$7`0_kIU zCv37V7X9dwZIv3@ZraBYucb zT35Zotj=0WI8#^KGN-dI6XklUUDaONW@EIaE_za4I+j;i*GaPNjeUb7*7*_LQ->qo z9!uyO9Kmo1LOnp zTe-Wkm!C9J4M#y}kzR)ELWb5mgf83Dp~l=@qwulCi5{O4o1ghr*PRGnwF$+S{wU29 z2Mx^%weptwUinvX=R`CA9F{eQ74|QtWxJ`o%Ot-BbG9} z#~pEwZLXt~r)^|v^y`?6F^40+GQn_6Z0>ZcGjz#NC2gUY;}KoMB0YQDaqdlyIno9p z#aO1TW_D>y<#O=3KRGpBa-W}Der8Qxn_9@fC~zQ9J}}PT!|xB+)hMsqk zCEM#ecQOC(mV2XnkUQ2r%T>`e$koAh&pF#Mo1GqIX>DB9&Z0LKrKmwyuuve;cgXuL zbx*3~ox)W$&X>!dB~UEbRDP-S(q8Jh;Dd163w6XIwiw3^=Pg%Zw}7fy(AnFW6}^*N z%tq^GFKK%tZm?b=C%#G7aovzr&!T2!rfwUxyXFE@^J)84S#dCR@S+w?$8b4(+f`25 zR?8s4MuyYKI!-u4oi^SwPbf?O{08P9CRqLr{ew4Yjeqge1zy%@sUJj*?WS6lIegod z{EAQBB`=mQ$}RcR66KgO6;iJz4^W;d|HF#5Qqx#%I6^zX+EYL(78ymj4+o)wSInq_ z=HN0dlfF_D)V^}fz()Tf|F}RD6ALde3(*^lk_#$(v?i9LlInUH5g+{|W`6XR$g+`1 zQP#Me8AoS{$T}ob=5)nk%SJtlNDTLTsyqJ><3b;lb;17v0|KA?6Md;EYky`;`2OvW zZ*}7r{m7Eq)&C|KFI$yDN|IVDG(s%mxZ|qt`M^xsCgJ~!ut)WXeivOmdVb{Fuqy5c zXvUmDWQgXHpS=9>+POWEBRGY-=x^2w?9w+eEaioQrVOR-a>(O@;p^Q*>G^E zr%+AW<2c~{5mq2#enh^A$>I6Kg>WP6N?2IfaCb9j4!d7WV{K?zXk6CjD#e3$ealiq z$*q&8B$rAFPaU2*B(*{6#?&ZZwcs6fj&aJe)cQjD*BrLyw78oZRN?UEM9x|p{ za$3)b1#LB$Tz$fJTyiiU?y-1LTrYknbKPZ`92!ln;VwFapO_+gN*Sdj%BT5Ixl71R)|o-CWn;? zyXcwbRvq8OV@7-Vlh>Y7J~>TFrqtP~n^PVo{rnL~oc`lL(w@`<{%^rS%6hef9&Y(8 zR&zf0AnSguc5|xOMdHR zv9PU){g$Jb>$JOvXSpZUo$Na7EbiECdnRrWdWY)iJ=LkQ8b}`)TBF&j%=KEo_VNsm3zU%fv*0> zzUkh@UXSm!Z&!dRjMifA`Z2NT7Nxbt6c4N8`p^V{d?H4@XL|I zV;-l`;)cbwkF6ecDXg6PwzIqQmNPS{#XiX%3Jc~)ei^^!Td(hh5N5$t#tdf+R`e$H+az=Zm-!e*D23vo#op;*8u0|}68Xi3{ zI&<{usGE^DBASQS^Z1+_>~3kK&?)q{-c38AWR$N2n)z>f52Th$ZIv49-QYVFcqSiJ zo9nTmPnHC0NRsUroX=eSnNvNE$O46y@RA9sbA6w^ zyS(qbBYkcBKL3G0&!AhLD$i5uYj$I4C`>2`rc~CNfSPMOZ&Rd{70LCIvnMY~`aNk%%JE!9#YY3F3DmabS-0e2NwLd22uS#svd z=g6BadzJK?qW6cN_ZY5I_9B8^$)5T?u}FNT_=v=`Nh4D7r^Y9%KZeIQ`&RG!pdWwv zG`)l^uls&j=7=Vd8>4ro$(L?O`mgE6#vP6x9G>hta6MVJFnRO3JnUR{CM^C z!sp+Iq}7Ws*?T`du93=n?iHx^mh@>8qvR7B@8d zOISsh*Zz@u_F2hqjStziqCuPQN=o(Q2R{q`sF$$!`0%8sVmhyid}vYxZ^vT+CTZXB;)7spT(2DC0Fnkm1hW-J&WUNW-Oh4 zZS+^yczY}N(Aalbj_3X(&&TY8Gqj3n6VW_ugnO`KmDQzn_T@`zo46{W;*adf-%~4a zX169cP2Bt~?3g2kuqQc>4j{ zbtzgJV6A1zWqeeR$U6f+dJiZo2@o8 zWMsz}XE&$KS=*7>?w1aUxvfJj_l*wvdNo{m#ME3@uuO1UP!1N7wctjkw>%G&2!0Rl zmnSNH)k;LtW6}JnMohe4f5lXb?y5x{u6&mJ$*+R;V0fUkzql_UwQ=g?)VaP#N-l9} z*l%gxq%DxfAHLst%26*YTbgW{vt;j>twE;MaUG+Ug%=46_q=nKv5mL1RR8fmO)Z%6 zzmyr?_kLS2=%1Dv^|M0!(C^g~^_21Qd*i-vo@t9$#P;^Q?o;6#qmD;^h?*a9)zi&Y z$$7;Y?s*%&H}Y%50MAIP(NuqGC*Lmr;K1*J+WsHj9NzifTK?2vf?5@y`u0=96+eCGmJz_o$QXtO>Zd$*{2i2`QI$-()-8O)*H&K&A^rWP+&(rLr>zl|TE z1j`Pgo7DvqZW{IRSLpB_CNp^olJWvP=Mxo&l4KIGuoAve^UbBNMt^O!`i-d;d6@z< zQCY*e%cQK3M+LhFhWiWp+Iu&oT2oJ@984*l`Z@KZx3Ygou&>@y%;YW|IVkE*L_JSs zR}Igd$R%m7XZo5YLzeFu+NAwE_EU61RJX`#;SRTEJ1yMQ-Rj@6EdNesH$}Y}9OD&| zCM8UdFOcZ{Su(Yd&&C9o8~z%>Rmv>Aw`Gg9I&*{vNSDQx;%aG-W2xtALPqeLll~A%pQ9xT(^#s2H0;ntj=%t_tG-!KkxwhTK*Nfh`psZ{PfadKVe;{ z3loZ$Qd8ZjKh|5p+_}MYfe~=~sQ(&8^q$%yCd406@2K@r@!f!iW(VUVmAc~E8#ND; zHJ<4e$vG2216o=NP+407I|$|>lQItLx#0(lv%D3~!DX0_X6+=)6wr|3pb?$nZ7c=Z zX#%?OhzvSX*a>!-74^hkp#;>w{LG6OOP>85eXtu;v96Nm-2lz~P6qIp8m6LUChsqf zf>pB7hHB&gBP#PiF}W?kb4|poFnZUr@Ce1eN7PnIgQs)>1?UfoI8-PGJE{V6Ni$Ph zy@2BR3Fbc}GXJ9_b=TU|*FEA1Dl(U;;#h=tp!q>ipSmc^pF?LapS6K?BTCHm#7W{p zv9b7=>URotp6h6~B@5S>Q1+Dka|LMUTxwk3sKMMr1MDkUN-|Zr*Wf8{^h_WLN6_Os zO$9F>lRWytaXAqx37**!{O1uhxl~lqwvwlAMX`DY*uh7#){e}CD5m?FiIi76u8txH ze#9)DI251r@_lxLnjE8=UIDxZ6j!Kby=o0wJ+MVDh@+&_(rD=k{E+rmD|O@!R3CjP z%qLK# z+FW1`VF>h&%HwYRx$c2+(~akD&F%G&3T_Hhz8Yvnv}vfZPk>)F2;S>@v^`U?wwX+B z2KOBZ<=sPwAQ>vkL;&8!;G=8wHq zl9ih3H+4C>fc2@|6{4cIh%=F7`JKA-QeisPAd7XOH4E?42Npokddqqi_U}tn3pML2 zcow_C+E#;6##6H^1g|$z$VR2<4te+NUlX>wq0pM&S`NkTjn+r7hO>+D{EKKEz@&>a zR0|G(6DGlv*bK97KeeIhc+AHzYKrKFmc;atKDq_G%L;P;8WrOh7>A!>O2wh%`wGU* zYGxPr0NE<9dzs8tlF2k@)py*4>$TEYdIzeFAJIf>2BRW5)SS9VUf2RH1SdFc041(E z_@)WMAN*aEa2&+*3N@VcR07O-%a7os=LbRSgaD+2rS$;$1~=z#Qih`eOa2-WDHN7gTi_Z(-_FnkrWdU_By>0rLxo zGt)yvdF&V**nwDOIXsU_qW%64EFs5tjDo)A=<+hSv{i(jw zj_M6iud`w&CAi~nhx*_Vb_(I>a;9VDt}y}JP1e(&2^tMpFOxym4(~le7>^BQ6ZUbp zEZ}B2!}l;%5O#kiyRi3_mPcEPaP#B8nfT8tLX^z+)Nf3DVUXpz{; zVpJm6P_54ZBk8-*lb?|d%Xmz^;14jZtf4=tg*;#<47j5y%5iyH@IjzbV5EPQFO9FX zFV%O~|1?lq*`ZTWF>fmCdrMnkfHc{;DJ&w&9rHS7Oia`0*r;xsKDMHnt2<*` zCq`RybJH$jRkp*tTB48DI;q{2yYe78gM26$85|R65;z~Y6)dWpSL^7FL(Q10xE0nC z6O}@cXLUnPIGU>8NvhJ|9eLzuXgwnm>6l=jU`v*k@KY;F-^7pB-onjLJ;O`gVjp_OkCh?H1^H#LRL}_6gHo_c zFb@`WJTN5CDzH6JFIW%c?{7s_9uj$1Qo1l5V*v5uBV!}g>!P6%Ao7Q)j%3iB;1WZW z<#GXebnv0Soxh8}iod4+B)P~D{bFbvD#i=-256-JYiVd}7HPO6PslN&kckBoTkil>TR{XaC#2)!xb;X}@k;$UV1R+JM$` zfcw*&h+b4!Xqg*IrGC0xS;+Lt;c^b;s^VAXD+}b_!M*`|pdPq?JGrZ3Q!}cUmEW1^ z*B8XKg<6^UJR9Ln`q=r!Fof3Xr?tB3Hsy(&Pd*h41}n&Kf}4XKf-?j2{rCN!f|<2D z#y3lSATv8JNqHf+llRCK)Dyah`uRC{!UHXpLc6KsMq6$O7etqBw5_4-3g~k^QG-1>UUW#E zq~?+;j$>xZPB@k^+?K6CQIcSEn4R|Tsh*A^zGdbUKHcmfG&P;d5~DZg*8E}VC(MC) zvjEmjdSNphyYtlCD-j(yg+v$|DR2u%8n=jf9@8ICh3K^~Qy|Oh$C%u3Puke^(t*s-UtGj!O`#AULeaAD$TW4$c70;Eh zn&Fqi+I#N1+BnZTBp|42{@|>AQHyIswEM)f&EP>c)3c$#r8`REU;w=(hB0c%0(r<0@`RJ~_Qm8Dp$QIhIPo1|;fQ|SPCN*?Krc$$yJAQn?t zhbrPvX6P&@6X}blcdK9X6rK^UY$Jw?g*##D0hm?$$;3Nb$tn_9jRf+9CRkqrtnJZo zWPjt0YBRyCHT>D-)abJjRlKLKV+1~E6u-v6VyTRBZau8%2R#M1;n|DAS-1s_g^j{D zAqf`xZ`k-wn5qNe(mWzI&q5rW4%S*Ge&wNhcbF_=J-K>T5Xu9Cs@n*#FU@NoCdHDUA1~ls4{cer^K9Lx{C^a zW{`v`?ALB~APmOAN^VHSsK(uV3ARmsYU~@}EcK%znVD&Zd#z5+cy_AMsboa^tw*hs zm_(upKgoQyk>`EnyxJ@~=y_{QzgrQMdSi%z7cmi#Da+*F8OcmH!H6^Un=9j^bHk%u zN{#p(zNJ1qEVHZJ)T9!L$zBls+2DjugrAXwW_%-9&i7!I9D~&ow6v#R;el|UnsA5{ z8iPW28<_lKIA<;3`AJrR?D!`$G9U6>Wtr16jhv=E9DzeHnD)aM?gY=HI2^(;;1x&7 zs;|KQc*0I>;?GN9HVxo;OkI%Pa0{Lz?(hRrCDICEz3# zsEF>OqoXk&L)nROSc=`)16~k|=Uqw$zJS?RXSI0N{WgM%uINM9Ng8v@qUf4>1*>!w-_g`xF3(Dv z3MG-S^lYr~aq3@vVU~|1w>ySanmop|aJ8Pn^jXF!mMA@&=emmI0t#k{;xxYhkXt$j|b@)bWFL$wofdE~f6_Cp?U)W>f)ASxw$= zH7MCAEYoCHno2e~`MF`ltnXl{{l(8+hAmuWE$0z)jNzyM!^*yd+nblKse(0!$avH7 z-CAOC!&tNa+<0?|&i~+hkA9Lv8FGu(>>VZ4OrlXgp$M^ zHgKSQ@F0itnQi#`8LaoXU*EGmIZ`8jMj=+~KP+?>%%mA`0=u)PX2-=wxY9@A+)bc| zx;b%o51!jp^fDDiPP1w!S=9^h?PSyqEAZ~Q@PR$y(5xeWUV{A}gPHiy65R=HRwr~GP*rrV!$bF&p}dTljxFd}8Ui*sn?G&ibZmr;yc(?I z4P2oKFu~4nc8>nq!!2NMuesC8@{x&7-W=GBsaoNpyZk%+u^{)2sX361b?nYBvk$!< zyyBLuXiw~X469q86W))Gr}M1h0n1u;tsQvdC|0!}ziZ4}$?&B9`qdS2lJ9EjsXb)R zH}N-3v6Z64Co_nPcd=RrVXm0Ua&|KQrl8uh_|+HAU{kWqzws0w$@lkjqVBOSU+AU{ zz)CyHT6O)!H=N7o>}Pjc^VK$x(6iXbM!xSBZj(uH9#{P0srBbRo5o%(XE#oxWl)%> z>;>wwoOL}4j%sRUw}YWLgLAZxU04D)wkzwC#P`~Y&tJ}WssoQKHyBYI{y+V%lPbf& z)!=B9gPA)7&t$6Qm|3)`4Ql0SOuZ+Kh|JWrvcS>t6OY};8rJYEW>5EOScLn)Ff4cp zQ%ma|oVWX+DKFpvdpLIuIQiXq%c}6al78`$P0du3A8smum?|fxB9oi_E67e(;PjW` znSbY0*N2T+jDP)|m7EIWZU+CaO|%F6z!N2Nny+wnOl^wW*upcO+En5CO2laLke_nu zO*OpEoX1tz-8}ZZH#U$1bjQ>g`%F}6a@4ExIhnAP0DMzZKQIpKe9OL@+BqlLiKAHk zDZZyd3|gM&YskC&#uH}a);AT3O)hA6cFwdvlb2lR7j|T-2A${QCcW+d@tqE1yI-is zo4j;WAATykGXVy1Sx|eE_iE}72l?+AZfA4P$8%3Fgaf?^do%Snj$>n+vBi1#v=P7V zlgF%KI-Z~u>l$FS@AEd{d^abnY_hLS=Bud~`dMAIgHHqJntLw^$DD~9h^Il(1x}LCarVSo}D|>>dV=O7h&r! zf8Bx}?jS2&woUM3%wq#jxkeY?W%|z}c((L#ffj>yP_GJZBY3eSV}>D(qsCtSI`xBh={r>`(MfOo+(}9xraZ4p5Ea7|A#j( zU};15#Sz|YF>jxYCyl56l?i_RTq4;X*us2lU&Qj?a!<5@quVpop6qr6Z)wiGF}skn z;jIr@h6&YK$DDW#2dv@qOhhgK0(}h@@KU-uOX1^YaSvo9BmZXPr{`r&s0ROz~7mQR)%E?{MnAeUzWe{_>C;FOh~;!Z_{k-D-#u?^_If)t9-U>AO`xA zle)vQ5O#9~?xP_*Q+Cem9(1`2Fq?EQyI7A(%1h$xdsuN>nE9F6?OmLTQkvcR*AxlQOuhxQ;T*4yua7UiB6r+djCBA;A(L2gOa$Du>?|E|JkT_tnU;YoCkEq?=|`w4t(r0c0I~+0;}{8N!(>BdVS7p zLB7J&0zQc+zQiQTR~8>}=O3JsFPuD)GniyVee zkLSq;aF=D^zDeNrb#f-muI|K8FYf6?e0w?UOk*ahIhXJVH{L0{ zXb;xOR7NjJRI8C4naTkhc&6IeSXpdsFZuLUIwikgYi6u(s;-`6=S$$*be_VTW0{Zj ztIIRhy=*K~I(y-V=|gIV2MoT(V9#xtqh&mcM}!JDMT*BSVdteg@L{?b(L>&Fe3 z$gTGQ5Br|m{3O4A5NTe*>dJ6WKHwhOiZ3zyX=Y=KZ@DGT;Pnf#KW%v$kVjM}>T`#V zV_&)wv5mz$W#j~x!rOq-5>NEOBmc?mUYA?&_s}xBu>;0IGMgw?EzWX`6MY62-V|;< zb4pY+cG;6VeJQtmZ+v1ZZ%~k|vkqOS*I2nKc<@%7rF~4DtR{rGu}0uqnzDusg?QNI zr`WZDaQ#hw|5K_VtA#jn(XZr9|Kqt6EhFJ0bP@14HXAyiAv9_I6Cl1QPG+y#wbGk2t$ zWfm3G*;Lu`Qu`SP5_6q8(onq#m7bMgGYzSw?gY0w&)?pqCHy;`_XmIdejWAv%zVX8 zy0LdK19z!5nVkHDHi(LwIUjnO+DSd9=AdRgU-c;ylmlR*fI43&yfD8_E_IB*E*`#dZex}Mb z6;SDCE^ZJ;{5h?$mRXyu-cqV5wPZc`MsBaAf&jJto z1wP{*sL~VRuJwsHR`N-$ZIx_gZ0l|Dwp{j6_I386_B}SqR$Q7TTEs74TsP<)zDiuz zitdxqupKs1XZ(x0SUD}I%Jg3Nn3`P#R(LH?yEpVhEYeTY8}5P`(Sw{hA9lT&s3M#b z^$sgO%zbj4+hQ8lH;5jx&iJWq%#<9aJ3!crYe&?)>S*P#+(Z5nY!j?Xbz(-aNbspY zgMXNBvhRkkwm)}ZUZ6!_jsL4J#rN8uS-z(<)ci(c>ltZ~G!HF}3{nn9F;6X6{zYRO z$3CE@b~i$ZY#TK(CQq6vY4*jAiCz+sFYJVS9*q3M&X8k{{i0M{EDqoAt2S5tAC-&` z!T3OtK>fhNz+L}zUvY2Q)Mlxo_m+3LZ?S(vU~sUAyivX**HoIQceNHq7Rz%XQYjI{4!u18*q*bpVhmQN>U$d>+bnpsh|!erMcc)i(Oqnv`{p!7*# z4zw<550wAprojRJw!Read+&bV5dR>53!k3)C8cA^my})J+x~~a4@wngFy@Cr(odVM z4-P#R3Q8sHrySXx*Br5q0rnO4$&MY)Gp@DnOYUs$fb)gpv)y5@M4cs%G)8>QjKx#< z3p};4Mr)&fSN2mq$qjPfNlj5QDtY7;!2!X9;CH#EI!kLtpZ!s6Ro1&1PG+Z`6v~OE zq?yurdY&do)1^mJKHE~;CEH`$pSDj@RjHJi%NilnhkcQav*!i39i+8VML33@;QYX@ zfIHYISS#oav<~d`m-dJId;1Fo>I5^(>*cF*qWndkEuWJQDy`_T>7q_l{YrMFt};lO z1Ji!E5hkPeETozQX zcFAy(F*z({LfgqJ%7qSEez#r_&$GS}wntJ|DNecpZ)Y#sC{4wE;%c!tr=yx6Smqmb z^c&h7?G7xdOIjJNtNK&k9DE#j8)zC#Ls!&dMN}iy<;r_`iTp(VLz$>f(Edj!R&jcY zHsYUW!4GIj)L)AD#Sywn$H`l2Ir9`RpFdd6Bo~(BgC|)Fi{2pAfUIYa-kQBRWBe^X zcJ>cz6@I~Ecg?nMvVFAAa!m^hi>wm$CNf7vbI)#PMaL>fxa%3CG~!%29o1~Pq+z0! z>iJvCZ=vz}pH!2!YxVf~RrPe*bme)lUtqgGGSE8ML}{$efbn%vYpoVj3M-PjRC{R@ z6H=@VmOEHfsCx^h3ChwJ7QN^=Y4 z7dybb$p>3*C+s$P1oN^_!i zs8`hkYB6z zVBq$Ud#Y>oGGzMS=|;<-i^e=l9x2h`bq75wJgeMeT#~D(E0=q+r%d?sh?kMkQGFtZ zM-&OK5w^Lh|q@Fa}$I8U0BVzMBZPFmc`uBDnwY^=6OoT#yVAy{G<$ z!kyZk8*&NiFwclXHZyfOR{w%7*nW6HB|u7Z!|_@U17aLJzM*I{#4{zbs_`F{nL|{> zj>Afsh6+z(IO%DaZ~q_4W}>!|{=eVVKh+C#N^YWZS%`Zm7S3Wis(0iXFeVgXDK(p> z;#BdKm>-`QVcTSDVYfQEIYv2ZJK`MQ>^4VrM}J3qM}%XiJ*WMYZ4}k@^0pt+eyN>g zXN_u!|6#E*HLM)MTw?Kh=z%>n0N04eS{UnK+bp8?T1~4?b*Vi*f0#C&C}KV}r`_}i z#!xBS2EXSicb|(n!<+faDP#tvz}mWzGrS?j%}PZ5#Zn5sP(QR9b_nl<3{;iMS<9o? zQ-JE;Wa}I%^iw!55$NwMhr8BCXolyCBA+}5@=%4i*3{LFz&|}tx&j|oh(G;7ym$?k-AVRiq&|gO;89etn$eFvg}>>c_e3FSwmyj({~BVVV|p*D zwWhM$Bk+L}Sk42>4YqIo-l0I(4YosR z{uG8*(o6hRWn#!VaIoGn^L(oHrZrlu#CbYR*I{;Rl91I}4-Rf86sW3Dk+NAI!7hvy z_Jh`DVP{f7#~#!3s~HQZig}D<*nU;Lu$~uN&PLpnO6=W8%R;@_snx`e7hprpw6t15 z;?s#*xLzH$@jT`@SJobB^Yon5Z#uzP%?gX|5AB7vSsw_C_dSY4hp0O|GG3G2Kj&|r zlPzQ;PbiPB)haqqGwDyXGxQ2%Vy{l=8;npWBUZG^(kFDz7!YbmZ|P;*Zu<^zd_?cNTRncg&;1R^ZrqP&_Qr-4U5PHwFHRN_>dnqc;us7L8`?x`JNuUf)h&CGMJ0f9(^7ScYl zB@K+qjYpbiT_F~dIx_>?A}tXMh=w&i`!7m!@m~3*q2g|;g7UBUu^8ti8un#l;W<^# zvT(ZZp*JxVv}HP0Kad^158tvYIB$@wc?}u@Gth%=%gvA{bcJce)zPs`hXs{{Kj$&F z;LERRC$(={E-a~zUO@kcy|}JkrUSee)#F=mFRyYUrfA(!T};&4;jyy8JT8I1Y6DYX zF}|`jej*DBN&mvZZpR+j;pU!0U8=g)Mcu%CazVZ>OG<0ysPaa+qg+?ffebAN-`Qe} zrcNvfF}Cf_IqsqEJ+Al8)y{oR&3V*y(e3wi3GWuMH{x0NZ(&YPZnxc?-F?Be#hKQT zYFlHwX>;4V+K!21gfyXUdL8YQIu*8PoLXItREH`>nxL`s+ z3~r^X*{MuYTYd*l+ze6xR5_dkL7l{Lm->oxI$_rWkTgjyK$ z^|UawTkyVr5#Q8Ex_Jx6Ay`(PplncE>l;IhspIdloHP26<({>=om0YwMf?|@F>EAU!-?*`p8H|#A_hka zQR$@%cA^rFh}*v?96tnH0#YJ^-mI4`(VZl%0Z4k?@E z-GThRT&b}s+f(Lwz5aK>6LKNBNN`c$eZVbeR%GRtI#a6(KmQHcZE@is0z{jnZqj%( zQkz>BiW_Z*9eG`YT|=A)>;_!8;?m#JN?UXL9(!|pMcY2HrnNr!+!e5(J?Lg$0ADWz z9#fHgcbC!2%$a}b*EPiITX7piQ|a!;1o0RwRwesOfEk<*^sb=gFbHy`Us!Mk5REv? z8P3aB(1l2&fxZoYw2i*$+-e6hfaYov9ru6BZDH;#l|^NzQc9hJ@~BOd)d%pESHabN zr;g(c9fr%jm%SgW#gjE;Qa{qMugN{-$HDQzRl!^GC~b;mo;X)}CoI%EDczM6V~=f6 zSdORyQQyPkJgLqZ&c?0+o~Gf&B3DQGqS{Ak5#wMTZ4P@JRz9qWyQ!m()KF|AeYJa? zbDZmJX@vz^HTiKMSMZv=MI8=ikf3!^y#7O}ktxknI(Xv(4cL=6vQzFB>>sQNL+~~+ zY(d`Vks7V5Mm3?bI87QZS;Yj)B=DE^!dmHH$9q>Ew?>cGF{vsn#7&~xHpX7xQPYvv zenpxnp2GjXg5_VtQX9UR!|$#{oGy6uf9`Km7Z!mS_{LtJ3gy@)Z0o{WrnN)$$Pw+}A!U zj$mb9@6;Bl=e#@o)q`(?ql41}@BE4Mj`j`;un1Sm59K;aakYyU2EN+_70xK(VrU&| zQI#!T>ug&O#}kKcA85NQo`mgHMLa0wwCi@sQPLhj9U?}2PPfHbJXIc{23YGD%LJh# z+IBUp`z(`;XJj&pHqa4AFOHcS1`etijD5pKP+@g7};vmk}$RHx6O(p1TdS*{YZ zmgS7-A;oeLj#6vEVOe0*$9hL7(3`NAUJ8>UdSD0JZ_?%m5p`6xnYJ_Tv`l<~wvua?{Ck zl6#?zep#yugK)IkPc_sTS~3}SRy`wYH&y!|C}vs|9pklE+7~Sw+Ce#g>H777-!sFQ zNVN2Xcqs(3*N2ZDR5^=Miz*CyJd?~tK)G}>Rg&9AaU!?ps7dS;k~x_tjn&4E&^V!{ zI9Uowmr$_oOaE3=c>MlREY_(CEvc&@HIyIN@=vMOpbz&}ow? zMYRSLx*_QE9$^L;NJYUe_)*WP4<6T_E;Ea;le$DdSQuIPntz0qLT+I`sPq+b!d+nW zZ>i>&KqGk}7_`TD51Kq1jJcGa41zq2JRwzU%nf`OO!p79jk=3$_B1ilzv$gwhQ&IF z__H^+c}vzeKML~`seib^)7|u?v?u!OgWlvS;Vj&>2-sdz=^!eHitAHhE1fH&$dGPw$yO7oE>;HDO=GC&YQN^^oF}7Mi1(QQg0X^JIv@HUd3f&AR3FSa zTUN5Kx3K6A3jra&wH=z6UFgfw_)~QhQdM}pUr@;0MAp^S+7oTr_Iy;~GwQRQjeCl0|o-W?75-^b_^DKVft=fOAleiqQyIN*a~*9#k$y zQEh#Rn%Z5ebq=a(*TL02WNI_Xs~1t}iXnE~$=_w8X7UHztVvXeSF^)2u#VeAm=lOA zXHjP|GYW&Xoq^(N7qYJz@tZ`3C$S!%qY%jO<--kG?(KUsEeBKQ&X-1Nr_G@>e$iRwrq$6VrOXX z<4H`7sbX*)>SNu5$v#b;x@uJW%;}5?RJ?282~0-e3#yZ0)X*cq2h7Pp5}129r>QAf z@FuEZjbT5T3U+1Sa%|z}4dwK|;_RHFzP*&$TC%!kexJ{_#uBDDPkeYf=^o@Jt18NE7_?)?%xJIb5nc8+U@Nuj8T_4VxIY0gl zx&3YU1}DM(Zc=eHXV-jzMP$GRGpDm|1rzz5D(fx~t*o#G%zEutkbuX0{@-}f`}qD? zzLPmcC&XIF)Gdy)LtVHf%!;%*hd6+;UzEQzdSsP$Yeiws+Zf3q}JUoNt*uY{=&K_7m*Qg7-;E~m(_G`|V7!I;_9F4?t zuoc$xu>^bA0w3Ty%+VOw(r!HZEZ9Hh#D?+M|01l$oODxNh;^ zXqX$OvY*GS9K-XF>9btM`2)`l{l~p@5_ZW%GLe6HyMH;SwfVUoD)Me@>?pU7m$#}) zm3SDctrf8aZ>TPo8({V4QZpY!jV&*>cncffVzI*&h_Mb7TozML(@KrMi|~gf9ev}$ z&_Zh^v9WLwt%~)AD9omUwA0cVp20(-5qrN0Z@CH1^%~z`wK-RfVOYK3xtnroeqs$J zdE@2ShB=k7BP?j2QJ!7xjUP9sCY#e~cA_O|&P>)=-$b(F2r3pOU|avqoBRorbrWlI zn7J0kP~A;0941FtLMC?{3}Y|R=|GUYDSUMgta}{yauT@ecAoq*%##CDvU|bq`iG~e z$!WC0>6p&nnry-wtf@I|{Q=MZ6F%I0x&z~lTiEq1Y^@}IR3#T|%}r1Xwqzep_dlR? z8cdx7#vrWnFIaRc{GFAYlgeP0R_fkm+21>`rAolHyT=pG#CBfbC1U7R$%%6H7;f&y z%X%Dt9iR5)+8V2sv@f$jo10aiCV(C4#0k|z^}W7T~=luNCc5y4BqSs+<^A@ zlriMg3yD|f!HL?=bJ+RXY^=p^ustK_)fq%hz68E-D~yNZ`0&p#9H5{2NFwb{*qM!=wFulO3k;py_?H-N z)WYoBIPl#GFdsU?cbbf!y9L)FBj;&t6>8dE6tzHGu8-RjSnAqm9w6nJ7YHQI-7{42j@OJ?6B;d(e=D_6K(-Bnq2!U zSIUCNaNrTV*x@bSygVynX9rraKjy5k=P)=n;)|E!lV0FYdk~r0@lI1=MES^Vzf(83 z#*CSnFsZWOFWwn#xJkx?S5APL72vj*2)EM$7o~?~7nZS#*utDvpPwC0%Z@A{lQ1WH z6z6U4!ZR5`-D5KtZX0+Y!}0d9Jf#^Mz|bdZsLAI|#7{M4RX1VtE;@0=P&IDrG-xEY zfbI5^nK32U;V0Zu;V|f$ff#<|sWX#{m*wZT##_GT-ygzHd4_Mw!;_B0$Kl+6@luO( ziuZ5_$Fj@cS?|xVJ5)<`;*j|$cMgQTUI|r(f8a!&hBfku2^){WnOhKDjYkQgyY(17 z^X)+To6(Ka8Flt?M3{%TU19y9){|Uk2N-HMaQJ;-tQqu^S~pOV zIL%4D;Bu&wkdKvkiXBfD%2;2Ct87o~2OKvX*Bl!h>m5Cu+nkS_FVV}1a})!u&SxJD z7Bkh6&!O5nOKHVQeC1mq9bMS}5Kl0*nYS!qv?aF8WK2c9Vjw()ORzp>qNm`dmUNI> z#~n@Nna-f%Sb#e)4i7g3Zt#4jDEQf{8}!%|q!!bj3gbp5C$zMl<7D~Zgk8j@Yg2(K z$?f(Wf7Fqli#z0wIZ-N*$VROEt`?c@M)Hp(#76(ZUU_4r<<@HpwzPo`fF1DdE@53> z?gn$#(t0fU8u3*Ryvk?zZX>}jUHtk4n`a0yem-g#hf(d^&Qt$|IzWBxH*GXsWtX*^ zuyztqx`-zOlZgD?dPOu6n(8)C`5oE;(EkPKwEU*MP-meKvrOHmZUoW3PiJ^KDsgW5 zy9S4vqe#$u&?FAeS;XzzVUi|Z6DNpu>FjHS8bv*HB=(a--^1bt z!c1F@g*|3oLmd$1oO%vwfN5YAZAA0JtmOywne^=4MpxkmJw0W}yvl=0pXXyNelQ1q zv=6&oi@QDvUt0{{e~%|JyQR|_gY`G8-2gf)3u{$iLcKtXriGeMb*X98lDx?o^{)Ct zeWRvf*3tl~k`1VE?k4{{LO+>>xFTx$#;ur1SQ6&Iiuyu5kAa_hENE_h9!Z)_sS&pu3i9J5`v0_L4B1l5Nv$ zDd4~d(Bk+)B(k5Ac`QWz2ThE8bRJd(8UMhO9R%C_s4Q0Els-xk^{qM#9K1T+iih>F z^h!*ETlm^CoN9VWJkWY;G4ZRoSW={$QXQuJL`zBH4>2bi7iXC7Q;K?5P-x1%ev~_K zH}gr3p*0dqM_^sFN0Q0BXM^u&V*RJk+8_a(vwHFBm2|v}N8Jqz{ zE6mS}qXTmxNY_uj5jmtn=ihqru0dp76{!YxrV_9Ob=|jgz$PmBKqPviBU+N)*%Rt} zwLjJ3hN$VyK$E8ndNmEjkT}V<&feXT+gaSDvkO_>b=iZj?na(ZbijV37S_$V*4fS( zk2k92xQOo8EUB4T)cOw9lw(9hjp4lBr~0{-s#`JCUI%HCb`V6iq54#JtLc2 z$F188jzk9dR_RctE>BLBIb>E6H}J8ay!<5fMl*M>X;fgBZ?Vhf46AtR_IuE_I;`i# z&ljPRtfMJZ0dF;dj|<#6JHdxWa{u(k^KCLFVJQlny8FC!tnn2Fz(+WVcfj#3Q6p}_ ztzh=OoAtH^tXwu?(4{cf=fjk^_ zoI@NU`ZIe2+v7%TY-z3Fs6#zcxwd>BGbYenlI(6^D`OB+FCw$dx!gTCfL z;BJ!jE$S2h@cCn?y&s|1)h9d$XS)wW;td_e+sFe=MVqN$1r@*pdZ4y42xeDd(9Ef< zo`v}Mz^{+PtX^v}o*&!*%eamAV%MqE$J6|p3!ob|av=>icnTizIsaneUn-MN4JNKP z-?SWOBM!hdNVsQqojm1KY$u8u%=?Zc!kPgKcMacnIXlwWsDvjor@5`cmNtM4edRl* zVnUH1bAyGXq- zmAL!zFHe*MUQBklbX~w{UeS##S)a30Ds#CU*0k2lV1{X}PV^&g5X0W5kNOOn3cEqp z%r5A$kFVX1r};@0pepBK43ozGV+}lXY{ar= zy?Mt=LeM&0sd|;QT00hdyAc1%TAu* zW}L~>4CGcc=PWG8`kI4|l))1zSZ5}3z`EFF1!9yu>}`8|&@#UE1)e{Wh_x?~*Acu< zB>luza)o$KoQIP!5KXUj%$Pa`x_cPr+-J_tOW__E`w`&;9G5BFF8||{{R^Y|BsBpO zY3oQPlb($A5pTDHbr?u|l!koc6*q@DOYmRbVH@k#2juHHnZ;?m{ACy@7r=7=1!Gx& zj>0&+&R{+|ga7siC0&p9%>{CbjQG}UtbA5%auj*|K~_bC`;Z1-Ymf3@QV; zO=0dfb9P1+P?2$1UwQmc5%ReRkbmUg$>r>Pl{sPAPLG4xrR*m6`^i&o!&A*8mOjDy zzXv0DPu6L638Z4p2SG7Dfmykzni%-*x8%SH>_dp0v=EY~~J#gjv!atGUQmZsq>{huvF6Ccl6kZokW9HF_IZx4T)};Gh~Lj*6^CKtjqpE{ z=uzlnR;0lNvk^tO*n>FMTVMrEOLg!p71)7lym12%kILK*Bl&4_$-k%b-<_P)Rp1aa zIHBeE9J6=S#F4Lp)2!wDrpJ#Bz>>1l!M&gJ^E+!~PEWo=JkX9?Ypwni6^spN0X$^N zL5%K%k9<%sN=;xgsQ;g!T)jcF%|I}C9JBw{28Qx4_f-d~c3oM?5PfC0h!^S_ z8d1q{@*`Dq=)0*tUH%o77lL&=Py3fBFK8UKpwj@;ydSK`Q`RZcXj?Y>aeH>Se>d&5 zV7IJsoOfJw%yi7NPqz(}x}sV4(pp`d4KKEzwTDpB(h&Uc3bD)vqK0}}N0dVAp}6uw z?T1oXK{!L*wJ7ZZ>Lx3wyH3;Hc$)J@4EOg?GQ|nR*zrVj8R-Iv6cz~r)unJz5dUTh zW2E&i_uLJt51q-MYFpOfuR9Pkm;67L&H^l|?Q7#FrWhJT5$x{n>$U6JEn>G`Td&=% zf4jQ_ySp2^yE_0ya%Rq)_cZ~u`Umhl%?!zOJfoGuRK0OTm0-S z`WaYgZ|QY7jr{a6^&NbtqiR#`paIcjDADjjGH3IN+SlQ&mE}z51!XA@rmz8fcNUz} zqbKn1U_G7epPzOXHd=i;(9HABQbu{M!hOl%-cBF;z3F$;YNd66mA^Nws!Lbq7?OO) zn$6Z2j^z$(PoH*v0fEbdUI!HnJ{`0$cu>fPkX505Lhpyf(b4C!bDllkI)J=tFq!^R zafj&w!#q}EHxlAJX;Uy3VLbUmhW#9#U0S?Fo#@uOPhhPJ|}0{^p# z9AdFrJJ~+k%lOoGj&@%4Y35jAi-xsS&g!-HvgNk5vVM`nr87(*PJ+iZMXRNzD{l8r zm)%v!Rl*(W$)SuFlJ!VJ10UJVJrmdcTm@iS8Dv_q{Yp&(4y)J?3 zhP&?kRKE08a>8AFL8q*P`XcJ~&VZQb#0v_Lj*^EjNsUM~S!16Q>5b4BeEJo+@)BBO zb-Z^G6AcEk{)w(1>6Y~Wr7liMO75QgZ*t${EGeT?PNd{d{gYbA^^els=wuNcPkn-% zlbr{goBXZ>&JKyq^fT;s=5Lw%WGu{!Ll>9%V23-%Pd==cgAG~7PM zzS2J0oD`QbRfJ>dm8*M?v*bDqFi}Zl`gd4CzEsYiA?t`>q z)U2F`1Cr!vMGZk|_gt5+t8;q2^!n*z)6b-TPVeek>*@^m=77>jU7(FOUWz#^zidg4 zVa|fSgM7#Owe&w9U=KPGR53U#czp1l;EloUgKGsf2u$<;=J&vNw6n6$H2Z1hJ4}#j zi@w5pIHe=#rj=RA>B-~H;wqHhG|iTlBW+?@>vVT|BUhAblq4NTxsuhfD;iV z=x>bo)M+k-BR-EwFRM6Zv#j;4<*cQxHLbO+onh|Vt=nK)E+iM;3=LIS>`HxdAP8R> zBDfXQNGN3X_oBgX(W@~RF?TWi>L@IDJ&=`itW*j7*uLaZKZ(A~Ak0TMNiRKWA0x-2 zI2uc_67=v18PqmJj^(L&D5IAke$TC61}QA7ZDE@A5o(93d!y+S^^x-sL3Py<`k0M` zsr(U)Ue~gBlIrvk&@8X zE68lL6RY8~40QDd4^>ST`3LnSZ(GhjdSfhbK@Q9@`MEeJrus8d%hq$%mXLJKv2cAd;#|51)Mz z52p{_^B;V*TlmN^>_AtdlH%BTCjQ`|>(u5}<53o0v=-jR3T)#ea^P0>MPgQHG2X2c zaY$Y4-a$N{K>YAa_-AME!Mfr-e_+yJGcdhO%&hc)%svEpn4W=2Ki9&risRrMtYH=J z;%oQEV{gD6JQw`H;G2MYO~yZV!^{t6?(sr==FR*+kzT-m({U-6^oIY}#^=7ldeS!? zZ@d5=RdKGiKK@%2+1$c-Qsue!*4*bDytFMm*N&_-&d6v-QC*ssKB1NHy*GgKd>~3% zl)*i4il2#wHsYn-Cf9X}9D8p1U+=>*cVSP8!yZ4V&Qgc)7z$2y1AhBgsyH>Z3_Eof zjzM)Wm>X1#nwjS7vwx zG8g`P1W5b`;>aI-zYhkx8oaAJ5kNljXK}>URX`zT^0g3Y-pEgjK+`YaiKlSQk@%~9 z@jXZIy$_$?kM-L_cK9i4_!xZUG$_h8qMvo(DKY$P&R}YQk8V!5o5rd8ACcNrUG$Bz*+9O8QnzUNvpb&okQ_qd8!ViJ={aFoBAGcVWT zm$t@-Z-`t?=1$gd&FjcZj={g}z!RWf8JnrYd<;R^BFvjNKUKS^EsM`X(-578zQL5$n-d*dJVF)itn?C z#=0QsW)Gs~NLD#^v)|DP^M-X;o{=ltz-L}Ui@gLdyu{C|*v+fxwM4ji zCO@Y+`k)igx7mHJGbd{RC#(}I(VlzE&Q4g_^H;3#BCv?T;0Mpy!izwHup1H{<9zX)# zW$f2I?)eaBVLqr?YtF_%*fHmkl#5*RI!<;!_NOpdb2#6Np^=90e*1ZB;bh(5a||-K zWw?$}oD;LV%`(pNV=6h`qXEzI^GVif7U!fU`oAf6(t>-c#kF?8LPzsTGZ*@SQ+t)G zScDci%;P5Z?nMUMsVo}HWN?{X3FscfPbJt5bDncs-gPP1({6Tp4m__Y-ZdBgNgzBT zCt2=$T-|Bp@da{p0u*l76#GNcf`;12) znG<`<@JT&+|Gr$eolZ99` zW1Bo8h$slq~sUXP7PS@q+_uw~2xr&c?9_BeQC)k(c?rI{FgOK{^d@W#&R%T3r zozI?6V-FVcn1k-BkM%Lx8fL$ja88HH8r|pmZz4@*pRVJauG9SHD_`d4Z$c`~llF$^ zeP@q<;}yQ(=WATSIk+ZfSC<$x$8r8O@t{vgoRfDodn|qARW5KSv-gF<-(F;}$L{f} zco;io_bE4bV6d___P029QH`}Sc~=^zKasCu*!Es%iP{-bX7=tY!1J7-9^ZJkJDis@ zpo{_RT~YoLjwgBs$zRL*T|;K$xz?xb*c-mTW)D3)-jmTu<1U_Xy@}wjbV&^*yR zI$_UxVF|mUhcY8Ge|X(vKFN>wH#_y^CD-*% z;iUeD^!Ed+9E6Qrfwb>Ie;wd2%Q?NK71@W+YIb!v%>4zhueI6PES!CF_I4!i>dT#G ze_i|m(h3?7ucuQ{3Zz`&Xm$PWI*OBEMSuN(ZPE; z1A94pmyj8gKj7i-O2&Q^W!*M#b?rD~ZL#nJxVDXG(=GTzjoIbA+(9He76|^C0CJj_ z>of1u+(Xl!tH~>eu`@N%tUb|&3yFWs`QO{QyH-eHPp-coS8wvKFK|u_r0F&1@;y>) z_Q&~+mfVazTENOr=7e6ylZk=zHk%kK9zU)g+2bvo;XN?2DzMrX;V94DiVkwIpE=lX z6VJ{M3on;6om|lrVxCV(+(7o>2KRUqT~-(GD<^nORw4v{G61neJu}F=T-Tjo_zk86JHGIFyeLiO|%uIqgf83vYsLqPEL3+bD zCGXg?g**u#c55Lfj zXIXcK`lJLt@s#M zSjqXwTU`)#fgDyTWZ)WiSDTeHyH@(c2>(i6>MgnSsYHqG;Z`*OKc9-Odz-;w&Vl!s zhy1s%@tQ2gepsY;@UZjY_q<2@_U3fQ;19cDE5F4j`38ezFHGFotXKeA=sWV#7@WTm zC}~#mGoA4e#v(=EIk|W7lb#?eZHQ(icxZ3QKM7>ns*r=7&Z?|q_c!r%1#9Kv(TK?4 z2YInNJgO0^wImA74A!uk%Hl3Wqm{^=9l^iLj{O}2tLqVw>qM}OdvH8I!e%=TpW>1p z%#QCu9t+}!yr#N&Jq$EK$}Q!kvb;YG_<>SyYBo#4W~v5y^@u!YBIs@%cv4Pc18m zMzEjloXLN|T`sZ~rk{L-GrXMK-6x(kj119YFp7t|ANlPfUr;_d40K?!I=tK-ZjR(&T(+k6<7*7`PK1^#j_g)tdvoYDUo~(Lt@R18> zhffngf+xllgQ+b05w=ku6-lx+8Pag zShysfmCnl-EHAB_ZT0MKdwpiu?xefOZreTUKx<~}G`js4mwQWO-H`33oWDGL#uD^+ zV>D1Mw;=fe~~=8(t$qT*ULwl9{{Bx<1Ls@D9gg`hnHA zk>PQ0syd(nHe&5_!;2Wts}7RIZb+r_O)C6XlQ+Gs*{D{?3_GwUnY@Wao=*I^C1`Hb z|6PHNjpY9!W@FcvqwpxAg(#CL|>S@hjK1~CQJqtpb2Ya|5zQ_;#C6a%HHEszD za*?j-P0&sKk(Ju;B08hh7h%06e1Hi2$hpYEb)H)UF4;vmK7Y|C{R<4fwIECl$nZRZ zzqi>t(>v5V%DdEi)oY{vItC`yOSOkqNq=QzmNHv++cWtt2&@`9A*^QR&6)FMo|dUX zs4MtWU>5%kKF6&MM1O6yX9)ACA0J(W#-UsWT_KcC0MVgRaY-5|MO&{cgriV^zn4^Dz1(oE3i_# zrH2b;ncEpakAi)cY}P#1)s{2zW@$f2;WjY1_4sLC*k|dS@~p^f5qKNTSnIpGqW8iB zu;H7{0$F;16>h}t9tDvzCk8x3>(Jq!DcJ(eDXTEuz@Gw89)9D9)La)wxo89S5?{PawO%j}lU*L{o@yBL? zzb@tc{eUHP3_PzbJMjSoytL|-2<7eNvDEP?H!y@3pMT*?|Nmr zvW)2t%)C%5YpQ-%JZl}|Ob&Pv5*L=7WqbI?@ZRASv$PI77xGV_k8d?wF|oS3)b)4D z^~9;aMko9)LHX77Peh8ICV48U6OEnHW$S;A4bC!tmHcP=xAY(BSI2j=Pn!Ld?GiQb z=i~)a6LGU~M$4p*P>#8Or}s}Qn>sP2Bl8Szr7oq**>d+;C0gq$Y?MdajyO6x=ld4+ zE9lqPH^SMBsnWSD6{Oz6S-q-ICAA?n ztCN_EQ(9T;$>LeeJkLaTYtL8DR4?2)NwXIXcolj$%iFA1BI-x9%DN?dYL>xaw}Yen z@7ni@3%u*o+WmPHAM<17_bES)$F2Sq`zI{5mdoOeG*-x`?DKus1{@4J8T==>QgHLY z$4qVhW$$U5WDT_TviQqUVgsY4HpY9{^TSmp{Zi@|@_`+BEK6yTIy`NL>$`ac+NG{fDU_lmhoyK@dZvAJZBoBWot)tz$HV=z z-^`IeXZD=kbL7ggAlv(_*~3Z(R(8A>zIckK{PnBHkH%k%eF^$H>U*xZvk6@iJ0#~$ z-=Y}Cd&@GP8vg$VZ4G%C8j`7Q=&|7JL7xKb0h-@c-vi8RYAatb+NySsniib0<&~w-G)RW+epfdlJGJF|%B_ z>PURFp|A;-z~?;=nD;XI;Y#l7t{SdW=|$6DF$KL< zdNKN}O{2G0WA(i@$tX=Ux={3W+MPMr=}UA&I3mUwx%5fuXF5UbfN|k< zKl7AhzHoPa12sVl#RR67__JOgi9#nCbMzV7M!FCj@Cxc4tWS*o&8RD`llIdkZyd9| zj>rypE*r$<)bPaWc734M3%gPpKe#3_@jqC@#^g>)7|rx(ZIHUeo2cwluA-q+z47WR z&80UN&NJKED{rSaiJ#?*R8h1WQ`Fm@dakT#z5A2>&&|Y#i7Wq9p;uy%yNPO( zUOILJR>{0H@?4JfIUPAIIhJH=9oa2w)6DyVRo}1HO~x0`sMM*62jUZd9RL37dy}6# z;~vIW`c)*+nVcy-UioZ1v#jyi=^qjNBJ@PqsLV|=CxtBx(=s*8v^->XU_sxSwjyF( zb)l>QXa za7w^@|M$MhKKtyYtw$wrPko&FMH%f0a$irMkk&kPUCN}Ct0{w1SEfBl|J$A9q1%ZD z-Yi~}PgvL3Tl&!5&9|zb;hWv}qEB1LDBBs!O?pwB(BsuL-uX(L=NhLmL>bP+rP=gK z(dai7(tp|fcV`fjGtlkBzBP5fjGU>9bn zfjox5+?l2o!IxeMi#Hkn{0mV-R;?ht9rq}$l)Ch2w0nPf3(~vdjZsxxA{~;aGJkgz z9sfirSh%T;_c}eh(`Tf9Og@|R>QC7}1^&GH(<9lM*4h1!79vOblnZQ`sYrO~2>-~w zk(nX`BfCT#4!;|AGq|3A9!G?nO?SIjrM~{t{&z}3n}pc-%<((puEc$cANo5sX+zpf z&nqoJs%b0ajP~Chm^s)Q@+Txc^i`&?uzZ`IYHcw%^YZ%O*chZS6Ye30KGJsX{M#r}c#Wj8Ba3ME_F(jRNg~e+6XnpX2MH zGtMoWYM~E>SWk%6HC6UTdbTiK@Nru4v~j6#Q@W>=OZhLQbn23{7Opv-Y-%y1mo&&) z(NW2H)%UbtFaKBm%l&`)9rO)$c5-aD1zEewtHobN1N{{|vUkc;&tCURSI_ieXULO#Hp6z_AWl3ol|_r`}+A7a-MbkvOTnHk!A?g@KApc zpq``ZX1sTXcRdzpKC{SfDdm*i@PFLyfDHfjz7nDy(pnmBVVx9WX=V+z4Y1|2m9xgm zN#Y};pJw+ScCSyjr+rEpo^mVYZfaP1KDVZ{(6dXAt-(IMd?o*=fWCqCg7yb343dNV z10w?N`7QMgch-0Gx6QRE(mruM{kRWmkGyw0Uf1aKvS~F^kET3K8IgJ>ZKi9YXO?%V zw%xcUo|ZRTt#*fFz2k(Vt7C`#rfoAF*j&MH*zi|E2qUfrgZFfI!fq=|G4wpmMA3tCf{ zl@%}NmTODn#n+%{or%eMgPiRn^ZgQJXCz4Ea;WT^$7?k_nVf5z0%G zz+cd)EZlB3FL875z_1 z@D9|z>otTq;ufi|?2)4^b1cU#S1kXtgwsd$Dbq(E!0GKSmZmcNJ3JGo5KXVwt1tzQ z($_C5Q`hRl_Nzm+gFii@T8h2-2rK#j+Ax@P!F#iA=XD3frL{{XwRI z7(^S>^zB+KK2#y^Q{_HAt6HGFW~yDZwt62Tay$5vLNxqK+yxWkFt6BxmzE2jZ)Wm+ zwLxJ9f@>53V_M9NwG~ur1<}#vB(~@R9^^s1?LuIa2ZeUJp>@H8Fmu6OZtMpF%LmncR5w2I$x~ZG#BWm-O_Kv|G?osDxll4|)4`PHe zOt7jY?Pdz{Z~Wr(;BG}=?VVwbcEN&g2A5|O_~R8ApErpki_%YJF?D_&`Q*aX87!dt zdc1H(94%#*XUfOvrE^i#H(~erf(ab$7L_JMe z?(DdhqV)t9oB~cijJ`0LK@anR+nxvMH~Uy5g5H0@bKU|bP|WxVcDM!3P(@;!+r0J` zoZ3~q-vWAA)`sC744!t1spR@>HDXt~p2UMr?*(UX4f^W>ML!IZ z*B$iFtV*3kmzK&P5X(SURb<9RucmnB_zj}Nj$7#xnFJ43w`@)%FNx z&~T!JJoMEm4fE$IwRx(Rjhexg;8CN&#A<@0m4^Rvj9kNO(9Kk2_#;)ir(w?j1&`%B z8PHtJ*YFT8<_7870YbKfZl%kR9e+4i$HCX;Q$KC8151fjsobg!y4;P5l)tHLxD6Yw zC|J-_kn~C5@q6e4sw3|s(V(NiR}L^+uRLcWgmvvLgo_8@2IrT`(~rdLYZyxPL;|wX z7HsJ>nH;lULlSI*cVNX^K_RAtm6iju{J@&7hk-Q!NpdLo#6twGM?uwe2sj@0Q6!p zaIRtO^?BCFpV|(EHQxl1o|~&{PX3*VC0s#swDRqY-mMp@1E`ln)V2#UgRdea(??Gw~!32w@ z&LIR&***~MSmr>+z>XWnU0S$`Dnuk}h#Yen=ehdU@Ct2u8nfZcfW~y7cIum68SMEX zT>tK%Kri5r6^5_Wm5!>p;lAACca6x#F$Dt#OCDC%VkCjQ-Ultt3d>{{2zE(yP!tm0 z7K_mkE%X%;uCLQ@EIu=B2-allkQ@;X%aknK_73x1dWSU#?M*zp@+=<`7e$0CE< z!3p!DM^eG6=P*~PA#BHQoClkf4txCx46#mhde}=YI1XgPoIf`e`7GI;-M_*?l|uSwvcoxt5raX*)^v8&-HH6Y6O@E&HZQej@#fh^H5(8!_0 z<#*s2rIRgsLGHyvSDs5yioDf5LF-1y1No zm~N^xF#aaEgnzk_fqC*yHN zb8gDwDI8}->Kg@#E*qk2%x*ZFzy-b`#R?f*f7UV@xv#@Il;SSXb&}5lJBU7p5Ca>@~!AaZ? zX7q*?HGBPhLc?C?w8xOuI)>&thb+cta3?-+$N8~JoycDHAiLX#>`O1SL3w1xWSkCT zcidF@yYaA!u*TE4m&;s5DC=bEiprdC9mzS0&Dg+--{qZ6{>g9d$-#-JN5--oclLul zJIdEJzAxrXpWzyAv*S0}A9FHaV?MJrpWmMSZwJF@5$k%DJKRGyxHAld{N#B|CX7ml z=>quJ3fb)^JbrLWZjgD{LiXl3?|p=KzJ)FJNB*jlMVf=`nEm5o$@q^%3QBUqV!1Dq z2YHYzSr_sK3s~`M?7bI0PXPB-1x;n{?hH_DlNnHnoV&S0zd+))v7i4#A6Fy`Qip&0 z^A7Dn=7ykwOonGJ^nf|3vl=To3l7$4vcHG9kBywn5v*_-cBv9sDu3?hH9N7Aoidqq z=DZh^S7J_4Jc$2J6VpWxD1x#Bl5P}_}@fMM1HdK=h@k%6 zW{{n;keeO0b7FI&)5~yY9W!E!9_&+P)+LgYZsohl)w+~%?YGG2`SAQgoSC9X`QKcl zoqX~~_IeA_a}fXMArk38|7GXzCKGlC?CL}^1sBP_j^{IK@=4~jK9k?Rm$i9_v`l4{ z8=-$X!GT%9Jsd`U-y*%)_@vhClX;5ckOk&Nu|vzz3BAd{`hv9nfJbo-Hg_<&Y_t1f zPxQkoR{1$nTpc@Cnb#`htqgRm4_8|mZ?7=xUL83Yj@BrgaWYL_PagK84u1*Y&P^tr z$*Q&RY_qnw1y?bEJNZQAlFTHpOK`f!Vk?@#l{Rak%??e>dqQuWVRaKZCC@mME3obp z(ZKVOg=ctGcR3HgU_F{u*Cv;}2k+dFHO$LuR%XpxaLzkr*z4hN6gDB%llaupSb$zU z!>l(n*)wsRfg8w`f=qikGcQ@k{ha;X?9my1GWCGj`OIXE?qV$#po?2!wIVs=MaasU zY_~zIQ%|%GnE~#xH0#up)tCU=Vi4yvH9R@0Wh+=^6U+)iI?6|v(So% zk@>A?{Ow4s$!-Ev5b?xo!^?h=!f(E*;ozJPMqd+o#J8k zrmxA_Gpkz5b3XI1t7ea`OYF!oG{8zM$zblnoZ`8VeYwn4-9uVCupTv#yu$c7PU77> z$RN_dk)@>4|C&Ix_s_jb##i1l44Tp0dhD<)flA#UzDJ*mJ#Yxtee~ zudgBA-=dd7?l%cz#SWlxrReAY@m|uvByUB+*hR?{btTn#p6&{gX&Ds=P?EE3e^=|@ZCR=DOf@^ zWJ|dVYg$){1~=ME8^xw^yNQ;r?Cj%bun1ySMut)>DY3Z zHCaOZbBKJb5Al#$Q;?tavtrMdaMlCKJzPLO@3501_R2|T>ouIax5U17_-F5s-oK5< zWLtZ|^O{C2e;2AO_7ma#qm`n@;sBX%k(oYa=m0Z;nIQ$}8uXT0iz@VDo~WIs=V)~@ zd=>O>MBFv$SpF0oVK<0wJ9-)|p@OM5GJVP0#@mQ|d1vt2soFF>3=Tsgk1|qzR^Mwm zX3cHeVEbjuY_DjKwa;->^6BlpDGX{Hs~)bhy5W+Wz^n$_I& zfw@D^BXc5gFEm^{UHqO3&0$HIy0@X~%r)JE zPAP6Q!vbz0-#*Z|Lc{{bgltU_S{vWB_IhhNn!MLqa5~!5!}wB7n-t&my+cF=cncc-;F#QJlM_Vn?2tA!dT;6%uJqPB;K{<*dvWUhsgMIWlY z#Il&3!&>P!`U)T88PoKorl$+rVWU0^pC=jL;}BWIZSVmf2)D#p(jo5lA2h$mGSBj_ zWwkY%eWd*#`yE?Ddzd5LUVy0-g~&ATvIW};STBM4jb_$UEc~`+FtJ|a8|*a>=o9D> zU0o~6M2BzQJ#|qYp(|K<_(C6<0TRtb`ts6GQKB}mkn{>%%uVz>j=ml}K{i(4kCYU0 zh+~N>%^oV*nXtGWk7^Lsa}XLV7VWVDlwpp(lH5)=Fvr<&OdnAZ-vs329`*A3)MJ`x z{08A}r$&O&6kt|FES!)H#O~{u%rX;Pc&)yg9xy-DD18W4HY=UV=c8w`2>n2@%LoqX zywR5^Kby2&+(_h?$9N_T6>H&%?lqnYm&J+98_jLR(2KdLa154_!Os;12NA2GTkup` zfL(py$`(>3w^}c5CrTL~t})nM5 zY{|A@Uazr=$)Kx-VX{PjfXpspenFsk2|uYb5%@PE5zVMVG;tD{H#x;Gkn}TnWiqy) zGLmohNlGQBJPqHh5qmoYM(1K;w|HYHzRqMA;l1ENePo4ZkRkttk7f4ScHo^Y)VpY8 z8}MtEf#0pt=D-L(NB1JEgkjf5srl9YY8`#JR#_e4t)O)@oO*MZKCARiaLI1diTx51 z-4I6KZGENC!HCf((=)UQ2tqXY-haXdP}DrqGZ2?-%uZe`u951&A-qVntX;B*bD6VV zp8Blj+}RyowUfBED)+TdaKdRU!_Ds1c1{wRzsq06t#^c7OyMuLVF#VcTDpfMUI*J`aV+}+`7LAru}W~YX04r7$QNBu(9 zI~1F}nBVOpPqUx@_k-_sjaO8{J7uz}kq>s@S7OPUAc!OJH|L-ut%9GCgqQFIw6HpS zh0N?vFQyAu7w;Q4=~BCsy*bP8Mf*T&-KR}_XAZ?u7WKD6;YdPTn_^FK$Q4mT-3S)&qq3LP@48KxDy zs`k-p3){t0%q5_6v=~LFhY7Gpw@I5gYa1ogLoII^VzF9sScr>eH`ByH6M#@SV4FB{9xX9 zBSkRL&KMiJXL}It2;UWIgi@~fN$nInVbv(|MM}d&FB2I0IA5aS% ztr+p2Rh*ALWV$lQtcoVQkKP-AL@Xv^{E5ug6Q2=#{)eVijLpQ&Ex{U2qD!BFnVGEq zQFsoc*td%GzB&qSV5O(lAF^>dVSNvW^Sg{*FJ-}(`|HclI;~)(`@n`P1vY;Mglxaw z3@zIct23LGz75O%Cmg?iCQi-$9tCfyDMX07h@;1H4rhQm6a^V-E(VDCz%X3g$4~BW zDoBbO?`;SY=np>IiU^=0{YodnMcxj7D1q}|SGz$~^e!g*b->zpV)E;1uba*-)9@1g z&^Q59nfwQv_ZE|PHsd?2$0NUN9cim?TW@P?e`61HJhS`S8`^xBkk!KW(Yo6DuXPW7 zDb1OkCzw+33vTFjB9nCp5F_6zxUq1=>2FTia1AK$~IWd zrXT>};8K^V&21lQc9@?T1NFub|z6Qq(|RgIxJL&1C&o6pD|TqF8$;V5UDmNf2Wb(kHR=vK>5{+;Ysl1{4B_co$l?gKGg0Q&0%~FlMg7 z=X~d^=-bZMaL#hJ@yY7wU@vOVW*6)m@dA5VBP?n3!TBE<9 zmA%RiWulUYxw6ANEtq@4%v|Q-zElbkHRV+k)tp#`l1TnOcBUpZ53j|FoWu@f>62wY zO9`^=d0;dQ#;=(xd-2NepfKptJ2d13P|wF?c`DQ8_Z-nzUi#s6;;R|&dzUO?anX%-E<|_lo}B-~hX)>jy6P26>o3W?`J6X0R)l(!b<^+5suu;k~G2S3Y{K!vCx19^fkN%Im5}kD4*= zcd*lbD1|kva97%DS!4CHouN2;j87Hc0e*}9iuy+eWDlGcSRgPiz!7K-ydL27@8H+R zdD?Nt-p_8>qHN`@IV^vud)Q74a6~%};`2f|=b7i;u}ol<>AtE5TZ9YRsWe>|JS=Y_-(0%(qOYE96qk2g_KiW^H2opKUHD^csCk zHdy*vPB0~Io4AE~q3Yb!(qi!>2%p)FmfSFN z%CdO(GvVzWk~M(IIwhHGwMvNtW2(Z`%8lM)Fun5ApKb~^?wHY!+(a_bftfq@0XG^7 zLYgn5a;bF&mVOhoFb8(89MOG#Z7FP`wO-LXmw8TElzpC`Ofp;Tx|seoZBN>YwAE?f z($=Nda@}=ZarahgtC7ZQ`I_UR^R3T!df`-e4E5U(+&a_4(A**Gf_+1-gftBO8FDgs zLD0HDOTa+i+xFp>5_EfgW|?UfEX#y1D^F9UJS^d! zbgF8C9vX+eo+VZx>bgbmjP_!8>8o7V8o(^&56mun0Gnl`C6gt^RX2DpYy}L z$Yi-Khs&*{oiHc{Ar;3!T077iz7RD_`#|v0=wK#lbEy?+!mOw1@a`7C^7Bzr;CE(W z;+3zf54*C$bv^w}+Q76fX~)xMq%UvEwv;1W(VtrxhXxT%Y zQ42O#gwO+|xDLLciTsO38q=G8(qHC2XwY|hTd#)?TpVh6z z6K;zerAWCZGa~cJIl&OO$i0}mR6yEHKC~a0brQL?g?I#)^!Kp9k9bvOi85BX1{3;; za!y&wx!j;^R}Lx{m2Zk$c}lnBSyT<|Q4*9WCe{r^F3NhdDqr3C-7Z&l_b6sUcU8VC zfogAkjabim+_Bl2*%?J2t5|z|=RN=b1#Jml9o#9n5xtwX1dR^d?El!egR`4Y4o4B& z8F`0zl{{?`X|D8BxU20^&bXhr#<<++<u>FC#6=l(w%5}+7(HduYFHe&m34@IV z_=ZlH95wL7D(bVf1I<rc{7=*g|Qq6jeSm)w=@s`O3Q=*6B9QOSDlQWFZS4WEj&W z_KVComH&r6m_z^VBJwy;m{MXKwkm?k$-3m^s)LMF!z-&z)D2S2{F?+iZI0Kcq0{zd zeR)$XFb~H|ax1;yC-`p zgRs}f!%Xm=RqKMBKElVCskfw#)B;OOlpac*`&0+-_0cBN-|Lxo5i`OH zc_nXNW=bAmwRW?cN4)*Hmmc2F)Q22pZtNWIPbPeZ(m%c=UBJV&)p(*ML6RE~T`bT$ zQeE0dcp+YtZpf{yyKIB)hP|u9OLuO;C!ep2DV-CYPw^Kcd_MX-_u1`eW&dt%X$ipg z8~`)&7j_UC#TmZBJYtvwROggYJ;ZPVXT_=B^KMitf~6$rWr-@1iEfr5$-7~h97KA% zOZDkl*MbOOI@9hxO08MpiJX#S(i)~0kyd%&qKb4hKD)nG4`6O!Cx`9y}^awwP*L~Q;zO~kRLFS5_f>qQ4K4xn$$j_o# zf4h$p6i&bVpHi6IM}8oevXo&~@>ekV{^b7_!DTDTuCBzd{0cVTpDd&Y-bWNXi4sic zI-x%#V(7@-oBciS5E}$jt=Jn+(H{@w4Cu>Fs-ydn&3#Wdm6GVso_HXpM#%-DA5CBI zS0LB5wU^BL4dZ_1qp$C)smzUUn9-Ac6EVZTVEH3Bv3=Vj%nVD?8WU@@dUdwnV_V#A{9_eEwwF%e1hrGJ0uq!uC}AO z5{QJ%dF8EO5^tn}ZzFMHPtai(4B6%M)w-%aXGZZTyfj^_fV^eV-9+|3I9n#ZR)?5p z6el2nID(qM^gKTV6VOVz119OsD?@XVD z?)-b64C@u@ef?l5RW&}7Wxd3y7y=ekjq9vVrS(DbToJ7ACNlPuS>yZUm%D*fMS`I% z1qF#fy3_Egg3#i3VdwUvKWjA7x&wcC1^<>Kua!z|+b}v`ULbb+k11v?(FxzQje2D= zyMyT0TLctk7&VL2!HACuF|b27;>{ZH3myv(iEBo1e}kzPFT-6_7E^?NW2-;OwOnqR^lv@mk$J|A~TE3X-)HyzClF0*^N%1E=9N>E3r~f5Ysng{yQPp8{th> zrN5je_Las;Kk&@nApH%@E^W+s%R#IW3%0tA|G%X6-!2{{zYbBV4LW+#0FZh#-#V}z4C!iQPnW^FoaGo6? z*7fO=(T!JL1OratO6M?3rllCmeKx>5`e@7(R-#vSLYFZkz$Xk@+To0gU zQ){7sv7U_9HF!DGS(CZK2)*=KwwH`#?>_cYYQB?qYadT&P;VvX2?I-C$ zXcPDF30@GuwjgIuVEHhW2Cik0P=>0&zrbqhl7C!;{xSOwOd?vZuSdaV_Yrba%l(Qo zv6U12O5Z?D#(MmeI3lzfU@$w7#gEv$Q({))%i(0f{a~-mq1%5!NuWYun^Zw;#yaOF z;^-pg=KS`Ams(JqD%=BYb{Qj(%u?caVURId%T7Jr9Cjg<9!N8d?Vtqz)4Lcq@N?G- zZv>0c1S>w=sA?kktVT!T(P+BJGB1=J@!=$AXC7WJp%4k&p2y9M3<5mac8 zaa{L*-VgW$ zwJ02ckI0ToFM;+Vw?duS5upm$@eTT%R{*_D<27V!1V1U7sloNI&W+GV^fQwFh*hZ4 zy#+q_nYeT%c~O{NLVZzUl3XcRQ~$DuEyZg>RoIzpI7gAjT*)QR6o=8NpdjdbLD2uF zoZoNcAcKr``ZBJ58yTaYT)|7@6l{SN)II!xUmZ{W)n;}M2X9=Yy)a_v5gUkIb zI%KTDo9?eK*KX)ZcpsXvp56P66#tK`;clj_RL9fXBMdMi^fK6)C&njX6E-qUJZ@Yh z!`#HkC7vJ;To9eoS}^-TzTo}$8BCF+>U9wvFLXV{c!(BFf}gvb$!kT(kQwkwo6|Qi zA8V8cF3VT&SG%!P7paI(q2FnL!JmvvTdWMd9fir@w70+-$B?soWOP6W-eZ#PV9w+U z@&%=ZJW?NNh_H+Ew2VG@CLin=QaRS}YCrTfMyC* z8HP+Y0F}FrE;~bpavJZdBPk!X^;#|C1X{Q+S+pX;Mk9zE$1f1&vTz{ZAsa*WY@n&x z>DpaDx=r-9i_keJ2J8kFWyOjS0-PI7}^J z5iv$j(Sq_lgGsctw$#Ol<{~tcGd92jkg~L@&?AW{wt0Nkzr(MCpgg?sZ3!Uy)<} zA~Y2fgj3{UB$(Jw$gK`x6~3Z(g2Yx>;%8J5^~1A$C)OorI2=~=6rnp6U zeD|K%=SUFyENG%xLKjY0ML7LU(Z&hjAn1 z^$%BtWUPCAW~spz#4fBLSDK*r6xxb!kcdb0FAg(~!EBfTYwfsBu8}GXc&4y=rlCI% zP&NBdJkA8ll4MzO3)88ExQG7xg5G}5nO-5Z7DJ_z$X6ku6{oHWE8L%JzpFPUp03II z&Li_*jGcm^C}tO`>Nku76|#b+OytxJ$7;94D`=0`Rg>IY6ZYJk@Oc_=ngVi#qbIjfN;C@`C6KOLiot;+d=3v3BRzE4j~)t z$S|haZ6iZ@A5Ury?7Gh6Ur*p6?xISqBR%6xZs%B-5gPm01l@59-7pzl8X&#FCQV>Q z(LZR!+t|3HWT-ab+3ps={IyWM2YY&r{=jF&Wx`_i{+vFY>okPZ=q80&haO{8BNb)+ zj|)feTpzOrQ#h@q$Uwe?0Xj-J$8NtvgLEc3Yfc7WHksvy$pitpip z!93mMw3B;nO9j$tb|)t}cmt~!#ZTdAy>{$=X8MVYBR3R9)U^z&^$&cYKJe$F!9#1X zey6b5kA-!d4`yrQ&A!1#mM4m=Eta95O(^x2m4v#~$;^V;J)VlFqG+54*yl8C$!B^@ z<-@+LCv&?F4ZG8rh;LGuIIb9eSuf(8`{X(P;P)QDlfQ_k-Gvp8L1OYz196&sdL!i7 zqhB#ZqJ+;dir=B3E`lG@BIW{^ro+=e>&LaI3EP`p+f$~VmWUk=^Jq`-!4Uxd|a2HI@_F!W;QPUvoW-z+noO?G4ROA3> z;-GMs4p8vEVL*e%o|7Tt<({2G2pHoTS2WN20k zeT_2wJqq?pVeHlt&iNtyv$yEo-k=fT*rX;r`#Rn;z2fw~ ztjTuXFAsQ~jtmxqS>2V)-rsO+bgn)U&+Y)ya5!V99+EA&%)8&ke%@l*MxywED|np2 zEU%2F+(jq9I{1bS*r!0&#+)FU&PwLzPEWX^rD*#t>hg zBg^}en6(PBX7=iQ$!R%&);WmxIh8o`HlH+vt5@*j<t9fS@(wU76;qWc?Q6_3NSs0%M-J#}jn!CC%bg*%WFHKz>K z1b_L779C4AI}UHltQY)&KU4z`Hy0k`79{2fYZAj*yuy2yABeSC?w!1QriO`D^_gD^pM$78R-}kdRl?|4x#MNAh=K~&?WcLG5OH08#!a& zIB8vwqaY&OqN*a%Ls-2@ z#%MYbEymLA#dq3=Hy6n@6(%RU5~k}tYE)CSOzil3IDw_fQdFecUlxiGaW+K6JNu%u?Th7oSwNDV0t3Ds1{p>O5~JreieF5_OZMx{l7ZuxwfT&B6gO znq6)qU8Yuc0ri_-sBkuGn)kA=?XYQ0(E^9i2_KjR-3Co?6l}5$9`rWFXS`&WTYvl2Yr4UX0gZPW+m8&h{#aTfJvOaew?>O$lqjQ$N8X-OF6w`LBYC!QdfhW^#c}dG(47U zWF35%N1F%Cx*R;-t6b2wuxAAsfAl-lQ?hEi$3ZfzVV_O%Zn|5NeuVN)yz-kXb z0~?(4Pt<-^fQ@$oebbGq^dS8KoWcoIbI*rAu#%rg(~mET{yA17L|iTZv8pgqf-DuK z8KO&a&~av)-($ZQ&O7#c);X4W*0J{Mj%kjKwpa2ru@hG0iTGafldDU|nC;S1o2Pb$ z(cyrFT~s@#_GbETb}-p=Dy!FPx!`;?AT#RK-h;`8lU*o4%smyhSs#3PAH6MEz2*8Y zV-h~OEQG_ryZ{Qf9FHp>sM{1{3XJO*ke>bcArqOW7*7oL4d10Zds2uf*_Swd0?e>l z#PgBjK{R$E_wV6cucTu0E53GK;-_NRs%YxFCKCM?=dA3)GCgD+%_)7GkcMvD(^*!t z9?^qAR_ZK#t2|;F`K5E5_gSp*V`9TZ9yf?)OiQp8l?8``U^L@y_v;x6i%Stt; z&%;2tPKUHNO#kS>6q%k_QVUE^1s~}lK0>4aBk3%_qsW#hJT0S9NFc;;cW3e7?ykG| z;_kAzyTjt{Zi~y}4vV{6Jd=^OdH>{n^AY0d?t5?5sj5?_!1GC1t}-C)li-B|T+cDC zwJE*Nt?9+_Vhanc$&OB_q;51dj@7IFa z79g#ZZ<@O?!#5R%W*NDzIj{8}+f4gYo6TC=TutsImoV3`9E8d6oqDkiMsgq(*GIw= zvc8?ko7K?U!Sa4-^ad>qAp7$hyR9fZ+l}C$+P-I;f#Ljn!S#h;r;1>|a)HoN$!9bL zvlk)nyAD)4kqF=o2&))%h7PG86Du8${k{p-xXt=!1N*+9(v%Fa?If|f)LcGmzD#ASlckC|M&1wm`i49Vr0`f?Eaei*WORnS zgD(~${w1sdPy3mG9!_~^LZH4*h06uFZSJ)riQl8jzFv;+e*v@CFlD&A&DrJThH<&a0jKAdX7B558)Qh}!8TLg1;<9dJ z^PY0<;!U?emJ3+*fjsX$)T?UZX;#pGQH~1I8oqLfDESZ=y+17Y7_KOmx7mp?kK?WD z<87M5t(*WJYse`L##?`3J+FXNW^vlvP*H5kdly)d$#7FEzze%h^{)}Uy1{yXs(?+X zSng)d&NW_XdK9~V6R3fpe#O)3-o3uk(D1C)_|4jHp+=0GnPO~bPmwQOr=}i2DCQx^{AkFD}EC9l* zC6;I2;W;L8l|_#tRJh3%d5M<>v4{OVs5P-*X7Z4MoT8VYrZ;5u$50J@MQ>s(`{%7u zgD07r)tk?`K82@@0u^LtKa0d5KBF{<{vj5`Ms$^(Rfxof26J8}a8iB}%Pysd)IZzO z&f8Gp4iEl+4$;~a_Wo0L*9-1&0J!5BxV-|*;&VhlJNb^TL@?9&_!oNR_1Pr_;QGEM zekws1>ly4(8+Z&0(KTsBWu^fiwdcKp*r>O7jYTkq>tMlKQP~SbQKT9+FCWjjG3?+- z{?bysb}h1$**NhbXcfN#?>wPH>9gd#Yhc;pSf!2FSrg|=zzgcI*NcMH24N}vvt~cw zwQhlwbz=EeWC-RnFRUPJL{ZeyD{2cjS(fTbJ*5t)2K^-s;s4I`37RL#-p7y*q_#_D_~*z%ze_eXUL&>JqWc8$3!DxE%iwX*C3YH>M7HnTXvB zIxNh&uEl;G!hKA~vfX4=*I-pnu*W?_pjEK?{^;A^=U$ZGJpcyKTK>`q_Q^)pYXmuk z8Ze3SgD-yKeO_@E{^p4s1ZT{~OATYkZsPyStYj9f(=eXOVgAlaVvW1R7?r6wjsSt4 zqzVv*Mok~Gc+;4swvT^*3XO>MOM+1!;|E8AQq#x@`1McfQdjs%l;^k1oA8>=i3z5# z!Yj$i7Xvwl(Ki!Il{>Q`@O}}aBwQ*D9)oPq5~PcKEdnhxm+sVq_*y$K=TSVz5-evp z`(gyPwSR_QNH1*f9-c%B_4RzjH<9G6igL}BK-B(;Ks7iqDa2;kL2ZBYTqaZ3@5>I3 z05AFHyX3&ijE5r-#d|sNdwGbCD`HJ5;jue&=l*+N3dXv@tk1jbtBH8iT3CxB+^fIC zuLLXl)>N6A>ug~Hm76JWLXHdf;h}95HS~0IOaH(&ZU}EhL$NcJ-p@Yth?L0aY)e4Z zD}Z?94!P@HSo|XFyGYLSdwfbg?CMSSMRxAC13gH0hya^X86N^V|7ny1|F>gjB!Uvn z?2MaaYVHz6#bGz@g3~IJLEVZTz!~<*Jl67G&N{tS{Qj2gk4F5Jc=pdz_K$)O^6N7A zt45jdJ+H7h*YUqsu=N$NUzzYO&5159a#C*cFPZZaOm43?-AF6&PX|GqcgW%-gEBvX zDc^xUE|Fh6#IqXBbIea}sTgZf5U$!wqWbwDwRxP$`grl8?6@M}O&@tpoeW(OY+nLB zRS!WtE4kk|utNkpA`Qj!vmpMTx{19~i^zNg(TP7YSx;YIlHuTY#PYK?pNA{ex#z0B;?| zDo2wiNhPzBjlCVrN2S>TeucE+#4Wut?9D_xZvCpWko)nLZ*Ts)8Zeo~*`xYjAYbF4EMFYx?1t|dEXak&- z>*O^X^B0Ta&9lHssY85liG2_N#%#hr5nQ&Bnnp98%wFRixvwyCIxZ~BV{Cs6e!4t1 zuL#z<2;CcgW&Xd}F`=xuFT+3J~DYW&J)ETvz?BbmHUH_q4+eDfJ1jC*iSj$r%Vus8mL$99{i zy_V0M=b889$@Sp7n{r}{;=9xES`Wdpb+>UxZ*4O*#o=h5WP9W=*_7@{E>s+ z*Eb-~FK`&QqtJaAAELosOQJ&S)}vvg7pFI-A$w>vKlKgknv;BZ6;91x;K+Hr_HYM& zYZ%NEZGzvM%Pv^U)l>)T_|aE?u5A{X?iJwaDSW*N7NIQmq%W3mA~vK8451PHgugR7 znF?__F~B~)cNf0dOI7|0NXxHv=$}G&2Mh6@w}-Ga{^?}X*b__m->)$M;=ohd&0kmWSRC8)sLs5uopXg8;i4_&d_<} zn(_-~#!qzwx32U~EhxQru0( z_%7IL7oXpZ2f0o*!fgc6xm5_P)sj0O#=Wd!kFNqZO~+o|z+3p|78bytF=4}CHO7EE9>Au(FTCXK3zXLLi;cxjVn1f< zHif%&i7Zf@P*o^G&e_M$ZsS_2W3h{6?Dj0!f#p2yPCO(3+fl?h1KBxkc=iRbI3_Gi z82C03D{}zkwuLACfPBW4CdN@=j~|rz;Qg>MQqSK?8+MM zWgxGS#4<~;Fjujg@fl}R#A{c=8<)ZM76UW+IRUxx(&f0yTAZG)Ji%?8rFlF{Kd!2S zg^b3Vsd%YNJTw2iQ@^UyJuKHwo}PamP%FGn4SZ^IBB`#t+Gp&c{8aYBxT|`6tvHq^ zl2;}?R1J2TUstIw7O6M)u!rkCz$y&jd32%*(2b|Egq8S#9mz#4%RgN(j=eCCIBGU4 zev0#d2|G8RXm%0aVGWVjLU2+)zNZ6ws0T4v+YHNIga7?H1MTrG<$3uhn*M*zsefiy z5uQM9FiRFtN=d#F&U=0Zw!G|{Kt9St2Fghe`YgNm13SWxbd%T>*NKrA5%C-XlXc;K zs^-3}af`it zf_?T3M70+ivJ^`)m%7y7*p!*72tEd`*?DCN{e%`_x}Y+DUd|IG*Y}cH;$Xi$Qp!IaD1I$Rt#OMbRCs)05b- z4IGQ&#E;|gSIenS)T8>54VJ-ip7IRN#USFf2_UnA?B?n`g?FHiXWZR+?(G4-!aw<` zA*XaGr_aBSrgILOsTP|X4{2CFzSX=*$KaulgX4h2Z{P`83X0lVBQn|Z~_F$}FqFVC}R)bmC zf`{6G?;gh$SHxQO> z47MhNG6(xng-YdpYF<6DllzI3K9MPyMhv}$Y`|6U&QcKmPrl;@R<%4i1%LiXH)c^Y z*bI`{g5A%`GfQCaS2lH_($fJ;9s|<6z`gi+WgEF3{|v3|`0eJrn!$)W#cA_TPnNrdlG%W354eb$ppg@!h>%k0*E3iNYo49@%=Ncwr~bi zuDvt&`_oj23XcPI*d?q+EBx<6QT|4m9l* zmP;MX<1LM>Dmp#4tP5;?>@V#e`&Rmc<$!~Z%#KZ{DLu1|ww<+ZvE)JDy^L5{=!il^ z2j6AG$=^Dxokf$q0UB1jylcHHz1_XPd*^zedSmDhy{CTHZtMLxy-(Tk+3?eK!Owez z0ir=)XfM>ix{+I%hCap_l(Wl<`N;#`!-Hz<*fe%!v~LQUMH^8%n@7)XNiB|^@MrXv zZK9)PKQ-Hy`e(f>aX?G5BA0vdZ)1dghj9L%t)Qk^g2| z)m(WIHN!cQi~fLiLN|C+3aj;zI>#@qExk1>=rg@WPf&Jk0y^9G(Nd!Yv0E8=P|)-JCMIhu+xQbYHcif|f>q@eZ#1 zIREycCKb;V_*Ctu7N8fP_hSz6O-x2E{V?l!#7HEA*dEK#4HVLsostQgSQYyh12*&P zE-gSKpf$CVvPNBah>NJ5Jj8o$hmBNNY$pwnOIi}Fb?hx28yx){iFT{wrDJk{H83q; zZa`yJ_JjSa?Ywo6MK+t|V$yS=h-sgSw-cd?1 zHBJ4bUDg}3e{xdCJS3D8|DoUF2I|xA(H-xFvRhaAImMvQauaza@oq_aK4XLy)MLj` z9Szo}X=OB<_Krzd8<~R@q0UrfGbfJwR!X^zhKAsjN`DgBu=3YC-bqG z7p=ILQfv9O{F}L!*rBJop&T@Se`~o8bFOc*DZ5um%$? zO*Y)*Hj`irSm73qc z`OBu+Hw5es%o5Zp=ty9Tzy*O>gN6o4yniyF5YxUkFtf;NzAcRr&%>p=rEjIaJwv(e zZSQT0qTD_2Ki;X{(%xuqH*Y2-NDa{LX}k3K#$+FQu0mrJ`u0c_=*wTt*%&D2rgt{=MzMw!5$DE(lOv+lML@MV|$m{O?!#mS^(YwVv+&jj5%v()S zl}GA+tvWV*J=xyv)X_Yq(?So?D~>|Jvn=XK9Z|$fX9mwmT=NWWyFX7;mcOBBEzucOtZtM4W*t`he$S!OvrsBl`fZN*!jP~ zVJ0wjKJvWxv2#c81$D5q=dr3C$iD^|@2PRtN0q-Sd%d3eOF60>X9t#1*QtT5=|OrV zJ#@_FqDEMRidRw2?<%w%auapF#-8UTd!}QD8^b+K(5>)*TVSh~V26+4-OH23T8A(5 z+wyK?JQMVKFeHkBVbk!(Il)CWgjV7yDW|!NrH-|kt%<#jqfEe|fPDd70*VCuaySBd z2lNVf>iB5?%jU4&Gk=HUFpc?o+k`(%Xyl?(^}AL>6SXs_7lf(1l|d*LURPu~|8}dt z)DG;!hx#sK5WIsmFzhDL>sC?RE#5;VeusEWyeKA+x$J`7twzV4AQlpO!;b7igftnR z-fOKUGp6RFADLT~n74JDKHtBTxyl~p0nM(W+V_^_6N%VW5FOQNddcdvrIR{QpX&0eI9>pxMzxSO0;W zs{1~{eXTd$JII zz3TzwYFwa=5_AE!rCVe(^H2YQ?XwAV&;u4g1e#Lu)LRokl6yfTW5EXb{*j1mE@xv@L?y-w-!EWl!V>QI-RbjLyiTW#V@iCce)MclA5D zr*2G)4I(xxglga!5Ox^71|CxoxO)f~+$!uO6H$TRPyxic0balvG7_!G0M8<$69A^m z49aLkRxlMb*OjbUD=KnRz!-jp+;ifxgY4;%pn~`8&_0~i{TX?{w=m7Z$%Zy2kK@-i zo(y(wKrUn%sQD3h?bkk9Pn2*7oB4qZ*c&>uer99?FA{fc2U+>g@EH)_5_X|K`?im2 z^*QQ!$B5PZ4Atpi@xEl?RuQwUAX?}_TpUeS)}OB~PHg9|PJSRmxqv&F1Fqi0Q(VYt zuLLTPSf`3ahegOxyuw$W1NB}fpBPN$t0hsuaQ4d**6}i#g8Sqgzk;DZv8oC=%yjO| zN^BiVKJo#3=NxzJpMRE2)Er4XIw`orp0XKarK6tDKSD=}jh} z0PF4VzrI6U8p8^66>%Y%y6l{bAAI&Q_T1k;`-?1aZK}44ZwP-s zCoGBo>xdV^D_@`%8zWkZy0mT0kGa~$h^j){(u6*2SZz7y(8I`i; z?!ZJ>UVpOaes#Y8SaBydV=9*B05NGGRf3QFtpi+lQM~_VcC<~9 zg}3bHV7dUFa8k3gzU#?NBoJX2B+Bc`d7lU(Gt+hZ7{**?a7Q~nentF*QXTuw$sTz{ zHtz>a)yDizzZ#T3O7Szu;$dqq=E=2Z=iTD${eT+~$yHm}`OmoGmE^IQK1jBtG`o5i zul4Lzf7ky>suoe$j4qtcndBMA;v?IW1@K2d#j!FmTz`9>WJPxJQBY$X@lPq##ujlJ z_G1q;_R$!2&0Kicso2Wxppaoy;>NL_nK*9>%%|$~lUxuxpgi$jx(BavJg*I)f;1^e zz9p5BR*6p7O*$12n-EEkoc>r}5>*PFN={)S4F^8A7cqP+=hd%zG>j8bfc>}`3voIF zYdruz-{u6I##Z>%?Vn;J2D5tk$jR*HD#npH9LuW(&-Vr%wmLh10MTg-`Re@SGt9m> zocpu*}j$c zo3K8e$+D&Le1~H}Dv&+S56%uEPw3Yc9f?oKjaTtkJJ0j?PV=6?-%h25x0%xyj&_4b z59jJebB)8XvPBJrdX|^zj`^vL_Q&(w0o@lz$>;$pvQ@RMYP326ji6D?U@hQX#k|eB zs!Qwa`&+zd-eOzps1xwRamv0Ay_c`H%8q>jGXpyXS_0B+Gb~NyXH<|PVHckh>xd6c zUybA1GiJHo@+|f|@^tiG@Y>L3As$n=qn|K=S%5e64@Pc!=*&`CCZd^8z+KOLplV_x zWhiig5NO|c!@6Dw+l@IDVEe34+NiWGg z9%r9K5t-bA)&Cxa&nn_8s$9k3-9AOHB#Sf<-+zYM`A4ocP)Mh4x1VVAlaY^gDo=M) zIb$aAz+G)4^NHUQ$>k?T8V1ICs@KBCmEb()#e3w$*DA28E5fCI3Wuc#IfT!2Bozi3 z*20D@Bhv4OmwV1>%#WYT1*>fX*t!xCZz=dH2MrUEQY-ArG|u@m^0@i1?ajzbU0`Bt zO*p)5sdlfRALKL{bSF8`tfC7Y$pM0bS8B%DdB~m`gs%z5_V%Sh>*p*lCR@`0tMUWB zeOJ80e2~Ip82w$S5U)gsY&msv1O4YEuQok2zr>4@#r(TvoAqy7 z1N-0h7WQklSlcM;ZHvXy!#q>&CHaK?^aa+XgC_!vJDrKQCA92XCE}o+Xt^B*-M%LB zTt;90pD1ubo|SF7j|?@MFl&vGc}wHw+BK~iV&3Qt=lGwAQF z_=C)3V@^ds-Qb$~fjTRo^wC9IM1-6ZeqlIkR|D&LAH~KAXhK{;#cY9mOg=6Tkb{UV zMQJ7#)Ez=4GC_6eWr^|~hP(I^Abq!z2SK{2WP%sx)dF&KetvZ} zEbC<0p7YSL9SN#_hfc^rl%J=g=b6kjug1b_{8Cdghxf^7c7$tr9gnjQ&-^d!LN8g& z2s}t*JjEHXNG5zueo$~B&`$z;rI7Csck+a1&<)j)T1?8dqfGr>Q_(wa3e&2D{z{v~ z%-tBRqZX<6GAfvU7k9|ZEQa-kt+RcJeV%=!eVpBEt7uCnU-MZUW-4gp(gr9+y@NfH zCkAD(j%e2eqX{*h{KGVTjIkCy+aPqr&q_Ju#`5pvNPO}f6ysi6Dx(PzXw6}HB)5?c z3bJX4A?vTve|qfAhVED)&nI`XyOPK5&8hTP-)J?Ap>#O?#q$+G|9&mT&hlkB$lT7{ z)SO60V59UIcEl!nrFW2P-U?IhA!;FO)E$b;d&PUhYiFXZgzk*0F4lVLeYvhwQxzh{ z;c_waaq}bd2{T${a$kH;K52!R2>N}+O57kbvYRLNG-HnHTw?DRx zUtJ=_zsabrhqF76^ZXMGnHOK)jjUh}a^Z7fb06VtW3c&u3{qRjLb^j7h1gLb_} z8K^eVD(iC%3;os~$*4HYJIqdVQB;HGn>Bfy947ad?lPr%3!M3y^wqD0fzSz+fiGN3 zUov6|(t_lKQ&xGvX#ZzR^(nSAUFPrpQoI~d90iI+D zm6r}gf_vC&4*CwW`L==nb$VEiF=Jsa7W@KJ(5e&vbOZH_r;gE#>YhwpXe+-@mK;Rg zE2!=b5UvuvtfQ84nm&+cOc?8gD$*2~72ov=__F{ipf9K~_&b`0k;&`>vTlL@iUrO8 zpi*t7g3}C4wGoc7e}{a^V01TQkNsjV_Gf>dr@A+Z*HtPsOToNdK;qT#hCAqvaSKJk zD#eKYkK?uPkz0$TUvLc3d<%Z|FHr6p5XTB~JCU5{zU+=lL{p_W14Y5H{&_D3T@Sw# z)s6&-ug2Rx11YZOd$WU+mx0&($}I=Uxy%E@jt7(71>e^rZn}?st;*eunS9OU~7=2U{Kt-H&;1w9N!wMVE`ZdT{2 zyO@_)gIwlCCKAYc74(E&=*7wP-@@8yVESMXE6g%_@^*;3$pwd)&zd_}>RXH1p4+P1 z8`%G|-LWoW@@svmjnIfGsaMnnis=36{_ZO5s_BY!|LMuAB&ZK{-Iq(uF8^yzwcNJ0 zv@NqOuvN89wbo-cZ?H5-u+bA7%=G!=T6OZe$?7N7rN)9adTVF2XtZ!9ae^0OJ74Oh zS(SoB#mzx)2Qtv&4DyYqzz4a}gj|m@_IdF%_dCwl&*;lM)3)kJlAfZva8Ngp z^+{Mm?Om);=$YeaQ8fNGDAYx+~1Ne-(u81Nxo76Y(u#eG|0E{B`6-^;--hi*DO`hd3w!1O4`*kSjU&bS>=C}2w=ATFKcZ|&8 zRDH!K{7u9=j>xq)e|HEOvuQ+Ho5-g5YqDSPESc~v!T7XTswkbQ&SaN55&s<{58N89 zz-TdC7)QtJSYsO9K#!R`^MXuxpyos=@hnqK6ZF50nN*}NP*+<_#_S0!mGvO@%hc;W z!V(ySr*6O`fvTDdO~t|L?`mJ(E>>TtQB1)-1E#6Ws|ULLJBeLhW8Z!dT}7ZEG#!ke z-_%w3ERK?InLpD7mf!Z$cHTal%Ek$Y-SOJi)B4$5RBk5LFx52v)GjEky>?HUE3fNb zdJR_(x9+*D%+OjGaQy_I9;&vaZyKb<3OX(8bsPxNYRj1OCFB;X+rEI~u&BF?|VuH5Hl-?NZXdIZK zHnuGiuW^}}D6?1_K6EbPuPtC5ldzCn)_X8Tpl>#D?rl!j40QH-f+^~dQ!I&v^!a9y zpBXF|#1&7(Brz3~{XpC&P8BP_bX$WReu*tVNfk6RbutA^e2(6;anw8)!2~RdMSqKp ztp!G`1CJ#N_V$|$X2lsY2$g-W!3`scrpDmO028OC>J0kxN2$rj5B%QX~zSEJPX z(;RNjA!m{jh4XZ_cQG2H*L;b&OtsXKsP=`Tt2j$7OI7izK8>vFe`JBan>q;BP;8ze z{!1kNM7&02ZL!!H9@j6>S6#AoL&*yr;{JETs$Nbl@uId2o$UE2`+d1}w(XfiPU;qgA8O1qtY zx(|9bK7MFaz@C&Rqmv6%8c+S9GMdbHz=0NFFnQOJFyjkTol9YINQ@Ssxz(TO_f^oQ zP+1tS7sV?6Moum}SY$n!(&aGmrV{1dqUJdU>k*0$ac<^|4^peD&3LA9Y6t2ziKvoC zaV{Ghl~GKIlrETcCX2tf*LO4xh!6Z6ydtz?XoKKE0Ri?t*7BB?4ONH6 zhDWt{YrB>>>p9;#*QW=#dbpaqeV#&UC~@^4%*2QkYs;~gua+b9FrG3;SduM`t&c2! zn=8mU(7&2Pufb!ydqrab_3h=_E_Hx1!t2B~qyG^jB3y=zl<90MSOd4?=VkcPj}B0&j`;$Pc`psI0LT74ZE2r|&H`{tI=t}5^(D$J$Ge>8A8Mz>8Th<>TH|&RbetXRYrQh`e zu4l;~6L%&(PWC1jPwt&mEcr}ocGqXmKBb3t9y&}>b583R>kZ2TOL<#g$B%$P0gddJ zEQQPs%`uh$%kT0jA=xObkJ2P9R4wZL)1BGXF#V@MvfS3%KFq$)UfGf0$QIDmam{wj@>s4Qofmw*n|g}sLnFAf67N0fUF3BrtJD>G z34HQjru@VS<0XM8c$N8I`4!W?!da2u#9~4gQ(e}hBk|WV@@N&Ya_hk-b;vQqfkFN; z?V;1{E&4XE$ZY%PL=L7iav!cGL0G^ePKoy}9?lGi4MC*=I`{mb4jL%$DA=#e_zRY!^O z)i75INC=skc}iIQ@XuL#XN!#Nnl(LihGVBWR%#}CO*!;D?gJ@D5|1YyNPdtSpSCA$ zR$4!&&D~MyqVM!o6`qPkTpVhba9L{7M;@>WY%KL zoy@nR!}RdDQYRb2MFg+>f6Uqw+SJB>nt4;j|fkQ z{p$O=8y$31>shC9UJY$`pJ%8yGrH*4)QYfF*6GWQc2vE+3yM)~FG;GUb_ zKCNKNhonPEamm9{3Oi?eChE<^SC%{h{X@dT)yN__59HPIEzUb9*UId*VoXs5vQ7?Z zV_BdTcTPPCe{loh(Iyer0YKF(K+;O#bX^vrmbA z99cHJd~hZE5VKqOq#y8HOWmLF;^&8-Q{wX{=1SU^v>}oU2JP^D`-tL*XD#RF+0rVq(-LF`gCQcC))ic{SRka z>hqLgDaTVzrS@{pauxS>(AN8wiXrA6*0Xj?V2-sjDwi3mU5PhOcN|BeCYnp@P)}kbA zpgxvKsA{@x=-O$j;-#2fX9v43Wcor^eTmV5nvK6_>ld9=QLyk{ksq#3XUki3YL}_g zl)*|hwI#aiqx5k7mNrbkX|#sbaFL1(6W#Pbw0&x0rHJQN`kb^KDJeQxN}c#i-+up8 z?p^iQS6?fw`)iGG-uVc zL8*6Br#MHs+IZ?J@oGK&u+hh4f^X=Qw!-?ED!&BNTo#Uyp? z@n+;XW{?M{W>2>0@IxW(1DIxLnyxf(c1W5KKlj&wUyfe`e~LeY zez_AGq&{|yQF80wjSZ$JVt?}w>w0@*ht&~juVcGqyudp+Kn_$`W~6akd3KBtz;i8;i@oQ z%lPse@3ceeDrKLyr8krJr{{)eu4fU`H&1xWs8_U?#5#kBpH-tVu|PZD0AsawUa9G= z;5p#F<<8@6p-RSiQ!{C+xu@-~z|hRrYzOn!{AT#zXvAlO+J-&(A`LXE_HBB2?+=rnWbsO?QE9FCJ}kV8-#ui z%yqZBYo*3$B~znmeOF-`PCg+{D~M9wAmS6*UfFU82-X%3#k5SHASG&Lioi z+y^{oy&IH?s!4CKF2Lbqk!`;g|6E2T3$%;!Wz?B-)}}~&8kfF z+;NX||8QUT^!C2>)>Cxls=Alfsd`iyDieL@1L@t+ufRYXpbypNGRfnTGGD!}J=F)m zk?l!uwqIjE1g_mf;@NdlI{EtEMDAu&Ro@l;shU^udX{;bc}jXNyW?G2`kVAqu1ua@ z-s;Q&xUH087Q{`a?L`w|ZAMe$C|tsi@P#_*H!-F7?p zc;?69RkOB@D4FeeL`rz8(3Sz`ti8>5#kIz1&&kwji4zi(#P7*7Q*I}FlO86AIpaOE z_3@^^(Qug{I^=N6QENWiSWd`W`3W-*lEgx=7N=#=fh1g!WVE zs+gHLvdDANebkknUcpt-eat=FZFj$Sy>oSRkMT4l7ar)X#!QuYicLFa3=q#*mST(Z zg?tU&7xpvaQB3cgmva7!jf=Vz@g!^6tRus^2d$8gt7X!!r6@@Q6MFvG{(0ty$sZnn z7JoVuZE4?J>FP@Hg#Bqqx$vS9OC!HVH;+w>$rUvu%i7Q@fhX+cEH#9)iZAVT;>h^l z;`heyjQ{ql=C7FeQc1VchIkBhkukzFPISuKtuO4M0lx>NIFjsFZHKI%Ez2#_Eh8=4 z%w6Q0;(ByM`oKe2Pd=`P(N2@qPD&o-y4UPI?EcgBAiaZYp}Uaxm(my8@VjrUkXim> zS!iDvur6><;O~J_;O2mbjy?9r*6QZA;!IyjJ&W4f`;WVxD>!|;bBeQEdYo&%`xEnC zV!;93sIP7j7K+)KR&tW;+YM}&6K3%qy#u+V+QcjCJ@dVbz{lxI9k9S=y^BvrdqN{) zcLGj)DROqjgzvtktm7zcICE^$y{(xYyv{S;Q^MnO_w*Ekg{gRdz?ppIneI8^dBx0w zHlDo9ff2k*)hE6_@@;E#$E?7TAzj1%$!5*|IOpt~d9(kDeigMPYEWeUEK7qn$(i(= z9*q@WI&dW4sVNQ~?log->b)@qq& z1>XqBZ=Y_CGnH1ZJJ+X-CDPfQG&Av9LY2h0l|kB3vV^@hC6qq%}KL4 z-Odcz+$C$1Q`^4O!=oK)Z=OjoibM!n45E@#;s zw%GpByQA_%{Tb0Y(|@*!rq)VHS7_R_q@6#ze33rp{22A+%lEPIKa$MO_gKF};Cv+hD=n7KlZkpO&ZEk@#kaQ)2Fzs?l$g7Pkw54J(NGmxF)K3eJiE&)*_DO z0eON}g#?9vj+~c0EN6@CGowF8ZjTI#nxAcOSoeUh;zVt`=V5y1)G7%xzjgmq_G8rN zY~Nyj+7eDC$GJ-BCFS?_px{H9wq^MdStZsL`zAUu!V^|IQ=yQ^z$X@)DagAuZDn$3 z(z68TuO~ldeNXsa@K?UXA}L?fF1RepXnn5fskmJpZs}?p>DU=CFJOeDqdlj6yj`Ouq>0)q!{LEU6x9UAANh6?OT~_OkM9U_Xt;=^!d)V&dJW)=>uJ@JhPNbT0UP3 zVTIJ>pu@@JjH)weFC%nhF`2HP!2K+g-h!!_!)%xzl{k z>aN4?q26+;rfq}MGf7xZgc&bwllF_<1<7=s?vFuQxH{jP-Se9}ultJoiRZodp>jz5 zpv^ain>vbjrG@fiCM7nOw@J6f1;S@vu%W3(yaU}^iL4r=tw}AKYE3PZIx00VEt|6w z`0$JSDJ!~K*`#LIulr_8W^01|aKO-@`XSXbKgnu|?i^b`wrljD$O+jtWcxGQ;w%qB zI@lHq2ejY4BV2t_YbUJx{{D;Y>#uJ$evXfSo_Hj+f#;MlUp{PWANV39D!g{K6Hy1F zN@nwf&&gae)B2D+fqgA2d=tIn(hH}xO)({f#lQJc;YXRDVezqvLb5saqjQL-kmf|0 zdXhLrF3q(p3D_2Jlo?&`?He880W%y|Y+Eh)%=6@3a%(xCR6+>y&DR%dx7Dpm18=fh zcCT>#K@9jbJs~~SW%f>0ujy-ic?Fj^-Mr1VG+mF~Rzu z+(CW_a?qKf&#Q`@XUXMk+dI-JT=~QetnW$v?nj6J* zMU7|D*b;9kEY)w={;kRub%(Z1zinu~hH$`>h-dDT6<KY=4&5mdt{V6so``4IR z(MzLFMiq%%pQU-I&wgHRWQx^id*?fEB)$37?q`*smwyWJx8tWI-b!ieB2(cDlFC^| z*y{%wq07VM@F$r&W$F|9A*5JHbkJz~9r>gwS%1Yev05HA{bpL7)U=eMsT)%>r#(!& z@3gp9yDi=f)Nhyq`8?z}HfTSO+hm zwKi09Ylrm8bRC?c^DNOA3@i5pYD&+jO8)^XV5g{y$>J933KPXy;v_L#oKHUfG<8Ar zpW(NTq?@uARng|;2}P&bkJ@PlmDh#)cr~gI74&16PY8W5tocp?kKgX7GKil>c#lE z6k{uT?~z1iaJ(l^ltM=870Ri0t>jm8pIuw)`_&p#Sh;WMImVKV> zwWSFYjlN2U;Ig(6r_cfF%1~+!5dNW?+b}hTAD9ZyX9qn6>)?>@ptHH6F$GSYKo7Dpc}`V*}YT1Z{lRgd90W`fzkd~j0 zk<>)4yC5Syn4e}YQXIJ&SDQ@gq3vH9v5O@LYkOD-kYv~ z%}j}DMYru6YDa-^A+J(%UQ1Q`J+;^NFgKQ9gKki%?qMWRPyJJSqW%SA?~PJZ*~nDF znbZlopoaNe$qScY0TD*M3^veTD(2u%Bb=u|ob|G-~M?@xaD3ye*l zR-4I0*Y%6cZD_!&58Sj^eHOg5Xy*9U;`FuRy}amb{>jXZ()7i}!Whpj6cUTVI?Zjq zXpXa-vShVN))$t4E#EEGt&^-vnMR+_GSxhR9Jh&?^hYdHER`%Sb13tO{;))&^?Jd4 z+C0OY#oSbG$;9S@VDv83qWk$4(bX{tX5}6J@^SiLTB!}y%4#LGIkn|oNd)FzWYor--3y`w%jh}&Tk|7U8?WQ1(g zY~G6&qJzTBgPt#Kg+ta0F4}!me|N$E+rdPs`tZ`O@;WXoV&(I(IG11Ah+^ntK;+cI0Kr~Va2r5Bb+9y*({tG(5mY9sb* zF_gNt8l&KL&!S868PVrWaXEYk8D>srF;QF~EtXa>*K9bfjO9!++-M20ezG27YOG*C zW7}k#YFh;VqnT}ywV&mHInMl%NuIwmE%HBUr?iNC(=TzF*Z^JhvA!z zqN}pDwvkSyBb?P(#YtQ{ovV7FPSvc`uixsEsrjD5R`r9Y)0JK9SEeWk)2WE@A4+0v zVHt$eg*A%GZgDEzc331WVBM6b>--_y;HgYVd`&bw7WURr>dephXgk&BComeTNKK_i zQd=VYtMn$$kOZQF6x382!l>FyU(j~gwyW8tYgv!ZN+V?o^WdteE!aIh)jMiFxL<*K zp#D?)z-%)UPj$1_L(9o5xhRx_i)pX$GojjIttxfqujq&d!l(At!bOK8Hj< zqeDC|){?$USLJ`q{VcyMYhdv#wdJuNu-~vZw;!^Vv|X|Gw3f5V*148z=3=NEPC^T_ zp`2CLr1MfKX*IRq=~%LXD8Wr3M;%VRs;PEToj^ZV7qydGn=0@@H6erD_ED=v-P}x1 z?K0}@evQ_$)PA+PJJrN=@{-wL+cgxNbTRCt^D7@cN--#54Q0<$<5Z+ED{U|n)1tMP>O{309$^&|Pd}?6ti}NCKjuC@)hp7;&=18J zzn14D&OoQhrb&(*OJlcpAE+LXZkK%(rfda zzUua1fkp7_M(RuX&C#r{+gQo``p2dPlzy$z=;y_dx0?u`=@S#y%F=!M6xMZdII3^3#F=3UufdPrKs_yl z-o#FHOs%AfR1J-vWH?8$)Xj_VszpDF4c?j?M&>*??}429EwBx))3vu&TqBN031l)p zZ~%<&_wYEFy+;p(LAOOTT^tQz>y3x;au18MnvcTSNylNFRYS*ZHN6d&w8>g`Iul!K zjkG!_?+(|-z`PDbbM-b)t|fgE?WvBm=PVU=^6478UEpFi%>dPWvZW<=yZ*o?$h!(CJr< zu8XOlpf7OYI?&-|q2s0*KJ}Gp5F87)ctYAGkEHw3Wf^WwwpO;SwWU$#ILt|!YE8DZ zwZxlOn@eKf4$1w|6g`KXpGF_!4p>l+uytJ|r?>=0%z2oxsbHrWzCFe_^e?M`Q!e50 zCUNgm=?gx?jx5TmeMc#EC0a$}cvdygV6Vz6+<30vf-x~opTG_~MIU<@b)yx=J3P)Q z`qOHF3%1jlvXT2gK)>8xIAtB_$FpUq@t1=?(~Hkcq#ti9HTI!c&Of=sHhi`kYB4Y9 zcIX9f(E&31jupBGJ8~>1Io2qRf9p#RLpju33&5$W#AhqB&k8Z8FEbq8WO(jTTwf2) z|3!Els*x4%=vVexPd8ajkVQ9;K@G67Uw!{Ayc`3T{sT@B#Tig!1sFRb-8&Mz$4hiv zRmKh;N7pVE1kqdn0AsP2mS0oJ6rW(q=M<(m&QO=D2dI8#)rM)w+7!-ZZ7hw`SPIu` zBl^8C zgA9AL`86F4J@HWS(tUhDR{EH8QwJ(ePUtO+(5|T1Oydb{Wg=!0de5U!yRHE@^BG*f z8QN_4HCwckoG>TcC=vS|&MxnY6&ispnas4%73{f3Sp7JdOZQ;-eS)9up@%erlQNlm z9}n+3tM40XAN_dB(Wpa6FqsR%Vm^VcMSr;DSLsM-j7G|2_GV9>Y-#LI671QRps*)+ zkw18T%}}xMYu?SImvkr$)K&PG8*p;r^3Y4w12$F`v`u2E5r5!kKG9)(pPJ-e)+P=d z{SrN3|72`0Q%@4$s@;Lv)r>?mGr1*h92hU+H$xdtFZG4tQ-PIr)v@yoP`v z{Yn&38J+Z-=pa$Bw@1|hYB6d_+0`a$BiJ|X)Me^9RU<1t483iq){lzf(hR+MR}g3 zzU2pKudMkQz2d#)Qe;@}QW;h%M>E?)(Q#WzFbuKfqT`;5k?F z^{rUxIdHxgVs);9Bmc!-9_6DhSkOvzxP;OBeG!{IMjOvMcEExU;uIW%4-|_3yQBZk zncU~=Zi=FF;kejdx&>cxi7c2mnj2d#TI|*#sE0hYp0)0`p0NIFy~}$itTU~htTyXC z%NEOc%T&u`OKVFOi;oJJg}RuX9F&P^vJZ%Y4q!Eoz})>z7xo$Y6v8s1tjwUMJm9aV z+Ip_4yq3-`+oE2kCh{HxmJcnE;#v-7JU2#5BMBC?pu6dv$^=@Tfgh*?he_d_MWAyT zg+H%BAJ-P}ekvW;UixcAPD~OVs6SZ4qjZlK#8(~ybJXR@Y4k@l#vA{Kwtaq1Q%QOh zm!NjBLHti_iJJ6sR487dL_Xir-_pmD%d*z|NbbtS^byo$`irmVQ(K6I+>sGk1j5+$ zYjZ3kwy22xABA|eI&36#1;6NK<5;o=;JlOJD2oc*lCi*r5qjd!7 z%usKs1<7ehAmY6sA|Gl5b@6jm`JGvf`=FGD^a`oq0a5=3s*TnTfcy)p9wiT!cE9=r z^^Q3Bp0#*dF<7uS`XaO$O0&zM9GTjP{p1*HLwl=$1A!fb#s>uioew-27#P$!s9umB zm_M*$KwU>2djnezYbMJ}6q7?m=9dr+cLr7ciOrtN$yvm#iR#?nV0?Wg>|8l`nvdy4 zUeB3WX9}VBxEV3_6rOr3p7ji_zBhQ@OP|P6?465#R;%eI{g-#?NNB{HmHNlZ`}!8J|g zX)oZl5sxtz%u0 z^f3+1g61HaJm9kL)M=Z7iH4KiISZd_BE1^L(BHS19??q>M|2zPJHlT61ZMw_uBA!|1` zYj0)CZtZI6YHlFkBmdSKwqZg237p|8caAOe&Z5O( zL}r^{)tkhILL=rLzSlRvg!U@^lpy7V_lQ?e8Y=6R_ev!3dT(_)sw!3Zy%F&BXBs^? zcQSd7doai!^ZTj_Cs5*g0_!vn%;8Wh@+rDbCer~w7NkFv_s$rR*ldw55s_@md!ZQ- z`D$?yS20l>PJiB1rnjtOvd0GTj`&DCLf_v&u>%?y%g}kc&C@&%YyB}gd99dv=7x6~ zgra*5Ja8qDr(f5-6|r+O&{jiwyjsyMH5|35nQ-juvPbTk%90OT$=_arFJI?7WgLUu zHHgThC2yOPecVV4*_fW205tP+fB-+y!!r<_?D^Q}Z{Xl>Snds2fJRK*NYIXeB#&z; z*t0>L)t2P*R)F4_yb11`%-Syk6}r%Yh~w|ip|7PK89Tc$8lO0tZqse(P-%Kz^d5GT zD=-ZIOyeHpn3{nhmhygmauPXVWcK6CE&&N%($}#!ExvI=Y5BaRwarbWA7R^PYiPe~ zZ|1lU!+wqMfRn71i7uN<_{53$!|L?Ow8ajt!T(L71HU>L z>?~RyozVk`Aye6(&iCmcqqe5^^ldFb-_Xa|4r5yUAE2FVzISwZAH>=iaOrlDSILat zU?N$te#Ajv>3H^_=H;btYYM)k8olo4z{xrtn5Ru;=q*g5QkhBGBxROQO9SYYuOd|y zJ7W)qqPjbcKDj4Y#PLi|c!pOUq;&>it|e<&2JieB^fj0h`;@6ib2ypVu%6|~e`F)B zxWrSLO5bpd{!p7nK6?WDVh8?iDJ)}!KICnv7hNO{KSlq@cp|6r?1XW`IqchZqP${) zmo+G5x(J^8k0|~#`zRax;3~b#amX0_CIru7fTN;A@*vQIEB+EJFB2sSgjcJlk)$?e7 zE2rt;8LvE62kFP?!dy$otKpkN9&jO;=n&mucZHnfrJE90<%AuX%*rl>cU8q`j-3dG z33P>Sx#@VcAw-JRux0Pa-jx9(tz`nWnNID$#WG|#a`2N!$^ZOhI@B}zxX!cBBZ*pP zl3Ur0pS?wNvs}07XNgFwU<3WVNL{&>2Jm4^3&ZKLD?)btIA?bXpRvM8+8}&m=4(k| z;QvTE2k812Sszpp@GPP^{r0-aN$KmIXSlg34m(UBk_DMUVU1cJeN`H0~ z_S$E(R|In=o>F002;VOW>b=Z8g1qPPr7xpNo-<2qH<^y@LjrQ6HEh7SBa=DW3NRPpf`oySCy`&&fYm#K>hWycyAHJfg2*tvhrM! zbUIx`@5e&p%ixf#*w3ZV-;qo!>L-;319OSPFJlUb*d@FIh!!H&JrQqx3A+kr5tSGQ z9!H=@b0If=5?NnG450`coQ8taf^1-k@y+iL^TD68<$5HM{cO}3w#TM_0xkTc>w7M7 zs8z)JuJXEzPQFZ+Vpsg(5O{o@rxu-tk;tdnOlrDeoQnh*NNo8p8I~eo=>uFm3rrBv4t)*Ax35pn&Mf4{dE*PBs{gRLuApR>K2dqI4sNZ1ynTyy z+{u-VY-~QtmBZ*p6MJ}{6CoOlpL7{-FM>X`t=P0}nCbGDZqPX3n9&nYcgbF&Zp_%@ z{IBSL?+RZgv$OFcTwDk4=*yIj5VD}N=*MeJX7o4~$vyG_X{JofzN$tP_Zkx>gH4Cz zn(|4ePo^3Fg2#Hso8(RXymg3NR7Gz0hCBS}N$pL1GKP3QSy&=r^YO+PKnLmR^R#fc zMp&V;>?&=OIVGGqq9%^DTU82lmz^atPay3;WR#w~@Jp@fbz4^dXk+O}K0u{kG+( z>OTb~AI4tr(1BN)YNJ4;%v7FuD|So)u9hD-i_X&`+@ZapE*YSS*fUGfff`j&5Adt| z5GRXd64i2G|CKp2Q^Cr1ViF11#SXHZpXgLxML$(5!+%iGKhyyhBbs&`7!~opT${U(gBB zk=m<=#N(*fHWc#?A#Phx-;4ixh)DudJd2ov)Pow+b4;q~Okec|WPWM5>Le8XgNp3z z%p`bAB<~;^Zx}PeF5nqIC8BzSc_oAJ?e{RpqbR(8&yav+X+UPyMj9E}^w3ReOo!G0 z?mGdmd^Wy&Z=apj5I?3ZpPdQp=V5mg^ty-;eIhzklkCQKCPo>Ekln&JnuLtn1%x*f zF=7K5TFFXeXA)1_oXJsQ2G9azZVYjubZDOT&`K}#Pb7DH%#)RZ-qzrGEyKzSfNl#w z`Ok>U&L+keMHJ>Fxz)z>R~I#8L|cw!R^wW1;KS4fr(^!x7`dLD$kvK^R6mw8?!c$$ z?;Vd9T2|kM|38O}(KBW-9wjon9LfskiSA?1MPv2$qdV{)CN3qxQ#QjqZvs}!Q983G zYr6WLczj*@($bhdQ-w`1aYPh{!H+r7V@clraM1+v0txg721rktgiwnaLYp~rB=aw3 zz>$H(%&uaI?M1I_CmuE17dP$9ITfP04&Y#d`&u(E;~ev6Qh?1|Z%-sjPvqDw^jBG| zw$*U;6LPU%=_I>^A8{Wnn~c|?#}ahWg<#`6HY7@I%|)#1t&=PRO|pEQTAHh-+2*EZ zt7!rCH0PnXxlm0qc2EmX5j|Q$H@nDuZL3w%D>WH8#q#*Vt`z=eIu z_!KH~ri!659&#<_=*-gY5eYsBKW#-KQiBf0GxIEBPFg2o(>;jP-KJAYGCqcm{xU%> zl}_$!d|d*wBa!+WnG@U`Ep`vd(G>{Y1fCg)3O-}Y&nhI-2C`?hkV?_?re|R$#wcR` zpV8Nah%;OT>P9+$R}#l<27UJ9wV2<=@p{S?A0pw7V?~}uLeUvO#H0XEu$@e@uz=H` z)`D#EYlp2?g&v6nX_g$!4Ask~UCiQmMh3JP61kwX#~5bpX<+jZ^pi#3qK$`suHm`< zVe8&DrUk|_$8RF|Ngz_}W(s0arlJ)h)9@VY`n6{Q8G#9WE{+^qN39W`KE&MBFg)gT z$fu@cm)0z zvq!5lXPHj!Ra^y~eKU4a6G&xdin83(q5T`YWg{J#)Fb&`rLFX?o#gG@Emk#@OQ(@V{aW6Y0a~ zFSVDxGUuck64##zhxM5T5KCReT4NRC3Eo?h=S(D4vlz@&$M0^9jvftP*W+H#@%K0L za^m&$g+gnftIzYhjr6BA#iur5LvO`fx=buL13wofv(X)Y>Y=APGWaOA`&IN!91^59 zJhK(s<2jaORl^IUZfow=f(nej=-%;o5C0KCzl^sZiDxPH4n2h*P7#rvN_K1nk!NB5 zJ|I)r5}hAKj^iC3fY9{Uu^5`cbA33$N~#_{VuOp>8LgQrzLz+zWXy%XdH^jYWF}i) zyi<*RW!<#PT+JLWpCB7QSz1Swyn(rbxgVSBZb`YMM0%!680#7Od3SQc5lmS5tnb1@ z4Pd&P2My8yuAT?xwowUUHwFO76~NR6{SCsucnnT1!VN2lkBp-?cPv`%vnQTCTm~ky zWyk8Mh^`vJvxXAW5E+Bt%n{jzo{8Y()j97rqS$Zw{RQZ1kZ_{a7$smx&-{u=2%Z(B1gjk32?l&Xu9DT-bUZ@OBosCqA$kx<3vt-Q@h2u+nM)Md5ucAws$anY5L^{)KaD!Oyvw68RiE z?lWg-1KnT4f<6l!MdP1@;Ypa`zU6#66pz*gP3FJ`FA5c1fJ$G3p-R|;V&7gvY6H9; z3-6rE)qX=A3BYANvhNoWg#;w_boel{*UCHBfg3yeE)MA`_McrN8(9L&S8Q(j!rwBG zD_!lIy?F!5K92MaVmsn!X1hK@`p$xDMq(kJ#YVYL$GVy9St`2c4VL03PEa3>xD89K z7g3aGG7Ba&PAq!*GIm#bq-0^N_QOP}t0H~6kn>%G9um{^C9fMAi^m2uGMj~71E}a2 zPdJ(LpM*=^ko8L@!fZy`hz%F*ku}S)+57UH*niR-3tnt+DvUG_CWgKlywyNz_2$(G z-iyLl$OaWP$M(L6^?i#uk0bFMGLv0s$bD{MLH;3Adycz_{NPURD|U5l<|#Z(pAnn0 z7I2?x*iX5USPPlHmu#FzoOBu5T|{(SFrla#mGyhcWaVZ%;z(+q#!@@97!GWMA5s~M zdL~{`HGJaBaO*`VzJYJ+!*lTT1lfCud!51dSp@vX`S|4smP)wyF4z`xOxE+F^LgG z;>3pDhEUd6vLw@qGc82_U4#Pf5rI<3QI>`px)BLjjYQpvm-!WF#6Yndp@);ee-nH< z6R)E$CvQ!S$3pP26G^laNp>9mUBnHT|8;*aEmA3-8bSG{zpDgGi<7>85@1`24r94aim#H!p%ghSiDQwR z%c1Qa*e8>T#CP(+D;Rof3j8*K-$_7DXbQ10wkbJ(k;5NAM1Kn0E^@rziJd*6hAIJR zAnwWoB-uWw{c^Pz{E z0*9_Zq!w_ginZGoT=vEz$p{sNLP6|4U~)+bq~LEKg{0zF1Ov??SSLB~;nVo-E3AkO zP>Zm@2Li7(@Zwpp`Upw03`=S{|89m8Cs5hd04SD2E-l9U7rWnnA~(Jv6|12q7NJ?M zLKmX)U=fz>1|;J{Y{EaxsuX(n3$eERY)Q$E-F=?jT6>Y!)9}3KQ3+ELD~W6v8tMBE}&0$Q4HlL)Cmz8wZzXO&i}S1_Qv3q$`Axj;8eB0NZfT6Y(FDEFhdT~~errL$LZ8P{ z%f1%w-44B8fs?mjq4Y=cErXwT@U;a?E)w`8p)J~RlJQI-uSAT;F7;++%@S&M2l3r% zDM{)hUy)zPXQ}0?BD*_;ARk8ePLOI!e z8>`;&u2??#l+WHIW_Amh#zU13B$|*)=kY)KqVEc#;~E3kxk&oOP|`epzn%A+uq5(< zaTkzF<#Q@iMnaK(GG4L;$rlKfh7lQ0;VJCsTk#Byu;>~AjqX01WCpr-89IIxx_1ti z=w6~}&+wTyAjuXXorb{|ZHU}eVdhi`p1cG2=>naO1folc^mf9^7zrI7L5FU{Qm*FH z^@Vw7OWxZYZ#5nE+IzU;Ae3~NuT?$@T7ox!80v{f*T)0Nh0xzX=&v`l(}@!o$G!@{ zw#^Br)aAE>`CCz}%codqUpz7vZzOhQAvA0Tuu~m7t`m}a8t>}Jt2aMYMoy{tvk%c7 zkI6Y1!C-l)vn5vF7$oCHbYeKVq98WeC~W!@#0QK-VVjc`7*F2q0`ou*(i8fgu8Y6S z&@@YlxN*@ZQ# zL1p>)K0owP2x)i=Op9rd2eE|2X1^Cu_Y*kbFs~SNaXP5eOb+Z8zdZ#t?B;9=a!{iB z{QwejEfQ!bc$|Vv5!1s*@l!AOZYf%O3lJF!_lg}%<*}Tr@+yUl@k72div_zp2R2?- z@O~X#xB$B=njDS6qv`eW8Mk{U5&29qHlQZtFLU)?7$-|Ln6Z_W8ta4P^9Iu+Q;La7 zrKm478OLEUq^DLvY&C2H@4UypS^^I?LbtX9!eY0x6Mwr8(%MFj)|eD)EHR<%Kh|0znsY`ZHG((<*0yVPW6hKqh`q3Z@{Tp`8tLHX679pQUc!wy`H9x91{xe8963oiSh#|A>#m$5)< z5ZA1TC0hXNHUqP3=3pDuLk={9e>=m^HSkzoB2)e&_EVYX%;n?00DL3CHQkXpSE1fJ zXoN#>Z7_5&l03q7bbTUv{uz;l1a?AfK?1Y}0$=gy{_^?={@3ul>8Ml`Pu~=3?*lH| zp(olweFDcP!TLJ5;scZ$VmaAok`ZF>W z?F@7}29BPAj9v~O&jg~Qfq;ofQ5<=?>R@y{dbBrmyoA$T#Y$M?o85Q^4R9XMZaY@- zBjj~fsP7Nj^($AWw-u=7;k=WOwHrC1@R)YuoxLQ^a~D}XkW-Z69mSy7C{9y?(-H5ZF%DrcR5f4mYt=>$#n z0=G@Ty@_e`e|efGaP52GC3Y#_FSKx+kP;y={)CjAy8XHL}0h=(MxG*r!g}%@@p~6gJv9qNb z&r^g`rbkM6xKlxXDgiE{pwj%%;76calAoKPYXxm~gDY!s&g`66P|RN-JPvM~4&nYf1s+=@)WJ+P@0Qx`D%3uPSP=ij{ZEf_w?2^@SzSPu&K&d77RfQ*r=m&S%^ z3P*H-`^xfE9$>Wr8s5z*g*+2-Ttu}uapsT6iYU%0rVxJy8V>ON8Ojoz^NcGSeXtW- z&c!Z6C;yB0gu&|-d6fWrIk|T*pZ4$|Xrub&C!fy4 zyJ~~)ra-4AHcu^hHk8vBX74PF&_3V-Z*5jP3v_lex^J^t$s z7^W2fps~_i8f#e_Hc?uf1 zj#PZc{ZH|X!O*nC-)kUQyTFHysJ9u9-OvEM^o9RM@zZ$dTvR;%pHrN0(LJP}h$8Ry z*-FhP#d{$U+-;f>>PZfo%=>)H~;$E%r zzr~zxAs+*|e+|wR$*IKd*qzwPvB>2tK>szm<|$Igg(fh;50P9)NS>TLeP*aCKR=88 zv>FznfKwuO%IM>`ugI-X&i{f_q;ii4Bw+?VZ|ADteRkv{;PRXAUjy+t@Er+UEAgrW z97ltxj>wlha9|;zC92lFJVSo25K0W^B@`CI6aL|oKfz5|@YxyNoEe;o_+$*6q4=od z91272AaR2>2pxA8wf|D z8)-1(x(+1OZfpmk$G`IaYe=1W{8sp6p>R@WEU*yH`-C~9*?lK|4h#xIL-~+9PAuN~ z==_FU@hg&B$oNR$DkC4xL5U7%Cl##ub3Pfk6~{6t09CyPZt1~RF>vu67>GLm6ySS} zC-{m)6a4a-GrmAxrsKRipAH8~g^>USfn6X{>KxF1#aU1AzVARnL>5Fg;w?B98;s+D zL;}AT`Y(xRNCmet++>Ge#MR7D{{KC`7u;Xxt~ua~yx`ZtPcrZchCi$DJ8_;|$e#wB zRmjdl{3ejA*nIRYG;tub;s@RQ;HjNR&l-3E^RNTPLa76w^WNaJET1e$grPE5Y0PyZ zfkY5bkt+4TabLBiXbmi z`3iDPOn4W2gCl@>9-c8h&t3wWE)Nzvfbq_p__5FK5VmGPs3C%vh*MXAd#m$48=PAxctg?Kec|qgz&IF4X95c8p$B2N zNKpQFa4R(MQ)J{m@c)%_{o+o#j{^dLsgRqcp|ubwO>8L^v8XP{r`-HL0*;U19)jB9 zptqYyM`4v;!S=d|%%(=R@#T!(67ILmdO{5r7rf)BCzZYG-i zG4e*|Z-;#Th_K_HLSeDcs-3H)2fFFOV=yO?kvJi6M*-+U#Gd?+3PS#+p%1b`{ozO^ zvFW`zCu+!7Px#A%t_tMoJEKGE!hbWx9;d-q>v*0sTI&z}&2OwgL(p0{ z6!`;M^as9gIq@ZAS{S$%w%{i)BlaGC1ZO5>={K(SnfC~d`2~2SK(7`~E+|%bCwq{R zvx%FCen7EXaTffv7AtEd*nS4hi{3S-ufA?1R__0+nP&Lndo!Te2GC?1=;$BLBs5q_ zs4fcr&ks#i;jdAivR}EEo3>FA13b;*MW|n-R=Z0M|9J2eR>GX5^aK#A)XY0eB{Y_dWx|FwSuc z$=KHy)on%0y$vxBQF9?Gv*zJ-j6wT|2sstoNt|yu6YEaD!WcoEEjEJY);MqsOH#dRKUpy62 zGqM@k@f4g1FXld&e$IE{=)|+kOwVNeh}`Z=T7(R_25acj6z9 zbG@^iEe{;fo==YB`8QE3^MKb*?65*W_7>L_F_aJRg3$T+kI2M_iI0?pwwKcwGEv}GRRy(-nz7pX`&LLTM}l^KVr&va0SwT7$z0G{iS4E?F1 z??|Nh5qS`imzn74L>Ef|YU=Yc7w87v`M>poO+9aCy`?r*eZi!ZZ^|@vEE6;>^bWS9BI}J|p|K=0 zKUOg{sSlGc2Ae9I66A*Rf6P1|LazN7^funR!$a?zUZ1HK&D9XKFuNFR%q z6vK{q;_XS^b`bgPC?=2SQZ{ngo0x+8lsP@m!P`Y*w}V4 zS}X+pxqzyKdt}<18<(-SO4ESCoQWP@Eh1Im2j9@0{FdbDu@nmIAP`#KyC9UYjZcj#J39->u zRPYssS_7!Gi6d8dkJ`XJ)OwvDdpMrzi6-=8&Zj@DnEsgl&=BpT8b)PqM(X5;vpXZ$ zkk{BoikJJF(=5eo1?Wsqwx`#6IrtD^g*`>c|n2GSjIfvJf(p&!Y#-&Df9%lz3aTdtTZnTMG!%deO%^ah(U z%v;RUllfW!+88#}d}e1}S*D@!WC8*5($hLw;WV-4=i`$q3Jqk^* zgxZ@~OhF1m8pJX=;T^LR>QUj675Q@=9onC~|7S3N6{>HFj(rH-7W2M^&Z~G*^+Vup zB0J#*Q2%(A?&rpI>+hwqsyKd%h_Q%G@AuK8_mD{Y(G3N>SAElv8nP*+rS<@hFQr~o z{!uh$g%x80QC2mp)=Iza>1U`UwKT{#fEP~CcrVTgR! zywQ5WR@XkjKG-+aH)e#7kzZLyZ#=3H#^JZRi)IPN`5=V^Yf zlfVR^NT#!Wr_Z=O)2DW^@!>qvH$HjRvb|&&6KXWn#-jPWA%*8d>-gLoK`gTRq~^xUP+gd z@}$&HOL8)MPPZE*w9OOKT5}JJzjcz`~PE1^Bz!aYqaP%Cy6&|o#{H>u1HJFp=yf1IeLP+chbp=PsOin>6qZSyoS=lXji9 zSKseVC)c-r^0%k!96C2VbB2}Swy^ghi9!AScUwQnOAOET)#`8ed6&VJ-5KeqoK`1w zcuM7z*D2}JrZGAHn|qDgQZM99Fcgz|%cD(Q%*`x4nbjNN*UZ0fz^{N~0h9e(*=Jaa z%7?wt+AY_-G*e2Gq|JZN{=Jw~E5$!8(Q(4nRJpFYwQzcOB&?7rhBnej zwzzI)g7bIN2a{$hYo5xUjiPJ}(k*$|KICV4Xlf>}Gk*8x_6%WTO_0*h?dJ}3w{bsm z4^;B0h-odSr-^sF;fiswq)1uVM>(51y!&{9`Mp!=(hys5U3yh&rEif}{(x1unkwm* zaC0@LHZ-UHWfPkB7aFx6-pzGv^k_83GkPGaA$R)gF-&MVtxjO_&r!8C)5hu}F|HUC zW3c>A?rqw}W{EnclXOs&kVYD}V_#n-auY@VI}02xqd__$-QQqw?ZDRbLzYdVGr2t0 z$TT{gCgJg}Wh>t-c20_Fi$?4`drzNt2IF3=uMBS_%)3}wIwq3U6w@2Eq+P+zrTCSRo zN*fH_fc<+V&ixM)sT;ZvyQ?b?m8R-m^&;~RJ|eA-sh63iBk2ZD4ewM#g0a55&lJN% z?l5a!t7M&reca8whFQaDQduUu#o>)DB#U2+=vFK04Ko3$r`X@!>HCzK_wWO|I|o*y zA5hz9=m>1OQn`DL?S5_P@9bzi0QU^12WODS>ei#W?>6w_P(G?{v;K2H$4`jftTh%+o zapT!JyqemxAJ|j%**RPXj*2jDKz?nb0>kD#hNoopB;pYqz{1JLWTMOTbRTE?cT47R zZTGbIR-<-rHB*G5sUTWL)!GB59=_uBmG2%if#flDY?J6a%OJg^W?jQu%|>i|AeEnk zk+%!MOyWQ!T5=qm?OTbf(Aq}M0T(gHje1r z5_1Xf5s90Hr8ABCkP%#CGj@>3v5Gu<5V0K*P0WJDG>?ey3A!3~QJdZu-KY>D>5uNe zh=h(~7i({(wPt7bj6V}apD_XJG5Z65F{iIJ6NGLtAvGKGgt~Ep6L?o0@J-VbSD4Aw z9-^O8@b4}Xi6}+X;|1rsL@ZGBW|bwLP#%xI0@5cB^*@E7g&x#%JR&yKjHuvhUQ6h> zdSS?qM?4Onat1ZcBd82-j*bs8CUEjO#4beFdJ;K^HALL;ys_^Jrxu)q1+dOlbKbSTU4Ok)Tbo6bf zzhMX%Tu41-HYx<8fa`d?k8#vERl_em099m%E@JhI^t}FM66I1VL(5P@yOMi~-k=z$ zVKICpe0c|abb-(4_|H3tN@gWSe;?h|n{KbmKCT!;4?zif3V!2P-@_7~3nZHmXRAf@ zDT+vwOvG~?^=&=jkjlW_$h?7g{OU+(YcEl*5aNQB;ayS1^at2o0j{Ia5LMBliD>Q^ zYG?;}Dp2G0k@=+(ahiHm&d(siaNYCGlOAp^2^I!|;r__)!#w+2;_{Vz^(T3W-}w>q zbP`)kA%^DW`9%~_s_=C9#V=m1%pg z`Ach{6pB`jC)#!sx_!)b*$Cr{HD`%joP4g55HewI^U+x|N7i8k@ z75G=^)z8RY(Qh;dEhsX!kBGVEBnn&yde}(Z=p;IBEcdL9eOwMXli!B{5z!X0eNkuq zjL6@5FgXAm%z{rVB1zLzy&XxdY&7&x6TO`SshJZxZbtpJ=*<~Vm0eXTJK9o1TbREk zQ)5cYE8qWv{8|BpUG;fkC4Djd#!zNk;N0Ao$1TmX^deGP9odwMt-hc4iwV~)xgR@x ziCazPs1eyhY@k1N za3Z3*g$U?#bbV1`*ejUB;blHSA?j(xhVKk~<$)4f`sQzo`5aY{wC|ycO+-igk|Ptf zTyv1Rd*Qxc%zg=Bb5%w2|YQOgp5!+2a zLq=p@B`EV>@-!kpyBnBa;Qtl8WX4xBO??B~X}u7q@*!}O!$L<-lC4J?by+XtQW z=1BsH_}!xpJ&_K=(LA{un?xcf@er7AA^JQBEDqruJAiU3)>&R~)*UR0Ns_7XwE`!c z;F-Jf?EmrI9k^M;>&`~rp%hQi4T`G|#(Qy=MMTT*aGE4+t+P~-wuS}`WF8(vv9sXz zHc(YxEQF@e{lDn3BXI2jo^v)j;y*0QAy8Q(=uTjAJDI54_zj}Y@FYKPr0-$~T;GMS zM$oXBVO1MCDupDQja3+nhN|!DhFk|{#=+NTe0}v3@ISjiO(N6W4jncWUTp`@h)OB} z3k_*@2kGMA^kUoaK=RPHz<+6IWil4#H1tz@`XxTX=hZp0fm-x~R0k|Z>jeS-YIr8g zkh^EGpl&c*@eVRS1bO<+=3qM0i~l|6u!m=*p3NtuDW<^TrYxZf9^Mf4inAc}A1_uYxP zipq^q+{sF1+dZ!HgUm@u&NCK{naDlH;?s90F7Du21*Q%_>zj}-yEvn$#}pZ3(OXaw z308nte(qKfogiwPe?Upg$jZ-xu3Ga{b&*MZc+y8u=pmxOG3-Kqj2v}wpA}H_8~7$G z(kCO-dLTQ1E;19S5<0UM_!~eR-RtAss_?{l?(1O2 zc1_?aI=|;Z3&qI(JR~M5>e!ATpZ}j*9PP{FWWb^_^SOuI>o`ynbpbiB$nx|12TAGutFXB!A67xHYIkSqUSeTjOl!a%Yq zFHv1KjO^E9?2gXZq4|J-A6K*Ro+(gc1AaRTD#?eo$;b6wz(DkNifJcp!SGa|)E~~R z4zGq|%V}sio#@bCe6DKX>@KtuhX!el&6~^ygA8)yT3ttKc$QABYfBcY(EBC zMTkX&5{t}=3=#D$qB7wvvhfk{x(Yr-T~uatO#z@Fx=@&jF--x)F zsM9NsZ78ZGM0ef{p11?_*3&0XbSf*l!^LgESa$S>s1V=J>j*To7z<+-bbAPh{AJ4F zW2EUwq^799yo;8wBXJDiIs_VBj)bU>H=PGfl@Tk@!7D#BkP*AIK78Ab?C5mlQF&2nuD)MC@hla>jV_bLreXTDgF7>7$ju_?8h2FxsuPq|4BTu7EhT8S{3;K zQ5PvH>TaPyMq~LN1b-iZ(?{gfDW1L`I=LS9%w(*il2B}4DEBgP<3M06YS*05n+dG8 zMJ@$#@0>`N?eOt^@Ue<&u!sD$F!cZ$mC*%xo}qmHBBy8q1gl`14&t+&Ia_tEq`0RDy??#GmO(z$yaA7DhXHc48wSxfv5IxB8Rsn zSIEIRMUH4}J3*j;@I{YxBOrtLR5pTxTe` z2Uly3MLeG84uc9iamKIQdpPtn4_Xja1QF=f7vNR$b;*SCw`fjU0C_ZlzyC&p93VGj z<9^lQA5!NSc29X=@dD^hMn0@Yx8w#&eUWCnp<92Xo~Xzz#ML9XUQMvr3|gKD zg?l!VWI z0Tp3^-$fQip|J)7SHTMd!AKNzmy4^m2Xj^7%H`y1r-H2rvb&q1p%q9`K`nx(a`XJ@ zq3LZ%+ICQWJadF*;UUyO+iZch#nUi@5znInyqt-%_vSSa9vI3MMcu54snFfP?R_Nu z|ECg^rGoY{7UNW4R~fFoPVIgjsus2&&NsjvHJ4YHs;wvGe5JVGPo zzVzP^?^jsi4E;JDoZHbFrJ<}#&|Ep5x+Fh~ zo=P#Db19TG0l%m?cF|qrh^Uurf$Z)9%%}0`9-J`@T&55KsRs-jbL}2rW&lzsJNOax zJjamIqO!j|{>B=t=_g#L1b65QRLsOlrvS5NK&%YexzqSN=b^;u$Q9vXihj}4)Z1?- z(l8U=?1P0l(EEirRwVgmJ6L+ko$b^@Bw?|)#ir~Gtd5{}MnS=kfld^D;SIDvEqHDr z_S-cuQ5dhJ4G@08ndV`mA45l9Vk7fYWNA9&l7htDiBv8IJaQOB&vP<(O5pzoz@s-` zSNQn__v!>(Rv}jv=3O;GU*K;NoecySk!xHmmk$GER#7(vXVED@UGyu$%i z4n!85!j8%aUbk?)u4slk=p+0JLoViZe!=@(jr6HUM7GoN%nSyF8iW8+k;UX5Z{RPwF=pJ8hJYdoQ?s)Lpe`D==mxsVUv-&@8SJpNVHO5Iu=Uoh9+|1*$sn_?n2qEq3^c9DHZux5NIz08v#Ub z>vOk6&fE<+L^2z52!DNzei6Oxzr5_pK#%6)#3PxrSC!}AgYFbFl0;o#PJB)a61WDE ztq+vc27NvS3#lhiT8ku{V33Uq&>ri#|3>uZUMS}%JGyT8s#~M5td=A7ejynpB4rv| z+vAW+L57n^$1cEdg|{lFZvmeaLCzc^j{Sk0TOw6VLC6x(Au$Z=s|7q*p5Hn=TabO# z&>wT)lCn^O@YynR{>ISE9ANntsy~F*&j~&GBiBVQXFm8@g?ioqwav(82`xPyZ|MLU zX$tc96M8^&aV_MNCwQ&|bV_}wxdfkE1@&IQ9+-;cx&c*%!$%{bxHfR7=wq4fn**>F zO_hO9M1cXY&KzlDdg6xwbMAEv>T_u!gdSUaoX-&1Ih3RK4TMs{=nC!(|eG51>r zwg(eW>H%!OanDeACJiaRpXYswE-4J0H~QqVgZXLixPD8ZbO`({#@_WH`NNP8lc1eI zEDh1Cl!Rp5jwXl!_C?`ZF>&)TG}#aCG$JobA(QezBc0(VQ6btEe7r_;2^}o#mz_Mx zW9;ZpP?R01_?hd)LQP$Gtus%vqntA`^yuCA$z-`2mO%EDcH^eUki|1Z;8mS z2AhAN-LE{$edJzMw33)ZxfG5TJsl^2_cdVX^+ux^W3fy=;ekxzM6=|pE+&-n)Gm3^?7fs8qh4{WAxMl9c@%WQ- z_(~_m0c2k>Ah{n8C=;|ENJhti?fj6Lc2}sq5;bR)u(4ZHZ5E8zRt*06jm5AYJh-8> zzC^v&Q+sn1F3HB3!hp#|sHzuKd7TKKLbk6Lu+f=wM;4irh)BX)G{qF^KpG;)cEZ

    ^eD2KE6C?O@`Ui|+t?-&5^N3{t_7SO zg`{nR_Hx0m{qXt%m`gc;nJMkrm^Y6}XgAsTvxH1GbE(!$ zXy?f!9?{>kovaBtmLcRAE)o|!$rOk)_^V%my%Xq*dG$%23`9J-f~nlhBS<4RmO_@r zpIB@XRFMWmO}+?g1hz{s?|;K5{(_yK{LRBVa$+Ae08%A@WE<@3u}JrqaKH_CaU7Y= z6t4J^%*{F~-p+%iQAG45wo%{Gda|`Fj;!)GwFueWZcIG+NSsRXiat6qB_tzx;1kAU zY=|vuK4&>%Eoz@D5c$l7pC1gJJ|w>~m02!**p#x<6NZf!i)1T<46)-oh`MXxJBqm3 zQ8>n7kht1&B;t4Iu?#V%L~4~Bz(0W6qSIt^22x-2fl0=T^eK8TvfESmZWUEa6Ud&` zrZ#juln_t$v>2Jy(quoYlMC!Ye0V$40nbr?d5Vc8mzZ^vAN#2`G}V(=TX5~hUz*CN z-c#i^nOWy8*nIMfEkyaWZE99^vQk*7!#0S@>O$?Oj*c>PVN%&$=IS4#LUWCDTOMm3 zVC`ZL@h=jvG@x)m5r578&Q{D8U|VN9V*lxvBj8)Wy8xY?#glC%E$K{srD4R$rZQb; z6@2jBIN7k-vsXK#6mr*P2lh?ZUUwhmC0ovyDsSBPUH6>j>2LM@y@44I6oVP3bLimNj+FPeT?SEZoRK+ZZ;-Vr*60$(q=xDoM(t- zS45^SVQQqqI7j*<<&i^WS$-#dkc>=|dM(#6jWJDNyYVjOO4+4m#FCaF&Bh^tdm_(| z!&_F5pI$+`qOM@uhLJ5d8-VgOEt@u8eXjIUiYcv?U&=Lg8Pi%Gl{%Qid&Y3RaYn>%2oFS_fq!* zcUEPl@`F9)iP{j)H*aPp7anI*aD5>0TsmedXkBD~=07~JMo`0`-hnv-Huz1qm$qNE z*Ygh#JP@=ycvf(9&>w%vp2u=sE-ali<|J$VTrN+YOjXHhsI0eB(wy5IBOMM$dDj5< zKT0>{kbAN#$uTx0CeD1F2JWuutL_Ip1qQ)}Jbg2Zm14Zh5DvqIoE_ z4h_sv=1%O`{D<9--`O#Kot-`9Eb-=LX4zbyT=-MwZkvs=p_=y~ng2G_0pEpED`^eX zery%>xNf)}xk|WiP|q?7icPOvai4SlbvIF5%2oA-*39Dvr6jV^VI&)nW=Zd*DooJ$ zYznkQSbtg@v%jmNwWuZ1T+Osq?jpC5$I92`RQZe?E6wJfLp=4_`(B-WtD7`ME2y{9 zAAH3*hErK&8YrmGiriTP#NT&;Tq?>>R9D?UvQa_4WN|DOIj#u59^bX=d$XEzTZ> zt(O1TOmxMP-a66x$(m-}YxP((^GQ>7kkW?-9WfRX2GXBJ*$bOGQFUL_kGFtBk$9mWqW!C1YPnG=2c=sb$4_AFxU)Me;^pWeZ zYXU!2bN%D`*LBs^f{nzAy4mx==rHB8F19wd6gP#)PvyOqNdL^~u7vgt4-Bsy`Ze9U zpi2QS{k{HO1A7PW3$cbCP9L4FY(Q&kLAgKEG^-mLF$p@0^wTic(_YQ!%9!R&HYevz z4ovBrx-Kosk=@ZWt#E2|%G2b$DLGSDriD27xdtgkv?qFbHY)rzbY@~uXQ{Qk*py(- zZ5?VGVK3}A6AHR$-)Q^Sy4}*uvXlzG1=i)(KbCdo*YX)7`C{fQ`)i+73)SrRpw`vy zqOKi|c4;S5C#OD7J)E}JvD8`1HPz^(f}M1vBSA8}%N` z@CF(X+lr+g?Upe+(~>{3&3G(+&3a{G__mAi;?U5SP+ z5~!7{fQ?Z=`Q=vK{guM%aq8E5usO6pdo;>XlVfAU(pYBL*QXvy!otpKSYe!D>SSBs ze=DH6e?_ZT>TjGbSGFevJHwu2s1RN}q(#6eo5Ng|n)|QTt$wWoKLn)&Zu6^c-tTRp zPIi}7`e;s%n8q37Nm1Xs$~g|D{z}P~`Z6^st()_MD_Y6UKD^D!VzYEr}9gN@Dyt*iZ*{f(`JRWVJGrWi(Ifj2a)H{Ox_SM8w zz*3jEMyzS8<(GAl?I@dB)>?O4ezTp_O!Z(3TW$MBdv$wO+X%}%(?_P3KW39#A$c>E zZCj|BXkl=8Hn4BasI5|Gvx{nx`co~$Mv=Z+C+(3Mpk7cuDfd-B>JndhHsI-tsnt2L zq+Vl1Ei$xW&ebqxTiqe|_8Y%F1AfT}{OK~@Pt=8;V^^2TOz;fYH04x-8lx;x)+>+M z1U^G)hwdFfMe$=LgW6p+Yb%-j9mfWsqiFnUL;y$PD{nUxWG3ryHkf~qx=QhEaokD_ z>ms&iR-!toc*FU;%)`_-Y3UN#VQlM2dZ=@c4kO!kiOMCse8y4=Xlr&>O{a)ADuIp_{` zh-y@SD3h^Dte)EJJ~mRL@Et$YNQ82|A;wr+zGC`f?rrI3d2S9cH#Nn{=h%&qVp?U6 zHLo|nHtoeqP{~_HdS790Kl2Vkn&vU2dP?YLX)f=s=4C&2B-_b?*aEeU>c9EaARf^F z=?b9o0+NBZc(!y3U>Mc=BRf6(%|)e13ms zr*5YPG#6H!Ohw~bJi5wMaP&o2x8`nEx@bf#_aZ&LK9Eg$ozxG?GG!Vx6sVL{Rx4BJ z?MTs&8rsOYsjW}8j5U?#IpgK$wjV)bLvw_m39X*4Z@?XUylt@kpx?RxOVG=pxS+tm zz4ra)6UMim{?tPD#E(7XX{^^%pSZ3%il==`$&+#=c~$a_WU^?5rczHeI!NY|Z^f`z^9RwSnK_LpS2 zkGz0cx|OAPwqZ@x->F*^en_VAbAYJ{ zv;Hs0W!T`en4RZ-lAGE6Kbf;0tMAvN*sbJL$Dy|k>MwXYr!vZY#5K@0&b7&vjxMRb z*wuOYYmoX{IjFQyt7<3p+-wUf#a1^bdGSd^HeVY<{x6{AMj6DNannmJBNJ^46!vgJ4`by|Jo}0&GP@` zzu9ktwSe5z6QTsW9ytq9-#bVRRIj*yI-?v)YL3)ZDN1tFm|e4cH>H!O8INFj zHY2-ShzMi2H=F0Bwp*>HymP&APH?7!!rm$p+_@5Zc%zh8qLol}wKm2x%ivGsqA;kZ>r~5NIyOGZ%aB~9s9;qxG(yyPmi7&P9-e-eA~etjxx$2l6db4)YlEHFLNnpT%Z5 zY<@|Hek%(@M{wHDlvsDf-Dis+-jV%3=2lS6J@ljMZdf;}2n=sm2;aDbok7Sk!VM4goS8l#EO z!x1WhPIxZr8oRLL*=`?-%pa!R&|YaZ_1tmY%8Y1IRB8 z_xwOUUBWkac*c0^P;qeF_>8@ImyLboU*^pAZ2lYlo?F9A6{YjCWZMu}Go)8Yr=a=v z5c5wqb`>+1w>Gu4ppR;`DMDgTfNrB3bv;sM4nBxk9p&!gy6*hs9Oc^Ks^MzwyyB>e z)*hkydybIx%HqApK0oNyliM&xlFf%L1=uFW4!xSPNy(}e^%S9J%fa+X zH}UiYqGUG>4~>aLbo@<;c;j8@|2qh+l$0F4$&*`*yQso#$3*G}#DSOK8&9KGH#_yq zLx{BvAX2r8iSWnS+5A_luTS={ft!BLPDDsn6Pw%*E@m0_7_UpYS{<#0Hid0=UGN0fqVfKr3-hDC zfu5j-Z0a4xKF@0CgyuxI8~XItT@Us>@zI`CJ9-R-+18VXu7G3ITnr+kl*O0>&u$@+ z#!SQ@m++|!#33{D+3`eu|HDf^O=YV}A4Gzlor-nEW2d_H6B{w#X*0F!S|I&MwTMGh z@$ErckM!T5`*~8mxf#%?Yfv^3pTjR0ns#w?C-q|MF zqOIrY(abFSODh>=dL2BJ zacT!m*DZ#4V>vmFJ>#MBG^va9OnN5E=3DTbhi&9m^C{D4HX!!J8?GX4U>f5i;_}tV zL@lN+z8X=Dr`Z2Hl@NIAnbwhd|0&us^|~@iiR3qTv^9Eb__`1gi#AjOv?2R>7mvkd zIBlFSRbra*2Bs?4k)I<^1`=E7k6#{OdPPiSujHYU_BL^H8}ameD`X+?PZ(Pb6Q zJ~183p)&RLG5QXC$_}2NY&)0OQMTEWgPbaR)rbt=$4Wa7=guXYu!l%_J+znCn33(r zb%_COB!>}#?bZ_fPA77H3ftiikUOJZRZD6y+6T6%Pt~euztCTG)eczx3+VoK>vfoc zKFM<*EgeBjbv~J&lUP*mkystsqEVV?ZeFPjGgrGY0r(|RUX5AxSJ~4vnGDbm@|2>| zDhf$*g6(7-sN~n#JTqBa&Thpy_*AXc2g+U|Hr?n{T1Hn+Gd5e#v!3Ir2i_v_4ZD-K`#WHsL1(KXSz$64A{+&xi=(n35F z$>$gh%g}vnG{A2wLEJ7#PLOk$^U>E;#*)n(C%+|vohU7pdzjvu{LNQPPWg(I*LcGF zlE`8x`M0;k&BwwMy*y|`{g9SfyQc1=XSfX=0J-VE6C1L}=^^?OBA*n|D>6@%Ff+@JlS|1qWDEjvB- z@w$QZh$QdO(U2J*!c5FyDUo9{w#OA?d(I=t33SyAgO~l?yLvg}3+(2Bp1QIhd^Z3URZI7W>;T>__xt#4JQQo#dd=Q$d61k~o zf})^+ zL6=B3NW<>V{`WKf*IWy;v+um`InQ}I6QgLv--G26v9+E0CRCTD@vLaT8kX2fTK%zN zrjPNWFA+ytD)(ZG_(?kv*xPZ;N8tROu>A%!7(E7(iP>VI7l>&+Dki=iSKlNbXH%++w6X5!}q*(}k8=@k+0E$YX<(4U}+6+8eE zx5=jHC(1cV)Ub^2UEry^vX0jiw`O9NsP5(N$!p;${}TajYu~lyT5ObS@`DJ~g><@u z^5&b{;Y(tE=c*lgUKHVB(TD?f6?GV&sieFPM&Avdqv7?ewo9W9;ZsIevaoGUXS zy3qUtHxjaPa?ocjwp@3S#n0*9c`A@DK<}f?2_2;u`hBJ~{9;DiM!LczxzJ~`zq*R4 zb(cS19b}ivcv$N-FOwbgZ;eS2ysXMMUTF7&>0Gr$^lHdzxKwPi70R6=`Y{#1J|66^ z66b3qPIHMYiIrklvqkiW_|`k*(jKthE*d&cMCb;w#8Fn<8OFXKmt}`5u2H8@8-DZz z;kWRHQJ`Dj$>q|PMu<>G7s!zMcaw61aE}{(ex^LsXWjW+alO$f;!?D9vin`>^xjdy zag{2x@7S3asvFqLC)`LC(Tirb9%lLW&+2Pt*5$OKX*kKJ=%9Rhw^_|Y)RtYT!fYem z;aHS0LqCr4D6yeximfz^JJDlFPZ`R~HAYM|_H-h8XlkSbWzEDiQTqO} z1inCrvz+XAavs`~Qrq}izd;Sx+QpCPCF&1`c|U-p{19%u3r1g{hk2kd0W98iMf6dP zJk7IV~$|oO&HxxT;#Ky&|i>i>r1N+rL@d?_9ijw;1U@(ykYt+el657@TUa%9_Xt zE$^@OU||#2RrdRXFyl@+V)giq8{wTdiOTlH**p37cCSuybshuF4OVa~I{X-am{igA zu)aFQO6fgfmN`WO3QzDF#oJWIxz;A(+M~xWB2W6&wxO-9qItSlYHSEG=H_*oL(jfIP4%=36x^tWPa6<^<1>bU00XUO3_ol8DH zkocW_Q?2*`QPzeml}V~^#;acYNbT~s<_7*zxK4zqrOpyhh)|BC%il{wJ#4meNpafm zikhg6E=ae-5l+rT**D8bzkz%?9d16z>dF&icz|4JiOyy@_en6}9+{dg#c7}L!~<$m zFZI-1XZ|JKqesyg*5`{lV7$&VdaR1;!9`g`&lUDD_3_4{RYhl#oA;#8EUuaun))N> zqud8`b|zQZ`J<^(d7qYAUFOR&i%NZ1qIG^c@0}1jmRdow1_=wBr z-j~xayHaW|nyj6;$b95?by50LUP|lY9_e?B$`lPRJh`w?&yTZBwEwnnlfD{H6>g=O z?>t(jU~Iulg^w2XMRg(fKEe~n;E3CbS7wb*+)Q&!oBeYjc_i62HCA+ZU+Snyl3lYW zXD`csKD#WRPg54iXQG3*@c3Lo|M`Nfxj5?yn!pw5Dmo7=R$+UJdgVoC)gDyM{FHj; zm4#JU$4{8px|3DFkn(?0Z=yHxshdcFH%Q*cVD?7*VjB9KM@uWF50;?IEfdo}CB0Se zip6S!8>xIf(Qm_5q~FUgxwELczg|YmyVp)e(sQQa@SA+ME1hMl?1?QTUi2(12R_xw z+0p7)zm(y-g?DzmI)mwA!`oojI#T}$c{o42SDyQvo>dnV7AMV&Ogx#~mbxluN6vLD zhu-;2)OT3leDF5TUOY;hIE9Fkg&PuHzogX#pc9ATY z;pw?WvkIpdOg>tNANqWe-&6DzxL$vleuqy!-2d>T!$pT%AGz(wx+7zczEIG`jPU7& zn+v;vR#W=-+r?jGeWlOKo5^Rv`vxAE%hbL$)$3(q_7`S!*UO%o>MNpOhA(cNNln9P z_p_bJD7s~#45+JczR0o|p6)~oFJ`shZpw6CQL^Y9p5#O3z8)(}Cao7o0w*|{9wa_- zDjqeGe)5S-qiYjelX=-W+0|1g@cO)%Y?^&Z&Y+xzIX$w^Pu-ZDX(Dz9mPV!2*-3r{ z627x(m~}j>70dnPqUYh@ou;4viib7j*P36jn$M?aVR@F!2Ws38@hp5$Sc@Z@P zx+ZO8zg^EvFBiwWQLfqVGP^2>DXt?!r;(-i!it~phA!mSM|jvL&g~IXfySnnp_~Tt zhSPlFGu&f3`LhgvuZT|!1&N<&vh&%;zq-#**?PlZ$DQ`rU)E*JYyXpsyiw1pLB#{@ zx@>Yh`~FFsZEe;giLt5oazD%eJO4jv-GRrd~?@k?N5>B6~>oliBBH*U#Q& zN>{1WGce;6am}w)#!aXH?jZXreKqOCaK-1MbjREK1FUxrNy+GcB2wGXkwZb;5c_0K*ryMsKeYNGE@M<`pFL@$zPlIKM|}%z5e8Jh2~p^)(gjZHxLA-Br}9s9RCvqNzMf ztMxLvQICfisJbtzo+B%x13vLa(E`4xl#G+kbnb;RcDO`k>*bzoR#$zdlZ{U6Q@p0}ZJfxG7)>IEPx26H{ycqgSmG7?S|anYkIcL` z@Pb$P8#d85y3s!JNsKML@cRZ+!mF{8#h$<=vN8I&X1qkK9jk_GQ1EeY&1rFW}uKeJ1+QvN{(Zn^whN*ig5t z-u$9-#aQk?`pJ=oN3xGJKJvD5<@}R9Df{y5ulSOyrD~+QnHQa%deUV2`%NnD%^yF87IcYy zy_7DZ5BK5=DhvKkIf222-xTJO!JijxmLs>E?{#L;vP^|!9*w=SoY`*jvRlXr-jVff z;(y77smrsw@sn)ktvNH*oFAiH&a&)*=BvD!YR-rKEWg((+G=|>0iAL4@5$8Db@gjp z^qPs~MFnRS4j>W6+WkT~XLl8hD_B}^S>gAj!OfiEzj9iDlU+bmtfbl#7oNf+vT*u7v%@G zlD{@b{OT5Ykk`|pAI9NwWpTgkR4-stG?H;uR#stu`+Zhs>s2z5-(_(=&s+Hzd#E2- z^|_Onf{Qegty@!nkkvHgnSy*XQ{}VwrJj=y^LOgL?8Di$aL5LF6ST-VKc{BSFWI%S z=kmh;keV<1>Q@~4I}-Z<4lzd8hxggg7m3Y1Ci7wf|J7B6)eFDlh5w*nVL?e5dHIDE zebTw`8QRgsx>$_lo9Rb-zXC6>V$~jm*LSguO7TxTDsysb@)uFRuaX}mKjf{LksOhH zn%uZluft?=BKt7S{vS!hKLuAkBHQ^rzJxZ#^HA)QI<(xMJ}tcg*XSu0{JcI1CDTpP zH_4APp`8YIzAW}Ja=Rw7PL5{@9II2zg~_Tq(9KRgoLbA%bX>9}A5C}9%St}QM>B~{ zc!Ou<$-h|3Li@}5ALK!w?AwJ@{X1(ps_#CyRY8ENG+a-_N2Cn|t&NoY|fGG`QUSeT& z<1;IUeQ(HTo-LbtAn!)EL=AD9zOtuRBo-vQq>j>y_9u?h_vJ-=@;9^0KIGZ|AFg>x zs#(F_>ZfGa=hTJ)>4e!&?$FT?GgqW&ni zbx{Mf)S>7OJz$2(x-02qX2Jiibm7tbC-c}s%}Jhb?JrGh-^4E#^&FqGXlsfjEWrD& z$E$bCWjQKFT>`C^6}6j2UyTVVedtFWMCxi%-4Cr->&topJjgUb@%xsc+78^h>~0KU7X9V>$Xbq z-%cjHjX$!sf9IbYrvt$pexusFM!n_VM^*ebIr&jdvsE3yvFe0I!RGJy7XEN{b@{J) zqU*a@;Gggg?AA>uU&ca1y>1EQ6#mKbxM%H5t;$+{xajzO5^q^UbhMh~ z6*ZS>dCW1(Fs*M%Q+}zdSz--%?$+`S#jJxLyxZbu3XXX$Tn>!s!hX+}NqvycR*+6Q zi>sXZJFqpn2S(4G^LTn<9!nd~Yr+pvlb#;anU3ZA>BT#G3y%B}f5TW&q&fKAL!?>< z+HE`EIhBv`bXP^SY@yZt+s9-%yo^fQn6I=P$1ae))lp4OD_6zjsWPe@>XU0RFD9mq z)K*#4ilnQJzE8~5z21dqMK#e`uBmTjS2^2z@XRirSe-pl-KWQZK}@ht^5WICkF%^d zrUMMYi*Dd`?Cws{bNB(=;RMn@CL$-SGFwFGAKKl=u)MX?CdxyPT{*7ydXhhGft?nsjcOp))&#`PwCYDx)%;gp|9j-y z2jtk@>Q}m=?rTtYm(07ncpe|Nn>*q2K=}J2eg6V`k9kTJ)Hhm4Acpug*QRvyZ{ zqBF^0H1i-XaF3NfK>v=q%yKXwI(n|LmXiF(LHpm5(8H`RY9?YfX>ITdziSECmUafw zPd#RVOj4!t4$s7EIL$ja#s{pnF``#7Pj7_!yp?JmVm96=uj%f!%E~G_lUmLo*FAFZ z;}SS~oLnbo(}H(1AiZ9!shm@e{_?FnJNhYfV)0#^!RaPEC)4Q>AJ7(`Cr|FcD zkTaQJ1u>uGTbP`3!ZAg468$GfMo)bahZA|gYI|QR^R5xk?s9&SRpd@CEw6@NNoUZT zvlBbWrpfrp|M+RngdNdKASO;k<@pEniRg_I6YXnfs@*G~ktS+APQyKh+jn%m*o`-A z6De2^#xX}BXs)@vWUDTzWXEmosT)1!-`Noz;c{yj(2`VN#Y6N3Ub$AMhJ<;v57A>9 zpvI{@6W!(3_GbC7MvX-x?+fVtKl1}j;Hm2h79%|WEijns91F$$TjG1in3%j7KY112 z#I*1NFs}sjH@dn9?7bXTJj{oDto6sd;wN!{LVm@)PVGPJ`&GrAQN@IePIV%Bna6AO zK3Fe>3$2~nN$BN161u#+fs33|)Ky-C9`ihlG40)5@bw{d9u?Mu=!$hY&%Uxnl zVNBRrB7$)zDHT(CAB5#01Ea=z9QnG#`L^aI!lNaG9N5V{i5BDrSeQaYCbU=9qKw5GXwe>Oa8sFYW0yaQy~!_TjX7Fg>PD z{qCj1*nqm(P`IU-asaU^N)$Gh?&XaBl24sd=Qt)?ykDlo0pp|2wloSkqDD3w(?G3H?6n^_lKr5z!MQ^t{;86}_c4w${stq9{-|ywz7U*ft$5N1xn#}da||KtZ=<{YmS`YP z=)`1!+Jp^>M6zr$k=&S=#v}b9oAz0n!o#ff`&3qyrFXw5mLEOl-vObRNb@Z`JBe-n zJB?(x{(uAdyI$x0dym%Ni)Q{Djb*Lg9qZ^R?~38h61UvJl4?c2>_GaQZ(UE&Bu3(o z-@ub2YQ@hdpI@OpJ`F;b`*#p(n@0cq%9#&x>hFr?y)TA&3*LK;Gdu|tV?xxgurK^E zb7AurG`xH8r0cx8kZ~8%5PH)b&Y>yQ6I19)FYf8n3)o+m;4|;QkeN>RM^wMj_djs2 zznnx!qBOo!CliT!2CsV1lcU4Jw|Mc-xbze{#J^sxw@Ap_P-6wNn!doHo^wvqVahu4 zC1SIMI6>5*o(qz9ZZGw${5Z@x0H-*Rd!g<`}GS zbp!RBt*tlPUCX)qiEz3B2_18lYLF6pNb^&y_;i|1U7y5!ielU&<~#j6%WVRhjmbQJ zg557~)XXs52_bSwMWdcX00A{a%DX>{$0`DzW^vx!rRjxt@HdmiY=WXDRLUfaK$VcDxR z=!a0|m-ZDC=2n1p%+L$2oDX`zpE{C4SNXXdzlfRf@zXb>CwGMt9dN)4;ZrMDpXRBp zaE^{nzXv=!!?$DZVZNQ_f_5u9X&w8FUURxx!;Z#aaY}|y{RP%BJ1I1nufTf=Ns*23 z9lJ+#i8T`s8- z*4t8N^bHL7#(meqhB%K6{$GPLeP|V5XEcwuKsYAlzG82&rZ22Ew8D>3-9q#py&1lA zkI)5UdR!@bQ1n`ChGLp|VoU^Ub*xp!kbDwVl+984MUep4z>Zc6keuMY3y}qIi zya4J`Kzf}uW@U8MnATDnd}6LrCsHIPmpzMj-x2LG4i4~@}mNT{gyUIAuN%U#xfEZ_8MNMV^v?n@vEr zH8|#=pc7E*&3>*1yXaDN7AT+R44T<#$lK4XH#{;wSp5?3UjwCAocgDv5=G3(5TD^zD3~ zp2(Jq$mXTA<9Ati^-1NAMR8xGQ=N^H?jb>U$>sW+$NEHhIo0IIRFjWzsp#%)yj71T zo{uX>i`zem8AQlR{pThUV*(MN&2_Qxaj^k2Y)G#D|WN{M{wnSAn-J~{|O6cI!-W3 zKG8?c;Tw|n8(9;RNZAqK^9XqNLw#+?jTLzAD17E4ueWgNsn!^}bM$AM0yn+@x2b$S zF}ppm{RR5kBl24(+fU4p`vYFAg>&J1sBWicz~WOuzZ0$VO&K@eTgf!fjT-pSdD}b3 z`#t|k^d7xJV|roC(Y+?ajjLwl=Qcbf@-n94Ibq{Qr-W{7!RzUsH_}4-umT>jqUb^v zSUJ<$M_B!*Fy{A+eUy#1qa#3ymn19GZ|QejYKv3YZ^x@(R7?PjImfj!89p0u-~%L1 zfom4S_|M!e9`(Jv?uY{RDB z=J_E%*2Cvb?i8J4BWtKU8jQI}+x%@CTPeB+M$e?^V3+G&p@XNbdcSYY$LoK_{etI3 z&xr&c8WT-JN|i;`Ej_WizomV9F|IHN_t^%bG50Lw+A#Y12R@BCTg6s&5Hy##<7P6y z25O6*sWF2tW>K}YhoJbEQP9JrZATpL7W6rQy>}ljRL9xmfkIea`JNs9@ly1gl=~Es zA7P8coQmj?cEHn;KCcFz&G57;R$oAaiM}0?r5Cv04;AzQwf4SM&9%Sd%U{5cZ}5{f z_P!J!o1WpQ%RoKm7thH2Wkbd{HWR*weIGq8Dmb}{UgdC*8g>$SX=R-6i5cG6%%{!# z)*J;L1J5kY{dRzFOucR7+4Z~{xVnzNt#!{8Ud!D#%PEHL7+TX#d)njV7lUkMyal!W zjMuHQuISmD>kjAGT~8QwYUVr-`21f_#c#0gGt%~Jds|>%G0QUiIa@NTsDk51&#L46 zud>oAV_MuL;L{ECVwPF-NGS#0DOeI+Q35k!GW50#eHOL1;#dPbeE>^qzw3$&-@z`62C_EgVA)*>@(sT%mXe-{=xH z(?YK~>A(0*miz5be%%?m9B%VkDhf4?74=^+s}XWc$I2XhE`6oROM`V4SjfBbi@z`A zU5UBJPvI*spzPJ6Hdo;8XUnfDpFB%+riR$bkmS$FjmdxH)vQY%PF7D9ZRMq|lNmNy)a4pJsP!_@`lG`2&MA6rPa{V=^5Xt3@-`zA|6VRCxx-G5kc+b-?L??)#P#5CxiS$aKAGXMYxX??vFx}`a8u;`VYyH0sKO@;jfqJi&0+o z)n3_^hvXY91hHo`*5)GcERq{m(+)yX)?jsZ;OBgqoQOG)`%rxN80$Fi$8r4mPW%h8 zjp1m25Lx^CM)DmrsWB8b1h6V(#X-R&xSvrGhoZ%-=okw*w}AhnJ5hmj;tI zcksg8VTD(E_2$ERE#p5JOHRcs+IL+!0fc`=S26pxP<*rxX|q^_V71kJsV7D$nXLy! zma2lzb-JBjpeJ<`Uc;L5F^Xj{mhu07U1+AWj&GnDT>y_O;es2{X` zAgMJDuZ;ZGk!Z58_!>C6Q_tmj_|gyX;}zMe`>gI^zMAJmQNQy3T^^hrUYpIP`$SFU zw{i?_6m!}Slh*U;{fCY|3T^}rK3052@ob#sGN(52y zx#>!>qWZ}PKS#%l{xBwq>U#31w58qLg46y+)80gehnH^$`{Uove!cMW611nb_;pB< z-~Eh73tj0!BTz>p5HAlqBd$|cHuzB0IN9RG@8Nj;RRZi+KeS0E$xs||tFC1|JWGu( zd#Q_B+LL&&R?Dk7S(ljEy8X0G{YPzDD*K%5Yjhv!EW2!1svy-e`vw#K&dmNNHCiUa z5&Uqt=yk_r1JUa*Wy5|?6R%F@ej?&{k{sTTc(S_5??2$l`-=9%lI=y~*&fnk=N(fziomCUQdmmPfuY-4;!Eq$%7-YK)chUx^pLY@(GDnkbWT za^kVX4bE+em6sE<{hzKZ4~xp(Am&zDT<{VY_CFQ#OL&+Pazs;!&uL^41q)BfR@^lD z48I_P6ME&tIP}x>%F4wnNs;yFc$F1QL|NbHoiY*JwxnA+hmYRF!(e5F;-kEcO?)rEcpi9t z=NUW5vBf0BLj3<%cmGMw#6di{q>8KQB-u?of*tgGX#=}v$lU8C>fBlk`nmM%tgf=Y zt|;DJ@Xpa$g~uev=grTbkaK+2-Gz4-O-NpmKe6OP`5m*rObks9%Ux7rMTs_fKd7tu zIQvjus$_{0zvc`}o~D;TBU6vQ$?lrUOTMC>buZmui;9e6dGSp~E*@A^QQc^wa9`mG z=0Nq70dZB~Plf+A7prN(nxlgYJ}tajzQ|DZMiUEK7w#^+MNZqL`qJE=?#*+sTpr@a zqS5Jt#Z3}Vt01`)hKp*Ws|B*v7O2gcoctzvYVtwncU7XK$uFH!6;e;|rsk8uX%u;l zPD`VBTVI#=x|hU?ob@&Ipnjst^}w+Z=eh~+97%?C0_$5yj8^i}l4RGlPP;OFWf{w> zj_Qy1beFt{cGVgUJS?xIKZ$e)Yo$@57e8%@#M>m+J*fPU_|X9|lIO{+o5wy7qh42 z=I8&EcSr8u*=w_($*qw8q};g%IbBn)CZA39%vqJwE@x`0M)G!c>>znG?XzamDz6Yx z(5bnoQsFr&%MKLoHHGY*!ix&tRFB-WV0ghGHL*G@sW02Fva?FTl%ofZj!=;`$sDF? zAkY#Fwwf8*MeR~Mc^`M!$#wYmb@1wF))=!Rzg7+3%|xZ266Ivce3Yt|T{3%|8oFCk zZ>K(2k6kS_H(6RPORwx>^@7MvUQIjuk4o?rD%JYvnD7g&qP4z#Z?Qx=)4rcCUMRQz zIN8JRlfpOSx25o+*18P+N_Jf@hq{_PxI6XASmv}IFumn0S=O7;`2aaTmzsF_8`)lq zSD+~zdcx}*(z^n=_9&|;dbfXowwvJ=kCET&(`S=R)r&un7k3ICR-b&|O@5!6zE#!6 zKWcnVm23JzQDc~L7+v>L1#vvxDn}go8d`3W?C(r(I941vJOI(rzb0Mm58A~_c=4pw z=ca3U)_nTHv*{CbQ2vno`JQ(8s0ntzq>jyLl9Qc1Ud7&w)YhDxc_Z@=s%zHj}zKqD{@@nRF&Fzu%nBG#KW?u#;en4GOw>3o%-(O^%)+aq$qz4q; zT-dMRpQGuc&pOrL^wK)#=+8&yA9?&}vY>9kE}bV%P*Xio&HW%5WBp9g{K;(DtPDpV zOzP!|W8~1*)`@LCLmxhxRgtgc>%^_eDspaKNVQA-LxVmq^=qnA_O0@;zLl$XAoWr9 zhV19E$E%M%lzbs|y^P9a_N?R}GbNrhA9Ri^ySb`2tMkMC}QevqjAGIx{1A6 z^rWoyXZ>7na#7N(()n_%lPcRzQ`a_FmD>ka_LH32JT-NnIOEUA;#2XApVBAFV2$dK zH))^cGV#cr?5c8Xrl;AgjdkPvCKDTpe&e687%n4qOQ<26AX;;syIcX6O6nL_DqTjk z>>3oZ#>B+ou;C-wtBrN>_%vOYw7;CJZ$oxp&6DvM?cq*$+dw8w_4{q~@i{c}pS`A_ z^DJ6uigg~7zdMQAw38>f$PBUr;!K|>swP(^|4D7mzB6ZVPVJll*)8Q84$qmJJ0(|v3^Fkd4AEKUyX7YzOT4I)VS9L2qWDue$fI$j zsfC{vzFByp>5sKl%`edx>Lqz!-E<1L@92X%P~1|mOK-7qg+Ds;fwG3Xs2gvb;cw@G zZhi2qjf358?HhT{o|NtML}ICK6s`0&c?3tlCbcScs+p>@^>Da9RX2NF_H0_lx2cO{ z=oO^S&0eK<)f(2;G?}$!QoE8Dn7dggu}t3UPW?A0;@7`}Q%bkKA5ivSQ}g!8IZdhX zUR_v1SGCS21b-*9wQu1Ag+o-OPAdG;Uk<>5%68Yk=xZ63`7&$IrKkNDZDl*f6VcwS z&h{T#SNJ6Muw~kyuA5oUF^{?_-_#X2{yG-V&A34ym~w|zbmOnP97m{xdK=L@8`CA; z$#9AyH2!_MDhoPhl1FdS`+1YLlXf@D`nZa{(~dm7%lk7(pRj|^WF@7uTATa(AH6)o z&OV>jT;K1v>7e_q=~R>1rB~8EOC}VWyg-u)1z);1WTtQpGMWJH|Zd+;+Iuv zsq2fzfaKo7$%Vt=!VogPeqnLJa$KNlVOvtahu&a^Gw<&zoLktW=puT^K-4wNbi)VD zE*wc>Y^A5J=M@_vzwO?{Y%^`siDTr}-IVMnOZGwY^NveRlrR56sz<7_J{9??8}!Fm zh{KO#TaTex)>Vc4xccJ@k_*(?--qJUazuOc9Ui9Nzv>KsLC1$=NUt)HZ7e-wec=;2 zoIR^Tcz0o1CGG=-9STn?yrXa`soP0T`w$xESh@8ts82}hiq``FxIL5Cmrh@W;x6Jj z9l>Av0o&kfQLlkKS39#h(N+4&D0`6nP3wd^Oy`k5_IulQva zz|!;^bogJaatdz!C;jkNCt8an|Aq&(55BcX>?Zu#eNlDQjGo_zulRME?t?UpQBHog zRlFq1d;y)O0W8Q$RAIrEpw-@|3hhr-t$k#=9H-yN_4MGOblq#^!+*@LGMl!W=i1RQ zr(MR!Q=9MSUi!^QYY*Hjr1!V=q$#}A(XsR*e)|)6v(C2PRk-w8@;O&8fMxpSyaU(1 zv(8%Lu@iZ&*P9vEmoK5c$l;r`l;_dS0d`7BQUA|<@*0bI1HKtP%Y&}|lBalx&%f9C z?=DY@?(5&Oa#FDFHh#fzAUy$}xP@MIncv@~i(SSWxQC}JboEUvooSx&j#zX|8@YiT zt|2q4A|38@x?N>9b3#qq-z4MDycT)nb_*QYWRh|_!56fbgU11Z#jQkbZEL# z?0W!PY7mWnIKJ>NcRv@O@8vlW?F=9FCv?xqJNUppzVQ9*y?^ioK;u$QwzVd~M4rM~aSdf#_ulhNBo zSrOqut`5t)rpx@u z7dO&9V|LGsj6Y$#(;LRhi&#RRMbEM3NuKx|47`hV)X$3B!h?!pVO!8p%37PC zpL$k&9xuU#Fyves>Hyyv#m^R*PT!Mfvz*Z`a`F($uj~Hp$dO*2`UE>_7@jfNn%@Aw z2mSsI+u>!9d>o8l^X>`n2W7bEaBF-)RPZuCcf+L<*j3?Qo=*pgIQ~@B5}AtI!DJIp zV`K(J_RvSx@)1n@%Bt?q=ut1*Rb-p3M!7%ZwJ{STGEEOy;R>`n36~z~-(SJFrqe%% z+%3m9AJZn<;Y3&9i|y>BDSzY*Xy$oR=5Bnh2hLcV4p7RgB3n4-P37Z_ku`Z7zjY%k zuH{}4XFrB-Jz~;jNvbXMxe9#RN30;`*F-&UL|3CadlSFIbdumtYs;f&p2}+(xt;4> zTh*Ogz|o?N4|KNEnnllz2~`z9W*f*%1DkF9fES^evvBuyDEfDIjwu3$vy(EF)Ws}B zp}WLPPqNcvML=Ud&-b9S1rA5XzfBXYp2M=$!Mbo4dgg5 zIYL^T?u>f+w5k2pvEGwKc5V<)I13#$_HRFUX1cP|9!gif#$O}nBH+;k-eBKmurO;ab;z7c0#=2sfMM$G+3{ly8R-pNi;IYbR3EZg*S9uV`QsK6lVlN8>aDt>!cGWwI5{LNDKf zV|3AvsVbXYy~x@k79X+8`Xpc;yZnG>L?8PNPU(A5=FuoG^74vAe5&Fi+g<;U*Dl;J zB9TYIq=Gfn2a{NH_(o0yxlNw5%vDEWToQiAHzGeZ-$_KacV$m2@Yx1e)bQQ9@IGQ` zhr}i$F0}?;FSfpzx)qam7LvGo?XAF_R@z_G^e?l<`PLCLN|wWfxS~Yn^h<*CF8aed zpKNz-|9yI@nkGj4(-MOI;L>knMZv&QZy zC*d0taG>SRr2)O^*v#tl=nE@BzYwI3f>BAB9W6wRg?FE2 z703(S$Ok)=ygAqY|8QPEIkCIwIp>OTf8i-R(EGJ8b~0J;8QJtb4%Hd70#3`($Vj@; zE2v`}-Wan6OWQ|KV?WQTMO)s4Cyu9eyo}@DM3z2EOZyZgimksbtUCq{9dBZ&5N{Mb+q>NHd$~t`iuD+>zwC`4A*|1 z1pgKWZYH}MIg=W2FbyLr;*pE3@GZN#haBrb>Rv+PoE`Ro~;N`#rNP_(ev~Dtjw|+Z~_b3l&`* zG4&SCv<3cmhF2Z@KC*OU8re=)S9jJ|;3((8nTwoxXI!QOtc%!iJ@_3_=dJd&j+GxZ z56xgmb9V{8QqQ}}c*|ezwHG!PxNfZ-ZLp@O_t>4u`nU*IT!_0>CYiEHhjp$B*;LE< zlo$Vwh=0uPtLM&<3)b9Ij)Q~sK(QRCEJZtOedjQ)@g<%3Mpo^kejcZ_hO~b>6BmuS zSHXW8!_JCudwyd>9EItN zXe<-R`s2XqZ)ZBwRo~+F5q19sG$Nb14T~shPC9^Ge>%g}pm+#o-$9RR>wKTEy1`a? zAAFt#I*0MDE5SLYP;3U3TKG@Y!bIl677!`MTO#)_ax~k3!EqVMbUcc!X+Qg%)aneE zDuIemCyUQ!XH>r@2T8cq)O`u4BzeF&Gefm!24UY+Kw%< zKv$)kd4xuSL=qhC!+Ud4`*O1=dWbvrrT?}Ujr@jnS&J{@DfUW<;&pV^55@X!<*Hb8bz^<@ZqD4{8|ZqyD$~qx;xLu1-c%Q2Z5i+&(-=ub{2L&Tp9izr@uy!S6vnY$H+Q^IlKF*^-{I7=K@Yr}Sra zy$Igl(Rb%sSIEGN>2?EDW(_3mJ38lz^tPayiCzP(tS>w3Su#GdccY(P)Z;XUtueXh zN&AQjm%;w;iSi<&EOKUN+sQzBz_TpIF*w=pFs~?+4SSn)-Ub?1(lUF(kK^I*c9a|X z-W<;uXLav-dgR!d$C(z#_e9EMFiJvie?w0 zp{JdHaIfvKbuIec%C0O=zBhu^OJH7P)XjEsPs4{aqPECvk7;Hd zts-p=<9#RQq{WVQ!-L4&`U))t#m*w5yW`M5vKW%Etg*_MHE8Y$v5ihRS4|S>M<*1s zM$2a;byZLak3i)3gzQX!>q`6k8dN`b=g=MA!L{GEo0xmq#fn;4TSt5D?R7CYpYKjb zX-Q#QdMYmyxb`n2nNq<;=5N{)rIuz-t@9I;9P>fwFiu% zQYo?%p7O21?4_%DTOXrWO~s4XfLi3zRK+n%gFw`nM4y}zXf?8&4&aW3{@vuY!)m5v zZ0=>G;<=vlG&}Egc>X_h`w<&zDhi7^jpyN`wP8j`z&*Yf)UzJ8&jhIyuGbaD_UEOz zmkl!}BiBZ`*QIu`+)f{7>6I6?oGAwMocQ2WQU7UZdWKkD9xiaZYgW@Ant8Q$4t?mJ z=i)cVtp+V173Uwep{Vt#DurNpB?UU0=&D^9^S+e7T`f; zbRya#dtnRtJWi~%1De{7qkiKA9wi-bC2?;f6UTy8W!mUBz5Q8 zb7YY%MMtxo{9EAPmt4KxTBf1mJXZzn_W*}m!R#tnb(dG?jQ$WcPHo`*F?Q9>na0_+ zv(K1^+SRU_cxowrot2)m#%m;6kDQZh@bi4tjF(CGLM$;|{Lt^Ip2~Oa3TGOxUTfg>N zOuGJq=2wtTAHnShU|3jwr;}yvt-7W6F;%q;O?oX#9m*3?wsNXY@1xw z^^Nq&sF?1;yA&Os9s{{1u>BMHL???8KOhTXN8+DEVPdh)N?(hX-k<5DH-{wXh9m9+ zzu|BxI)--PsjLf={+3m+%pRtRQ!f&|UCBGVh{TV6c5PgBDM|Y@44I5?rRa(G| z`0r@+(OaHFfu1nYuPvs{-GIMb$M1Cu&tT-oE=IK*$&#ztDh+hznxQ{N#pGqWM807H z(J~$SyOL1#2e^2D$F+GgYVF%Zm`1hK`J3~w3}}B z4vM?c9UgYyLGpD5d%_2zS2K9QYSL$;vZ)8Yw@8j@FS24^)@kyAYFd50)R?L*DQ8U1q@1tKcsV~Ok@I?XGm)$|CITE5^EiM9{wlV#NJdpl{g)mSg)n|J@)xfA$w$CQD~7$~g{?egpi<;4rW70GD+pBN9nde}?F4`BieMnLX?EkQk8c zA>U;#Z|dcVqpYXudR4UXl%eT9U~#f8yj8`DEBQ%@cy|zmj@dt@P-S&o?J*I?w{g3u z;!rE!1(e}*dti5hRfd%L6c`TFn-6EnU=%&S3B zi=rl`$KFRy2`$qvplD3?ylS?^Csjyn0I+zDsQ@(10&(gC8=YQ-4jFg>zkH- zsHnN=UqyQT&pP_9cx#uVy-Wcpql<10-MVtbunUWx7TM^HKU~4CYUG4^iwks<`T2dK zNpf)V2c4!S`}s?huUu-U_d}AMldC~%veWpJ>{*S|eI+wxs3>m@-Af+TwfB$0Y=8dcRS$W zQ*oPW#Vh36J}0VPT1@$pqNBRm4AF=93;l?5bd-4##&nT0aR-ae0INf@ja~K-Fft=_l`d;3vu`_*A z#>eb*F3$7`9ilDiTi@NjPy8*X=mc3N*CZd-wQH0ZcoSW5BIoa15tGg2=pedKzRnDh z33e^L=T`sTLO;2hd&NuT0+asGl;c-;=Ik^5ICR|1zHy*gAITBz6b=aT+~loX!oSNSga;4R_E|PDewF$mf}~p}8c*|HzJqNV2eBBIiDO zp@y$*B^^0t6_8<2wdUJ+DvUbg||>cB*I0+VBuIK12ay{0jJCOvr;M&tXjy=gK zsh6{x=e&?}Fy}HE67S^>lO?f}&hSL`Td5b4eG{i;l`bxq?o{-4VL?G}c`(12#C37O zr}{vCR(M@eYneF{iie2U&oa5_{^WvWiPR~2JoQYqH+^9pN<1z3C!L^+o&>4H3fM3n zydR`JU*Ko{W0qSnR%0v z4di9Msr%m;UE|Nt@`!mp3lCG2StUp5kTFPd; z2yV3^F~Wnk7p7H#H_fc+X&%V!B;3>TuQusV`G-v0VRG58WkLSKKhRs{_$_qdg=FO! zG6a_EKNOSflCoHzldE19g?!D+c03Ar5#DbmFAvJt{L2}>f@8O0AFjrKXVPu%rHNcb zC+tne-iniswX$ubUUk}O1uOd<4a|XS;k^obt`rM%2>(Y?#=}`Ud=8;AJWa3uuO2~{ z>2p+0M~AoJ?MWy%y5rnWVr+K)uVgwS4R`Vr=&2{529cPrZp}m zo5M?AG2^9)cvUZSK9aBIE0S^{N}J%_6rR(`Jf|rC+Brq(b;5vS90kNQfL6lbC%Bf?Me77+W1s8G1GCFSBrm=p}EJD!8@a1F^0uy{ zeSD4<*MnT;;+gE+gYr=)>;5;9C3-cxWS^bA$yV-$jxsOC z!R_!bOk_ozRlFZh{vA)rMx(uOn}K#V3(f`Qm!)Nw!Yhkd-|J!Gmv%GV_eP+}@PGv8 zHQEYSi!N9-aDh|6UCy@icdue1$<7kJ!vK`Fx^jR z^fU0q!&cSaihA-wJr8ppMbj}!H!2~c*GNp-*nlRFWFn;%@bTI9H{WO5e7DH+OTzf7 z?jN0~N{SQiM(?vd=hKYOV;gEI$7*Qr9Ir>Y&*74fl55Z4vnQa!^S$nY4G-cS*OKbd z&1R;qWE0`Y?|A`o?_OlbGI*Qi>H%+0~6k98zBx^xsDa+_PeC>5u@SguCTGwLNm*)|0 ziaPJbWykUp)FW@(>aTWJqQ5Ggp0v`n56kq5x%R8uRh|pE7#DcNxwbeFdiYWR;a~u-*kE z!5?1V^C^AkZ{gAY4F0|8_mxh)uBTk;g!;m+_IPGwA=J0ue|+~#_&H5{s~elGnLL-W z{XL$&OHN#G zdc+vtS%luV%h_75?(AXGZaugTqmx`qYiPhH7Slub@ws)SH>{-FwS!Ahwf+VfJexOn z8F_gNSpI+)UnxuLIM1q-d9@;Yd*SP|NZO8kD>26)tc_35{&18KxOkbrKayaZYX#QDJ@d>9J z_2y9*m8ARq3H#=efv@37cRQJgT6~Nr&B1RP;VWlb_hTUT9*gHCr+fjdyU?e{;t(@% zujg>I&}&+$q`Da$q*Yzi!YS&5_8=7oyJgmulvlTcui`@36tS15;lz{p-`V`8$Drz` zVC7DFSSw!WCo@>So|Ig|L%Rz7%%EwXDUgdxBs8 zMk9#MRS)4n4S9H*!ueO>8p?dX>}o7=N4Pbc>3Qv!}~ZdDQtww56F< zSEnI0!iA$dNgex(ysACUKl-=ix_|twj(eY$x&KLUt&MAs`0F3AX1x7GCf)1K>{gHr z|IBY7f7mNBX12nsnLf$FRT^gOic%okmKIWrme&w=*Coqq`?oR}uxpReP{0~pzS3v`9~{?Y3Qr)rtjdw4W#%W954E{EQFE2 z<5~^T+0!WQcbM>7#xL-ae2#wj(=qN8wSJ4(<#*G$`Z(K#C}UG*2WxQbs#blWGkKJB zi0`xmxv1*T6O&v8;_rH9WDvfAUVbFO!|S;t<6SRdmmSe==-vPJnfei)e~vD~hxHy_ z_yZWNfUS{@aXz}b-tJF_H&OTAzz(C2$O-K#7M)K1uHXZK<1QmqHWY`Q8>Vn}%n6 zK+k@g#c?}IdD|)viG`o)DVIB&`*E+)xYd2M<6-diaeC~TXyimZ=M+0_36mmnpMXD+ zS$BxMuM38?J*6~xc%gmX2qq`F{w$b&0o**z=Q*y7-h}yZaVOp!6DM|%fF)_pzmrKJ zk=D>iqKCs#8d3r`S`62p0r}`9_AX3();T;08~d@9hS@mS`hH;ksqPi|8h^l+m>{&oexe)P9P~X3#gFlM)HzVkt!!xqY89-im9>p3Rm9;%W}zv$dg=Q&(Q;GqSZHp?H6Vs zRTrd+aEVf&Ro=VM1*1ztbk8W@F$pTz=$UWe^pUkY+HMv)rAh8K+bL~uc5zz2_%u3L zeEe4!2&aGg;h22wdSA@9RZ_LYjmYL2X&SB{KM` zd1`HUh@MRq{Vm4{9_IwB_}fWdr@@=^P}ng(ISMZ};%PfUBr;Zi0*_^Q=U1e4^u`Em z&U0ntK=yE+F>~u4di7)YaSxajk;|}1q6X`W45x@0Owk212WKn6$~Xj;`>e4pKG4WF z>R402AtL9$+S6*>&v`^Y*NBauXca9okjh5oTde+&XGTxhlGar} zLt)MA_k6f}wsXryo4e3X(#eE>qq98(K1N5r$+V!){fyu-d6s=G!8a8rATzA-_O0jW}+)6-fzSX0A{&e;b3UqLE`9did-I|rw@hLpJ%U+ta2 zjHLO;=E^4L%j% z>TdM?JG}c5XZQogj&X7i!o62Wg$elP&;B|KH+$RZPP3XVAhnldDFDHMTV&uySK{Ta zev8EZ3>`*Swy3RHj&8p7nr1KSJt=xAugh@oz@TDx*auI$_!W}8ymeIcTM~AbcJ>8eyAf2kxjIfI&vWZ~MNg0=8679O zoG!Abbs1^W9X0hsOZTCm3tUwMV$m14ntT22n&`Yb$1D1ESM%3@DYGT`oNaXxiNBL} zcRM_a3Z(s3AKjgz^Hfa82~DhW23{xOv7u|Vz-jB_!6MNRxI zGJ2!K)IRsfx33D|Ru0z;stpfw^cvddx3GNkKq{agm0ZRCj~*RX+ZD6^5|bU9ez*4V{-0KvPoi6SRnOU-;eJ&-^9VS$bUIBvZHE(&7<&cR#rx71_W`U~ptKy-O4S3g3CpzM>oWGEj(9h+e1Bvn~2|#dNVGz7-lsXpz-H zH!2m1WQnvzH-R6;RuvMq11t-Cj~>Q-*fj%Ky7$7u=Dr!6FKVS0;5Wa5@*(FLJ=P+# zwt0qbcatemOHj%kb6kJK$=7h2!8^k4$ieN`c}4ZzR+#da_uGAEe}Ivb@# z*YwDyzlqen+UW;w9P2*Y;O<7yDFxyYZ#V>|g`TsE{3&Nw(F-hUGV4VH~ug7PlnkuK|g9I_GK_RBvA=JQ5h8Y827CX*K6W?C;LCwE2?&) zPje-&GWHyl9JX!rvv23@^Hl46LW_Re>AX$5e2e|}w$~Um7QNY`cUyG)oQ;ET1>3ZL zmxJtdTr}#UqGwWQ7r~?d0`+(;#Vr^3iOPdL;J6z1S?C*|`-u$M4gMcbqnU!wu6E+l z32Yzegypyny&RXpr|9Kg0~Un67a8Q8KsWYy#MxD{@~D&v-*77stYMuIq2Liw zmFVTXAMXrVQEVN#846zvW-sypyp1D;-zYSqAN+jh-3quCHDGau-=m!HJnePvt1=W3 z-Df9fs6Kj_{!C8&;@ShI@r7sOE|h)@tPQK}I2iYD3?{m*hPB)U2A&HB!DGwXTP?e) zo7qJpJB%#ByYTdW@H2An%2;m#G{ee>9PKI97NB#)j^? z$oXv|OM@dvC*|Ocp;?uI?Z;YCbP5hmQQPlPHPi~1Z0Bswbpj_z>r`-glP4)T>My^0# zDO4Ms7QXfOAMHFU@T#M%@EldM%N};r&buO~wbvaZEBi+$xybL0;9E%XTJ{~jiI^pE zitmTU8nZI;{B|+eC9Nkk)Zj>=w{Aii;Ug~zzB!&0`dw+%7d$Akfp@~Bcm+l8_5WVd z>@fM(68Emo25bHG{7aSBg5EHeO-noa!7);jx>nBa1{iVkco~ z6?ooSyNsC>$HVz-JpCVgoQMDY3Ett?IRHw*@5B0N;BL`tI;y~<`Z6rOkg_x2bl~ZH zt6C3s(LuSA6KUy8Pw>4Ou&PPstRgG7MkWp#y*#6yJoMC#v2?9S+tp~uELe?nW~1zPD$wu9KeJ)Q$sB9<7Pg})?8-i329OT_$oG9n`ItKsuL@D8gqvg|jK z{%hg$0?&ymxqxTj?myNOUEZU%G%5sEx?{xf=J|GH8-(5(dVcir+>MedIqATwm=Y4+ zx?}tV<}E@ypYn$eV&ly7q~-8!k!No3Zl%+GmuK!n6cSzABReK|R#o3_L$f#!?5o&s zR1NHc%Q5dMET=W@utoM%BiMF6*>nj=)yi<1(%=-68e&FCXsH3!x+o_+MTfvTUc2nH zj_Z2y0YqhD^fM31-3VMudG>KIGO`kmimZzmbLfH5KRqfw0>cYIY#%&soS}$jR$I;v!iy7?e@(m# zTPnO~p~ZwxR;MeUMtAnm@`H0%@qbjA#&^TFSk)&d+G%Gp`9#0{3o8o_JHMXaEg0Xx2pmtGY2P)x`FYuhFNf9lU0W=J0@=g9XBJD`my4k zWCcD+_TMb3J0O$YP&sP`8|f+5QdkwCk^KbwOb}&vEJl^J&`wS3*zBn*o#<+C2#;(U z&sk3%{_E+=b>`9ecPpHZs-&2Q5tePp=g?}8I>nvdRnJhT~wO-hV(d~8{P8_oef;%(;`Ov*W_ddZ-=#jNyd*lH{_x6-)j?4JF zy0a86!HK)Th%WH_Vp{ESzFk7JYrZER0kg1>SKtmGl1VS2(a)`EVaA%7N*4U!=`Fz} z^5~j_Ncd=iN+Mh5WcO-e1?^x~_|r}Vy{>*g$5UfYN_+SgHPWFAMepDae%ivV;IPq& zAnLEfYT81lI6H%hhn#jq(q@BHR2EkPkFa3ED;X!B>pa6_9oow_kPRG;s^(VK6xBp! z(E1**TLN#tb)Tp<4NvSL-`b}W!fDpm9o9w`#<3ZAwZIoTXMDxctNSGH!yA8}>%ylR zp10_nTqsH&HPm5G$0>xrDghc*VQ9=*{}JVcK6MyZJmjf6Ks2gtcA%!xB-Wvf3~%lp zVe^;BoO4a@wqj%$6>on*uh@dy5By+M;9LRkahVWAN@G~1fPg6jPvgbKMU;i zcTX&Eu@!=5+0(MwGp1J&Uj^CSbs4d7+Pd# zv5_MZKEnlg&s5T5B&?m9IjK0Exo{=AD}+uMYYWLS#Z#l_<1VKX(UPz&qMOV{Fxc+O zDxMoU{~@>@Q-8v@R>vwr&KIM@7UbP2;NJiamj?HIoMsDt85(fVd&EE^pC{H4l}q3I zZH%biyI>IdPx$Lr($7AnQ@ulj`Wjqf!oygfjPps%RSJ3yoo<)&`W?h-!r1U6UBvcz z2>0p;ck9EY^H}-e*^U`&p^=^mn|r{UsM!zxQQ4Yf=0Gdh+zz$Z^{to()yI9pm)gNk z^vDP)9hTHt_7?Z4zb0-bYPz8K)i-=fX!=)xOVh z?%`WIhh1?#%x_>94Z%93Wmqf!&Wnhc!ykS-3b*$-&m_!P;~7(}WUW&x4}*gHMrKu5 z4WZA(M2$sG z+@0`+g@(4r4!^;Xwy?=gbsmvJ)6kxhcy}9_AXhrS%DfA6z-N=w-R7x z{~uMn!T-IyI^s;h^INz}eRA@UHH2JUgIoWEE+c<1I+kw%?T8)z4I9g5I7~#qB61W~ z#?lObiD^_Z8S5{5D@1)Nl6AMfW(ZwI?e zcMC|TAkwY)BBCOo*t~**s2JEHCMLcr7ziRDiU=y*(kUU*-E2(HoO9-Wf3x>q=d-vw zbLO1q`SAyXXWi!wc?=dfWBq{NWUub zsS42os6-K#X23w6!aR0aHFc7?32tuAE;Wbi9Q3>b4p#P*=E$htUimVzJM)37qU*hh zD?Nck)x4bt;jT7(F8_GBRjyNT07{W9AnCji*{aDCe~``fA^%qgoTXS?YuT-Q#5z&w zR1^aSvUo*z0pc&pV!g^5j_}%=vYZbA+aPptn$N@RU(k}DBM5po7Vurs-c{fs zOUz_WiQia&ph#QYP?s3VNLyRy3ES#rSv)uH}t+-E)a zKg()$qRmh7lF4<_2O%KZiC1N;`AghEklurq`7C9zZekTd_)U{FR8QKAj6 zUCY;{)4n8Yb)cvoFj|{(qczJ<8qf_~WiM%RRW@)_w`{Wz2xDKDFBrK8+e{ojV7{~k-7jO0sD-<1rw;88M`-$Kht1~fyi zG){dM0AjLMuJc{dpYoq7@wsGyV(5|p^0)_qz9w*L>Y98W$^wuy^K+#?B~es4eFLjf zRJ|W-Xoh6;^G+Aue+Y=U9J|r(;uW1^7l8G*umLth>8nyav;^K<0AJ-n+m9fh46-=3 z!C_0eLj7&>`Fk~gDFUgvaUNt_1E5nKnK&ALy^H5O&i-y7w{A6D`L1TWG=$z-vvU{ptl3C9d&@iJ1AEG}>d&)PL8fd+3n^3t zt#(&FwjB zXkM%r|FLXRO?2za{S;$-mCVPN(F;$Jt?)Qeo9D@dG8(deU*)O4hb$_g-$Zfpl8ho-NQPCDNPM?(%I+&R~4~Fojv#SEOlbq zo{~oD>?QA_Dz0?CRTb1u*77f_J^`&On^Cm{%2s`m*o98fmym*$peo6&UC^D*oJW91 z5j58m>}XzHMI>$zD(()1l&vO8R|bfpDcNwJg}1aFJM**G;o$jMVAzfCb!L^XviFC1 z-v-{w{g$so=dBIMj%EC=Jl-=Y_^#zI16x*h$QkHwBk!(Y#qz7v0FIZS#5M5KQKY#t z6&4}04{<+Dfyx3)`_TfbYtczjb!HEeZSp))vm25}`4KVl)(hdlP25%4h^53`Zs41j z+4n-?I6uNqic3n*RA5))u98HS;DcHRazAy z)pvxSCP2-~;T#DS{TIK?cjUyq$={XG(RI0AzGd0k$`{wvF!>l`Xc(QiYe995x#NG~ z_Db}{+0QyE0^cgEVgnhdFF^ClpltPq{3&G_AI5fh3aUTKnW{2bYs09QKW}&BJX0R~ zQ1*xX?-o$~hHozbzO}%PvW@e2H2~Y1-X`0*7JMfQ@41wXqMeLjhdL$d6nhhUYsIb= zFYjWVU*Q#bgM4;nAZZ?%ra>vrpARM0A&YYo6-4*&`6pbv9bKxE zRDGT&jkKS?HM6@SnxiS+m$6u*1Np1m>-jHyB>kWKzq-2=+Mf<=RziglC`>0{O|I>M z-0cmUpp5{0R7RC=e|Vz_?^l5qq(dq~^L>F&IbZ7S*$YvZ=Z%vnxXA`^KN7ADu1ssapFV*RgWK-!Lh~`z>sa-Fby&ks@Qg`RqAv0@$~V+w zbRz4N_o04@T3Q2v+I+U3r!U9qJe9)7DJX9uQKHGIeCN-gu07Dkue^3b7g^+)ZsN&5 zBKhSDQcocHnlyj0Iv6s*@)c-8Ik5xLgTqqR%SdSHE^I03<3arYZuWke`|L*UDPKgM z14)7_RCwG2WUjMzRc&JHFI0eJ5REtkR@7OPbD-HpJi8&dbb-0f9PTQ9+B|6y;X zcsxIcr_|d+C$e2YLaTiXjg=06R%Ep$aBhgFYYJ>Xa-660y_4+fd#;(wzps+}_aq!P zf=pxeQu>zfuLkllR`p%#9ZeFI7wiz&ya~9s<-T!tP?7Z&FztUCxEab=zGC$7P!(3GR?X^M(ybB>~9F)(X_Tp@KbXju@#!w zgoaR#-8v{l85@^Vn9+%?9q{YUdGQl6!4;|3w6dRwe>_Z1@Jp;+IdXsUuk6NCDF_T? zoj<3lVm2ZtTB6r#rcTj{#f+sIV+Oo10LVTKHkBzmh}{;!s~3=@jnHljfrtF$I{PR; zQTggYBzhNcCfhz8dVGVr4!xdWZ;GciWtTSVKga6B>|_o5U!3Bl!>n&A>)Zs@C6EV- zm?#J85}ef=$)G8Y|#deEnHiTnv)ly_)R>gB06C{Fnp0} zrnb=jCi01t$)}T*GPJXy*y_mIuYs`U!0v%>Jy1+-cxNrEnux6Wm=(VQ7EZx?5omY| zSH@Yzb3pGYV5i*AF;u$G1}=(f=0R-%?l_vP&2?bB8S?98WQDwO7I#?z9*e-vXXKxc z1ODxim-#@k6_l`_9iHJCS2#Jg#BR75Jt3b(2ktR~>)b%HC$!uSIiAhS#rbL}nAie^ zdf})QK+R0v@A#JL(REyT2zXBh4$84qX1=_9n(C-LHuZ67L(Sk+=s+iGO{iAxat&yD z89eeSyLk(^jQ|(D;QKpyW(W@L(Lsa!S67t`bV-CFL!7HMYTk3sH3F;b{u@)X0_^H z8%x=fzhh0@O@D%?;m_NE?o*s(ZU!T>_-!D+H$wV}RtE#It*J_dT;z6F_&))6yoMzp zJ+3^=6X=t{MA!SmeV5SNUHQg#bjv|t))%R;5WPDB8+#sf{0*}2-qe{pPF>Q?{QfCb zIU9hD`dYpOMkm1;zX05`X-p4@?U{0!OE!m$}^U;-;U&Zp&*L?7q0^$XT% zE3~ER=T2Z7G~fiUeE4V3DQ_c5D`O8xQf)_?oMOlAz|gHorH{}Lk~;}3Z=H66$fv|Ds_zE<8LTS$M$4ZPa`IgT89VYE&1?K4WaVy2$Cm(!pIO~GFe)3NGBS7?T=ggX=t82c27~i~*?FXkD|r(YNNJ_H@AE9GK-QyzC2a8`^p%I{m*`waH24Eiu9mXBE1;`% z{e4J8+2U2yrR>z1R+0$>pYa3ER6N|?4dI1f;GX5kR>5}LD1>V%{omXeFgD*Ot~KMhJxKzY@X$4;nA`3+~l z?|!7jE>!Jr^?5s zNg+{A@zuyd+Xn70@VWfTvR@^%BxTnH^Pm=Nd9r-JNSd zKdM9Aj~(+3+`kp-xWfG!f|Gk#mm*clf>o!JUPuoc=xC~dWcyR}#j8&jzh|Ih%_{gF zIBACNOz^I`1)SP<){4l5|+FGZmSVAHGnneBoKqP%k!{`~>`UQBsLROKzdgcr;z`}i1i{R7f72p?snViJXX z@(f&)3;JE(|!mK+#ONEycOlhLOqxr2PG;_fVV(i$kI z0fFtT@hb9I8L7&AQpS|>v{m0X61z!t?ho<04|$^8^XI{DcW~1Vx>eSN>NNWB?&r|* zyJ$?EaoPiq>gX`#G@s$gE_kT{+OQ@PzZCwI&r(^5niZl-AoajfmtFZmb+T40O*QYT zk(H;_$vO|?^-&epa$uqw07WWp_5 z+yKQ_Nm2UJRSZyQmG+ z26ERTq>g5ND_>v@)U_GvmK-<$4=n*#s`Akpy&B)k1g1xU|0kSF?uSR`VlgS=IExj? zQ=tC-pCiGP@3jqD{{y_KZ|2YZrR+B453b^A(vgaLCE zllhxxoyOwT2|_tQG3Y`Wnu@F2tXTR^_PgevE6-Qcm1eQ~wa9Sw&`)C>sTg{>3u!HQRbFKk zcKjfE@Bw}%0ozxwF=r6dy9L>x8uwzbtPFK^sSv*&;hE>ymE@#!RVmV`F1TzAPL-ph zd@#kB)KzC0Pf)&!Y8{omsZR2Wt#stx8Q@2ib!DtqJf*tOOzza4&mV;2R0XC7%F(y7#X6ctvcuzKi91L__+ohtw2U!Hyh`l*5Rz7bkc)~YgtJ$OUq zRa=X+*SSFPamn<>aGCnToZNXgI~s5O8uMfDxBALN--cj}v2hbBL%YF@qM(%=`J zZStYG8o)pq-YcPGO-xycZaf3M)q%>#AQc~AXWiMQWVGt>6%!AocHWKqss8kKV5!L_ z%J%KYUjyMN)g!jz|EebL$o^!1{J`B6-TV-$FJlIPa>C9KgH&r98 zSfpaEQ#rkUfQDDL!ZiMRpU+fFuG~e{p}FyyDfeC#>`CsaDoC9!1a-|gS%D<^f;-7e zpNqzGvj_FUQ)Rzk-V%Azj=!UD(=-m1uv7SP6%NNnY;jRcD0IafZ2 zmH05!Gzw|0***6G!G=&_B`E0@?4EKjuj&fg@r)K=MAoGtYM z=)skZ*@Joo9R-40`M>niIpBYocYo!ZJAtaQ{}-~>?Pzt?ag^sph{>Y(^1bfj&JLdMh7S}imKM}mH#21gS3xRfg4rTC{E?I`a1YQ@ z9lT<~sxeipNI9_Twa^cm6(6boQ8g*)JgGJER8eeMtg0kE0k+lgO+DvUBVE)7?q4*? z4Bk>JzWreNacjdsTMy)=- zD^{QhFg^LLH`o~pMxH8A?p>mvA4UdleMxd!Z1 zeWpyJgZw3GD9;sMHP~++P>ZD}?Wrb9IStJNQ(Rs58*utbv61dO8d#XwpRn5s;F8ZELTt1b@5!1NJl zTD3?!*oUkSWtGSxlpH<9@4L9}6dWk5T;aL{ybh&ghkVQivP9XafAQ46_`fn}oOl5S?qP)h)=4r=du7 zLKNk+1`7lDzK7GU;?oacDZK#KDvMZBOx1(i*|{=%G&ih#=9%)Qa-m%1hjxPUbe=f{ z+~mhmJa`xIUk`oOOYLzWGDX!{O`$_+zphxonvtWrQf2owWaSOPN{r_yGe>Y&zQF-_ zss~)ti6^O}g{IGq#$Fu@u4}Ru)u0wbPnxKv83KFY;uXlGxp4SqFtU+Z7tT`wb(21HZp_~$Ba_-~$Ev)1ius94o8c6Nq=orlt z(v%$4_};?`)o-COx~U>ikw+*qrPJlnG1zrKaMl!FbRzAfzm;QWa|hWo%8^iyNmZZ+ zu~Rxg@#?`NF0R2HHGL?Tf7O5BB+t}L5!HhC0A|u6%DC3VIeDYhg>5Qck`LM85_YGo zuywqvmn7XTbc!-z^T0tJ)}YEYRV`-$OVx**KwfLc$~t}v@~kFkVolJy8Eo}quT|Nl z`oy&39uPO{s>SzJ;bdZoO78p7Um@g<{N0*+CNEJXD6Bi&uB(*ERh8?6O+`19b*5}P z`QQ|TNQe86uufG*=&Y@Npo-3`%3IQ)CimB8PFB*EU0i0Z%5FXjR6TINw3579tAMF0 z#d6u3`roz!Vr{@sU3RT3_OsB6&MD>Dxym6@j9p$L#eAfTRD%%!;?hpaUm66>j>6V_ z9bOy?OqxNB9e_&L6fb8XjgF(OZ{SLKRUb#f_k&+N?8*m(m7V4V`!PcM zV<%~dGzSPMp2xKfQEhoDcA8S=Dftk*-1Ce%5u$sX@RS^R9Fp03E3s9)K zxU5Bs9e|qLK&Wp@W2k~84Z5tucOK>$s$WazJLg%CvZZBjD?j=y-&QuBYQ@xLNO`N$ z29kHD_*>G=3)QLmTN&*ac+PI1t^6$2iYV_&wczR-md>h`8?20MSvi^nca=|7g?2S{ zUrj5V3!an(DlIEXrRcmYH2MC-OWCYc8bZ4GBu|hA-oWpw8CI@OdCur6zOC$H)!QmU zD4Sl@2eKL(z(dLxR;Iq<$Ex5~#$yAXqS^pe#6^I#mzn6_r%olJedUd{LL!S=G*eYm zDph5DjNR@AhpII@%WhS1qPSE(JCwgoU0)Rs%H-RsLh@oIYf8c)RxjIG@t>v0D6M-D zv~?P}rAmKwP5KeMZDyVFXlmM>I+!Rvr@1HRcxEwomG}Q1C`LS6ft_f+n6e_(GeXt! z4S82{1(osJ8Tz=H>+k0^I(4qU1N;v|kLX-kGerS9zm|gCzj?w{{#^|Orf^#RhUY7X z>v-z_>KrCJtUY#lTkfyw-tEYVh3rHVJWm1-)rHq%Md}MA8>X1=#NeQdz)P7^;wt&Z zb&60OoX%<5*=8iwPe>ij&Q)!;DoK?;C(BATzRCdCZvEg<=R;XUq7eBWRnK}2dXYWZ zfzL%{(yV%^N>N!B@+|xKMn3COG_fY~Nco$xxANd~c`(X5D&_NcI+9SC`ic?P=WliL zQ6~aL{DyLBd6FKO>M7WZJJsfS>OY}OYSp^Q8u=0oDV{AAtIg(DVstNWGJ`0VPf8QAZr*iK%*CJ&BV1dkhZJ8CM=N z`GS=ft4N>xDbj@Mc`Lvyvgq`FQ6tCDqCK=Ru3LcxzZU)F%)^UmH(l0nQ~*R z0X=1+8r)xgxjNiS`P5NXwt@97WZkOZRYai#e4XZ=e(+z1Zwx{)~JdBX)fhpism&PPE^u^v!gm(_QpEUnXr3`qm?h#01i{NXpB4$@wjS&+w!B{ zS?U#_3S8l&HY?D-gJ`8bcw1m4zc%As3aW>X_7}hb{XfIf4H|Y+eI(MT%{^K)v2h8 zqgsmIYV#b$)64r>=uT(2I>7f_DcQY)?*z~W1wdLBsOr~hv3}LhYig$Eys0YZBr8$X zg*!$4ML<&XJT)6vJ#SU{ptGrZOY8+2f5NRxxwAUhm?__UJ0y*&*~`%I>H4rduiBR6XY+@!A_@0)$r=ns0ngEKn2o?F(jqUk7`)jV`r#ikE)SXr*nqY zDa%aunXEqbN@)t0_kwckb6*G6fuPuwR}<*99kxq%wAWpHUWr)iB4RB|fQY(~se6sG zj#YQumQ&FiST()DlDbbQYcvZ=D@0OB@}>g;_4d4r^~g7H9X#tqt(gc7flqnw2lW6{ zHvr}GtJkMsF5lA^oQ0kvO86bT_$7QP?5I*lbzHhX7@lH(x>vGO55msE6!K%40SwGr%UNS zFZWb+dja287Y)g4aiKVFPs%P*+$MqqZpf-sG2J8&1{zS8_3|i3SD-8%t-L|` zy%n2l&Q&^%c7kG5by0%cQ5Ax^fOX>k(oT|@%IGfQKB~``GOKCxDGCI(cC?5@DDL<@~}s8<3W=Y*qsC z_4v>>bEP;#zP}nNd@46qc`u4asvb_$OPla{Rn|R--Kg`w&b{h^rV6I=3LNDKNQyUR zHL{=;FVg8qF*`{-L0vW8nNZ9??pmE4#95pCH0qkCI-m^hA#J1jB~>zN;+JaAR7RPgaVr`%!2q-=!8(+13A>gAr5M7ZCjqC#stB_yn|ERu( z>iMiFeF(}+hi_`aABT}0v#|U$t6IIjM|wDjX_tN(Nmf)t+Rv=ij_Th252n9R#V1`s%7V~ z-W{pc%F0)V8GRzEmldYD^g3fJN~z8m(!&|hjpo1?@I1|}RyO8FpuCpv{L8v_a9{Of z{uJr4oV%?82jzXb)a^m`^Ec>M^=w(o_m03ls!>h{FXy2t%`{d`hx*BCBDi`!sj}iE zkP!uzR~1INLR6)EC1t4I0`;obo-90d-dC@ZM!-U~dZM>hTr&)g>WNHOhZ^Zm^?_{$ zM06HXzP+-vPasL)#l5~sP0nAm%IWcNv}J2 zf^yumSX(o&S1olv6KHzS2eKn1rFA#S(h5+0dv=-)c9dnjg;hj}Cnz4Ku17itE2gG? zoU*%fcy4)jDAoCOPkB=ZrKG4bTxBaN2e>*bQ^t%c6VyRR(Ur$J0S$vzRN1J>z!!kG z&ULDb_#gQi)6q3Lb5c%Ijt!jvMH+pQFT*S*55_G zp-zjk|CIfyiJ0o7qe)7_uucLsxK9@#-wiug^`h!RB%4QflK)N}GgNny1B5iyLy}cp zWZJ{meX)-vEz~DOezm?lSv^-I@ionJ1N%7#%_ehI*aG1 zKk1FoPTv%N>vVRW-7DMhG}k4eHg#xAlKt8US=0jx%HbMSrO)L$aYlkCoM*SPYIGJ; zPYDZ3Rn4ZV*5wI2#ubXt=={)_mEMBBz5`lR?{8%bN+RnFsS{#3&Sag1xwB-FXmUF| zA`i1V0$+d+RY9kIP&%utn!mipP?q+Vln3K5Qsg>(FV0hr#$k9;ku&*$l~1YKO!c)^ zhb2||$v)F`+?{+wU5o;(MEwgD1yZHY2JZ3~-@3*pn$EixsAK~ZO~h3PxrJB=Gq4zb zV|}XjJc16?n$;Cwo!6vWck`K|JGzTJ2&$!1c807E0~OVMZA@8hsyJ5PB1skXR+K!_$zD~o%6-!*KzV*zr95x4Ci3{b8avh`Z}low zW|pi~d6_g%G#|Mu|B`G$Me*dbQl^ULoyc?Ig(mbp$r?>k7v0BFc>xXJx?7P2Iyb0e zSR1ZV7bW?;G+jqFdKlk*SGcL{XvFvNP=5z4EkZNR zq$BnFDeHF$+Cq8yVfH3_KveU0O5?~nUX-Fab>~u7)y2H8J{H7M6}Fc$M*uB!_7`GIN?GO-%JNtcv& z`B84F?4L|lqb^@r(7p7TqNYv3ntJ~zcBhFGGgzg%Eh(#F1=N3q9g8KCfjYCu0#=@maMQ3C~Kh|oOBxy zk-ig!DXUEt=US`uud=Jldzh#*fubd=*zZcHNfqbir#5-RRo(P2@KV-Ld2XrvkeXt6 z4Y{VC3kmLIFilRCS5wI+dY#O$i9~8YLXxfF+OzDT3KY?sHMM2s9r+o6-JlqyY}DFF zAN3wrk1FYNRXv{Jd#Y|$lvz1lTD^yOiLxl<7tyrsi>!1nw4^%AhER%nPs=x;igD%k zsbb+C^h`r&p$T_Vk2IY^LMo^DGZny577(Zge5!M2$x?N_*J)gGSKY^Z0@YURPG>0fvi0*E z_4!hcbUUD+s8?IQtA1#jV9g-UO<7RBGaVh;(m%FM9hV-jW!aAGP zXOA~R-?Hx{19bKo0CaTjm31=|cs!D#ik?8O2sp^TQVe@7{Qf7fn1#H2o=h(F%brfw z;VZnO{x|dSqWuCzuK-T!pC-+r`f~O5QPg@a@^yAff}Y}ZpsYbz-16yHOUalfXjhuv!Ki0b{%gB8IlMhXCjFxchjvswp(9MUjl^P@Xb_Byn0{0o)Aaly##nDC(B16QE|K`~)Pn zu}67q)PMaT)S)xCy5?L24}LtPTiCa{tuEyg^$s`;Cgl&7&Ju-4va5H-M(((pt55Qh z$Lj?5Q;$-MYjlPUvd$e`p~(=MKqAk7HC9;OTlf?^RyLseSKihH!)Y&qhi%o2*G0$>&&cl5M*9b@XbI(eiHfZeMo<0Imu5_EoOtyNQT@VR zFrr#}^_maDE$XSFu3hR}s~)WKYoAO>5@iLI#~hV!C&{Z5hE9X|?C0MU)LO$i-JlI0 z9v97!m5*^RnA!?o&Vl>pL5pj6s{HoK0%*am)VHV+I~@QANp4p~3aEZncA&ies@qpw zu1*RPihb(bp;~8k9j*oxR3|$sMFX9nN@<8R)~>D$2C`iqCi&!R0NFY~Nm^Z15Z@y~ zmmvpdBO7+IO34awq~b5?sHC|y%lY(sWbJh1hAKnU6LKn+*K)Y-3g1xf&3lX{XW-z>oAg}gHaoDN4;s$)-g_O88VWBHzd zcU6_Bjw6aDs=Jab=L*294bOR$*AOTt24zZj*W|A1VA>LzzXhsq4Fu&CR6o>gev?#F z=0aW8qVu`|{FLRV4l&A0mJg+V3YV+dt)>zjV4uo=QO~oV@#g)Cl$}QP&kRMBybtKg@2=jA%Xyx1?9TyBt{{Vdl#n2MlI!q%UHCL05ejE%p1W z0ZjL?HrcoT!p*W_bqbQDq6mmQO6p3w5}IENT+|Ey7~j|ioOEJRZ2daiu@TDBY(hyz z#h)Y_RO2p>OOXAzsY=@prG5<+Dq=bfTKpJIwj9`q@&tF;Ldu=exl%TO=F9F+;p=x` zHy6D92z|;LS5FOPgQ?50(oN zHJ-{rlfO{bPX>@yXAgBels1*MrkM-X&}*vPQWsgp=){%hQ`D>zM|t-qbzSo#1w3Fw zRIkXUDojO3>K-Xw;7{FE(`}ShBYQ&;Bze`s(2%AF`Y^AR`<*Q4J{Ck=fyBO-$ z#7Wsb<#^O-M%|#6r*yMylAnOTyos70q+9|;jZ`xh$9ti8u6)VLt5#27QLlWxnpmP* zH0e{>_J`QvQJx`xm*(y$)*E>a3y+{u;>MLBQ)Cw9HtvT);rrc|}RCK>2-r}LM( z<}ZQwHFHIAJoV94&u(=yzs$vO&L$J)tB+D^4NkTk94EXlgcT*!WHsm z$bwRrKV^|`X2s%T#jGR+&vN$|_ty#FAD*uK@a?SUXL6uE2kO70VHQCbsyY9eXDRFV z0M}@u)lb|{XAb!!WW%erOT8bI$)d_KWkMcgotm$5glFV(l>?2c9DMPYIylyV15|6J z{L{)nwi}ew2`s8BbYuSN2{zP+Pq`U4aGi2D4gPkr1I;UIh6H(#e7rl6cb)ltGIcDI zuuWR>eRUbD4ZJ0*+VWDi&=9nVa;t}dtG?)byw3a!C@x1jqY$Mhs0>oqcBl zXPsM*Bhyqz|2tSO&u>%i`bVtoDEHKfQN76a@eG{>R99Z!#Y4Fh>aMJe9d$=n_GTt` zmd9RE9reTLg1#KUza7ERNG!y6kn0Zs;pT9T`ob#rNiidJ>QsmFT&z#!^=ckQAHJ_{ z)Z%uXFjYOF2t{M`xoYQ|LIJJ$t0Grv7F=7NbPrg*3mz-)8mrtf^`Dd%MzyH-a;A8d zD5kpgDX&4bAO>}t%D+)Pg=Tvwe^0rM3xUT9cCJ3(oB2yJNtr}Zd?V_Es;ME%(F_aW z8g&9x?}id!_6>Br5lbkS-ANxPJ3&3JWqT^uLm8!i!!2(CV`b1wU#)^Nq@}j-zGerj zUTZgZ*#m4fB}o1D)agpy*7vZ6ZG7`bYz0M5HD&k{yfVs_+zeeP^II7M@^GH#p6ckM z3fv0{@wLwBK@_ z`jdw`>Q$!H4sb@r=wbdHk#Ro;wrsx*=Er{LH4KTLxp5svJzcmnkMTqxwA{MC$BElY~RkO zw^G%v4jT8+MdfZfB+NkjF9r*WqiO=oYVQ9fXW-Ak^BYt;-J0yoywL%v3OxA|$5V~g zmieB&c-C0z7w==H(or|4YoadbfoS!-e zI!lcm#uj6vvDSE(S;AML^iJ@U>HyR^a~D;KO_G1xKij|9zuTLcclCsv4TfK9BNhLG?ms5$eH@k8eSq$pU}F&+_cO4cOjm|J zyg!Oh?_zyznAQECWN$EZD?Jp3^IN-QLuSf0WFGA;;GjpcQPR!)=tfXNr({cJw0Gn? zEt0O}aeJq|$G&2x^X^SxcO==`BfyrXiH;>t;TdS_1MC{jTX>Tve*{%Xwil;7ShZ8u zoib;VRGsRSsM=G>e`SBkA1}XMJ#?J1JoDkm32@#8CbjPeR^qtb%!R+jxXXChxYKx^ zx#`~-0pmvJ0OvE#cbq>s7dl@8+q0Zop@}-K>MoCKmvfPGzH^fEPUo%8-p(%0Z0By{ zZR1hyQDArt3vQl)%Ql|4Mb?wco#sU^W^-jk%M~JD_rH(R3)Dea@O6W)MFH?d07{iQi zMjfMwm(w`rILW8BBgrKC19W3i9OMq<`Vchuh?E6s!bOS!&qE$8r!L}F@Ya+SR!E++ z|6*r9L%Cz^yLsJV4}qGi+J=4II%XZWu3JItw6&h;@PF`k$ckG&JB`m9*-h+j_7nCq z_7tG;k-gkrVQ;jz+57FQcF@jb_H-3`s-!XRx+zrlG%xjP(d^RS;D*)EjwZG1Zu3KiQaMd}1Vxr=7K3 zi(GDZEw|TgyZr7u-P_!)Jx_aH@pSgAb>HDGau>JYj+QVK4+xN25oz{P>JgdNZ(AsS|8Ktt{`pUZ9s%F); z?zfg&7pxdRd#$zBQLB#qHuTaTe(KL!A7D=|c=QcwgTJNw-z(^puj#VV+<3wGke@+D zbtCAgYIHPuLXElnzLTA;aYP&qj2po0k5I+)bYzb8qx3b6Ev+OzcLVE+W&>ZT23+s*&jrh zJ_iqXB5|rRy?i;)eA#}|{sM}ciFfNYrj=iGRDtW;I}^@XuFmeI?o3Z@kJppudB}6f zGs1h)Th~|Yo#c&p=6SyG{N~x|x!{@OY3+I2y~owbb=A4S`LlDfv$k`Ak!QS)<*Ck{ zn(FuklI?q>e9&5A&9UaOug|RetQyvz=6mJ`=5OY9^9--Q&HtH?ns=FZo6nf@&C}*? za})o%t-;n0mTh&kU$x(}f3h#yZRxw9JPw<_2qUPVeFX`#2D#-h>Vns*Mi^;{l-^LpDWGP$(iB&+W5$rYrKJkt8cu7P8b74HwL@Y?90}hP;q6e9dsKq zKQbRM`&`3S!H z*6L^-Fn5~g&4g*22h6b9$*OCAk&GCBxsQ3veEWR$eH*+JeIwIWXY9#J&uNy^HajmX zn9(C6ls++ieA;B+?Vb^?KF$bGxXC%tSfBjb`YF*P{#I;WtZ%$Hep}+ML~f!|;=Y8N zdw*|b*co=Ib%bZEus0>oIc8u93^Cq!e1(ik+QXnr^&sn)eAoU5>YjD2pG9yAx>%f8u2PRQ$L2x%eZA(!`4-8sEp- zN7+a1$DwU?T=P3#O&+r!v%^?2`Sx)8W_!O?$?9xAm$(#P#A|B&k@#Kl7vsLfti-dX z-+JC!Y<*z$vWl#ytcqZJIM$`IGiq{=w!Av z%zDupZQ15*Xe!@ynM2JF%^%^H$=2sduWNyCWk!wcqgfwhxYN%1Zcn=*YkS_V0=poV z*EK6U?QL%d?+))5-Y4A?9MQy*s5`nYHrxE3RXfZr(e~kGp;eJv6AAlCqmJ`KW3R(y zyyZOVKJWGUKle58KJRV>ta`aZuCDIiTy>qBlKrha5(DBL;_KsgCfX%N#dBguB2yym zqTj|EC90d%&36;^6K^Drn5A}g;}d7p`LgpJV-FU=7^5YY#0Spf#^3Y@yK27yZ%;)p z9J11^Ugj5xF^N@)x6Mzjr|o~S6eroA*v;skc+u$QvRxhBRorddTixqC2fdSg!+rC- zhdskR%RJwBTlzlr)$k=eU%Gp{p1~6FJAX7TIlfEwwCh_P%_WJ+i6;^r5?{t0@x`(8 zvG3x4CDxgf~leGsb=+ZSCE z{VqC#*C)~UqjRGzVwK}JC7QrrBdmSa!}e)=6q>cXlSfNVg!8d6Mj#89U_s7FwuZWw zLfc*JZ;*ZUt(&as)?4P=iIMS7V{KxSVvoh&P4q$fH8HcyzsyzkHs?M5mf5W;Y%i=* z_-9_{tWVM(&X|$&pTZ%PODnZ4crUYphZ&j4;f~tI`^i7z4+ry#pSku{$#>y*%&p0o z-7wKOGCPzN-DFL6pHAzWxjJJ*T33I2|FN`F85=U^WK8h)b5C}3w!gL-+b5vI_SWQh z@5l$iYJm=c`N2)$HPM^ng^96=V~JztU-m7=RM#>0-|ktiG0xS-Fy~d*Z=TKGq28+Q zETb~Pr6Gx4iRp>Y6Goy}d{XQ}ba%9E%oneksABFl@3elk2HAHfPdMf}Z*q6{wDdgg zp6m)Z-L6kuH@T;{eeUY6X3lqwb;kdkldv-`yB=~Ka_)6bcWrZj>?!vA;MwCo;5z8+ z>AYl2HCh<=JNhPb?Q78fn8bj@1Bs!DMv2PEBQvqpJZ`m4KIE8SxjEl(z_Hc7!{Il4BXpQL1NM2-BxHSAhGCNA4s+~ znmK=W?r~0Z9>NxU2Jg=bWbI2pcAGujHmw%c93=djM8(7h@xu6)Sk?H;@z3J##5cwp zCGJSdH}kA-g73qO-2AQ;msfl?e`MO$ zWY5?m5l`&>#EQh+XvM(pYn87)QM@7CGda#P&i8=lWn-Rw&K~NV<9j)CarS|%b{W6< zXZt$&N2Gn6_Ox%JtFL``>_n(VaCmT8XhLL7^s(5;*qP{W(Sy;=vEGSc)-TCT#?P+v z?){!uy&ZkI{-6BA{XcjM+?J!iJ;O{l2bc@Zx#sD_OYz;&?;=H!J0n(PZ}eR3WPEGl zdvm@OwcmC0a;|oL=l(HYH>YwR4mfpx2STVgEFwPLH{ zA16LA^O2^e*%FH@U~RIB%@-1X#b!slM*oWb8Cw^>oQRt9?P>DSgtOxf ztw-RQ>Ba%0Hk7!;^{x9B&mEo!9P_>VHqWizS>882P^i5pz9%v$+&t`vREwUE-Wi(~ zyD@%WV!heczL8URm*i?rPT$&}BwfZl=QY6+@?h>djt|I=Z+uQfl>B7Qd7FWNNv zdbD}0b$oE5jrkUq(M0n#a~AgAv(`=6wIAR~y2G)}5k~{k6*Jk{u4gU59;}WmX=AUm zBlb1>Q*4Pp%;H4H#4GWf_@nXX6W^L?Xy)@+)w|60iNUdHkw?Ntcy_o)Y_B!T-6v~8 z(ZcGdYyDQkS*a*zpRb$eobUClVfi@~24=tRi6usd+6L|kt%&{*{~`KpVC}VvS8}dT z312YY%xqj?TEV8g8?zRt{q3)wekH?`IXZ2u+mrk_aU#Ceq>HvO%Q#}!j9(3|E_vem zXV*8B%n7!LIqdJ9?|KKPZAjmoo|m4NcEs1s>vE4UcG-K(wq_%%Uh*DeB09>Ed^}Mh zx-v93xIcI`^mwFQY)Im|wZd_;>t;`5?>VgNC!EBb?d#aZQM0-|f&Pm4_>L49_mtOk-j4 zA^RhA&lK~`M60+r_8zj@i*EdMcI@_eBEBxM5}ak*PoYDev*VoJKD3^*8e2`Qq1K<)?^uIN9hH#UA38Xl z+U@N7I1js$pV?cjmFBucL1IF@T6{t5-dIkocdRb5GnD8Ebk-*xO=Kim#wWz4#M&jQ zB%kth%l@h2^%`H*o?D}F#e(cj-UjZf-nY}g%&M3<+;b!`B2cBcddb#czj#k8#~cyK zD%)8+zid|gG0(`HeT6qw++MIh_v@^QnX@v-W!6lu;HhPQ7HJl!UKR+9kDQNZT908N z+!5~(eLZ|2loNhAa!Y)OJ=!(HeFR<{(SjBdZG1wJ#PxIt@X1V?_<|W^yw z)-^saG0nOwIo(m$Si>1MJK4B9jqMrH0CvH+4US{#%r$EkaSyI4>`|ZjrPK`)G}$|vk6+wtpA!na2DtnpA##L z{So~m+Ay|1wjo~4j9bGUZr6LB*L`FBBm7_ap7r+dY`Wx4sL$i{4}I-p+cCSM5iv#pXb>lIbyzCVZTH z9rnF+Z~Y9v*@Wac`)|v#zOmm*PH^l+pEXWivUZp!uzdcT_&zZ^@oZvB;=RQBL|3!W zy1{-ZIn^=FC^ITM*Kwjbj~t3YtI0GqqIjnyUef3MqYMb z7=P(}afirR^vsv>#!+YVa^g!@qpTMT+f;e9%Ci-lRrogNxvYVikERE`3yuDXaiM2Q z3rc=1X;!u+kQqt~-yE@{cO|x4?;stXH6C!TaqaVX{g0=uN!#jw*Eh!dqo;wV$XyNX zb~1i%q;Ke8aB1j25hFG+{#l~H>X)qTO!M5}JL?NU$AwmgSuOD{ zXOduSe5^P+9XtJ>=mW8CoLE0M>)DMQOP#lPdicisulWD*5B9(5yVv`R`)%hvj_2$F z$gC??0nv`9@C;=;X3!;L71kd(_^SEZcq?z-a5Nt=^))qkIFrYGV0!r8_6U-AvBX`({x z(?~EpDAFX_Dt0jTPP}RS>)7bnhS=%&LNhCQ#;D^S>GAU4Bi`qHTm7!|w&@rAZN2w9 z587+ZvP3O&mD$8HtUl&_aO%y8VdhACkzQ?3(^ zL)MFly79fSM)6;fS9KEo&^Y>U3o>q^-hm*eNRUYvdH zrLsODGkBmZy|hhPhscEFRsYr88HJIe^M%6;)>TMU7+P>X|5Ell{!5N{tSB_LY;ozT z(xs)}mo6@Sv+Rjrm&l-azmwd@E+zrSHgWoOLW?I({9Wcd+MOcY*5($5t~E z{U>xh&@S*{V0>_Y=#NM;cG+y?c+NG%`>20G+O6s1((N>F+F0L>o;RH9lDAsJN#t)8 z8_3QogiC`Rf;qwWgA+o{BA>-xGG9z)J43D;Jm);Kz0do)_~-hc_s{c<_ty0kx(B&_ zbuKrEeA_RXTjDRo=0+PvZ;Bp@hGSC`Ut8ysH#u)}7kiT4mweCr!rqo%k7tc*v@;3y zkFYygwau!Ddhvd-WznaiXQEBx`DQn}iDSF5g6P!qSSr(vTE^3k|0EY#2NS*HLt{Il zH^f|668D+Qt(N#`F5x{I;+UG8Y%j5*=6G{v;!xZh|0vce_HVRuv_<66@YZ0jz%OMV z1|AGAH3xZr%{x$aXq}05(`tQHrF+rW1#c8~DZ0JFyBRLWia^)PE6yD|f5VmUic3pp zmbNJQrnpJji3q*Y{FSm>=h=DZ^Zv-+RB%_tzbb95xU#~%S-rfi97_|gMo)!|!0zHh zR~KF`y8O_Uw%5;<{u5dr8)IE^yx~6U^Jgr``Zwpjyx;QMRCp-goqI0hE$?N=4JId$ z@UT#|;AdsNQg_KiCEt}b3cnCvXg753b8qtA?_ZI&DgB3xewmMF{+n@Y`Yzx5ZjbS# z)i=I4ayj%~@P)wavQ}j;maPdChgL?vO?d5i@{p0|{;zkte{uTxj3+a@WPXzIZ2EP7 zd*3sjDXuxj_sN#lGx2vLfnbHe23WJ zyp=rN-1oW;8?PqcFu#e7j*JRFA08Ds8ofKdIdR&YVzoxPJx)|=Dspfu*8CsIfZfMF zW^J&_@aLA{1v_XiOMHg(Y#jeQo`yGZYUlD=agpT`ycU~aV>FfcU-q_if;)&608-tr|iYj z0VUqzDc3g@FDg47el+m}`@O}p#5X8CH)~RkKktsb!MR&382M>7EM-aP@V3 zV|Iwm2(J#_5;$Mlw)Ck|cfbyvj8(GJou7DOesAW-+1a@te++|||jrsGrls`*{w_4wb& z;FGb1u?J(@q6Z^a!rjA1LeoO`h8&^qf{TLn!*9mcB=7e>kbiI0TWdF|JG%Cm8gr^8 ztM#q1uKMvxOLI25x%&K^^{zks4YH=?9WESG zc}&&Es?M*xx9IT-mvhc$)bst7-{VY8_`&!N$xnpu~$*qw)C1+*UqV(UrPdO{t?)ZpsysUr8#OocdA1R(! z<_uqs)=6}+HYYDS$9jjRm8O@aZ@~)@@_yny;*E0}`rR|by~Iz9Q_2xmM#m<`+TR)pcVGX6jJny4bFJKWbAQO*mNCx%zI%xy%bFOwKm2#V zT^1-=Qu0yhq(GbSU9lHUV(`Xe&W^4}T?1S$*H=!bGu^nJ9B((ay5YMBCz_gV@U^5T zs>gd_q3nvT#P`u9_I2z;?1gx(#L~og^ES@oCx{N*XSR>ejPwju2uvs)ShBKsdhwsd zJxflMZVCP!a~n<4Cl<7?KC14U^^^5>)IC)P@-B+`q&R2lth3D1PB; z*2Valg{P06ZE^XPlDgsh;t8vhbFpt`*7f|`i>6lWS#ep>jYaPk78Vrc)z13Bzs6nL zIoz?-u4eu(dVg?J$=K@?u4P_(@@gx}P)=Ujd*w>;gP~LLrO7W{J$(PoD9U}LpkBo# z74I(ETVZ$Z|FSaDW1hQ>WoB+{UN|!}EikwAP;uh=$>J$x>ESzKx8mK*vbJJDMBS%- z-P5|HeTt4Kr1a&r^y0LK{nNeuJk#7qTyva1Iv%q-nX9m>8pp;)4@EXbKExaOqd7JC zfU~yeC*ReycQRhe+?Dxa=KUF8rv2k>;5u%fj?a#K7@5nE) zw&qm3wc~ap(>dJvFwyI&&Y8}yoWqoRd06Mlj(EP)6$z zALVc2yVKLhH57k=!|rcxkAKezeP3*E{JTV5JP`#}H~UQTf5sYTKQcnLxxDWB?&Gc+ zu6*Zd$N!Q&>@{Y)#Ln2SQ8V&Eqd; zrr3+D(dNZ?v)Jd6ufjis`USUP`}8QjaINRHv}@Vd?zr~s^$$z3gMY-@Iv(>4%^qBE zW91grAE|w<{?W#Zn?BfdccX=Mo~-&_!Asen`hPU)$9k8XztH^jkQ3h?w~zmIB5`W< zx!{#9!MWxr_c!THb9WXFs(Mqc1$FG&Q)+mt#0pFE8|FTkS;=?Uk&~zq=@U8?{5be) z;8NMAWe*4J;PCMN@Ezf$q0d9{(Dd*F;s1ti3dRFdL(8I9&6wkJx7R-{V`|Ri3a=L} zspzkmSvWrb{hU^rU-?cMV-mwd3rc2Q`{&A%%Udsvzcljl*{c^zCWbb~KDA889qu1} zU!>2?oSwBY>$R+gS;sQFW@copOnb}k^Y`$7<$uh7uWzeoi2H3~tG{vjZl0Wvk79cC zxyU==1);o9)zFbpr-&mqJ3igqWDhc0x-WY3(w<1)kUlcKXWF~I)}H5_W0PIYW6{q; z-<9Q+^t}GbwLh-?dHtD^n0oh)AWUA`);^c|6;8v)ecs?E&o9J zBx7^rw&Jyy8l0bhX2_`zPfR@V@~MjFFJGBg>WvhbInIc`R&G{NLFLPp7gWAj`GqPw zt1PWtv*=3hUl~>WS3I@d8;t&T@A!nsP^f-vBqQ>9=;y%NvR!2p1HT4#l)YN|RLPem zr%Ja5hJ`*37f1TUS6PRRe>^v*{gW{!`<}cp1#62wuDH8sQ$el#zB%hN2KcCOWd(K^IoJ=;sUp$$!cFn{S@w~VdYZC9zt8v^JkHl)m2gW~+w@KV-?zToH z=Q$P|)17a)K5&2G8Ry;XZR@?{uI6fJMC^HHwfJk1W})hVPf912%!Gp{7k4lDsr2^1 zxnQO6|oP4~rG|(~9GVy(~mwRYhjhr!so2u5R^JjzS z8lP@#G=95bVg2uF-&w73VTG(&?#|Yi;VmWIuQWVA`gGRG9Vfm$S?TmsXBS+Uc=h43 z=CNm!uelHVXJ^*PD=sXnJfYgOYM)jusDmfWLU6f zS&P!_(#JSUl1Z@Ho~sI-c2C zWP>(sYBRN&nyGc`)V6hN+qP{xwQbzGO&jg*SpVt&?9<-dD!Vh^yg2VU=iQSHY^Io(z2MmT8@WdCztlf}PAARzb?#Rn>EWLa zsa=rORV~WSPTgd)yDE76VQ0cOMIMUU8?`9%bNI%v%H9d$Fn$=*j2vsfG}dY9N+RmP zw(=(6Jy(<`YCe6Xal>2zw0Xb%1!!Up@TdORE5I;_v~z(y6Ky}V23j3}KvuKf0#kXe z579h;Qw)33Kmp%XRd=YsbXTO|zRMf_>WunOxcGA?vSJ~rqW=z)? zqdRG3la|k%V2m{4jeYtCt+m=+*(gWKZNQzUk0u$|K4RIEb zZ|LgWQQ^8c(X-K8!*}2J*0<7k+q=r+cV}{aXLK7QjO*MNNthoQr+qvHKdI) z?&8cQ+83PN#C`G?HH{vMdE5}H5LFD5lLuh@MbcI1I69urNnfSnfdxyXNuI&{{Xd{A z)2+v5PxFECO+TScP}eD$6+`|dAClL|ro2XJqV7|VtNqjxYOGpSZLAhnk3mitCqI;i zNgbrT(!x;okUw-I^iXP}4AUwaxzJH-55v-Q7L=Jm1_wpdh5U&NV{_<6T^Lb`)EaUB#@WcY(!Li*C=nWoBWz7)giGJ?KgF zPs|Au9n#)mjxY{sdzIo++hDoCsPs>1<%vd|7bmg3*&vg^w4{qtzo5lU#_1%i&c-Qin0i22 zuAEoiD<73himKdJziXEM%Gh9zvZmSFot;EHd6CQy-d!Ww$INA#Go$IH)LXJD*&7(( zRBM&-N?WZSRvyZqBqNkFv?I7GxI8!}7z&IDGz`oOgayk6GXxU@L@+kgR(_|x))$(J zRUSypL!S5C^?mp6idYs^Gx~Jq&shp)t&^op441hxs)!L$Z1^cpoGT3skTFEMIbJIw z&j`&89uLefZFM!R(=4p}tZ@WsDYK?y__ne*eUFiebvK|FW&YvtT%x zi|74(B`yp5n2BW5*r8l=&SdYg?YKI8Jwd{2kC0hd$!pwowmMUVg14Vt+pMF1P{vD@ zg2U7Mrb($LaQy~psc9JlSA&P8i^^;5qaoVZkVaKtVz{S#ch`FHi96a`)z>V{Fzea(pmCSQYoP6z16^b)!%y%kgML}I9u%f4y;WAOS>b(L~P)}>ie zHR({ONa$#=OK@cHZg5&?fpkcYQAcWdjW$*^(SzQ^O>t%M4EG)O|A~;ImS!v)T`T%& zMn2=KsN9kH!e{$rcVE|Ct}fG!1h>uj0!-qpvPL$S@u0i9 z_q1=CzhlIi$SzS6qk2TWjFcnJhu;b7=DjEm6q<1jm@L$2M==BXZta_zRV}K-%B!VA z(kywtQeK^^eo`N49SpxU!|n@ZL|dW}=G@Kb*{*VGL@V~E+x7-J?saVbWOLK zfbrEHbK^ZWlqkeTat3vSZqHg=L%|lRx?){*Ts>WFTpxv-d{M3)vl=lVzY}YP8DUy` zB}^_YjSES^vB7b8no3H0Q_HIyLB z$`Xw>lFa%}035P>d|6jW+~=9)tK#1pUMr${L~{6&@C@Ni{n^8scw4zQxJV#%kk`aoKocB%_1OVGXt!VhVYK@-cnbJDgXrghj5Gu1~Hjt_iM-!V$g) z_l{{neCsrgJtSgXz$iq|@ z+Rr31Wq{oj;pTC-xS3oJE|Po74rX((_n25_6nfLs;I`Y~9!J?(t>)%#qnNQnkH@rO zzji`QysI>EMWEiwr*MjEDN2}b>PYKz(N|S3ECF5n_3TC zgO+L=bucCsN42Z^d1HlH&-#k%B0KZR{PZg35lip|gbA(`(QzO3l<@BMR`jj)&GVi2 zmh>cuw-G^?b0yjF^dR!Mv%$8l+15uB{j;$EoV7vVR~|D@;N2wROxJ@_Lj>DwIyld< zU`B4iXXyh@%NAs%QdU`WAWYET>x!PuC@Wb?zV%y)=AA6Mel_UDMR8+9ZwBZ{b?cG*4O|Y`DFX zV3X79@TtCVL-_;zasFSvF8_y{${k~mFuUkJR5M`k3!Eq5GyZ4o2k$iz*X|~+$aBlE zYJoF65UkZ9c0;?2t$|J05cNnmOjqU_^^HhF)=%kU(9!?YZsQ(r(WYn#S^?bulj>%5 zhPoF&pH&~KNopU>tBaUazA&F#VVH}SqTbNu*go76{-^M-_}P8UQ`p%XNrBvGR=vm$J~MWT{mMfcuQ-|9#%a&4vfdSIM<))<4kk*7Mqnz z;6`wbxhLS?wgHbKnaV(U$PZ9almahp7b@H$mIsJ-Q|qi%1YGg4_6&Qwonl9V9iAP^ zf?m+~{M~Ll7*BWz%-2p}bH5~Q@+Y3L3FKOGJ=q_;=$hce&cyF5BZrWsFfTlUsbdRp zHAjHAIRV#qHoiw+^D=l&bM%0AT{{b;YPGgPJEl>3O?{Id(({AwwAuJ*kmhe=jZw+C zsdv_YXic?!Dz83NvZx=_Pue3)fj*h5>`lZsDgx2*fv`|)>#@8Y!dm-3`|E@k4o~z8 z{;$5K-X88TuFd=%HWO0^NYpvIhE)tzak9QZAE?jQpX#NJ9Y$&M6PSB#K+2yFzW*mE zmUbXs*!C*hYxf1KI6vO+ddx+;7=w)SMpH8rp2BxPD@=l;JhY3+z{>1St^`oo_532f z4Sx-%SO(7W(+Q+gkD zlI(>Dwa%(!KG7R$FO`Gxd#Q~SCDoS(Nav&)vLOeRr|L{i)LZG*^_=<}ZMv3K+oaZ3 zGpU_aQOkrGM|blZW~iIVbb2IrPk1MO_ej1Aet*Q1h?vNhk&Pn%MC6aS=V!xyd)AA! z1uqw*Q%J?hV=psX8NI+k z(Di=$X^q$FsSA~Ja*C8oiVJNDmI*RJEszO#c&ktusiAyPIiP(uGT4QQ43wLx!Oama zi}O94uXQ_;!ZU?;_ID1;>pSJC>E7@13H7)tObH4*>g{#r zRq&xsDcfXG-Xfg=I#Wo#Di2dys9tT2mJQs%m%6UsL*8?vbDFP>)Glio^yPX*BgOb? zmb0fgJBTIZztjl26tj%k4n;*{=qfrhLz&k2YBckhX@+yVjH}GQ;F}6th2z3>p}4RO zbA`6-Bf32mL0)&p+fBgB%4^m#mzlbmU@`U)yA?Eoy62oC5$%uQZ1x*S3jeBZmSeg3Mw&5ZpD^& z%O&MqQgP{6r~&%9-N8e_x}g%%b~&3`TyJYG24k=obpV{hoLoD;hfvD3&s9}CDpo)p zGt)EHyUcej?3ce-M5oBusAW-oQKeOkoE0%Eobi|OmGxv19exe_i|S95wTGE+^`V-j zOp?ZaEyB#^d z9P38kGKu*}SD;T&9jSa!1AO~G|Gcns0O#~OcCH zn*u568`BfhhX-y4dr4cB`uZXBv<(LqvNiRVzRNz~pSZTWX>TrHB&J(U{j&dL_@s!q z$g7cQk*gzjMsy7C<^K>i5bK`X`MQ81bzZFGYQvXeGt&dfT0|3PIC}UZ=6U^JZJPQ_ z$*#1Q=Sz1&r9u;f_XAY}Ptrr_I|BEE;fO)Gl_QErjaOHx;o5So9oWt#%!O}K%n!A@o4*aKkv z&O!7nNWY>c09$ARoxmDrGcsvEtFhI|>SO(5`K*;@nlTj-e7JGX$Y@qJ<1jT5&DX{$ z#Q(QOU0laUK#~80W~7Um8?2~@#&KhdQQkPDM}akVS19QecQrX`dj;J_`CUA`Tz6( z34a{XG%_V(P57IzJH9NwsosL#1W!r#71t=CEg#K|XI?_{5=J&B;+&;cO)%@%$it=I z=%K!cDoV4Z4N_Yv4c)R8{4bb46fISkljXeN*#>2wvQim>YB*g_Fuz-k&t^ zy`iw?5gj5=M^yJao+;v3S2kCIE1S4loGi|C?S+}y9dd)+**s!Y08e=%B7F*SS6_8E zzSAnX8K%YCywi&VQQ2YdaJpj>odtY+6Vs3B zR1c;#_krIa92VLMfB2RBEdCImSNJIGaODtRx^4(jd;?ZwrqIpkigYIW2b3gLsgY!5 z;e$uWlaOvhE-bKeGN{s&d-)LEE+gs@;%X*zRTRMn+qWy0!*T z*gRS%^^me%IitKqA2m?zk1pmQ`1zZl3_5CkH52t4D?E?}+tKC*Z5J4{ zH>K(5+y}|)mHFxg)X)`GLM??FdY9Hy_Zm8Ij-Pf$Vl}jK%b+3KO5`MegGc&?4r7~f zUHE-`XMQ`Xf*{)ynahW1ct`MsTJVjzvTQM?3i^gWR3TuE*Qxbn9MQ?&s!dIaSDv)q&DZY*l{uC;_M#gWj_OI$gG5bzeGpZ$( zdU97OhcpIg#RB=0JX(pxXANMMJk@HAJ~rB1V_&D7D_%IyALBm@N5oFv zto}3MJ;V91PVNW%XEr;xkWX+e5Z8$PT~+y+%y{yhUCjDordzX}!BF3JwQXalwoI7~ z=EN=eu(Dr$tGw2N+lXGJtJY7SVr(%JEgD;tQ;2ZrWoAS5lndH~-SlH7#0Iz< zd@G?8ke5}Qo2$a*;!6tY!b_oqP?X=pexTjdL}CM!o{w>Dh7%`&6)E6cT-94*h7m-+ z`V|w6s;KLGX_ER_eWn)FwredA3XnttB@CgO2qg~()og13?DF}IMr$M<*j6(72u^Bz_uvpe(6vzXCL#;kI-+1IRP ztkI4s|Hzl6CsH~2fxK1e0|st!eXAZs_af^_dN-qxImDuzr^IDyEOU`9%N64`vKsS; zxy0t@PYHf8Nvz>sjJetg@W|S-nbg!;KfZOMj%E z!X)!QX_J&$rj-a>*<`IHG$QM8)>`5WC7F?^qid5iJ(T%}y~Jv)iyO$LbN%>4V0EVi zUAQNR!eOojn-wvr)&KM59w=b85G|cI)?9Ncc!y(AOK#Cm>uvOv=nI}I)0LjUHy$dy znos?uR8acj*_|qNM@$S!adI^|R(>U|mi9_x1N9zMS#QW|&O&>JV^ej6y54Hx zzamOUi2kPDM(&ZKDH7h=eiAj>*NFCE^Tlv3Bb|#{3S~|OXnq+wK*ZYBjAiNx*#dsH zLRq8o+5;dOKa@ktzsd+@j&fI-t>y1cV-s*9u0#97G0AK*;gZm_`jYBPBXd&0}B`4>K!(FO<#NX3Wfd(~$;2`qXHv6I|RO`+=m&H0C$$0rJ}T-(Kp?k(xonL70j(^y`uI#993sl<|piXE522HflR{y50$K?CE(pu?{d-76EH2OLtsb82e%p>YU z@7#`f%YJoV@JB_Kj4T{3_)^58g3YD!8%4%<(tp$c)mzmynAuBQv!B=}(QjrXtCQ!6 zHV!&*Jyy-53|Ee-mKF&F;X z+g(@Gg#G+vt{dBl8AA`DN|9@vcUE?DvYtuHsV-33DHD|`N=N0h+*E!dotDbrIbqZo ztsIz^m7rW5VBN4{pmlERlptOZjiJ0<03FFaxX{?P$L6i4#%=wk)?7`Hr-Vv@!Igx% zby6TNc=?3U$X@F-u&)`#5I@0dYJ3cTX!}->1o;r zbX5N;Q8m!^x0#l`ml%s&R+x++Pm|f{jOySqNldk zmymk>VSmTHr=&~1)*Wr_cA@`rqm^3v@^$^Y%Mb<8sqgE+6JYw zv@;Ns7D|r(^D=2qQrz#6f3l=fL0YYA$wWBynAi;!^E4unTi|<=;XzC&Yt3vQve=ol zs4f16-XHFY-s1lC5o;p$hAH9~b^%$#?q)nycVoWUUV0_%k_LyG2VSJjNL!meCm1jN zD`&uU38>-5R4Xs>kZQxO4idcGL%F!yX%E&e;^NAPp8 zN|>+o37VizkV{F$Lk)xF1E10krAjG1Q{SY`2zHX6s_XPK#sKq&Il^iHtx|Pr9h(D?>%|l!^ArtA!1Laf!RyvZkV@5-cPy8W$T)8r!UCAr)yG6C?C{5YwVZ$N@YVR zn7%nRCOPbP`=nb*`Tmqi{*?SErFeR6slSfBrgT59Jx>bPgeR^E?rq)*{>qVcGfv7> zJ9EuU12YT`@8#X*n!pK6dFnLLz}ai9FkY%_B`&Zi<UH#qyxWeCNrsD(-H%# zL;dBUDr4lekCOS=GC~jWn0uiozgP5?4qNX}3-?FPKqqDTuX!_y4LO!xMg;7))=%@C zF-~W+f572-AAA;w4^$6?0-3-r{wNPrOX)SuG4=uC4i(2#VUM%X+!d(rdJ4^5UBqVY z8}3!^rlQHeV^)wi?KkF~|Ep0OLKV7DAEg~uE=t8P8?78#D|J*#Y0vc^#w2sOIlvrf zu0#){V_r55x|j2&VSLtSYP`zH$HBMTnEEj#L)y$>1+6}jg`eR$@ALZddM>%n3(sA) zr%rh13`v=u#5{?qo2gHv?kT`wueuqb$7y%fjZn9}QsR{E;O<6*zM!LC8kik?ge5Lx zlvUbw^AJ&j?dePV1eTA$4Zzp|rIDL7J`nr+qVqTg&YcK(9wrec2MOvfi!!;}Hpw!z0G{ z-*|cVC}A}>kX^xCqlbavDOnYbA?iKpO0Z4f9N5d3fG^An)|YB23$*O!K0BK1N6%%W z`22z)ymj3a&$(xNC~vrTzbD!KTx{j4#2d^`Y8~;(jKA(fHrPy{CXv0!FU zz!9^e8{CJUB*yGun?!l=%F@`JT!2krr!h0=gP5Q(U>g0g_dv-sP9LOZlivpSrguv{ z{O9&B?#Jt|$zO7RyY=&KN@pdJSmjz3mObKZ_~NiOK;pOvf5vN>!?F#?@jU08od0Bt ziSFjx&K`mKv9)#*)7Ys}x!|Vs&1pMR^QFxDv-@}G&)BrAlA-3n8(k>kt0z%d#eHNCuvOWL%zkPsk=g#L%W_&^Zfe=&f`9J(4*kyg z=h&Yw$*#1R;AdIUlkA>U3w8^f6NZRxcMbP2tecAQp7I%C+x%)+9^WkYLP2D!lFhBf z+ElrD$O3i+laTyJ0(e;KEW*H)(eb#Hu&EI!h_-U!cT^c z@V*me{tMHJDn^8C5vs3PbCdpB&7iE8wujZ#<4e?}*b{CWNRjBiO_ zt9_sGtA1*lVv?;qVNs(p9myP@$;kLLBRGng7i8hHoz51X{ZckhmO2>@c=FOAJ-fU+ zG#gk|p5TMD=P9d_ndH)c_W%Cy+f3F1SGB=}%#7jAuv?hQ>})=_JLszuVMR5Ho}RgN zOtnlAk1hKKo|d&;=txTVZkdJ&b4?1;?T*(_)r&>kx5q#eOVfmi9Vfe(S5 z!K0zRazX96=_gCF*M-~eiN16GIuR!#j-d+Zgh@kg5iC?;GxTBE=@H-nZ?+6QQ8^Y$ z2vkU~m`(+%1n-6tK|^*NbB7)XY%l{oojOi#Bj!7! zZJ#yHn5ZpLG&~olLg}IUQWq&ZnA7KieFK5C5~&Hv^M1EXdj0FuFVF8+$s+@kwc1ox z&)}&1F*~!?$oeGaWadtpYh*r@X-@Qm41XeJj2hW85l}*eT>S+CSpLLL^%Z%XmaE*W_FQs2F=eRMLIbL(^cMWh| z6WsiDdaBdTsI1tvAwosD;!7HCa8btK$#X-rsNm2N(}03RzW@$;t%Os{+2- zspM3)%NfDfY#d6&9etLnrfg0Lq%=*t5IC%yv8(ZS!pdjN6>}lxOy+5sPDHni9-r}k zRAj_)-$6I+s>6Ay4^SKS50(uq3s#dmtGCqS%3JAsC|3F@chgLxv9p)HDwOdxjHnYe zGm;2z;rqw)#ht;^)wAC{NNgvZU{6y$ob!gD-VYrQ1k>99t6mVQh8a&MbY#Qivg&Bv zFl#zp$%bH6>|u|xMc8;|5Z#Wh$N0E#VYw?pEaZyfBbj@|B5RReUOg)}mOtZ;wbUBu z=kR>)x15qk1zhS(*FE4;JnVd0jJHCVJw+;K0n`X45T+<?gItnGUVyH)*Go zD)&-zYcs$fq2%Av8l|#6#@bA5rq6ROk@hwS%j91hmh8#YzheUC?Nikq;CKwTTBEm`0I!IjRDdqS4(1B+XZUE~ z*JWK|83dlg-q$OKN^K1?u{hYC|okb;BfQihe;KZp1 zu|WKWJ7a*G%*XW00&4mj%;;2j65+{<{$b8~<3$bDrcRzPP z%pg7y{`Qx>d7G~;yyZFW8FkX`qigb0OcE}IzDf0zkLnD)usH*MIse$ptik3Ubn<7- z_BKi6Ah(l;$p>UM>J9aPKFoAy&$A(RJ^PGlMK>p}+p5uDTcF&RE21AF^i{@FbG`Kv z4m*+dS}0s6gNbBO8JP-fW3CzR@O6d4!e@RHTs0~IYy6izLboGz`#WZw1N8R54a3bw z))4S{a=?wO2ek7g;H+B_xPNUpcmCZO$6_|0?BvITch)%n#{T0qEb@dS%shMg^ zOuKsl75iz71m?I1d|*lcqSx z+2-FKu{xq@con}KHrD^vf6s4)weZ#OB)P8hec27v3+JA>Ucam@(chT^ow>mER*H%$g}Fs-b84DCElfEi?^G6Q z*NiVvY9+v_?j2mQzCn4M-MR&2f4DK)+=p869UP^0*{|(s&QT&Kg-A`Wqd!yc$-_jv z^T3KX^BCKK>}Q09tLDrmt5P%-MLvLM+jhGUI>kxkMd~(vg&EBz1KZ@el`IW@`fzYW zvs0bnnKK583f+t}-x~eQsTS*Cvn%-q?iF!VC#nUNgStd^Co991V>Gx22gnBGIHJ4L z+U{oUHLIhqdTvZM@&fbS4fXC9?KE^q6)GRd2nN#4 zv;t|I|Z>5dJK@u79+zvv(M#jH|=C_)mw;@%`t?>Tc$m$i>sm zoW906^_@~*i!_Hi)qoN8pvyyRm=_K`Vep&U48E5IZT)9_&Sh*rF2udz?sCnz(p(*` zGG^Ot=;p*N^MU38yI~TlliSu{rv;|lfBh3{6CS7#OPGSOKtHQjF{0p&wbJ@z$yPR8 z`O?HiRB3U{NNCN+Q8s}Mjc|ZFWlglEV8T+`xkU`47J_ZKic&EzNr%(ITC2Cc&5?s7n4&QB*^6FV6WZ zIG(q8Co!Ua%=ud{g1*I2rf^ER3 z^Vx+%{9`UJ*MXhN1gRCo4y%KnrfkBj?33y@BH=Uh)V_yPoCKGG{P2|c1~;`QW>u?% zJ;9lT3C}S+)1&a2Cpa~UN~BJ9q++QNyzoUjzU%-t!=p@){tX^<#Yy=)hUMI|1YUi_`fW`I9 z`eLQQLF7Gr&j#5sh`5J{K5&+B(Ycw1Y&eC#D`4}ePv*icZJ=`>%4-%=G(RS0E$pdaJuHTw>{akXGQnl8 zj@8*}gxR|rtj|qA409{x#ghB4JA?OqTPdUQAPOfwXlEKt%yfd zEL(stC_Lb2bDNoZ^c^s`A2GYQ&O$GjUyKnOy7u$?nTJG2vzls1Y0^98oL<-Z24}-H z_7zJ2gJ};q1;0!ht}A8jnwScgCSQ{gR986eE}{p5vp$o_Ll-B5b{4Cg*%|q{jxz%u zu5}3BIbp3a?-|XFvf%JkF&97q{h83n%G4I{A$Op*Xij`3Q|MH-3G(JQJ|6GzC7X|} zkGgX%b%!iR?t};SY-b;4W0|4PEd&g9Hs+Dx&PnJh&cUfO8~Cm#z#r{{sFxAWiDk(} z1U&NXT~<0!@U?I*t7ARDyRL^n{RG&3FI|A9tFU}7lmI6E@N|Fre`&AN!qOL|4-9k) zR+eJb9;TP-#m{vco)zvIu0LE7*BD5K7=9U@+F5@iUwzL7*GFzF^}?#8w?HmFYb>?f zgK0I+iASES8q+tx>3>5Vrk6A4+0MKq48oeisp39iEPIapWxdynYMeG) zpJQIJ8-wSTjW`6<+kqQgNo|JqL%(M}vbzwea2qU0oPnEbCR8Z}s23DPpQo}?W#DM3 z!uK;0Q_lM2D6qj>JGHDb#!YRWwnHBZZre=a9{hZYP_vNtp>N`5oW;;>|Su5 zZieqU4sj<7JO^4M;^hH;G8(UVt2AO?7VEXy+uUb-)AK@GpQ@Gwqht*D(%VC1Xlt-~ zFncg5up=-j@K2ygATl^fTA-D6y0Hbsa#-QeK%lq<>~nsc`;2dd|0MYH_rfB=vU-QX z->9b33(P9TE8ntsys9q(Qr)?z(pGXR?}pA7ZBFIU^F%$7TFKYUWN^Rsbgk# z>XIGld@LBi+*nGU5 z@oc<>Z)ho;tgmn+?11Q99)7)XV5OD8eW_?|2Y1JVKIgu1*jNE~$Y%O}F!BbeGn6}W zKG`c*mwUrirJFoQnh|Oe>>c0(odc1fc;%5%hK%7(2&;u$Tq2ybmoUwR?4DcTPH*xb z^Y05w@NRUaFtx~!&SY{H3D-+N3rv|~%bqiS* z^RqB8o#)V{=uSBM#lS%ug?D+`?nNvl3y=q$EOrlMy9V%gZI4c2FZ_mSdzMwxwDi6D zR7_c)+ppo1oDmL|x!{xjcV`wh>X5H+x3AINsiH(C`#LxZFU>R7E_)F?O;q9$_CU9@ zufm;agOOmawZ7XE34vNb--9E_Ft$6}gWbj6VfUj($-xYSKXeYd1np46s7FBkn!;PB zq;mnD&IRFBS^}QqbKvsc6dpzO9TuQ5&&$De#|`>~*FXoK8;JEiJx zoQ6#I-MWYK)c~HbruhI-{e#&bOoi*6J}18;rZGCPUD%t zRG5IM@Doq^ZT!7BtEOEI`+knWPcj!=O-o`PJde)H7|c=j3EPZq&$Ooxp~fzXT1O-I z!|^r&nfNz+w42+z?f&QpI>M7V8P3+-h{edsrd=8C$<5)*P!)A#E8;8E>F162sQud- z0o>bmrU~}vIHS681w4TyOp|*m)s?Q8XD(DuD3g>f_&-M3hl%n%sh>1ZDgnmLdF=&o z%_YdW)xi?|<*cFJvO9%mqVBHe8R5Q!8P#G`(KPtcr=N2Rj9cag2j`Upd1a^eWJM! zzAQnz9pMGjbfe=~lfjosfY#x%dB?hDw}3kEUwfi88Y+a1<_+r~XD=~_+zV&mILy-f z+KG5J8rVafcf@OWMKa_Dr>Z?0yz;j8IA;ZF_O(Q7?|3 zLfr$`Y8TwV%Ym2i9$Jx+WJzEr`-w~NJgY-?hVyg<_^5A(Zlng-SsrRQc@cN@l0D68 zk6GYpqZ#Vf7vKQJTQlG#ncJ8Rp6@}e*x%_Ou=Lj8uJ^)^ZrTOZ>E-lfcpg5&q&the zSsElQl_H=XwUoiyNxd_8I4$8DbX-z1OQ zdk*vVPvS3EbJr)~wonLp>0j4N;W0m#OJKJ$<>39-5?8}(?{e!AOhelgPKUxv4 zm!GXSVB_2C%rS6dIJz6W8uxy*v<%7tt8+? zFRbO(C}?K>Sd(zQTf(PzHg;-kaY{MSaL~_1JSAq3BK+kQxKd;T=dw8}?t0Mc|A11j z0DS-zA#^@eeq@IGG5ARRqH@x;=)rVvIv0J3Dh|GIXE@F_ zft#z14yh=f@sgdWjc`634gJV{ZZh9h z80uOn-gYN@&Un9ghkAE-j=0_K6tR^1oI8)FnTPYlx<9yz2sfb;oW{;zr?XwzZ%h#G zr!7hB+khtH1){bWb$4?+FLVRBt$x5BW*NJTFmU%8SzB<`$H1ZXG?)jJdCFL1oJG&s z2HdnvV8Hi66}Z-@W#&XB(H831beC@z-d*3-ZKi3`WwDJ z1!vy>sKRtPx(?kBET1@91V^hiYWt<|gKvow-Gf|1YGh|bp;S14o`gPe9Mz0!fXI^v zp6{zs6P71mfa#qXEbW=-!=iEC8`@{Ag4R-#fx_fB++N0@>+G%1(}x4Y6!oi^Rrk=Q zYD>XFAEH%&*P0u%>p{5kHIzNTJnPE6pulep?fn4xrrchs3zy4|`h2KZ4_e*OwPCj@ zag|(7k6`a|*ZI!ETcL&P45E5&_b2ypPgCzP?`iL9Z(Z*@&p1z65AC_{ZsFbq72FL$ z0&kt;7ji#Y6C72N{zj%^x7bwpZFhFMITAKcY_MLTbK}jLm>J5r<^#Y(>0>#VA6^FI z?-$teONzy4%ug%W5jqUFoNVa@&kF7 zd4>+Ua_0 zy@|dETy#m#ZDcpT>TQ9zRM7UQt>I2MOTD9hg%0k7Izp|X#(?ejT3N5GQm!eiT16cL z_DdN}(nO;J*e0C444W$EIgN>8`fLrja%W>Q(I3HZ`T$I{8FpRtbaJC-ziKIF zthpcl&5ObO<;*W|9bW>>jDlC-IQ1T8L<_Yft*G7uufAYl{ilzCaw#hmL$SzzU%}G2 z=ClUp+7TJ$AbA|ld2;PHQNdTIhrEch2gU!63V(zt%tCML~P>&iH7$?cC6HB?51IOfI2@!ljO6 z74|l_jBg@*1^d30YaMtZ-NY|q8Fy~CC0-Kuh{wdOa9(K)BzK>yuIrc3Q+R}aP~#H0 z0i1`M!JcDIK=*o-bd$@lPv9(YrdIGN++cpieEpZcU!SA*)SK(EP+CpXifZ8+yas?M z)qw&oN$26_l*=e$G=vT(i+R|rjw|6u2B~j%#ywdI)qYKE1+bj5II&-d667lK2icQ) zN@WK>tRvWN1rc>9+5&&V08aFc@G^f}5}3x(`SV*(uCib8?+L*I<(Q&+F2=f_oF zr**;kFRKlMR`Q1SN3*qxxFfIh?5HCS8K2-$-`l(h1<-QT&~1Q0OLlGS<$LGU!Zs0s zYz4OIKyZ`FLqS!IY>GNzJhIVSJmZ_mGUR7$m^nwhLf_t0I$E;xHK{+17B+`g&bOX9J-4|M;v-CEcJA&SgxZ8m$ z3taUPGGQDkVt3Lj^yHOM!z@Qle+n_oK;=3WIR5qWjFl3a`_#Bt8 zZRQnpYM1c(ioFBBv5_Tc=R~C*htGEry(f*`Kn>wR|97`e0+c(05ZU8_E5>6!)EjQ~ zW3Z3uBu?G~)I8hJr%uQAovpw_FJY5T1=Kp{uxDsEI=yYEH?KioauZuh;+&U=tPZe+ zop?8!tgX<`rsJgpmwN?tVIi0?!@xqXZdHQDEjO6wZSnsrthZKPbZmRzw09mj_TTLY zAML`}E|TID0K0WDzT;f{JP+=CJE3{|0nJ-HIT82Z4621OsD10>?li%Z+K=pjT@e9d z1DL@1;I#jDPx#+0#Ubq78V7CX8)VBtc7JT1sf%pMV6(wjeD9}tci$}*{_Q=msb)R6 z+_&MF9Er^-3vm)k;2j)<%H$`$YZGwQwjt6-Qe~*#n5dn>3HG2anM=RIovq501md|4 z?Ae*9;fLS^_n=1s|LjJW#)-c{?Vy$+r;est;Us58HUEfQ2TpT4+~GXfZV`<*e-77q z5O!P@!F&I^9egxUfr^+WJp;qMFHopT=p;P0fh$U5e^NBGJ`aE;Cs-|^_ldJw;j3ZT zIMNS0N1Ea5_PElS5o5oY&%pQJgKGLIwr*6#?=Hf9`2r15lwBNH=Po!aXGLZ_25q^D z?RL@FZ#En~+*qaMa{t8pgI&o{Y2J_pvqIZfm`py zo$!-Su&3eg7UjR&h@ydGd$1i<#=bBf_)}^8%HO;32eEWMlo`FS;U@vRmHzI1DdY%F zsx9O7!M=+V@D_KV06zOoyPf?DsL3Q?W~;GfI~p8kCBxJ{eiv zOJ2u(H6Naivv?ZjV0+77<<$u|r~iQxcck4GU2+Zl7>-l61&V=qoI=5VY|R7ymc@#; z!mRwbmyN($YJ%r^DAfP|;+Z)J6vM&}3LY431Zv#oc5| zvmuocTYgTV{vLwA*A_p|K>nPC7}x|)Sr$a!}ZF*7Nvm7%h;N;9@~m) zJNckFz6H(MI^eXU@tpnL^;Ou`!K!?NZ9lg46EP|kRg;OYz9BAT1%_M%ah$=uyn{P? z6?e8MG_Tij4zfYHo&l49f3V%=J`}GdfRuH>uAWL@`TyO=oDX~9s$frC3B;q8$O?0a z5%_OBqFOWjdQNQNdklX2OKcnLge^Ze5xeFA$N&4M3&FEI2LJ7XtJDquVgvZ`&cL6b z$cl(81<-rsBa2{P&Q$UR`5yJ$YjQGG9Bl2@)H_6?8OX)gsSLM~>r&R2s1VfJp z7t6vXf}F@B+nnOaeDle|PFAQ*D_FCfx=`Mhc6@Mz)lh4`w%=pV$yo3Xf1-|fjT*8T zo`%X!V<0J?6!H2IIClp zFcZ&0Mf_U0vjv#OL16sPu#4d(&Ujz+H9v9oGa&}t#s(Q19Z$Md-9Br-x34104M&YS z3|k@A|9{`e2y7z1N+!`Y*fwlVCV)s)p5DPU;fnLs_?6rPsQM;)$$_TyX$h z_GKX19mql0brcDN>NKACmB@URp$1)zGwnyc{r9yHch9sl;Mo+>E#E}HTm@G$8aU_% zJZCErQ4T>vHXX6%Kg@O(0IQmA6@dEvH}s7zM57MqL}l}gNL##xgc1uhTuiClJ) zS-?&u8@%2Z=&GPRhpj0fAXW&;h~gZN(-TF17CifypB><>=)4m>Hz*zk1;h>G8iK^#1Q zD}NlUnrU_{Dve3VN$0W2uQoXP8?fsu0s9BvJDo7M{EY1~M^Np=1A8iif3F*I=_llp z9*FRT9FLP?yWs<%V*}j=+`%yW7;36M$oY%0J*Xr$RT0<^H3j;$VNe79gCD1`K_Fyy zFpHS7p#h7ph^-)^x!u^R&(|Wfnp%6*#_P2|VGS)?U zv$)Zxc?od}`kAk<;n1z4Zz*JK(xdgM+IzK~ngNQpF4_V;%q#$% zMG~AQ${U&Bai5*|K)yzn{|ZECZeu<8zk>A`oE2AN>X(W+!$N~IN*R}MO7mE2tOJOiv0zy( z!a0e9p9pUswES=rd1vm#dnJr1+7k5beW1NLt7eorN#5c`-t=o;&L6Lkahx#7(=960NAa|Nzd zHoGVvo90=c$qCljvwEz{!d}O~*=yl!z zqdyLxgEqu7>}MN94d^AF86iJEV50>3^W_t zL$DEW1>Si+q9vHf#~6iu&8^^L;VN(u8-fN4orP(_FJZAOr`Sbg#PY5j!Z$9NEzcff zy23NGC*6d4O4M?$1AX*hBS=YO0~jTfwZCk=Gs-==CAbXXP^s2Xhhc95@z;%Co2-`t z)AJlqdCjI!Nj*ll6oqNU7ED$eGcDN=m(EXvVyPD^G1Z`lE6vvy1__<`?Ce@=I#!3)W?0Gp0cu^{Pfs=)gakgRu!_jFA<4sQhMm@JKh<`%y>RR!-!x^FT)` z0MF}H}Ca0=EUW}7moNf&qN~SZj{kh}(abY%mp%#g0Vh~yHgR8tNB$RfI zcAaw7b9EJ>`6Fy^_)lIUH)4LMB6kn42AHwNA8n_)O%cFbn#vOi*ha9&BH$>K&nXT?`f>2k)-!LA;f^x5n7d2~Fmo3|LHrsn6!r1F z>Z9siMz?0Fv!k)Egr&d0+2oW}40*p2xH=E;JgQ(Nlz@7CxpCAyZT*j%#bG z9Vbni+O}=mw%vbiq_(Zpwym#LnmToAH(2lN%=b&5JWbnRcV_Or2k&{$dpc_4ndaIM zK1qL6Tw=)c&T)c5@ru(y60TA~IR>&)Rd`Bn+ZTS#6lT3P)H?gGsck`HWpy^M)BzR$ zj`WkhL@({A!{lg(_V0+)?5W|YInjw&mRj0jaqseVMa!+Vkm5h5j@D``T?2N9?}&UE zaV$8_+R>6VV0h?*$oS}(=);j}*x2A8cuTcyooq8~Tj}*|WDYd!5ua%n)JDEXp1ky# zg*evOUG~?o#gZo`$}@+6ys3d2hUl|GCf1 zy}`zdrVEzmw$Xttf{F#T4*X-AZ(WBrSVhZV%VJ9>%Otbk=ul2ci-fCMBY%68OKzhL zw7{Fr)6~_|k<-2$g|K(2YwQypwVflJ<(!8d6CKYT23I;ysPD6y3qDlZTu9V~z?cH1 z5%PDXl2J8^risRRh7{#3?8?%{By>0`n`#?RQ`4M8wepzsRvJV^ve!x-8G0`4mADsFnWnIdO z)I`T=_hfc^VQfQly&D)&4)K`rYhdGu`cc=?)eG$x(A!!npnYhzsFxY4WvrCGQ~0W& zfi~SDfoCqV4MmS3&X^>{X{~)W&r{gnb(}36bL={b(3Mi>rc_Vf`gho$d4EbJ1*G%*{GX z4+gs#W@=B)M_cOk9jU?(M?+wvdR#T)gPMW>ob+cU&$+LKfduVgT4Fn~5fy?0)XjcL z7r~#Zf+5D!ubo2)QGUytWUrJNhTufH&>u4w_8ZtpdVg-}rnR`2O3cX1 z(cfBkM)wuhc=TUpx;~*hFbyvJK6j#Ph;y60a%zs$r}mq!IzEU0yPlWHsBd6av?GU` zESI$XFMNG;WQO2$H3Dato*B>BW`>(HEXq_W!{dmHK^{w>Nk-#pf$120_2#B$vP*mH z@xs>cW6$R3=SZ->OD&ojn3_5DK+2rt>`BA^Zb%xJ8t!V~%>{>U3tiUV=nK27<&@?c zdN7x(uXP78KMEdbLQb^}4jdUYGH7AoXIr#&mN~{`H(oM2jDL-ljq8*yQW>Em^Rzzt zGWwc&r+eav_&>RNxfa3PIAYIhZ(@Igw*Cd@c4h-^a6fb#sZDM5mQb0Qrf*{wOf7PL zO-w=Q|C&3KeXEXXj zzra=2nF2dXTaV}Rg0%1V<^YSog92bnPZV}>w|la?mpeObiSqnABbBvtAoLkM39Mg7 zH0Nr-?i5{nsbgNHFYdm#iGPm}&&;Z=@-1Pz_FFG*oF1Gh{kV+j(^n07W)3%Iv@{Eu z7L_fdFS=v8Ho;L=moXL|aDCGgb8$<1Q;ghRo9r3u%;oHeqRCs=8dU4&CVxmqL*4Nf zm7w6%_9>%NKRO<{d%)DWM0S1!2G9uDXusr+#=YiZ*1guY)`ON_mPeL&t7_{NkUd~6 zwbBOYk#{k7N7ZkI;UZ78khDPfjec+p>N%k(5sh+WV8`6CJM9G=6&<VLs_Rxm&2q(!W-tkFW^*NC5Bk9?I+9Kz{I*wsA9fi=G|!W{PfJvyQ00N z!k1Gl3bV@q(i|%mr5=@9ys&#k%E3^>v)Ho#UashohkDj;Dz_ zQ8S1c<&Wf;`GnP45#gM%dr0T#8kz0U%R(Dl;teOw3qu~KuaUWDromAULXxc|Oe3f` z-{QG{vlOtHjHiTG-oDPK_EnB>*IK3tR<@r>*_2WlMboI%amiJa?j_AjjYf4lj*KLn zO6L||cTW2nshwesDbzC0Vzvx6molfAJ6pfnegs6Ky*k>q+WO5>-_q4Q#&pn_-*}bh zx=udIj1f`u<6Xx3s(2IKE6_+t?>z5l=J;lxL`T^ydz{@&$5dO#A4fbos>40=VStCD z`q-A;QVfi0i?~C&EI(0>8zdNBrwupgc&leTNAE^QQ*G00-f1y>-788zbV$E~(yRxM zd_2JnH`>N7xE0>ryi4^9atDzQ_fzH&5t z?KUU`|B%iz&Hfoa@F5zY1L5v{BL}+xf^n0$a}3kC8v5=rl`_)P*nP+K%Qe<*@kDrL zphEQ0xzTB)o4F3BG15JSzNz%iLXQ8@36$a->)GpzA{uTb>QZ4@L!GIQ&|R4p5R$jB3p^jF1QrcwK@U?574q`tAhTfZXi|(0y4d6OrCJP~9GQLBQ5pTo z&RpyK&k=2}nrbBW3b(hnce4+%ud{b^jCRg(4RE(c!Q>3{m9jCvv;(+XMsb!DFP}%1 zp|K&x@WD`pX&i!Svgy9*j%l#zE6S3+jcttum{hnEP03o+yA!0rO!BNI4kAv8B16xu zouCT0j8*(i4gS0D3EB{o;Hc(@+uB>t3>H^bPtubF1F^|Us>^#2 z?tow)hJ~{S&Tt+071Nl;Z6b%sp|3PPYv^OjCoHF$95-F{>DmYodcttqC2S;`eIgF9#D(3CUXx*Cy0DhHOXqns^j;R8?kP)ba(R!I0UlxDp(<>=E{lsWy8$ z`}@>YsWVbrWE&EHmifqK$iwV-yGUWkEIP!q)4;(hTX>I9vLlx(miT+$&_1z%7}=m=_@ ztUUyA4baX~-3?doqx&@z#e>Q2qjYM;xh6WBIQH4c&{5xjj@d@`g{kjS#-*G}@ulW) z#=4vPCTM?9*iP2nu$M%^YRVl_DJm&_=5#}YhFMBlW&~!BXd7KH z8P*;azbV$(#PD4nDj7uwT&ZEieipTtcZEA6+QnBKs~i&?MVJXx(7qlO>cy#6`%C*i z$2jM4=T~Q4Iw3QAT6;(N+N&k~&9HIc_u?mUK9=pOQpT7OepnaNHq&17XUl5qKsB#LQG%6oay@w$eB8t6CJokF`xl{kRY4uaM^12(3*L<0YaWyL zDKoV@F!8wowTdcK`A>in57UmJF0)MNf699&3du%Hj+Bn_k$RUw`BT9r13B4SgAt-%d*TBO;1A^NIt+!#%I1TYs zR}U}&D%kfQvkiuNUwM6=eXe!(1}VLh8>Bo3D!LKSQf4RFS-L7W~g%V=q%W^DOf(b8E{(OBrh=@aG$r<9wX3)U;GIzcBVU#K0J= zz&yHkVlg3$uKGW#&o}`h6R)1Q%evds-%_3#2QSDME|UA#a^y!_yr*-5Yp6RN?DY4Z z#>`$Qiq_R;dJbLm9Dane^&ZWRkMc4@OVcKJe1$BP$S32#YqBx<^(bihWlJn7e8J|| z2xE3JTtdn41$ukksRqrZ;$1{ffS0n&7wCHp9^M&t%~+y;iTM9MD|rVNN5?cw9{#U^ z+T(B#{Hat=qhUn1WKGKv5!FE@td@TwII=^XMg=m$UrFumTMuKjtLM1;FLPgFQD$!NA1eXCZAv`UT$( zS{RfksBT~nkgi7m*h9aqv&}P%D^Y0Nj>hO9X_`KiIQle^*JHT2AJNblgN;1os=}1^ zZ|3OY0dHOBmO4;6%i+mAYEL9rKldp^0lvX(geoU%h% zjsE6tWvn3qHd8xOF%%0s%O$08V4?j)pO9Pp21?OO?+y~wU9Asd-bO9MX-FlC1L5%2 z@{C7ixP>ReTbpXeC*LMmw8`G1?69-GN_2Ug)m1vN-s`;t1+LK$X*1P{R&q8amtl$F ztf7zL5$u*Qr5B3-o0-lt69$M$$uIvU2OJ8sD?)S&xj;1LQ8U{|?KdN~^9ohc|4_;? z>AS%CCzB7#V9V9?&&(>U3ZG5Y|H8Xi%Vg6Z_{o!cL-Om2LM5!RnN{(O-hh+Rc;0OhUHJQ@ zu~K_#)fuIA@Lw(phhQeOqqf^0HhC89j(?#4GIcmJmAE)~lIc)sy5c$IaihsK9;Qz# zZy4xhDDlxGZ#R&NKHhiUzOc<|f;M$RBk`Gf$Um4J6HA@57WX?&bc*$*NzxS9sqxZq zI=1~xshonoWd%bP!*3cDRV2288R4}6U!`s@o`c9AKg+fq#E4)H@E;x&}Y~w z^rudK5ccqG>egH6w)y}ECw&@%LZ$N-b)eDsxImQ9Yf^i9O_y6CEf(uB98O#Vc3w^P zUtY8^H*t+;)CFn;F_G$<1}AqC9Edaq#B<`@l_-&C;gnp4$G;n%Of}dRSBR6$^uu1G z%J_k+XaF;O5moe&FcB0g-$V6i`g+cTZm)xxKL>=V97x?+^sI|h`+ZISMFK1I7A~kF zO{dP?iJ2i9{W7L69wQediW>~4>3^~g`tV0S4~y$NjF?wUNZZBSsBJL4 z2Ew%~34e43tGkzT79eecL$HnRfOPOcj8sjw=w zYFmyOz5jXJdoOyAc~unHzN78!psTSQ^N7o%7F`w{@i@338MI>5hd*oG^&N0n>T-@h z3n4`FJ7EPHnABELnh#s!11y|NaETAWWEmZ&B^(ix%n@PUi@C9M$Lzk6|5JgN1(oS97R^a{n(l z3l+dgkLVxuQZTPR2pXNJgJIQ#FmrJRGt{5M7m9(w=LBIr!3wvAGq46u)_m&q8)2R_ zrAAyw$g5{18qT2kwIqEcHN_HiS6zjF@f!}pP`Zx_3zzxnOEBIplOr`}eG+-^2`Dw3 zhud^ZDlR$L2RGn9XMx{z30~wVD%5S^%w&e)q;XDkn8D-44&+tmIs1LZb8sv6axITh z$6f$SB!MgYt-YlF9|8ZY9hmPRy&<#a<6wFHBKzA2Bds&FO_8e573$=HFd@2Pg+ldf z)JbEg^k;!5G?_a5docePm~S#a(NAy7=W4`@^FaE4=|*;x1k2(%|E>YbEQmU~$m@Ax zV>#Eo$jwv0c0SXW5zn63569({_zu=YC}(yqD;!Qg;%|Zc0v69t7;ja1uCZhuW6|tv z0E4Fn{EcF$ujl2uerbs?#L96emwEO}h=2dWS$M8VuvNyg8|LwQ4Y*f<`+mw#hQcLl zz)yw4kGl>_`>1vb^!x>PyPP;`Ih>{SC^Og8^1|^t0$(Xuck+2Dd%BDE72ZGu9<3`3 zwt4y)6dxaB^PjROHsdqOq9dl#+jJBzVx+iIjDz{Hf?fp?Rm`^R)+un}3UFs}?3X*( zgEw@+tre;ZemHxF^q=rFv%&zK#MABw1~QX<7!I#;9&tbdcT<#(l5;3{r%hwZ!2M*U z?rh~rY+w@k3b+huJM9$w{@(1z1>jaaz{6T$E$)Mq( z?9x7TS|23Vybe$8E0(Vc6SdcCSM>WrHSsq6N1HVV6H{_=p9k0>bz%Q)g#$T|r??Gn z-B*|xSLyG)Mt$!c{G~rA26qR8y-2MuqmUPtdqy~TlgUjpveTCEv#I=S7JV@t6AyVd zZCU$Np1>>iXuMXDU9w5P%g)|Lg=#48*;Q}C`?S?1*3FK0IRa0s6I0yt(QoyReK`fS z^Wr?Ud_0vAu;b?POjqI?*TPuNEl#0wUPrHt{!eN2sTYZjUBuJxc;-MF^RAjBR{M``xQJ-on5-oH9Mz5?GJMkOX2oky?f-3wzSH=}S@4uK;I@tl3v7)HWEuKp>zd>A|E zsgOmyDy-(tJs?zdI87~xX+{XM(SV=96KToyZ3JI2i6aH0ewQ4-G+HppAm!hL$$DmL z^9uZ?P`F2Tv0jC!S`>q~brMf`oQn8iEd~50T3F8+ysb@5>sW(wXLCE6v~$ex{!+&@;iDzf83$i+DqPsF#3qRDoTZAZ&-<<t(>8LF+H=2qgn`;<|0nI3Fcya1 z8W1qIwp1U3KR77VCC(}gPwXhG?_uZRFJOX=5X1CM{>NGl_S_BqES!o}bVIzr66s*4 zKlE8tYzxzkIUQaJdxYQrp)wdGtmCQJwGOOP9QHjc_u7{gNsBe6>d&;kcZj@>In715+uHc6 zo9I04qOrfNlnV$wTUpHt{tJwjcj2L0~U z=?Z8s#$vT55?5ZUH^%c`PN>=$f{$Q(^PAEukPNLQji&IgmAvcsBYu(kceho#E64=(6 z(mru7=jgkjgTsvxev8>fYWOP5Bk3o7SaZeGs0+1Fi)lxMI9PGRK=}>}4`3vI(y!2+ z|54kC$NZUQIoTg6ehbg4K0Ve`(H{Lu&9jNWiGBm5CmVaOCswi(Hs&-{F@vy^+zw3x z_`Uh{|M1Y`scYS4ddU@Syqev=O>aTWFN!Ouafe}#hf*PH#m)^wp`kUi6lP#cdUF!< zP#b**=Pymu@-Q8J-NX*CeeMcJ^ib~VhrW{Dno?pNJrbRO|MYY)V55XSSi)F3fU~e) zGfTg)Fx#;zgS2njI57VDSfMbX6HLtxR24D_#X%OIU^TuHbBrfX7>yk}O|`BaPvZbt zS#y0Y@%>GBu=#{tL{WbImVQClFFp}+!REZ8&BhmegN4yXf|agcq)#_13ZS!5K(H_i zDTE$89glv5PO2xGz)1~7MQ<9{n(e<2<#3x4*9 z*psS2R!-PeJu)o@69TBaA0hsIPqg@!d#$fk=49Q*kCh^NxvifN3P}HnFSVoUZZ%RL zEVL5Wl8HTlBeRKJe2E=8O{gh85>Mh=v%m{{skNb>uK_XpcsjZtYD?*(KTQ6m2~EYr z_ze%UvEuPc!Pa3Bin`PFzBBMZFSIM;;6I;I5T-p5q|TnR*sDEHyyckc&>G{HJq2Ka3;UQ0RIkK zcsEmr9%>qx&q&Tldph6+AtS3@8SnE`NFr~YA{=G^)fbYq!Z3-^J`~4ewR^xA+=OR( zFFs{H(lWgiG5$PlxK6i_oJHWdJC~aI=F#9d7?6G-bO@W z+0X_U#%XDV%0of4Oy;vg>gl>rOPoh`oKvicvTil{wm#pzes>sNWr z)~Iky^5@_cRta~ppuxgWZLn6Am0k`D+79Qv8&hq%fw7+cpm$S#5RKt&ZjbZin?eB5l3F4zt2=`!u)n$od38m z20QkL$o#rCR88^6!DtPFL;f3vNH@;J1w7w3&d@>B-}eePwVdcfb;OSD6k_2y90nx` zAvXJ~l^0ISC*^tiJm&c1B@QY;?XkUhn)>M|tvGvbDM}WB?3VI)mWE<^sigRZ{>A6` z?&kQagMswyx5f4u#0A)$ z8q$BlD%AN;;X``U8NwP0ue7z=ML4cE#bqEdv&cp#6ZzVRdT)@=jL7UlEr`P|y(66*>?N|*ImFfRpg1#x3v|Urp+_-+Q>*KX1y~8| z6k&>ZMEK?3q8`+8q2gVD?>h8S)GxbpQq~HkuyYeo-I)tsa10JeL$C#l7%42kla_`t z-WarMJGIx|cK9+K!<0H zm|w~wcA={-4u*Rq`RN91*JR-N!)0wzA7zdLrk24>@^7a@h{Ve;;8VMKUq90%tE1RfhohE*G%Jx3xq1 z04!E_qMwm?wCUP8wBthXZ0CfU(n-PX&!ep&BDjLz8BE7!Td=i-Xr!#9cVGi~#&f#y z*ApKm)5Wk&kK_GU(V5m>zpJg#1~8$fD=~9-BH9RgV5Y%FJE9x$YSpo3YxGZ?$jo@R zOk{Dr!9oUOqnGOq=-OzDeY#5aDH59})AxOaUD-jOudQcAvf=w46Bp{7>>#uka&mSD z5+COvQk;(;ok0XShKz18x!d40H82z!Cg_!QA6exrGKizZ?%AnPZzaR5E0kuZj3=%> zteJ_*W|NK76jQOIXSD-hjNgP)AOg4bL&Re-y#7+*r%+KWi7m*WHx`P5?#;yST>|lR zkrjMIh3XHnWP-kdpL!;&ARaD{C5{4>K8V_2OMQ>l6$ONGtj8d*4NzM8j!qKiT_fB1 z%-(sdjiZM@ncVHU{uHDmD;dNYPESUjR}$KAO|Xv{goh$rGVqL-WQt9(rcu0?Q7^CU z*Nd>Kx#*D1#VbVVAJ{R?gmvts?m|Db@b@w&Y_f2M`0+V+wwhSi`zBe`J}dINP< zGbi!ELcZd|{uUOxbAFrBpBgASK{*o0@aE`Mz~x#Ao%Lqe=$6DkM~K@?3!&`Vdql(C zKnGXANEiU>6CivdbDap=^fj4I5qxQGEP5h6Oh%@Wm51H4hkT?bxQLBaI7%F`k!-Fz z`DJ&a*m-&=^|kWE-!-|nk9h65JoSw9W4XBRYG6Yj=<%3AR`m|6(~9-U$NLSzzJ?1o zgzca<;XH{V@S4A}cP-?_)&FIQ)tIW2pw;1PIXHcN*>AVGmkpec+r)7Z^w(78^p4ZL z`b#qZjoi_5GB6+K^Dfyy90t zdWnZQ6Jzua#O7=1+={0A(GNzgViPCu8~2Dct(=>*D;dW*SP0&;nXCTBUhYa4&;zoU zKzxTwyG!h_pVvK%a#{iKm@jyfso)b6c_RBjLfv|A(9eEkcAa^bZ+Zprg*IU9QCQ>c zp!y;{?-|#!l06y0-sw(^R+4qv!M<&&mnDZA&Yh*zLd(KNErD;p&tCt-nM>P`1$gR} z$ob2&dxr8_jf6Z{vI?NIgZb)YGMEb3lX&jHBuwI(^NIUF&ZdL!jp2$vV(I>4?TYX_ z1-NTF+2nmRi?(1RZnBTo(z}<9*G?cim;ox49h~AGlR1CD|MGuuVpy8$TKA zRIYUk=tXT_dp1^N2e0sopZSAzTuD#VZ(h%b--;tD+0XAqlKZYDKAQBeqpu>@QGu)O z#FLyy#{Q1APNDK~kSMJd`?n8Yo%;_RYDA`(fz`;%SB?0z_w)pX6YH*}%jYAx*CWkN zzPT7wZzDZIJ2=yg@e==W7sucfbYVYD;TaqN;nCp-GYyYg@pZE8gY;m2#s{X82GX}o0$K7y zcHSh;eH7V!Ghzsp=ev+QY0Z^a=8RtEnkw$^KqJX ze>vyL*Z>1xZA~pTO-loHC!XpPu--xJy9wY+)p`1hKvWB24ccM{I`i)id^Lbv`x6*t z0`YHJA65lD5$r3iD{nLzzn6IT?mw-)({xB)fGZuZwL;-+9Sr8?_%Sp*$ckiQ@Za!6 zXH#W34-0CTzXY8Ls6c|aPuF}z$anQ2+*@0Gjsm`rES{9M$z>Feyc2%LW_q2ADG!wn z)F<6aA4O7}a(TFO$LO6oEeFW;sRT6r*O`9`9zYi{j-FEo-L}b`^l>P7EhbLZIp_U3 zd44)&R$x(jkqds(Y^*~~tX3yJhtt;-Ni`#hd#KHK9}?k3f-rO-dSAxMB=DOD$PpLt z#D=oIThU7V%(|$06t>IHy1n@KQ@^>Fb$s_e>$;vZwVuCkVTI@M#HX;v|Knt*A%=y( zb0RrInL*u4vY+a}KRSfJE(A9$w`gE4S5{c;CB#U&d9BPdjHB9R1=C5yCLb0?<2zcj z3%YWuw}Aj|;OR{z2Puj_cJPjC`B=yr)+76`!-@uQr(^Kpb#xm(=n!@HA0T5Duy+sW zuB=3T;S8EBadbswhHqVl?uJTK9loK0IGs+}jr1}M^|$u_fk9u1iB|2@%2bo;Q1{BM zT446)qxyA8?c^^ zDm&r zed|+%9X!i0tk0c)Co{d!o^w@>Yu|z8dj$&FkiC+IT}R<7?y}ouDhEgYVO_sL=(cMg zh}hDzCuC0EWOi^{R1vv{0sJE z1F=N`Jm?UZ508k{->_qDz^0B8UU6L>tU)DxC*5^%DEgk|%%zYIwxDMu!M_1r(B1xR zd~QU))>~?OY4gTnsKRDYclib|LD1p7?mgh$g<|SD=A%wVL87HMH$6)`(1l;_pP+vw z@;t?!*#zGhT~Nar_&LW-9gQ}_VP&$xYFck@Y58E@$}GA4N>}Bna?UW#c+nWm#2B}f zAkLw(cNOK)NZBP$6NZER-_i%M?ycFe?}-H^?FKB50CxOOtX*M!4STBq7P=%@e;ifB zM3i=V@)V1*HV-%zw>WF9$iy<}136uJL66(>CoAVfA+lM>lj=wmyn*+rPBd_uXeE*u zbv{V+St<>C$SMP=l%(;jbgE~+>Hl3#Z2DT;ru9M*PosyjJyuF!KNP@|wdRB@$2&}- zo)IF}rMe-|__K4BwfMf-^+z)ctnHqF2X5(JOL(C^Xx=-BF z3YJQ5&fs=@Yz@>VALF&E(<|8&Ov*wwvjr7GSu^tSkt%u&ib#3=3VT`f-&GeeWugEz z=bgS7-*VqF>gu^*SbRl+Z3?UQ5*3flXlO7%0_~XARBiA0GlPMeiS&Ph(k2OZahY7( zaL`zv`N$cJjSY1S>5RdqC#Fp19i}J7yUe9_D+LT!>6J-_k#zx0jFR$fI?eM_k1K)R z-2|}nSggo(>MVBR;30Un)A)q^RF00K46%gX3=zLD9=q0yy5c@`odV&455?maz$?44 zn3>q2hyG0xE=I(5owJm-Hsi2bUh-QnvF~AF3lT|qEM;0J@N+)iv%+`ScQuHC&_bk- zy*=HHzgV>|e%&7d{$gSkQ_yO9ph4ZK zLR=7i^t$Fm0jeaLH&v(_tN)aA?t&=$IfV)0J@DQ8{}dnYp%GS<>ehMkkwv`9Ve<0% zyk0fVVI(nk3K3>5BGg#Al#L)n!Bl|`QiCmu5<)C}x__Cjv=&Ckb|(9kr^0-X6V{&Y zs+pX#rd|W{2aBM!nAdaDbI!ZdHxzbNfL1~;OiyMiSXM{zKiJ_T4GWCVj0=rF;RQT2 zj5AhXPTXoVSohOi9L)@YUvTK+QNA8-c)-Ml{_;y{E;<2)rTgr|P_$G(@k(}7u?7+a zYD7-&h%BP@!|)cP;f>GHerq+UBd#VE&j)5OlD_NSSdvsKJ7qa%HX@RJWCz{JJq8jh zScyz6#0XQ_58JuU#rPTe&8TmwtgeJTT8STRgU`%M|F1#~?-{4LCw<)MP!4&DPFV_l z+vn&-+)lT_Lpnor6fxeRZyW~OWQCT4vwD_z(m}?#i`>33z2`1C8kwc^aM@+*r=LYH z=j}BLNAE=k)tTMsa%?7B9*QmPLghRi`+gKL!Z;M7>)<6%Q`y+d)sM$gq}5rs<7Imj zC+eUIzp&7)z{5Y#@7Dqrud3>5it1J~!U!6#zGw1t3$%_V`u_7ZVS4l=bfzTlZO?MF z)GB%^c=CIOc>d?f>RszCHvc3R z%?EnfnCE#B54N8ZlE|r=irQgCtW_>*#c|p>s=gmVm|b9q4Y4w3v7=A$W^qKG<;Z-Z zu;st;Mm4ZT2YBszgM@=a+0e2m|yvE@WtwGZ5y zdz`M1bezXi0W2txf2oN9jw6xx{|~EH<^cNRG*-1HUxE^XUw$g<;{vF)m+xm9fCJXvR?)WHmeV%X z($X}9IrVfIDu3j=(g3jw2=M`8<33srCUBdCkR=OHgHV5_M{+zNjMHKHpZ@!J3X&7;0Q!SxmlFqpcF-Y@)#VPCGY`%phi*(j?HmtF}eVG;H_s950W`M ziD;9^X)?3c%rK`;Vk6s3)HS^=S9%glxg9&XU06fRxe<$z4sE6?gr0zH_QD|AxD6Gm|ik zF3`oEkKTctwp03I@tIuSD49Q+lgyLN@#g*3>Vd0*SA-l3P6+hdO4w%Glz=tpEc;El zP=w5Ca4CM+@$uA-I}-;q(g$dB)l=T|D3-5>4;hOu$&F&D>TjvH12KO?#it;c?Jlti ztP^^YKo-}C!(fu%gc+8^*)$W&)dH(1PX=Ve;x#04-bmJSS!=?2cO~D>$KGv&`a%d3 z;qQ?Bx4?7I3C#)If{n~ZRFXoTl$+UAr%|_lJ*u0f>MCix$t&bJPY2fJDq&6ble zumX63B305f^z8wDy)v3a9iRCQf10`98)hmR}4&ZYw zs;n;Tv__RCGnPE9Ds&can~QuXC%EJ+@|xP@rE|z$(sJtx*qh!U*PqCid!j)ejGeqk ztl!073|->faE#`tDm@FSOc5F9&Ej3{v3RD@<6evTuGie(>8&<+YNCJ8iSCQB>Scch zFr*>UJ^8Fs+EBx=+_27A$(&-T0cZJwt-q}>nqnDkKds%ZV=M{gOZ0>eHlH+2Ag?MExoSZ1udIiWy zX^`p5_~Kk*5BNZlC{xy_6RRv7g4NPaX&P!dThWBcB<9C&gFu3U^u}Jd!`IFtldnpK z6G?6_kO3?P4@#TwahsTU3z66)yzVvf%Or4>ga7i$7sNitiIuzKhwXTUE9B&3Vcmvn zf7wf;*k^)&57UR|!OM(Mt1#`|4a>B=S`L=xII@KOXpH=a4?Kf9z-x3!K43i}m_NP@ z%kjZqkNlz;*v2f>Qs;7OK7Bt8T2|7_8=I*0m!Uz#6Jl%drH9$fB--8GRxL?oNjOjQnyY6(TE{H$1oE02^n%Z!wF_C3H`tE)aqf8)6KS-8Yg!7jjye5wLV<@>x z8UyD7(PI~r8b9zST{F3PewFc^u>N&YBCtf`&J zwttits3Poz>pWB`uRM^q(qCQ(rQ1~Ltkjb9<$s*1ttiQKL=VV^e; z;9@h!oBkaq{auyNml%eYMIuUpt3Z)15m#o^y0VthJn<#u5+X?04l;rHuyfv`Wh|oe z5kT#Dl=MG%nRVo`D5yYXZ zYl}5N0OP2LHpkj)0486+$z5Drfc(9NYu^y&fgOS18gw;D4)l zc0J{?vXZ8fgZ5ko>6lnqyo@ESL#`ABhO!FuI#Rz!)wm25-9%39Dl{;P;_D`=&G~4n z_MmTkFH_^2gHI%(;gb#xnYzRX?TKXu(J8V|{S9Yh7-&=k?1MwZ@L8#>#*(8CAg;O& zmu3L_?J9YJL~llOX$V(8mKbJ+v>R^u5R~+WF?)D7inZfe$uTG%m1R~{A#86=__t+H z1`b0l?;U+ZM|hRJ;J-D<`7h%~dr|w!3Fq?+SWY2b2FE-KYjUbK9LrLXkGk-1i-C=0 z=gz$3pkE0#qNr-O#&>om%YZ8Y>UjcN{F2!46{=GgP#^pTOS>#Pr55pfDOR!wYt{#Q zaGfgPady);qV}Jh(lqs+hWLUWpu8=Kkh_CIb?1u8gA2Kd=iMNe0qltYd|VhST!pwk z11I}8KEXp&>0}370$F^6I-W>;T>=cMGxyY;SItjmSc6v{0ov7<|L6Uu2>1y5v=wW0 zfY|#acIzFdCy<&;IU<^#Y1}0+jV}EBNV511d zUxS~VBU`+O4ZREl^9ujJ2d2=0D~-adF_4tqu@k>GjyQiZXLT|5VHKWjC?}~QQOHpC z^nCXFIneR%TnSx(Tz6%j$RKuc46)}Sst4PNN#_$Q$AGRK;op7WQj`Jn%nL#!<2eo7 z`!77@e*Ez)PEssS`ZP}_jpwnGOzJNG{tB1QKo$NPA6LQJUa$tQ_|6#+)0KQ|;W<9X zKcTCRMcK>w8pXA(!b_~e50B!f_H+HqSiQT%h!=VNw9}Ww4$bfnGblvNoP+4gil6&H zj&qj^ayTnu#5=s@9S?I&tNvZxK`ImX`0aOOYiW0!g_?pwj{S$c@4>(GT$*QqY70D= z^uz}fM3V|HcXSwg(SZ1_8Sk^2dszMN`Zw~< zSGl|I+*>#iA`Fy&_*n&F$jM~-)A0|(SoLb`o|2q}2!0}wXYqvR@PwzK5t*i^u9wDi zUCwF^#Tuo}A=|(+Tf=&;z~?;%tMuv-tb1p!b2#zUVE$EwJvEwqbO&~H5qB_@{T0JX zj^Lf^;V}mD|Ec^$A5Qr!er6q!)O7wn$xgZt!tt72wVl3}Q zDfQHN;=E(ny0N@sR-VZhcJyQJ^(681bgDqj;1|{4?A7NCdZ`WlqFxkF1n@=+!57!y zdCg^I$8%EQxc|%7#{S!#1BsD)u>J>mc8A$d$EnxuV~?KWd4FVAXJhXb zu7@qG#U0flcj&^CEXh|paj%QH(j7eI#+>L9WC)o#$s)D4@9e!@ScFn|?5tRc>UizS zoQPV)2sN?phlx!-vyWb~%BQ%Bw0hAt?&>QR;Rp8@%i8Y#hovU5&kXDzf&KE3*Nxzv z`|#|maXmHg$;G+Oa?}haP$gOcGH{a5Q>o*o@fkjXAI0$$4|1h*K*7#oX)o~1m*PLt z&hvfn>Q^8a*V%t*+OFeSukk$Vp;QtlU{QjIyFc(mF0i9daw4v9-p>3x?~{lC+VKp# z5eIa`vUcL$isLuiP+6G?29OA6&CT9QvtDO7q36kxpHU;ZidTD$%`@1fOEtA8{scVlhtQy%%C#j`7vE+=oiGkY?kD zaH==reL8Ywzc`mc*bNu2lvbO~#=oPn-4(fqK#;f9oHf*q{@KbS*oc?BpGy6^7(bf@ zM5#J|=ll0N`Ty<8DE3)bY;axdk`o(}=O3!6vr^aD&HLC%OS!gtJo)>e7CCvZ;D4Wk zs2v)3#auk2jKUWz)NQQ58CVBrdCv#fmsgzfa9+JRYd@OxslsWU4ombcd;B_R+*Zyy z^N6_W)1Yxp^bq|wp797D$HCtJfJ0>>J6^*Xea{(+03qtbRZiuMeB^9DWT;&ngr7~+W8e8)kc1jc(wj|!e7L~?( zO(pKC#P76X@2+Bxe*Kp-?BUEvqR!Q&vHhnLlVzcTc#pUxjm7^TcfXmtbWzEu%||gl zvU7D8II-KXn)BiDTw|B}=rS3Gy*owTpA1hk4?A%KyBYm`SP3(*N7;1`n5_p~t?4Cxf_z%|SH0ya2i+_q${fSS?%SQ>Uz!=U@BWyt{JZ42!CT*VK z4|d``R`xx6s0e$kHYX}8`#6~$ewsb;fPKA^Yg~lS9>{%sfmPm!6N4HMR^cWd{wk-g z2Pn{JVzX7~8vi1@Fwt+Z8vgAqKCgsP-kWH%0`-!a#M48l3$=nV)e&`)Qhc-^`U-<_ zvxeQ1=IKo24QqL?%G49=#7+kG%~$fJGOSK>aF_}_hcII9l2r7}*zJti&4O6(uAGWR zSjRi`>m=bh6L`99_EFZZlrHKGk0r$mjtg zxOH$i-|)_9eGL8i7==ApjGr3>W;dF5S%|%k<@GP)19uP=q%peoV!1|PyX$~DzlR6* zgqZ#~eBBrjg-KLPs*!yz;j7`iZ()4b9BP4+P$M5l7Lr>TLJ#K`C5pOD6KViEL5_>k z9r#2!rA$|HDWB<+yvTg`9P$RVJm-oi<$>jtfsYzPrnt(V&VP|SzBJkdQN+WH0O<2{*D0LbYba_F*L$4jbgi-J`yi?`$FTWd{;%QK*f`3}T!45)QkI0ffX6>LG@qf9O- zQJoGUt8Ay9R_lW#B9Mw zg|+TPwH}r^5&c7=_=5PHZ2Sqt+Puc*7J&=70uK6eznfgo1Ge~sN}0{O!_yoE%~9xn zT}HvCi>t7!q^pDLfvdN>lV^jsnYut@-kLnqu+6l`(v=x-wSpQ1hlK11NeOupaww#A z$jjht!O=lsfgf!fQ37g$R>?f$O_Y0vDwpLH>AUy|9#@byRsH52;0bZRa=vr~Iab-1 z*!SE0=x;qkORs{fqPvc#yLX{)h&q{Bv^FBW++;9c>DJf=mbV{mtOR+k(%lec+;6OD z+F|-_%5RP~|2EAxg_>fFN169@hZ%6Fu)`y4s*PeIQ#tj%Zx$+`xvA#9_nxB$KOY>m zG1d0~?^n+o&qI$N<&#(5K0cfJRNe2NrFDhZl}j|zEj$3uOK~NZ*-hioPpV~{gN}!Y z`cPX_MbmHNW@CMJW+%fw^|uZ^UU!S_x|x7 z^|iq!jHeH%6=>TmdIT51%o##dvxq+8UPNCdxYrWo#M!CftR#c(ApIdG(y{Dm+=F4v zKMF&ia;EROx48F&r>^IndoEKz3@Bn{3p5574>5!e58W0zKeSQkr;s`!4T7yfy8?3B&ZFHFVR9LMDuPm4ZZA!u zJLuEr5Ltgs3`_QMKWax_KP+`*|(^pd&a}#rM^CMFk({$rG!)0Zg z+)MJ&E!kBw!C(Iky1s%*Lj64_JZ-#g??c}SbuJzB4lRc0>pb?grv4Wz*^E>69CR>| zF1K1lQ&sTwVZ>j%;a@%=5-5s(3Voxfe4aq zC+a7)-OpVuT&d2(&dvO>xxTtGdn)>FFyTExeq;P<`CmX-uqkv^RFg4|W$|=K5<0rJjc37%e zS6VMxk6UMmnORJPrO3lr6{_=(QEMm9ZAJJT}!co`JnHN<5=m z=#l#Djq$v8HFUmt+;Hu${@??SEazr&!QSs>qgfJq-8N`tWaJw~9Gztz@ z=A9%e7DE!+*&oq;sws64`@?Q8rkjWaCy<@wA&$NShi(z*<`;0_qV$q2CewX_mG~VFadT-KVetx33}yEVCPICrdm(+={Q)f z#UJSp0?(YUhN_d8U6bIAWwn~2D{zNv4L4Ki4f*yI~iN$RQfe4Ww8PNZgb*wxGx z>T)_Wx;DGcxkq_-sT*~_G~Z~qJPJq-Y7+7`#1y(eBx^|9;0;0Mpbvrdf)ax!2Hy zS?n9`&j%a4FS~FPy5^hZ($sDOP|Iy-E^m2i$zqMRzOr<)T*HH9HT7Y>=Uk-+D%4e?{z4`l+p5L|RT4$rm^;#mOinDl$hwdIhPPSVH(k zzf(hXgzt%WxVIhGJq%6elhh1@h?i1`jbdObF87sUww0TYT)vaOhD3$kh{|%1A;G1m zN4T2*uut?Za<_KPbQWZWURP&`Ym-aD_OC-px}{fEQ63eWC}m8?E!Ax$0&WE~4tyNg zGRPJ5JGfft>adsL$q}{Eb&JRvwjpS_b*r(eyqqq9-Bjd9`;Sn$iSobo8NEAQ0ge+C z+OsE>N~)DyDRr3Rlq=qI%{R<{4xWWZ&!`3E4~H~Bk&FXPQ_NK?&n+db^H4C|YOQFU zVp(ZkXXV zKAhnwxQ(${nSzvR8fsFYG%#&=z_ouTmYl!i1RtVg)+Z zu5lJJsfT?F;9o?kx7CSQ)U4niXWQ<1s%xCKG{o_QBedv(tZ9L>Ms~Lc4k$zMA<(YrTMYMhh>P~om74dDF64T zTK-=3t2xl&>W?;97g$)`m`PKY`}svZ!K^O!eT3B*;<37Gx?Ikk&KAxf=V`||M-Jye z*ACA@wV;qenPCdEwznO(^|fV1!MAU~+@O?@y5S?zB}Pt)+8sGRB6p}1m}uT@NJ4?8 zHCNx#ut+&4MGGfX)jiQMB4t5Rp``aoqf)xsgPiqIU(~T%H(Z(BztPPbtWMOrpzR(j zdlbcZ6iuKfrY+_@mQB_oHiNCFwT301xrk}9(PE46R#d11G{BtOdFkM+OP775d(p~`9E z!(7Z|?5;%0pWq{EFbCGdb92F>+71J!2VEA4@I_z4;vI|P%Qj{!bR_et4LY@)KE*Ha z(5^|Bschb+{<#?BN1~c}URW*mBkFxZWK)zWHv8nFZYMQ!GhuOH0Qg|OfndPkv?(;dB;b!r${Lmx8Meri+Z;x_ffxX-%6 z(3w8PMBJ-AdNcr9 zH2g&H4cid&L1Ub8viXg*HwsWgEoBXLg}%N~u7~z*sqa%|#}=oI?)x&=MaQhvZOO+t zZ52~>|4*+}g`}#upZL~PP#{Vo_c4i zQSfkFbV~_vPQKFR^PcYFjnWV~3RU@Jd9nPPv-X{A`?B~H?xKy3^WHq;+xh|KF%(44 ze-|A2mU@0rsaxP+i>L~@L7=*UOtaY!5TKB7QLHwKTc)OE*7CHJz2`k46xer zA1Yw77%Y_5muQN11vcdjtWhiYIoH%?=-X~)t%mw$_~!WLqhFtc3A($z^H9mr$V|5R zPWVPMH9DQQ4wLkrx^lZVIZHS{I{rH5J2$#ykI%c%zf8z2M;UG#+A(1^QT8hp&D8=I zhOLj>n7%;z|D`J#_Auz3&1V_QeDpm5MT4(}j1IXF=rs3|>uV3axjkRp3q0$*#dvDx zJda#89J5mfB=!G$<8Qa*W2tt>eb-HQNzZuC3+5`U@&)_PYg>f<(jH|sD)2ka@69*N zvF7#WM00h^WXo(~`;(SSmKm0_=Juwsh8S6rLd5OVk9vT9MGMpQ*&6z;z6zc@t^ijN zREn*hR^Dzt5v-uIZ?t!k=Qa6zPtRI!5mnSS(gofV{gbo~kY#jPjG|llzUY)(@@1t2 zI;D-2=ENV>#X;a!1wa9Jitka?ZXvIf>WcsA2mSp~Y^?yd=ZNN~d+85d8h^Fz{y23Z z6<`be=c2q`VX+KTc%OotnyIAkLT@h>MWlZ6XsM%E9u1O-AR=dBK}d8`OHt$AL(8&CN7v^JTyXmB{mf;5qT@%D9 zpsmwj>xb$sv||3RCi+>ffR$}O3iSbutK2bX~9IC*3Mk@vE*$lEU9pp3^Ot&|B zOKYeEw=GI;)T?{k%Ny>%u!v$(pGH-B|Cz($nxrdOZ( zr~d0)3lNV;4GoRWA-0wQvjU#m3faPJ7j2$^E(i-joYf*KVfH#u{kbZB()t!lAFP zHcd2rG+r?bREo>h=oF48pL)kBxyJ;mRR1Wonr|&u|1QzhcxKS{KtpYwH<8RXyQi(k z?9GN&;CMK1FV*5Oxh)`sH|XP_Iwdxc{=l*+BDZ8p{5d&9>8wlzbt{C|c|$B{6SuxT~AjHa_f}WUvO1`_!?7nui*vD08K$bTIdVM>d@Z zcc*$AwaSH_nr^={!nwzB*U_7K4rkHaEXtWpaz!warI@FrcZTnxdW+qbrkOVjbpIp0 z-8JOlN^!$H!yH2qLp+GaY%q;k%niO|w3==iOBvf5S}A>Gv_qs1=nbcrdclL}EV-B- z*I!sfPV^8q!ZR@0FVrKR!069SfR*acZ0al{=#jfl7fWR<)gZoZpoZ269P>3b!==>U4^q*43KMP;EbyW*S9LZxwcqCgMa-dQQKOkSn-gwYXV__de7oTCHYC5DiYDSc z?`Ujid$ks;v`zit|E>Q-_aat4B|noNDJP8E%{MI_tTn9#tdp$eZAWaTfc61>nNDLv zQM!h;q$Sqe)jZ5xl9@4IK__O&vCL}^7fT5v>35o`Jwg@#mG1>??=PMvs4u^BO>)h1 zy~Km}ad&aoXMSx%xWvJnk__G(-j%*u%sc2sR<}`u8NytwvUJ&Jmrh6(P^w#~lr zLw-i|@Jh_U+AoBcbr=lKfSr#P4}sIype~r7i0%?N-WmAAw^@;5OosD=PFEpLo6W@8 zUcN1?N*CshIn*D-a}CwUT*DaOKBCb!Fs|aUt`F(=Itr!~jf(se-wW05UqBu=n)*z7 zAuCM$2J}(<5CfDk#@R4-Cz}Ujn`Bde(@bYp@slPR-FHTE@L zH)!oi|LtYf&cyF@9lq#Ciy$xG+!fM z8J~seyf?kCy$^_Pecnu<0g)(1zO{TJa2z{4})KMashqw%;{B_9z zSAI1%uAefQD=;WGU|V&U6Un?j(d#fy?8(Y)gAcNw%K0zmaX;bo`~-s-3je(cs)j|- zbX-9;6iqac8=mJqIGx>KPn{q?>O+?K33h5fe`)`7xSzw}hS!8m7{GkJ1Ly~alV5%i zR`{gpMxgmygbtKk#(~Vd-ocetRSGlvdM65< zz0qA+LZ{S55d9oPlufig_`NmMWZt1$lE`%q<9%PN6^Js&F@}GdI|2I5&0xh;PpIkZj+;hdnAUMxa&s@jky@pa6 zW`Yi2{%tDeqvn<0qq`W#OxusF>q9Vt2zp^^iCd|g(}ib0!e6h&5+%XG`(l(d#^~ww zm!RyXCW051CBki~SK_-qeLES9O-5_DT$8CvH-U4I119Gwp)z-B8TFDO@>H}b2grZP zH-0iB2f%|or(D4b$Dq~Ul(`M@Jl&t%IuEzIBfXhz!1z$2;WW+zVV}X?RRH@s58Lk) z`C1on2#d^lBm3AIKQ#>&?mtXW*@(X?LDu&x{gmC|^S>l7pG7^Z4LsL)Do4#&+svFo z8x4#WFfxX*Vt)`@%@C_dU(q(bC#%dDT?k7&2NS5%D2J4HOeYwkmB`H`Ka`U(*t&r z%JC+=_;!593^2Ls)P;^v?QMnskm!;7+d58la4WrCov899p}-Qxlel4DRKn7v;RJu6 zH|#4Dhgwn#yNXRrVeYh}4tpKGLOHlYN13ZT16=42a@q{=S&R_q8xCD!`?RBpQf-3sMBuQdO8UU*QZkhs*qd z=|1@#(?0&zO*9W{!HBWw zfEfoWl)|_OpbyiCj-l1?arSYqFM~Af0pa^XAJ}Q^ryv~{x3aTCm?T?|UCSeWrt@nT zr|CLAc@q4%Ogvi->W$I#t6k$vz5(OC499U9^^+WIe_YipHpGeMW~=Y$))B zV*=-(0*sg^aB=>}9lQ(1@eF&h38sh}3pD{7RUhueMQqe0YV-rCzW2o*qqq$VT>v3C zg0+rD*Q_L-uM4d5$y5!u!=eb|z72x`y%w&;Mc79_c;*_cKzE*WAvpUa_+!iAkG<#C zJJEm3O|+iO(>)|V-Uu%LlBbT~EM^lk@jEYhvIl4zYD*AN&{^WjES z{Arb*!8iHrfC_0K!j8#^_Ke|s#vm3BWFww1VfBY5^X*sP`4sNdP!1$?bwmq)?bOZjs} z$P-i8Q7JvzlX=$}SeusYmk-->ihcaf{>b#(9B03mv%>>9S1Anf!g#x^)E$b$X=zJN zJBjBW&C_+|G%mn)p5bp?;Jw1&-jw6BR>G`#!a99NRpueg?cJ=yMAo+_)&Tuo)*~J4 zEGPWc6vpZ*bU=IYIaR4Eb%X`ehff#|OQp+C&Rsn|u?xAxpFi)wBtEgJU4>Q33m0!5 ztFoK5i=&Hn2^^N4oXvOW-fG-Q37%YWPVsN7({L(RQ~C5xJVSq0Xd&#bQ`GG~|Ktqb zVgFL*$=~269_DHHvu~@h?(4DjySbMMoP>CInHTvR>-iU2;k&y0ydm{ixkaqRU1~Oc z;kQ>~y{f^KuFKi1K#eRt+^i};IXDw|%40m=R!&^X1nf9CN%vvi-Qg@axm7{#WO?pe zHmqh>*iCggfsNrZwZt2@Bu^Q^j_%-VF~8IMr)}*Ht7;ivUpNsBxLE@^HShS>ieuwl z!Zz%|dutfawv^R73vcc%Cw(O<57E?XogkyvxTBS^g=;wtk5M){!JBpBE&t`+8xlcu z$989=3%UU3cQ~skP|dE-=}Cw7RU|j?3m$8?-3{fskwh$=cst8#NL4Tl{!BLfLTC6z zXRr$No5F33XD{<$YkIK@4%VLOhVZ2Rvxo30W3kS?u;72Q9{Z?7-{Uh=c)clnz`j^> zgZ|Qi;9uFW+sCoAtyr0vJl|(@ANTNUci^xMWrF5obgNpiPCcCap~IC$7{c9)M_ z)Tdi;YO`^I$MB4w(RECOUpA464~sbWrFkMh&wL*YBqQ%whnTPCHH3Yjt6ubN)(hpR8e!*IlRD@Zc=_l|#kEH%lO zSRavH%#3!N4!`aw?5EbSkl$IC?ay%E2Gb#!n6k>8opGG`Tmmz8;lYjcXtgaR za*KuQxiofW@7CLaI^ll`&%u@-`Hzw0Sa@o#8g6MDb+~N`e2~~Y(iZKa>`0s`Oqhu%o8^^-=G(Wt}T|JANJ}D9%maX z@(}A&&e|&G;?#W=^+%RDWe(^`~85XOy_yA{d8hT~ZiQ}GI8&VYFSjV$=PbbYQ{a`rX;dTweMwxH`i-E?C6=tKh zI*e+M0}u8e``Zg;V3TQslQ}!I*P40{}q{tbs2rt&UPf~CMUSl{~BqBjz`tB88_#?kYFt{vx$YJc4idOLT`M>4J#9xj15!lEV>CzQ2@;V zYxt&Bm1t=zYKvX)7aNJ<2J)$=VOvxZo>*}rdhb?u^9>>N*1NUUTYw4ZRd{N@4_46CAj`NV1sR$d9!NyVBbzLSQKZO#Ty z;WN!Dc0Kqz={Ohh#5iN%I~*6Y5mnt4U)o2^2r0d2T5ANf=H-$00qZ@H{SvVhI+QJm zCAwPe#i@9xxpFZu)NR%f@vFVlTqx}kQi1qBuo2B9SNZOlz32SgI|@h&`D;+)nsMmW1u*4SY;HsjPIKJ-A`l zMV-K7PO;LG-*y)(T7!+2@&_pq4d|U#GpVUm-~7w!C1jSST4~JDa*Fp_Z&edwrR``m zbmg8t1yQ(cml119*{sc|A?y&|OM|WLb{q1LkHQbrDO{Jk5{*>D`uoI=*x`{@yj?~v z&;4Nfq8%>{w{u#Bm^Su8>_~28ibw1V)>b0ail{+9BZhGbzftE|C2p4PTBG#)LbP1T zeqh$HcX3V&f?wno(n$}PKo&zB_SuRR$sU*=K2xHn!%8EZvKEPL=mOA$F4k>vBwU); z=yNXMZnUv|OqY=HgX`_D?9FEDqtqJS*k>_{yss4#%`!?Oh}pV;(D>0vE+E9~%k4Hw zMdFcD=2z<4-OyG$N>n{vdO~esz4RZqYL~FdN=qj33vt0Gq5|8BCi02oq<*qQ?#Fqs zm`&_OVZ7U2GtxdoU7?mZTTdmtchnSqwW}H1WRJSf(9Bq2hoVT=49R{gwNbVUul3vZ zOu4??!QN}U5t5|i%;4N;_LhE=9-FVNJ>V2)gl^^}tGImE{v^EM%$ApvInnj4!Gcd} zB{=K~rXtReTtYo7t@Ty1;q&)rT5Nea2MCIWO3WSmu-@FR<9IHlK{xy_@wt4|K4l!Y zThra!z@A~OBm+*ux0}Wap^~&s9A}G0IpLIa%j!udZ5g^{`d};STMx~3=+1P3MdG(V zN=1YtdTwigcv|jj#TXB*i(;IT)##*efz?^x>a4XEybjgQrjHR$$n>Y^ojG@v#6Q6l zmSI`v2p7fnW)|zJv_skeCiV!9Q){CCv8d$73WKGWXey7eL4L78D~M`7+fSvCG|37X zwtYYuAfz@ciQlQgl@ovAOno#D(62nu>aF|i0**!Y8GQ0}aTVS#R@}wy9B4B|MrdtD z3k7WrZqZ|76gjLP^lG)4Ls&y4=$d)fC=2E_-L7p8L^1k4R{J8^K}P)c93u26bFp<* zjuz*c1KH(0;%YLjXd$;$glcms5r4%hfe_nIh&Ij>>7^CsS)+~L?9TEFA;A2qf5ahb zWy1_ElMc!A(6FCoCCbF8Rz|$rGNG0@4NmcJtB5pF+^#>i+e?yoAKY_|TuUkr;#J1X z0<-2R+BIQznD|CsXf-mAODXeEtcB??cl5DZK{?U!pEWQTArx>l z7gv~THLrY3*KwY`oj7w#&4P{z5^rzGa0CtrD~-ICD!&v(E1wS5 zrM48lYrV9fl-nuWLG7jeRk~X0`amx?>iW|C37C)J25$@+rYXztbG* z_|H(0xCTGCguM(Dvy+&GnFg)7agVWBMFp=_!4$FaOHjstZZ*d5jS{lh#jR3g&aJUJ z|Jv!L3f#jd<{rC>%;Zn|wNb~uC9U8jIgO3>NV&9--7I2d6BiQ`RJBgoW2`@gPtr$A zHM-iBl~uw_V}zA~SvuFOTlO1k9Jk`IZGzHem#0ZHz*Os6yfJFV`M~m*iK+1{v8c-h z1hnp{Wb|SZ=5zbKkzgO?e*VvXX8Ns2DuFqKH|7MZlcb7&8q2H?@B^OHCpVuH-bdVx z^1>>iD}1nC)+Urfj}vFbg4Y(3R!bMr@u-Kd{y^5=6zf1ihh>KXKN%*kKVnrh3l)j~7%$Ar-Tvu7#$Z0JA zMs^z8wv}pvVc#V8xN3RrJ5&{>!bPpluU6yUPD3Gnx_A*b&sI7{s|n%O9D6HV57|Cv zI;d{^PUle`JlGhqG783x(S?5rZnBb`_AN1HV_3ks!Dt4GF7N@}o)5Zq$J)fZ+W6M& zC@fS31si~dr{5ll`p*BXy+qEBiRtHo5tRZpsV`*4&Mp>zkc*$P7Jyqjh!3mNe>RM` z*D0K^rZcJI1@YtdjjWCze0Q>Il3;FL;;lz-HR==CjCI@(Ag{15SZ-7UoZvLI3a!PcZ-l;wjva zkW~>M^bdd2wob4o=h^E*!W@u=eXK$?@UXo^Phn*GC+s@lNKA*}t=E(Be`V!*(1$q- z1osTNhKg10%oCLXMfih1^9i-#mE6uhIU%vsDn_HNR)!fj*QsD0rZUr#Ji9YkZC!GL zTKqVOIHU_q>HEBKK9HUQcRh+1E2UWkUJeLkz z;2zyvY3Q=;YyD20WH0KC4XnpRdOcABOzFkzM`aiNIVJrzZWTZ!KEikvm?EA`P5!Ii zh;D($dOJNoGdaVJ6?%0ohc+Glk4I}5x)nOD9n$h^-$K#)T;sVBZFRFdi1X!!N@d4J z^^EH!yvB>3Zr%!EbHWCP-S!UiPW6uR#(0Z);@uhC@yyt4tN!LFs5oUqgg=MHJ!Nh; zcIoSw#yel5aVr=W${DH{Y5pK#!7i= zlHZtlzd+efx0~wNs_d8hQCU48y@ERwMa-5@fB|J6q-L5IUZMd*rL!?YpRXU(2k0Z{ zqIw8#XtHq?-q0g+CEWa-a4NGA_lWFFB$HX%(Y>)tUa!=0JX4-3y_t7B&@oqOPKU?F`!xRw%5xcZ_GL`=u+_$1gpjK6z651Qw z6Ff#=NV-sESYDU4!g{Q((77<33S2L0ZxgU5-RN&_M*X;`JP>YuXJwgkABCTeOgF8h zT!IHNhKea{d~qUqS__oFml27-vBsk^REUntf0+h41rBK`Gr^F#M{Ut1+J*AtH72Me zTa~cFPO$MZ>}pM{;S#XAV{}Q^q7%3yHQ0+NJgSmH-ToQc+kM2mRI-kOyW9aQ-b6Lv zJazMieDy#@W&*Y#rEhmL)!tqBmTmYu2N`AlGwqe#_abYQwtS->Q>!5RKr=lsAW0!gr&CctNu8tRuhUy(h zEPb6G$4I3Hv*9Z$ZlxQY4C9$(`3OzY1H>AMczv&!NT#QNMC8XN=0GcU9zE+n$Xz>P zVGn|ltR^?zM-^*2CvG}_c0Bds(PZL7sP@djBVK~-cn#0eAA3<6oUSPUy?}aGb$r5O zZb&97#1}*lYCT89SxmumijTpHHnBft@pA8|J?8|Qze5dbH>+R5K1P?>d)Q7d=**~x zouo$qmi+|k?ir|Wce4hgf|Hp}&)IbP#?o7#L@)1GI(F_cZDKdxv>G#<=acJAq?+3R zwY8bdWzS5tue_K~^nq{pgQwg@%ulb#D%h{N#68r!2Y?_4xo1UqtL9W*8{nr_@HA(s z;ilC6p0Lwtm<3gaT33o{@89$=ej(#X5AW;&yHSDY^(b7oSty=fpbNMN`g-Y6AKU^@ z<~;j*2L4id_<_05-pd2O$U)CQ2!>!AW;A~=7h2g_>m)q$Sg_95;yijzdci~tXA-R? zPf+G5HDSMOQf4VlmBC7^GEC{G^ibB(N$YUbS7yk4(3Nt@JMaYGs9on5pHb0N>E5l6 zKPXBBbCug>;y=q1oz^DaX-=g*kyUR`^?E$9OFwE;^TF_Up|tUk2&Ff1(?RMi(Xb8t z_`wzAQ%#Aw_E1^x%QH9S?G722&~95pzxP0XyZ~q9qfrrc_1$oN{xkD}M~r~KQ4C*m zi0bZW{*8>BoO;}@$3%8+FR7rB$o%{4rIU%ENEFxB)fT!JTNL3GV(She}{ zDSF$JIeR@(Z<_+Yb`tO9qw6z*o0FHi=Wyz(6RD}c2N%1{iGGZQ-pJQYqQVM9Da(ks zKfy)Gg`(Fs7$EJXG4Q;q^0iEQC%uBBs*shrS1Qrp8t&nI>a$hs6#dA7 z_;`_X`j$TY_pHNRbjwz;vl*>`>A<$!r4Mi_%6;Qmi8xg5&R~6R!~YZM`F_b7{e>1@ zDJw5ug@}vNQJb1cbeD=aHkP=o4S7i}v~j=E8PyYotO@kK9Y;SmyJXQ*5F?e6BBcu` zqQx+yRFG0wA(LSVw5EcjGp9vjy1~C>M(3$D58|C_5PxQaO<{s&r=ce2;vo=j2x z8Hd%0BdQ(BZ7jvxwWK~*f&P+zVB{QR7DLJu(iSiYQgr)D;x97N|DW5cVhtq%oQ*vn zMg+E$SdR<_Hq0H6wbn$OsXxkXn6ucr_58ez{B8-AvJ?fG{=9ix zYW~B>j3%?k^RbDyh*R@%x4QB4o3VP=s7aI-3yH6}Pv3+dC>@l<^Gu;LLY30d=Q0H* z-T~$dUx&YyqF-B`DZS<4KG%|RNH0;3uFY3}w8Uyq(K!!lI~W_o(k4kx=DvsatIpt8{CyJ%h_rrZe^=nNM0AHKaz^6SnTj3cU> z%q`u(IXH_AO@&R*NdB>dw|RrT6R9_spgOxA)Fy=$@K%t;vP{&;57VO%9H3_W_jq=C zH@9~MUNsJVhF-7-Cz5Zj=D+%ho$$B$@UE5NVVE#RP0q$O>K^;3R!jmV3xnbG9nby; zC+PrtI}5zCHs5XN(aXEJe1poUqV9nH^4oJ7W*MEoG~7UekCXW{3?NFBfx`=MJg1TDJFbfk_&%l3b8 z1jqAr6y9JhJYGe<&-`Qz^<%Ayl5Jlg9@#;bGYf0G0kpFPypv&A+^*z;X~>XobI!(7 zEe`*=t2gj1i{Zy~hPl*|UF%EDtS??K9cXo1eBdvf?D9nOb+ApddGoKl`**7p_o58B zd|f;H>hPI+|&pvZ~I|zea0f~W?wH-%Xr0(_JpN1gWO~<`}u%KqX6oB zPe79np{aiuox!|6J8qvzKB*ac#DAfxT~vO;pLi?1!WNH}lE@2+!Dgz(DgQ;hfLGkd zUH$;eVg|8ZYb;b%5Q3EXDWkEhL%289!LS7T6)cJh)O#C{cb4MxUE%q!a8^J4^dW!o z+;7;^H=IR6ZSr>2Hi<7UVwINj>$zc`6e7aN%i5$&7oE;mf9z8%)-FY9D~`|GO*Z@D zC*LQDs7a#w-Q*{K?I3Z?R5J7J?BW5`^XjrQDNLaS@KLsaakRxd^uy0D<-Ac_{h3+T zrMA+T6VQUU8G!9AN3HS^^OQ>CO;U8bQZkOKSc1yjh;l>@x#4uAFlL%lo zXwPn9;#KU{URWGkxDn65F5LJs8K0aonQ<1o)Q|f<5C&w*6zdcQ(^q=2UQ+MfLax#R zyIGK^sFApdUh*^i>Jjjw*06R?;A@=l0`6et7Gk-pvPK3U;y}PS^nH|?#?V$_9Y4dU%6#-;Y(JwM9_{?DDLJq@6i=j*SJVmZdE;9%;N|DdaP zoH5SStT3{*?qU&FE_^uRaSOHk-iBy&n z;Kk_N{^#6~4ftxe-3|8hYr0Lp5ZRchNe@K>^|g5!7I-~Q?FDj_=oGd9or((j&keBx z)8)$YL-a_Op;*&Vdco~y1~1Xg9AdZ*!Xd1gX{Agk`i_dwJW!F_Xf;oyCpeqwA}Vf% zpFI!%_LEtRSuGujBG&LU`>8lb5%G@WY2R_8VzJEixiO8&5pSRloDwMyB0|ZCl8q1b zoD=AeW`aH6-^hZ3OMCqwS$faV+MpWTK-b{5$dh-+rjjMk#w*2)VsqzkAmae9~AVa*h#*wxK)K-<`J$ZuA=Vp z?vifFJsOpV63#|yM#p}+w=|iqmM8d>1oohSv5XFp-Sm$R)G}-Th3=toQydk6=UN^8 znLY_Na7Q@pL&3bFQld?GzNVNPMdcaNF?@D!?0yZ{1;s!!TT;vJVBJSKVhO7MjliBt zXb(c8L&ZWD(H|HQc%3{VIp`nYPv!sY8|QQQjwj7Y+Llz`P6=N^bC;#qm?N9P|3w#fHwTCF{d^3KB12tVb zq3l)LUA=oN7KX5ZSZSrXUWnWF-i=-Y&PZEnJPEIJE&?`Yr3?z2) zbxbZCDsPmqPl>8>U9IOyh{ziAFnw6&rdj4?Nt?Aqw$<6=a=gzTk@a+j-_o3qz80~~ zyUm$J-fI1$8G$YSrb&S#59 z%kWaX!%mEoCB;@&s99Xi=mD(DF0}A|_LdJTPUmI?Z);C8_giNvb*yrfS?~*NuT|5i zqKyvb4&-F5BKff=qp^OCF6(mO4gXTz8D!_SBs0pWpx4v-hfW2ff|~*j16FeXmeymN%@4M^oNsr?<|LkOE@L1@f){1VnTt+)H+?L>?j8p2W16<2J^THevWg`nm z1*4Qyhg024^*TBtN)Ny4-RFMkeC-$|#|in&OxmizUVmrbw4_Igr4qL$OiQ?tFgNjD zQeJoGC6#gRHHx>Dlk*+?VTC&hpt8WxkkxM4G0ljz)Mq zmy~4NW9UJZu^sV0ntn6BWd8c$Yv*rSzpeke;p^^iFMjk&tm^adxt8yL{;7c)p&^n;!7V+|Y`Kt_K-x$GBbSLa1%Dc2y^PM7AYL09F0;Md6SV!V`CCFxXBgzurRTJqCC`OqltN*Xk!3R!KLHoAnFYVDMqs_C5U zKIeJtmBXt>G>*&=wKnQNRF$atkrN|YgsWjwJUd+@R8?6cGJ(|iphbpe1(f8uz7a{A zu)r1Q$*+q)tl?jj{4G!=v{VZ-wwu2)Pxm-{i5lq3wN*YUH65)S9UZ?rRywwzdDRi! zvJ;MQ;>gTucD0(?P#vrGqnEX^qpLC-b-}_eLQdsTpEX$5xah-4tcX?c`oSoF+j?v0c?%-Fd zz}>7S=I=%#Q_os!n?o;9TDuwOA1D#notzp=Sp~oKkME@K6#F#SH`#a3*TVnAzbW~2 zU{Gkc9?Sk;kZvkB)%EVC;hCd*#S}?fB;A*EkJC@hsAfKx`Fy6W8GcE(GiGV3?oso? zvwLQ!S>-Qw9rKdr3AXS@B;82pmQW?(Zv2ZM`uEk}^Z(coUo){|(*C5&N&S2){OJO5 z!L6Y|T6w*)QP(VlmplWL_-`={mZpp&zxtng%=w?Isb`0Gepq~1&hSFv>BHslM^x1w zc~^N%S0-mI$0TN$r_6>=G}7wzLqhOda^2*I#4WA;hx|>G(+6G!<^=nrIpH??nX}<3 z`Qf7nVHB^IqUBpC+Gm$^5Ih4e?QKx?mCQ9Lj$X(H>5WuP{#)*(TxT-oYNeR6Np4K{ zsu!L#Q+m;AxJ-xh!%#%1P;gKnl>9b1GB7AmFqjxT722U)MK$jUjFX$N1-6QZsGhBt zXVE*jSYCrtR3-T&a}$b*Bfu5rf^n@QPsw4frs|mAxQ+f#BRz_~`3u@2w7d?Xekq5d z_?+9pNkJc4US|S1gS&zfoz*U_mtMe315fw4m?#ZXJkA21y5Vgin?>D?8W>$9b<;GS zbW74LPWv*Zdg?T(ibwB`Y#(07Q^b*b)7uo|bOSZH7RX?;vw2Qyd_Ec*1xIFm%QJ}y@VIE{92YXNbV7A=>RP`#zO-k?b zEM|G_LYHU19ZC2u=NQXj^~gd~>7EknEa;x(S?WCy zHZGz?)QRY}snVoc6&;M48D&S_il`V~&s*2s#ktBMDPQQ6cuii}Q$MN6=r3gm9t#`| z3UJqksj?O6>anW=h3Dr*nXefUltGLacNZKKJVdHX~Lj z3Igkt8SdopQB!N{eB!L@TH(4)SNUz%0#`X^QQlOaGv+Frl1I)gdBqP@CZnuo<{)F9 z{xABPGKv#-P&zyux)S;ks><}D|A?vuv?!`Gr|$qWMKrwz({KKy`qzw^b@kC%Ji)Ci zOeLcXIK~6{nKIvzQ(d60RS&9{)CcN2b&c9i-S7CJJ@c@j`YG!3)fPO}c&_;yL1*PEFK(2rqNFAsf7!f!gm>N7B%B)v6V$DWy zDjo}2zd&BF5P4uL9|8PE2;~dB3 zXK2f`0pFZytwW*iD>doz+I?m?wL{~rH)?jWzFT)20(itG{XTE;o_Q>p_4`_~)(@QE zDf(a2`1L79dsL)E5PQphF1Uz?YbgbpRd|+L_QF}wHPAKPwb<3y)x!1J+0;2e?FJ{~ zA2~*@DAf~d2nB4x%FT|QLs8&_R!}<~8iGPd?NBbhs)x2y8Skx~&>VU*eZQU$g~e!U z@{uVlTRw3-Y?TG{bL0l&K1B6@7MNiNCI#nH=A)sj!v4ta$O;R@#RSve6qhnY&dAin z&O~Rb*pZ3uIay=WVClu~YkVRHuC9B~x=YX$rlod9h2e?b!}!l=NbZ#je9TRCDFLm( zt7u2Ipk`8^o%;r^*N|?KZZLtfSUuoIeAG+mi?rOR0e430mBB5+zk=6;-q6&L zmz$iCVJzP1fdO zv4%16^&S@F7ru%zf9;sw-pE5;w-$AZU+F|Ti|wCFBoGZ_VwS^p^i}^+e|2tges;ch zPJ;t7L4CkV&Q`vnkI)#6hW0`gyPdV%d<;)w7V+&O@Qb&hZJ|-2>7i4h@91{aVq)nv zEk8aYQLkjI<-VskYcZkeyV-!9dk?-k8#dHPx(6G>8H}akHU{<0icFR5iP~l>qK$cK z8#R+EtDha$P+!}wB+A8LDt-d@{*|t(e%4iLxKlVauTVDq61t9!357CgxwMv8nI@=c zOlQ_k3lN~m)P%p&XEqIV{crj}qNp^d7Vl9{iGgR=-rj1xU{+p#u#wp?Xx5za2Iw zd~Nt_zEeeXjVK#22YuA3-gTZU?(41z&Zmy5%2z2KMfbCId21_r1ZTBT@H8%F|H(@7m5Efu1aX_5_D$uiWK&Vtu$l*(}5 z3Mh+|L}irYfun>v5=;47eT0u4qq@{}aC|DECpkenDl$(OK2&$k&EcOqTi?MohiaYK zu@2fgb}bSmE5Cljh%@t0N4P^*+)Sz%WkD}fW`y+Ob3OE?b;dGUAljo*nZAsQU1@6g ze=!ltr>_Ige8suTqCE<22~7!g0wD?ummtw2w9``DH=MC>Ep(X}8&w zbDvMw(J7qD=e-O)4CUg^ZP5<0ZrMR^59lFuzh4lp<$+N)&Dex`=s5gLg3%gO@uQgz ztkKCHc0uF+26=h7RI3uSM*81GL>G*fa8(w-DA@`}u^3f| zOmx*ZK@FiGIO`HPD?^n?M|Z~|hpKi_2XXUus#A#jA34rBRydlW+E`gx2M=hMR8!i8 zw`+=TDrC>MKA1JlIMiXPqSv;UbKFgIwO>{50Y(MLVNrUo-ZyWg#ILQy%>BLe929We;+%xgI=mSRFgMKQAEx@xjqcH%#KBlFOCY# zGM}v0S8qAGINZ$4?yP)da$`BEFg=@!Eum=E$(XBO)?Tr5*HOSN4`ve{%owa6T#hVh zb1cCiEa+#nxymv1Z3sK~+>R3*FfEpeA7N}QmTYN+ya%+S2SYp&J7@=ZilsVH{0f}t4)v8>JLExaOl zrGMmhj;E?Xp7=j^gm*`nH$sb87`ZfZTIA};osoY;K8ko6elKjR_Ye~#pF4}F4V3rH z)a@k{ve%l84N(t-hH)c~1S5jm0>1|i1*&4-tY9H@oSfQHtvFg?CJ3}ZENsEz+D`?k z5LMYr@Rp`g3GPE@`VqJ*L*T+ZWTi#;JR^zQ&%zM7%lyV|(qHff>M`@~xcD3O*KNdW zv#5OZgWFlqoNT-&lG&-1(_HNAIU<w!#7){7ePJ@h8f|0O5Y$SQG z-T9SHR8QNbL})CK0E!rw?m&H4|x;VBbn(!&rish?fLVhyH>{DHV(I(4h(_=bwu z-!n`cXu^chQ`o)9_^p}5i~l2XtV6u^+4zmSl9~$XT#)O7;yNi@Y3FFC4q*CjlFQ|Z z_5Kw$GW>IRh6n+* zgu+87gHwaUgZG04LVZK~LJ6U2+GZ@za^1yDgtzD~pGP^SEVb!F=)(*q6M6}s;uY~u zE|@?&gxByOGQglHM@&(KT3;MF^+D7|&itHv(UA(}d;Zi~DwHX6<)g`d+tQ76k?9gX zEO}pYvq;ot)1w5n8pYEXvpp=2QC1`jiW*cAH(+CL(b0Al_5Q`mb0wZ(W0J#1j=m8^ zp3G=HO(YuK0&^h?{KrT#&Q|EpBnloX{0rey4WZ*}2h6yOWChG0#^3ft*TV;6L81Gx zHvQ$3m_*(kZrftC?B2kPjU`hR(d_)7ebVB!D7_}toh=}I?eIPcMn;&A9u$ODkXw}{ zm#;+)dl}!usqEI}?LyQe&%&lXL>+$!T&r|calR2h-9f`9g#&mKzU{yCA3h|%ea^2x zLD^;+5lJZ+l=-mU^>}~7+zl7&ccSF_LT-_FQw};VsrQ^SUESP=!4>m_e*syX7||^v zE@ERuvxrUMtHQQ;R}kAaaGh1FI!?+>C3=C-R)1`C)~hlf*d5x#oY`3Ptq&zT0;>Y0 zg7br~P|4dBs;hm}rs<#bZiWGCbFjG(*4S;BQ7L@46|fUl!q-@Xr7A!@`6)_ih0!4R zOHkk(ddL8_OYu@Rxf<-g_3}C91Tab7qe4DAlZ-y=L8&&)cSkOkWmM+r2l@2~xRk#uj>GS^%ucHGs|L5#g z2cD@H)yd-47kuhgbDjAIJl1vQM(o%eZ1V}`pWecv6|$=Fr^>;ZEx{B@A9dHi%`5Pj z*HA?&Kqr%hEkA;qUVG+z=R{dK2KBuRMDlIH8~*0^ied%+BI0WfdNGV@^(;65TUpCK zoQf)NLe^42o(JEoJr(D!gQLXJP=@Q&EJa@e1!XwCFzDL**RU=167Kof5 zu{L~NSOf0|_eNK|TF_BW?oHR6ZFVp=YtMp10{xOt`D^+`|8c4meL?v8_?sn11nvjc z2G@knX*>1R#z5w}zOPd4^gFY!ui%qhMceoac621l^{4Qnk74?ZLR(j~x}Zon&L~8t`&GB7Q?xeL8Mj%7 z&*qt*9O;MDC#y;yq(yRN<${tHecNeFbr|G`cAQn(GVLS3JW0ApR?!!=h1Vz_ZZ)fs z8;u|;4I|1lP$ z;Xcow5WAaS)-~& zTH#y5N_kgN_1fzg#9lA8Zcy4ea6OSyqsUiIj)q(E7ZY4-!rb$~$GZyxdJ0>85sUo+e&Z-+ z^xlN|^<4_F-#32FGqgvRjh#yn~THggB>x_9?V3G@Hm$#t**Gdcb)2WY}gl zbU$W`)1>io2c3m$<8j$fZAL=gpOkiWi;8$Z}g#gY=>0@)aM8afW5V^ zp$(zA_<1AL7B7@Tzet1;izm$o2jV210YE}glWUE=3zSGC@7HX;0-D#M-lP)r2^te=& zQ{PIRKc-L2w3yGSZ>Q=Q{a2(vJi&X-UBOvSnJR|UliO9x9?X^O^fgSZlCVF1M11S` z-{Wt`_e^M)xG^cee|2&QjqfkoAIy0#X7?9*izVQIj6nJI9(cer`8e5Sefc&T))}cg zuLd7$0{d+Rdgke1g&g61Mxf=GOX;9=MVqh-sN5S=9RIV>qT zq0vEi@K9hzU~=GIpcymE_lA~h=X9sp!_w`c;yLNF>?C?7UUs&3opybM<*<@TEwi1Q z)m`+djFihu&(KxqN?%GcUbX-k!}s8J^g&k#Z!-yIJM4zf*s9IO7FZ26!D7D%YsEUS zi6<~qxrg$XatSRyAGuaCXW*E!nQCqwkV{+MC-;=AqR2N3#?%F@+5%YkW59qNRz-81 zaaB(ys~Qq|8yrJ+@Cwv#a-eab4pWh~1PTWq1mi*(h}5U)V~jy&AFDat+bGtEW6?Zr zF5i(Gq2j25-IQRil#eMw4X7CZD#uCZQIbnX2WnSqp?S{m81?jd+==F)E5XXBM?VgH z31k2beim#IN=v`WOFfUd3N4olVgvbqR0&hN!aYyDQ^LPRl#J>f-6WNfYJBSDsSBlU zn5t5=8nrayW7tQ}0asgUg?Gitc8p1m9L$%z$X6`sbHYoM^RN84_QMlDBR+qEl$biH zjBleqCU89XoAyrcY(B7lMR&0Vdh_X-Iz13PBhlehMfE*8s+$~tISx7|g6LgV{zQp* z2D+KUiJVKL>%13z^JB!o7eM7EqM0t+=`4D?jG6jfV&3ziN}+vZRvCkZf^&o6M2kJR zqtEn*hLbuz)1AbE@)%{VBNm3sET`eD>}u`m>`L$2>P$;Da))D+GMo;ZTquA1gI32F zu$yD#{mYo3lOrUC3K5Tf3w72W!Ji7F*Ps!R(^78!K6(&$f+nvhn>s~!c3u)1MNq9~tf7Uj366c`7fY&HVJe3+A zUJPhwG7-;5rYohDr=VQwa`aJqI>TM-TsOEclifExm%Q)2$=>VUbl#($9iBftN8MFi zH`L}18+GCcI@>*F0evZWSL?uk{w|z|ZHY@0FD4dCnv%3WDVr~!e?xM!pdFf{+r|>B zh;Uz=E89wLH78i)NmmAUGk1Nr(|yG?%~iv7-kHYPoP55cGL3pyCaHlq43(EJ^rVb7 zHIRos%zjH_yg;{pw|)XF?Yf?zXD3f-fzI$K<0x7D@4QkVyNt8y>t#Jm{=}WkxHaxdMaF`WlHe@sJqV&5JrP01b3T>Eg zvkE@oa-yHV$;dkp^Jk@swF}PhCVMPA`q!wT&XEr&Zylf2@6KDUMedcb&cyJ0;aMW8 zM|=w(9iBb>PFSw6$DUg5vCjRDzvXNA{>tVXZ7((A4E|+F*%QAq>*V&28$a^Ke~Q1J z@K@rSq#6EIfz_dn`rqbN`?R=D9^!~`_I8bSH}ZV*H1T%zHeg1_a?d&UIi_a$)kMcl zWsTfV8YE7mzxNJIy+-CvGO@j!<9y)Tg_&F61!dUF?7jWy@+OiA71KLWtMIV*Q^*U4 zp%`(N&hc7;OT0xs+7|n~M;YWOtZr6)YA&bKc^xF|qoXAgpZ<^+f|d*cajtIVF;g2Z zJ-s#(TihpDGU!6JJ8y7eFn{PnXb)AhxyBmviuD#ni*Mo&>81RqQXgyORLg^^Z2*hi z0jAwR&8VhPKQWE0yyKBlTY1N{sp9f<__1@vf9Up(voAB(>kTN=68))`3bx_BV7Fj_ zpcU|`HxR`kj+XlTAL7|HBvVp!Tku zvRXUr5@Kg!WCLa5-HyiU3U!$}1T5=Uh{?+2OE8t9Ii}OUMRVl zsy7_uF1Oa!b#`?N-Ckk+@>R~rRm zv5-7d84BO!U$vIAt#h1nm9w~WuNtWqbu?$T-QQvl;>YvMnrp7dfx-KNBbmciJ=m3q zA}zRMX{JbxW-8H3qYs?C*<^`>Kwn~DyJw`rdjc+cURaU$P+T2O*FnI1Kv&5&zTd#u zECZ)_9PIp9t08kDr;@wDS|RFRK=(psx~QL$y~|{gOu<=WZctK`)FB}soJ_D^h(SURz9;*-QR@FsUBZwhW;K28xdj<$)d<@1UQ zOBi&na0RKLWOpxu{}b)Zrq*^0MOXQxv=WQnml|Cq>mN>V8gn1YMq9w$0yX;^Qk5=M0!UzCLG z1e^U3{ZrLI=EKm^ElHnECu)N|P#)e#Z)+?&6MJhrMw_`ay-c9wHyb>3Essf`__m2lZlC$VL(uo{|q4Tqjh z8yLC(FYaLALZC|UX>bo!n#^Fc$wqB!wf$O179UB|!PJ{7XO%q6#4P~-d=Qb-X1Xjp z(YyAY`|vOClZifnBGxf;7(Jaqn8mM+=X`(s+1XhFwe~6IaWf6LRx`8@eh2mI%Ves? zcEY4Kd*H}a!QKQtJMQV-U{$M zp3&9u4Ey>44;zUN(Hs!8bWA9Vq06=fcB&a2DZ4=uFVUUZmh*H5_Q(k|tz*O`ps}6p zUDg*mB;UiFC}Mo4PPmg38`7&9T{yKRu>~&hn-f+osy-TeRs-l~xX#mN;qRwM8MqJV zQWn(Zvw)zaqB3_IO}5_jujC^4I6@c2JMf;4VEBi?n1*pSI%7+xQ+;elZaIhUwOsg$ zXUt7{WS1wx9WO4C?$WcqUOnTS@9OT(@44cc=dBgCHmrTvd~avZBj#PTa&(Z3h{vp2 zhOGSx4p7DKPr8{{j;|MqrQjrP35*W))CZYy_7U-u{K?VJ8Al9I+Ov_Vyg5D1-NjtF znBj02+;pszQ@m;qu*NgBs02Ku{(6Ld6W;nx6!@cw0kZMkh&}sHA8x!in$mYS#)>5p z4-+l%8(KwUl{$`7j*@C$wUe4b9p`vMg=-+^DGw?z?W`Zh1>MqmP<cpc8wTmFgx-J&8r=iWO*0|J_q&qHKq+ zoff5=i&)lCFmqnf>CzdE&6FA5SLvX8N(WYHqU)2?=Mqs7X$}@Mk`DIn>_;K6i|zdG zD>xbj>7jUpjo(QhdVdu58!`=`52(&AxJR$RY0opkV+2TO6SE_`H5bqKlxoc;`Xi#~ zidls^NGBqmLG*r~q5ttUy{xyuPv(Gt-=!CBF#Tx%g7`P5x_a(sFIPwW!)I{bf9aY( z#n%(?-FM)N*+CZT(an<5x!asRxZ&s{q=OmuKU0A9G>MrhEvN}y6tYWcmC9;gS2fQ# zZ>6wY;R)f-BhE!CQS~BEgg>Pk*WUSu(m=dn*4A1D%KC35jZZ9_P%-}dk4f=w6BOUv zTov3y-InXEa}=rvn_+5YB5K}>hDth=0e+*`bc%Tu z>}vwGrYNT1tYKbeZ)Qg%2S=01p4Z0e3yp8)Y`dxGAm3Z#Sfx&8BJ6ut3HV<{4CICfQVP<(_4V;i6niFP4er*P|-MernJ{$AQ%6Q+lAj?I;mAAku_yc~~ z6zLy2#)}aZZIJ(wKT%t~f~rCc^^0R5_c3U{*0ffD!G6Pv3}-gNJ|k!>G+QwP>MI(7 zhnc>XUS7f9d@f&LCiFIR`u~$!OV{b!oQ^U}9dz@$(*t~ho7e$Orc~w)CNQ@prmM@3 z&EYXk0@Zy8SD-g6fkIU5Z?p4F=&M{x57b7wqMG9+eh~ZK#bbEs5Ys^P&^JP7bRxP7 zrcjL<$P#!+&G1fZh#oJ2Lli=JWGJ=vk6?h$K!vLiEo`D|xi)d(8|rm3m7yTou>$iV z2h#zZg}DJk$uGW|1L1O&qDQ@fxfk?yu9blrP!f848Q{-;aLx0K_T~+{81^FUG(S!V ztLGidbhXEh)pBuhzV%Gc6nd6C%J1--zIb0@|8f6}uj&%?782-BTA-Ylv&)<4I2$7l6zbbetut^0(!%4Lq7NlWkHOkzcJ$HOVL|rtrzo{1tFm}MXMbJHi$O1QHpSI#nHq$Q?xfqFL2lY|X@m*I_p>q!H zbDAthxO|R|)wf8n!BkHjBOlQmifBY`U=^G{0!#C~UK~5}I`wVU$n$psg(HjzJ*d#k=Xcui z&fe8_#nOgpwX5XE#!+pmx4!$AtC_16S)o+dvGmREd(;~L(OTihYtlh@sk5myT>vWH z`y4#2a4Lv)G27)P9p9sro=OR27IC6X#uaUyy37086YLq|wo(Jo&HdKB#Z$z)9q-y- zs}9yk5v<5wrt(buXl&VJd22amiL$IQ=fHP6!(@;zN-Ji4zXjuDqe0&&^2bX}V=Phk zHtJ8JvK^T)a=_b6%|+HXUf%=u-YI#h(hxr_&|J|x*t~-nRzCAe;(t@9?~Rr}&`a0N zm_TjIP^~IF{vDsZ204U4@~aQf^E>#n6Q0{;ZKmFwZop8vC};&)&2P>PUMqb3B+xMGMbm&o=K^wLMRm10)FgVf6fHBE#IJ9BhXRK zQNZ!sUe=yyonxtHF2M}(L1bkz8;iB#YBuU!O`ce4L4&E&IqVH!j^TVW#yjWl9`PvL`|=7CbRKA)={m$2aD^V z_7{~!r$LJPEtSOL+rr%EztIQpsTXLWe3R#5aTTZTa2Nf&->F@>PM&x%9jzzHGsb}k zHH+TG;?(B0#9paR2X+~{VZ-Qg&ItdNr}n^X%0*{%ag)PzmNibHf-NiAE1gQIDntwx zQ5}|@31<%5V$MiaG}|sD zN+8p}z%eIh7=i3h#k=l}$GIN+CNI^I{XmRfPCu?j2ksxL3KpX)+H<0_dG9;E)i@B& z5AnLp{D2AY(tAD;WDErr>?WRkAD-_GcE@NcHRgkWw*}INBLGdnVcs_P|2F|zln)Lq_#y&og?5u|TjYA@%asyGuK#T`qu)nOZC8^e66KGyw~ zM6=cGFg?e@u3;o<&*8ZC-XzajIvD?9w%|eUWMb@{^+NQkmgLG8=n39K4)l~1h-5V6 zIf_Cr`(P&Qd|Eg7PiC_ zBDJHi%+^z*w~id?4A33Sa#Lb8oAFRRTrq?mvDHMzU*L24gNis7`9Fr5{y9{6G^C@t zFTJs8NX}`@Bn<{#pcY6yuR(lU%C)BO1HLr%%GKymZ%uv7JaA(MVx7m2JKLx+9W;P^Es;Ao1!7?pgcC_o^`<{=?=!nJa|`h+VAlp;ngDt z{0NUH8rlklBI10Q4egLH^+0Hlsc3k^gr1XBF1*4bnM_UEV5&iC!DS%e;Te6WWB)j` z*pWW>T7n+sS4y(!7xiV^hzs^7ic`vVkW7TrQq0`N)QWDR%E*=?)arjA^Cs#+ierQM z(}|pm9`EYDXUmWDbD(dNh%jGA!yS|w5kdGzDU6L6!tAj+=8yDQz?t$XP{R}9zX@77 zx{kABCzjWWGHbFFHq;I}3VINmD~N7=3sT?+&>_Af|9Vm@a+>LHIq4p#j7|$yn7J&+ zP_a^l_;gPF9KP>RXk#(fx$xNzpn)HP^H2tB@B~+~gC5u!Pop_JSBFmVaY*sTAc>?g zHOYh27%i8i!f6f?s~4H=wV*21LE{ev|Di73^n=K8)kg*|C3<-P+U*bW&nd1ZQ-5>; zN?gKOyoA5aT)iJUpe+7nP7o1yaze~j9UV$z)6m)^c=$6^6a~hZdQS4Y1upR^^VXFQFe26PZuZdW@Be7=} z@tTGF8OFam>&5VldV_Ei$US~Zy_LJBW|pC;-%prBwA@1{ta+XgiD1v+c3_L&!qk2hHXP-1-O4 zcQSTDD7!r$WT*Uc7i^W)Xu0Y53nS#ekgOZv-%t=PcOmPCag})dimg;A|KJq|CPpiC z#4(~&R$|io;OKYQnQg$*Fyle(q+%)y@sXTFByxbx5(DOZUNGAqf}0Re>@SL_2nDj( zsdMo}UuyxxOuoYt0$*nltIUeT_>T@4iJFORAmsR?mzolB3E?@rQ_oWjOXLS0%LL>~ zA&@arkxdqGLE7=W_wl=vhzwc5KaE7PG{yt33rDo$&r)dXW*}~qVl~;J-Y^i5vQT5v zlvT zAu2-s=6KeglSwM)L2lX#B1b;rSM`Z7e)RQ`2z2-Vd_LbFeAP$T0$<^b3!t?`;nyD_ zHoh94z9#ZDNiWZNxu`%|jV1IMe5^-Ai$=k#Of7^z7DJi;^4dj=;xi~Dd61AR(SPgF z0Oj!fZsHA;L|Ut0o2uB8HK5`h^siM1&+H`@YIV?CqNuaWESso)zX0mcOeo_jNK+4> zjd#*3@Xn?qf1bgIZ`iv+^iot}=axdrU@^1DKR|tJgO_>$3n88={-@yg#Gre-VWIwp zE~m2_kzmwXsC%@M`+dc8?jW+A6HR#(|MMd8{u(@KAT_HJwL*i@Hsh!z+J_bS5!-hz zyEu+lPo75xOW|61v91&;J1pYDfyrw;p-+rFB);0l0;|1h|i=ETgrhwn*bN@ zfSv|}J5zwXLMymnDzaq-FM&!HMV#^_*eboK?WjRyKAJV2!qQudWZA>o7W2EgNa!=@ zrQY0gyR<>xr<^psME@MK_OqR*V|F96G#6NBSo)dUnDSFeEJ-28JW$`4Fkd!-==VmV z#c^n$!JKSMY>Q@Kbyb67-_vI?j(XOgaz-rKD6mW0D{Yn5N;UcfFTf|U;47VGg>lH) z2~@I9M2dR#>_~wUP*7jEYdkx#mRh=*XthV|LM}99C(eHsv7^0M3UjE!stx7p#4=*} z>}o!}8v9}-nxhjqU4Ik5Xu{tuc(n%eV;M2^)AT_d649H>lt{+G+0i1XNY?v}eRce}MaM!h9zy0vd>#K;x z(gobCJJ8G>YI-*iRg0iDU^@0VI`USN1$)rMpx`* zr3*Mgfsy}}xRyYf%!H4YpE*tjXiJIE%2h0&IDQv|$J3L2=!Xmuh<7>~_BvAOkFR$2 zDU-*7kue34yydavLg3F^oKFIuN`i`lcqcn_l+G2GBN?+%OP#LgXTPGM#cO;!omoc( z*@GNpE$*>%M_B0$>NiL7iB8ae0#VzM*a^Fc-ETqqW}{AK4H=NDrmE&nc$*%Oa0e2p zOa#yFGPP9Guru>g)$d2FIRi+ZF%miVDG;t%Ht$**#v&u1n0TQ z1BzTn8En$;?CW4^?!4rVc7dtV9hy;*C|`8&W3b^1!3P6a&2@Gm6>A`#sL?|tN+s}e zEL2U~k!1C;1J`54A7;m&an`SVQl>Dmg9b?1@zBF??8E*Kw#o=K&<=oUr<^zfC z3BNA}_U9t}R1e&Any8lt+I6yj6|oAepz?%qw|Yp3rclfp*7*ZG?wp_~RYR`lXD=@! z*G}>08E}1Ol0&&bo%2jQ&W6xQWuoC7psdOKHVK`o^X+3;%Q2kXDJVAq*${zdqr8_r z_ye5=@hXi?s~{2LcqV~}{0!RpjE?!r(_8V01#M@-9%_#Las*lX24CU~p2RY6i8`>$ z|3Imek$3Zuu}9ePLvUC(XhtE*7L9xjV3&*FZxtloCJ<~!u!AFzJe_#P`qloutP*nv&p}xNS+fSa)dL%8H$3=~D4qkpkf2wAytHeeg-3A6H~68^zg zDh(6Ba5~DaF%JnTxe$5M2g%T$I^e;LF}f}C$*n&ZmWDUCmJq32yaFa!w}ZrBeYrq(taw`B0Lv? z$F&a2Fa~=553N@h$rTT^kHe!t+MAu5j~6I( zq0a7kpx7iafk zQqxw3s`e)6nW}h}HKC|s%=g?vW@I%UMsK1eLqTTjNz|zeQl%i7R10$D6PEs7{H4vD zSZTQH1^YW3Zj1zLF#?(qh=FU^n`hMY-({_{+1WnqcN;W86w$`g(D5#C_Qbrt%5ZrA zS2znlPKOJ&a}EnQt&x1A0l9=yaV;Bo=pQ-zm)K!@*olLzVlDK2k~4V-)n`J^|HY{YJF+EyR&gX%05;Vjsw##< z`FW|deTJ{`iwc_(=(3)?KOafBgKMng1m<)0F;LALC?pkMt%Of=h!qv`S+oMD@EZ~- zfzQOC3C8pE`FN6c{Cg*N5%Kvad^?F}QjtDiI7=pcuqNh_pruB^uS0mQASj}|&&tgM zAEZE$H+Y)+=+O(%-$^vUeXjKwI@`tSR-zNHvWH?W(j%UGB2u*@aC{| zv9EZ&MUMO;hg=8BJdTZTBWm*ixmE$54TpEUV8hnpcP)9QO6+ZM{w&W^7Gl3gK~1yp z+ZMt3QTY6&IfuERR>nZl%eck}o?$GKQq060$vaDVr!RCUq)%TW6V>3PvRr!s)c7Cr za}<%j_V@=?xz0SOYC9NwGq^`x&hv-w3{N9H1iI)f_CA`ETtu9t7rb8{S-FfIxl2_1 z6FR^PErsJ_e}MKrVhaYKlXD}{BhcSD(9jV+9S{ixwPuC=@WKZ0gyq=xNT{(mpLSxy zq#~aZkhuHV)dT#j5pcAw33hEE?v*6qN~+Q$ir z8Ih;q#XWp$IP{+v8D>QSg`*$pV|o399XgRK|H~c3{LFv(=4IBM8;v3~RY&$_Etc>% zGW$WzV{>qkY-S|DlCD@DiCxf6mat83yvq^?7d$*O&o43A_3b z8to;TYZUL-M8{Tw;xf}GeF=@JqnGONX)z_X8Yi8R9VrgosqF1Vo?s!IIg6)X!`_^S zif^#bUZi|sKIO#PzX`3MV|M{V)2Ap&!C}ah8!ziqm zI?!4TxF`h96}Y^@?@Q+8;`(ME4|Kp6Z3-udslD&us%Km)oHb_mX`TO{$orLh{lq3Z zg{82CegBF~Os3~W2lcKn->twC*M|x+amMf9kViblT2Ag5vgbHIH=(23@ZxvqCKFE- zz_*$~DOHgA$*ljPk9!iSU@64aav`(xvVXm}W)`0WFUu2T@Y#|1p{V>&9wmQ7Rlji6 zE8J@jRCAnte&^daKcCMU#Cn>+FGHcluJBSIdiga^cY%G}3Qupr-Wi9Dvk-~1hW`o7 z8~L5kKTmzQq6s|1fBc-_b%VP-;0iDJ^FF$b8Dh}nY2@BR-Vq3-mw57@>`6hM zqX=scg3I1=I?v&m3~)m&_#g<5C<8~<O_ z4u+HZ@~@I;G6f2>vdh`orDFW}L1QI-pDoF0{$|xzSl3fdNZ9SkSTw(ogc9GeGeg40 zE;u-8J2dR!Uf)^mZ=|e+h;jlPcbT&~%byRh%`NLqsdR?AAecPE7J@j$JttIWIKV8_wt#_MqLjl6+7=BwAWy zwYS+lAz$A^;osOzF@-!M&nV&%CAq_IsOKlYF>#m5+@mUYD#jYaeCx>zt+ap&%dmn1 z?0+71#toH+Lp6E1PB71{Qq`BqcM3I$bo}j$A9?qp@BJtI`w@4DgZFQ+Lx;KN3(oN` zc&;G4@dF+82ubdu-zyaA5W6PgHNQA}FDtPU9e&6Pz94y1;rNgI`zd$bi98fKJAvV`vzf!bV8yOVy~NEQH_R|2BJ?|uye)H&S~r}Go9F5H&;1_ zZ6&mAMyR>~l$o9N*pPp5+~qj3U<3MOA<}p)ycWZ|u{_OdWchiX^gR1`8cy*L$u9sM zi?~xBPE6Q-$a+aC7gA;Df26Z&x-5@A5o!C?<_NOU4TnE}Lz~@5w?ho$rnJYTDqt1FFpx~1H zM(6!^d@BRoVCN1FuAHCy)kH&=gDc*$7Ut*pc;=^Xr6=Jj5u;fKFP+9ObMPHI+#3Sj z*Wi0X7XQVaDsQ-eRJT zf#lD{e%avUpM2^D*E!DX3{?1-)!pM?`>+QN^Z&D4IgxKz*p(7+U?tAB5mZ@&T^9Uz zhbNEayI1+vPxM&=^63E7_Y*EIz}^P)90BahPp*8Gb$;Xi;&TRa(8W3m;)Ay3e1!b0 z$T<{euM6}1={~;J`1BiA_ybA_{@No6;^P<;ZcKY~578$GcXs`?6_35xTF zVn8e4+I_Ho`auW6tNF=Zona+HXNLMLXn%IW#@UFy{pq9VWY(CB{EUFt@^il|P*pJd zpAFg(G%ol!6pko~e5=d#Tc8_q^ZRgCmw{6}1wRY>c{lqaW}}OFKeN$qqp;~lVA1`{ zJF%S7c2>NH-!0)4jeRtcxb;|YNtVH#$9e9|KIs<#CF-m$l$8{Lo~!U@E~u(8T4*qF z%+YAnZmgvQzlnm@t6}N%U{A}ko>#2p9M4pq-?V^^nn05^*x$13x;RPU?UlwW7ZIT6 z&|Vnt3)>_+Cnn;Jx!8j$+^G+7)NRDb*Zc0(of%NQIIDcD$-@bxuv-i&;C~TFpdh$h zgNlW1^M|t#*HbtzL5Z38eFpyTWM8wfi$X&_BWLx1 z^AM=h#aUGb?okfyQi1h}>*V(Fsz4kUzCb)2^#Ar(sP75#@aZB@Qx5L`hpYLsmmVnD z5Bey9&Tq#)3I6JeT|W+Aq&qfBWvIIq6eOmpiIpXyA9F)_G9Iyr41MF-{_wRt~>~C4? zNvK^0%NZ4eYwkrT6^51B2?&Yxs`Hh9JZwS@7s5y-u~Jo5v%S;(t5 z{CN*4ah9jO!c$&DPaH(@Z9s!uEW=ut#oobX{XcZy{73e^39 z=T2caPO&G+$Wwt!?#JGhL26cK&r9-T4&+=Vv~C?%R+`_42^=9jdud+f+3Ww;$b~qm zdU#s(`G0x%vpUq4jn$?j-?Q_bEYO<9^~FjQ&P@0!E?#2NjYO@*Rpj>xR`Q9bdye(; z5Q^FceVxSyPvAKo^4={xh0E-T0T=tTOL%iZ5_@?P$0;hI^I zyCTy35RN~?szr_aRV3F#ytLd5B6a}RORH_LJ3!ee`ENf<7NY4Hlj} zlHK^sbx!krA!$xQ8@u7Yk6d5m|An1-3d(rTv;1Ub4xYCRt0=}Pi`g_Cc@0FSbmkmO zvL~gnqw4S~#cH!bKgD>80Pbr;YJ~DUIk;zDc30R0A6Vfrq`^A&E|!0v#0r^%9$U^k z0tNq~?*u<`zK^kbws5wwJnJ_&K!MV~`=0&@6nzvPje&}VoP5U53%=pvX~W@o9cuo> zUOeGl;d_8L%jx;Cb|=3NXB{ ziGW`MxQp<^gm0*7KgbU19fEBOir5c>#aqDDS*s1){>%;iC}~D@eTt1<)jwDtjEVWlYx)*WYQpg7@+?~ieg@?(cV0O{qzK?wT^Ni2_L_P`sQ}{wp zdHy7>^@aD7eA+G_=N`nWi$HB9IOEUk)?HS68!7-aoYkj5-NOG9(dwUkS0Qe4gB`pC z7u@AvUw8r=E6oUhSXh5LpAx$k0+$x$jBMCzDeT{M{Ec<+*kN|&HhlerpU+&=gNDw? zPO83rU{*D2DbH$byr*+QANll8bhP08FrM)bD-l+{&|DeVYmMuO{NfE>x452_E0p0( zQh4qn$V}!~bDkor`;({1#2WvwUnbvvt9&mOs!1gGE3{=KJXsQMWO^z;*^wjlp$cKk zWkDvTamC_D-eSC0p3et!ccBqQ<}8vGtDJEvc2r5OByz{4xPqWSvEuj8+ZUcyP_LPF zSAx1HN#$j$X{l+gQc6moF0z;rEx%+k(JW~{NS?i!8+wp4-%T}U8LF$k(T4$Vkx5=d z4y=+f40N7f`YmG**^hs@M|pO0uXK%UO(9Nuy7Z0Ny>Y}qkAMyEn$;C&mb;aSrtjr> zWOvf2g$iO$&I#7|+gMCRm`Ps6dEF*+F@ikWAblN(8AU-mUCJb%D^#4UAZtB?T(?fG z$O%1=%B1&Lt4-kj65M?*87z?xvygGgP95Y#sv((yO$086{rLmW314g^c5^nk?SDiq z!>9wuMwP=YC~6ux#hm1K%Q3UDIGO!Z)F6G+M(XQ1wf#g)Lx|A!rGKg>93wIvF5?js zaev`;j3(xMmi&GP&gBc(V>Y@SZi1}x2%a25%=&@R9*dz8RVi1Iio=XtU}2u)nhWp* zdXcv*#0VaF6#?LoA1ysuCsW#*RxoV@vr4 z`}-H%C1Tny$ddL$9;`r4w}#3ERYWn@_b|JD5_xcfSuc~Zf!9Mx)u~^Qq1%FZpJUmx zoNzz|bml&Iz z<+T;Q_(I-wGV$PA$eD*!VEOa!mr$K%G>0axQ4iGt|0mTLM~>A)ZN^;g(ZDBdBcaS5 zOpTI3MT(GzAV)q@&(fdi44t7Z5hEXr)T&H>K{dRo0I2;b@#YrzS|^yyTpQi|2AT5$ z$#4_RdV+mh$zEkaau-7{WI$$@;7*H))khO$>5L{^O@+unxb-r!s~@LY5}P`hwY23Y zA9mP#uJizzv4I_3gT%T3ANlbNb9uU|Jl}opJC?jaD{PH=cweJ73YzT#3gdI_m>JA0zzli*ymCRKUs+@v2*S2u@eReK8vLskq zOtfn(bhwC|UT*T;Ezqwbf=~$0;2$y%l5?8#8cKm)5N1i>-w@VO*zw} z(4L6nnRsU&X9^F&F^BP3&LXYman`+|lh0Jol;D(xk&ivgc{GJv>chEb=vBIiWl){> zPQlH0`0Z{go#Oa=E!yoQ>wScdxd?|oX3snzMHYd}I~tjh65XMWFiu{~f2+f+zSh(W zxIuxgD%YnMuPfNCQT&u4Ln8wdHaC?`3c^E}0U=v_=P>qG`-1e|)5r@RFHFCo93N^W}}h?Y*nMve7C zDm12(U%pLGOgUs(Yid57`X}bWP1WO(<;)1D!m=xs9MARKueSjtKXuuu+Ax_nC#{L})!N%6DmJa5+ z%0J9kFUX2sQ=OGt+w5KBNdozDu4gU1@0FQVSemY+;dH3g(yJl8nnE37UiD#coKJy( zuvwX;G^Y}IGpFzyRE!z)m7JpE;0?aC1=@B}nR6W*pbvC?ji>3(YYcUD_t~j`>1Gb4 zdwDrsku}MW%|^oQM&75;#eWugo0G1BH{=GVAcqelwZ_7|SE!8mLND6|_G}kBnhs9) z6}7e+0Q#sMOv2{;?4f&b7C4j)6G2uH0bE`N$$^{nd)mT19Jb9)B(h#56U~!ck^h=G>d9( zX3h&PPKvpKCDih-*+xG^9>q^qq>{!&C{JhR>ECK^?;)z&pMjaXo~~*;SYV~pK6D(n z)q3bm^Oi<3F84o=As5HGucK)6TAqLh&bQ9;p*%)Dg8 zaP_&`CwkmQQMb4aK@DacBR^H6SbI?;I7f|SXOXvCcz)= z^nCgcy7(JV)i#nHEx@YUfxI~s%=O&rXL>v9(+^!-uMEFtL)Q3l+I#U!+~~p^RIB}_ zJNh{tB^5QBwp6vK8N4gpbJ(#*bPIKN^>yWTJxDv7b~o)W*H70G_fqwRO zq$H)(XM)R{l=~?;|1?Q$=<@f(s^LZrxwUDw<)tmkZ>)cMz}&ztfpY?CfWE!lp2_yb zVrE|KR8tLQr1Y2m#rx9z*i|9z`k$*QEmP!_@+p5(vZo$oibgk&0!qmVtu9nwSUMvw zGBvU!Sj*e%I}As4zv6x;9qIOmw*J;f=Ax!^@))YFCzGkJghdvvM}kuH&XdJ+)14hG zfhF|%G^T3yiB8>v)CFA2;^2T5P}1ec&`dF9G96~0m2t{8c|91r*XW3Es9m7%=sq~< zq4dN*1wFStDAW7ttL^FS#-H)bhw`eswMl5NFI1%0MC;_nDsqy)TZ8Q+=-?Z(1P>zl zHc$)OjULenbh{l_CxS`V3iPs^bn-hux7rE9(Fo5SW(aveUA+LZpV_;Kck42@pd1Lg z^|kwYcd46cimj`E){v!PlQJ&JI3T=mSiKD4U{0+LRYFe%9}4{F+-dJ)si15%eye3Y zfv){2NxwWly8j6O`QTUalxeAZTz$Zj`{dQtS^8?p-_*}i*A{Ot>NnK++P`Vw;-Jvr z!$BniuR5>VS6TWg1mwX17~yT3ZX z7^l>*6t}grI~{!;>m0F;Gmblsc*i$K95YQe*iYDIT2suKOv%zdy}EkSecjbJ?MrHK z+K05D^w#c5AaLfVqx6oNM_**fV1dS%ewy8uch>*e;%wRMe{8jES*_#E2b2#|R%5be zSFd<>ySJrZapiMOXZn~ktwGwev=y%ZrAL8%9nAdlcG@)kz7Yg^!#!{_s(@*@ivF~+ zN_OQ1T2PW>u&(^+GnfpOB!B~RjLwDbYIS5^Cgk2LaEX7@>;0XnQ$y(}?LakY20Hbk z=^+>ldi`rUfvV8`_`n;@6tP54F_0^!xSzZCddjLV^aDz5TQ2_&!TmDq1`lj}hM%E9 z8LDL%li_mc>)`BxNq)ihx|ZEa7HO8Y*;ChbC}qg6updXh&H8rrd*aVw$+!P#Y31CL zy?Jzp9BIyJtK(SZ*WCHexx;^dK%StT!TUo#g?tG<8o1HvXK!piD=pRbdzz&;NIUXp zddi{X*~ylaw<&l2^h!-iEtu9QZDraBS4Q_b?=t<1oNV4;19{MS#=m92%z(ZDG5%Yf z1^mv~i`X7n&YNSI!hIeYyGU=UK6iI@bxKW2$&%uqvN=UgjY(UOKFu>$ov4qJ$|(0t z9`ivrTqfUr&C-{b=_qa|+l~>1#aeyrt2SGxViWMpFliZuPfs1DENF z!|dGZUn#IkkS%C=z$U){TYJ+JBT2pFS?1P3b~uyXE!{tTFOx^+xy})K{N>u~+U!bU z&ecv&8LgHSZK`N($lQXaezp7_Ix0J2?QQHq_B~7rI&S>~-q!$gH&duGRBFs@wh8W* zuFp|v!oNW+=1F_Gz_#nDWu^vmu6i+PY`s6{uIFlRQc zk$)Qb^qy*hXRrGah|4|GXMxQBBfW>)>e=VX!yLS)-kKoe%m7WR9oBLraR2 zZp8?8b~qMyS**bgL^W2>t-1g|VUY1Z(DMe;b^g&CK|j`Wbqqc837|wcu{qw-_nBF5 zO4m$YeDZNrdoR(G^(RD=S5y7v(A#U%=o5_vXSphfPagLw&rt7H?Yp$ea?-DUVD;cv zL6-ymb1w3G;CvX+FnC(X)8I)#ul&pU9klhf)H7X>D(i~3t?OvYhF|@E+JDte&YHTz zb<_RDGsk<+Tb(Yz4|=*}0coWYh|q6rZ|#Hq`huryV;bFY|HqED)*w@TX^p;_zQc(i ziO$uwtE;_rm{-*&eK%5OVp`6$zG)>~=hL5hdTDQs!iwEo&2rqbkqJRPz`)D|vQeb}(*~@o z*&tkLAjP!DK4^i>ItUa6kG4@SL8Y>&Z;2zKA0tK6FH0+Mwg_!8r?fylsTQDf zqbho8482VU@Rh3)Z#;%S5l!vrRaPYCn@3UWD$zC5iiqG=<2#(jbUtuY_NdFifXhj* z{%f$?PqEJcWNt`tWs1q)(#<;E_CNb6`+j?I`$gL=TcWMF{gF+zz6CFF zGN>@aNv$?eu7j7@Q0hysMO`{KT>4UMo`(4Jqp6eIN8Q{fDYqO+FMbDQ7U(Z1WcVqmp?|LL>*a%%P9WQhl%|&@3>>Eu)GSUQBGH)j8FV~*rFeO{ z63REi%>&HS&0Wl)=JDXD9|DPK89#lw&c9#*FH+utf6@uu-pcfv?xx?!4OYx6&bA&= z(E@m;v&r?oBC3!HeH=;rEP{BSpIiZqjKAnTs7lY$JSur_;dyIXGF>+}K#Zz|PjenE zeVM+VWpoh6a@}Hhj~j{VEacRq@#yapaZZMsP0UlM#C+DrV8QePuX#8NC$iivEhmdKO&;^n!o+tCA@$ZAE8;_JP5YZ|}jIIR{xO+s53&AT}h!@o$O4NqP;A`S@b>Y?5OqMCg z@2b-$+LE~4DY8Q@k(4$_ONZDxvTXH;PG)0H-Ez<>Y8gw2xs*g3JcEK$n5PrVJk(!mGRU7P#0xS|OB_pw zY%es^5hTT7x;!e$ZK%f&M1G52kF7)li^Jbvk+AuQvDG#XL2a|S`$u8~9_m$evM>W! zZ9eHSPxKT7#z=64dU85$dVA7jl}N`D=_FMIlZmVC0uAsKw%-MM{jM9g>7Vt7rcRQB z7|MAUB?=n~O;_bQEy!eSM7EqI_BRQcS%rw>NooxYqFP<4w9N~~%n^D8A7Zfx{GbQ= zO!lXl(Vd+*iZ*N`jh0^_4--KG*iK*HV<}x;%k}q3E_y}I5CPYiSiG4Hzciw)+!F+9~W01Hu_?hd5p$H(*=e}2idOJp|yL)9riauo(kfA3py32 zNEV`t`G~t;My4+@{*fNYhA~cWL53(E>dT}bLRv|jLN2KWK6MZj7|#CmVz)*S4gO8d z`50`}3ds2z^zdW={moAG{1DJ6I}urnhCiN@k%=NZ;zvx;kBY%>$e;22y&Hf_gT2EfVWZXlN)Y44IlI_M?y|wXJ`lI&*A$_S4B3pXivZ-mnSIRDdu>tbiG3kG%6gkXjt_MgpOy`tDDMs(3^+oqyqyOs_{+REFD%hNn`OS`zX_u$NEQ3U?IIb zAz;zSauM@O*{-eDH%N)tCW+pP`c^rcDbk2g%Nli*${@RCH6rz+^jK!mqqH}ADJ4JW z-Os4Y&el`rL6O@)+0JVA2ZNysy-90~%4!hko{PZ2T9t)k|1d>?e)GxrrtFc*crPkm=?XgPt?^r`s8a=xO}9mdKJ8mzZDlI#q(cC=U0DU%C#XqzlrInsv7V&$R=)gzomxqr}O3~aeRymJUn=f@T zQ5B>;#*@*cXrgE@CB=M0*7S$o-_lO&UBl$vukFVQ8w)aRQF*dSHom#b$PJXzdKP1@ zmWJ22URsXzvQ{!Nk7Ra_oi{sQ0vc9kcZ(xg6GQKNz~ z0NZVv^h3l1^m@h_WuCM{y@*ZG-Q3P7ixeIrT~Wpegb4P00@LyeNhhJMhoa+^Y();D zO^ubI@*nL2Ua+5Xj`PjQ+iLl)otcAEQL4%e z!q0qn7PfX4X|U4NSgNJyr@$EBPo{5;z7mS+%hc2PdTIFp`J|8XIenEeNdBp}1W~+} zv`}efLyRAwq*cXAt1tZr zvddW`2O9i0wreoi#D;t>BQ|6M^wlGMzL8H}iVtnZM=dTL*7xFp>@xb1)i)uDOX$cYnTq+#{}}po5`d;V-LOLBhMIJq_yx&B&zlkMZMvva-n{-&nqDq21}Pxt~a={1`BI(p?e(HTE>`zCW$Yhk9d}_YRPr(8pMknukl+W}C_IY{n+`N0Mjbjz4&kTzJMAsoaZ4#%<*e zPpI;hq)t55VK7f_lO3EwPG~gUYMGG)^`O9E_(e&^Le?IR-*kujG?nUO!_49Dk5~h( z@Zl`n_Y5|B0pdSbvGrC$Ctc{$eFa+NEGRORCp!Z-+<-nlAajdBS%X2ONTzzE8X4(4 zMAk#7!Vz75qCadWw#N(d?z=hp4N!uCf4>gfYZkb%p;Q5MVUHu&^P6NnuMly%$TMu_ zboNpYwUv_!1%vXfzJy)x!nZm?!)FO{a5BZ1Q<}zSml12&O+9^8Ji`ZM-^y|l_u;CdJbiPnDLS77Qp^`9Gn(3h z&zzXZo34V#Hj%6EjUN_ErY=8EyN>#|Ts(!BJ(&-EOvWN~LU9d;gD%29zBN4LnBrEnw%Wz{QU^%lgpG7Oq(n&%Y;;#I1OrD>(5O zp3uRS^57fvhHl2O!U;&m1*|8Ub9is;gI*`VeHp22%Pt+DM&mL&Eij{B`)V)~xq|4g zTg)oDVtX8;{-+4nXz8n}ld0HQjs!c+bI*iU1J1rS(k)DiV}C_& zQhv_)4!iz?s+T43QW*1xrr;rnDs~g6Z04y8@LYBHDF@vYg+rR~1mB>q2x!jA=OtDv zqC{7qR|gUEoKVGS_*2wM8<MXlFHiahI#!32jDVWYRMCDR; z_|y-{l9@kq!LRw)U9)f9iR`D4rlKPAD{JfmZB8H)osnF2Jaru_;oZkjz-4O9P9h!r zq3Y}WZvnL+3%K(*Xs;X3y@AT6O;CCYPj(i0bQG^$AZH9^Jlbgusc+5Ha^bv?FT7WCt7Xebxwa13r3N-Sjz-^z-N>J1N9;vCl^ z)f3U27l;zvM_!harr;wiVl|I>f;Ut~$jIef&~{5|b2<`p^5@L{uu3PAIS#a+JnU6A zb~a2J5B=srYDG$-3uX(n;TBa##18l4#dV@~PhfavL@vKa-VB7x^C9<#;?FjwmhTaD zh-a~dcM%VWrpN9UI08M0kc47y7DIw|L)PUazO;cjYB?lI0sfzhij5rDPj3hu*-sosb5PKv8@|O=309DHXXq6gmy3PD|os>QgH;lAU?R`947Q z=9h}0Z*HI=>XV_l$u2KNPV9re5Y{WWyOH&7_r5w(gZ&R^O{0#~*(wN69HxO^w3e{MBs`?5ra&h06ade*AzpS#vLH?2=kOX0=J#T73>y%=41p(`Vgar~{v6}aMZW67g2-h9O*033d9<%a^CDQi zXFwLbNKXGc`1mKt1msuidv`FW)W*c{`<@@*Tz+Rpc?6S^dZo8cUq^{|b9YAfuXHo> z*mHTZsUb#`Vq&6WqV<7gfw`qQr**93OhAK>v!NwIX9S-O+!HV+V01vYfSdu<0)G12 z{SWx5_J+15)_a!s=4z&3d9tpkHJLEh(NmnsvWq=U+>2c)srUX&{u7cqDNRbR?as~Q z+04vMHBph&UVW~8*K<<`xJ&*YlR3Vd;>-z50V!q;w4Sw0vb44AvUIX$vo*C{vEHz( zGUqY*DG~A`mdxQRz=XVh>I$&b9;$uFqqG5EY!81Pz+Wp4 zANny7AR4T_(xx;ft$tJjO!+|!R?Ly+>!wW1%z8&$ZYMQ@53#AP6UTOf=l6=rx|>wS zd?6$G#y!v7n+h%2Q_8cHtu)n!_q&x>bv67_NSdrE6v<8DZQ%unYWdZ*>s(3soByR z#QJ4QHQN!YVjN+^!y+@Z3F#K}Dj?o}F4@PFz+u6Mf(Hd9`yX{Yvj$iSnwy#|__|BU zyCjeo`@uBJQy!hnYMd)MZDZP;wDGRd?q%K=~+QI{})coOU2I-D(@on5=_h9$aK0D zYAx3@$yrK+}hA=BQ{i3d+612F;IxMZqW%b^9= zaoy``HKx~m@#d$d>>&}Ew`3RVt1c!VOrbKVp(<=BurLim{h~yUKdMVI!O;EhrQ~#M4 zd)tY&3ZUNdHrBxoys^gEE!Xk7t`G;lOaAl{8Po>&$Q#K~lp&+qj2g`&%+(t~j_o7! zJ{pqwD@P{ijQo*o-UH4gKpur1bKJ-eYUC+uQDx#uqp`*|5=F13_XUgmrMDEbXLou3 zFn?hiHGfOJF&>BK2)N5ln3EIDq}oZ|X`I&wZwT`?H!>|fvo{}n(vmr=i^*lzqB<PD&pA&sHCEp#pEf5_gVMsD(|*+Ma@2AT z@_!xBD=1gUOs3LVLvsa(1cv&j`t|fH?dNsWcJyPOUvH~y>1Vo6+;#!p=qhS7T%3Px z=2LI>Hc<0J|G&wzTD5lMBYT1azD(VyP0(#bUL|8Pod-L}A$I{&Y$8_P0knEN)v0Z% za?6TUFFIjHVaL=XPgaqxi277SPbLl*z`k9>uH8r{fSDQ1EkMHV&P?CF?Co;qF+3$_ zR$l3@u^EMCPQm{@h~>JH zJVFTPe+)#-Z{Q^t)*h&<)xylj=zPVUzshy^mS?#`Z0SnS@1M7hMRD1 zcfEhB{j`Tft8-B&nrP}|e$DiwX4cWRy>{Kv-g(M@U%;DyIsx7N=Q>~co%M_N%i(P1 zv^b0S#o2APQI_WBC8i>#e9W1@&or6wM7+T#)?Xnr=TXm)f!UN3ne27I+h2Xf%qqKf zMlGRE_PRWAo~qE%UUdpJ7-x_+dyqP9u^B3o>k)l_Y1lWV@QEv-4dxTSBG)T-h95ex z^L^y@Oo8pkzMABfWGi>TU+d*$IV;HeOF*{Uj6TkSFS}3*2HQ4@8QY_n`*R*lJu+{y z8*X?ar!aXmRsJGdm0XHTUMQzv=_`1q{qZFFV__OtFFVl^C7`9rpg!02mV-M?-Y3+r z4D+;Rx_nP6T4I@H9P0hc#{qwM#vSTsByUM3DlcV{gQn#Gkx|rq)dNq=Y%Bn|cp%T& z5Z(1R)ehP9Yt-hQXMXJk(8%9WX;zy0=6LThY9209+uT;Yrryyq(F@cV%SSU`wG_8r zvmUgau-|n&_G{04wQ%P(zcI|}+wEs|I-J}6J~~vpVGFdiVy41b%Nk2RtfGddTTG|Z zkRCBm+<4<7lD#MMN&BKzhhdvE)Bc7whEoHziZzTw!-|<=MbHPq*k?WUG~)m7@b#W! z?H|yo5GOOPP(yzgpY;;FG!j2-8(F(roI?P3U^A(rC_omVEcFykkx`T6-CztSQLTEF zi8`67f~f`q-Azs*2JU%*7gCl?+bBGT|EP2{$Ba4}CNgW1Lu3Y)#1Ba>fNza>=o9<;x<59WXK{B|%2 zC#QoYTDw{MTTffRF=gqhWw1Hi^pmQm*6hm&Bts!7+(_5zV+$__>G~!=T|kCkNR7}9 zwYm0%TEU*Wf<3zsi+LWojI(&Ulkgx;QD@VFx$bcw61OFm(UjK=&buygm!I&)ed3YZ z@LVFmdrkn|{sdXNlH}G)vhyv$E}uwmg@;`1PQ1!qL|%^Kixxmr2s8!{URiy7&8Ku~ z2^{GS@aivW6JoKb*J=jZJPw-Afz;Bm(ArRWTo}u0ARP-a>OQcwH>z&631@fJHz{C0 zlfF~Pd`{K9IFXP_o6DcGj#Auy`g0`Hc6U%oym$jl>17osT?)f z$L&WP2OZ@d2kk5DKkfS+XZ*%HhdS^19dQh^_qN4bW2}#?sn)gDeU>WbcT98M#5}N1 z@>e8JMX9V&i7pK)tf>R~Oifu;aDJOGWw|ExcR|=K89AFp+6k=yJqI<(Yz?9JP2jRl z1dp&cUei@L#R{Fd$$`xzCKrK!dJms{2-bd*v6ATntwGz9u~p}h^^Qct$5YAE0^e~W zQZOGepy~K*WANXvaqfM2Z9{JfM9O6J*(6SW81aa?_^&mHJ01pKxhIt<{<_3O+N>b+ zhTs9IRJpvR-X)0@=c3!BJvAm%spjvA?Nyn-N5C8Runw+($0@L>Px7xsPAgu&#{c*G zerxCj@%n@DkyG?9RLCsmP7AS#=Of#qkQ~{_hkWORCZd4~;JIX>kL3iXlY_|wYpD;v zh@3H~FdLzc*4F7ysU~;_N@Z`5O_ze1+tj+pw$8rEKGEKfKLZ@Am`pz1udd%ej{n(5 zf#TlC8pWi-!q!EWvE~e>uk-{ppu(gb6(f7ebo)~g(20CVVWKhH=)ie}U)q8b_@H-S zZ+eq|c#7Y!hvzcWGuH|`;t0>)9*v)lwN@Cf_W~2%S8#&i{C*AIdIGO>yyGX*AZl$s z!aXDDrAw0jP_Ho9H%H+*r*iGu*qf&u6)+ zM%~c`K79=@Je>GW4xaQ0@7&u>KNZ#yO@-Sr&z-1AtOPP5Tliy0^-vBPCzkr}8f>!a*6WwDI2 zKCz9nN85|pL+lgmB^)70xw(Fi9Z&6VY(chF);)02Rcm`|XG;e2HMDOxwCr`H{zf^S za|$32dI+@lY3%M5w8=reDCp?7z~7&a96JrZd{b(420(HCXyXWE!7^%?YOo_~sr9)` z#n(J)teRj=Td<&>5Cix#A#3#4Gt&Qmz zEyKHgIgPA*$Aq1liE}H$sTr*90&86l4{pboSVV74PW>rbX(kpO`B*hCJsz99|6`hB z1J3IUmf}C2_MS1=B$d5px&m72vq2Kx1KpOOXXKV?F%!VU?G^0lwjK10eX#AY&*aR) z{Wd!;*W}m&O*N+lE8;2~ zx|6$KfzxB4>~-*JB=YtRUdb@(%ub@29y96I-|L32c4Ei><$2~_;~wXp=|13oiif(* z)5F_Ejnr==jJ36^+Y%S7vETOY@BKh3$+f0TcS|3+sAXKUwZXFum7<}Ie zQXl!9Q@TWE+$w3M3-6&kG9f>9MiQE?BC?&SC1i1(#61^cBcw}x$?rdwFXH1j#IHGpt972U z-r}#^MPw7Lu3*504qooWjv^6u0-ksXgF z7Gc5{o=24Ly!;#QD~y7S@s_i6>P9sHm(86UW~(m+F|*-zctU)f%ctso8JpE#WPTm)e((lyq-v zJbgc=khRn5Lf1~>hUcgiU&*@$oy0q}zp&@Fk^|{2H&@1)meO^ag=qR`YbKj)`(#~d z&BO$~q2{WljEYlcW`;4As>I@S%Wk2QbE-OoUeN5!oLi{3AhUf^YEQSuQ6-9r*tdDz zG2JkYF}+k;Du<}4??R44gHAVbUemAuuX^V*)B7)P9wyh9XZmk%?>2gOe&JJn^nPYG z-%hld%d^T8=Go-V=Z;ICp58p&mVVAP!1Xb^NXIgd9 z^x;hYKFzG^B&{KpAw!8F_l9OqFmX7_vVr+h7V9kQ|8aB{U`}0I6wZ}+CK+P}cZcHc zPKy?&xD=Nb_u{U_9`074P#lUB*S0vt3&mw5o-0Y-AHMmZ6lRj#d(YWt@3q%j`v-G- zi^Zy023aPW`xBr1}gzt)PpWhv56FQ}I(-Oi8%%N22 zv7F2F%+#5Qgs(VH4;TxhgPJ4l#j*byov}R8AE`4NQCU{kS88|A4pmV)hkg#045kO3 z2A&1X!Rf)$p^8dh^^(>;oDo@K_#i~ZchVx+FPAhX8Pnx;vM%*uZfONjr{N$XQv^-e zES8ftNG~P3{7ITGRb(b^5Aiezp~X-sI*UmYqr=~U2E~|(<>+)BH0)+-b_*u|E|)9P zKj~_;QIBqw9)jT(5Qf8U?hZpU1%%=UYOL(x3?`KAVh+1eU#3|>_{yur)ZA(tFsYlW zsx|?MDo!01W-i?{vZ;|&MXS-jIkYP3RApi4PB0J%`fdJxzWlztzWKh9{)FHHB{AGn zh&Ro&Wp%!e&ytjorF-g#Y%j7qbM($OAmwvne^;_&spXvf&`>DcT`k2V|4-!pHvsr;81ClrXLgwxb_AzQGC|GoFV$Kl!O{?Yx=ebQ6h=kvb{n$=W2 zf25${u`obdEDtgUjh#$OP18&TOjC@9<%|h{ z9A*NKhGj7pbh8Br>EG0&8=2%Og&%{m7vL+PchvWjF%CnGo`t@VuHgXu~oy?3OV;TIu8 zOkyy^HRC!{0)49f=00fL_L`m=H_N@z;wK0e!`;YF;evWdO;Oh>vz2{H8@0K%LywD$ zXR_2Zv5>sY__t}RxsauxC1Ad0?q>eo^oMaHlcjcu2Za;x8J2odP^(F zI%Cdj-_no00pghlEzw&}^J#FnrZM5;4NvigUQ}P8y;O6mtCY%0G?YbYuPjzh(c{~o zcGvO{-wUFNTTW~qPma7nzok{so~!fe6RuU-D!W3ff-3_@{p)>myvICeJT1IkF?T$x zr3-&p&O6&D{hXynO1snr*=ptNlDlW_yE*2j&PpumoM0&?rgo6%|dV#N~t-FMbg%H1SW%q*MvFw^W=?CtNL6kMeIqOAf`8zC&GQ}){Y)H>b1 z%#j3i{fooqC~h~|zP5;_o)Wr)$TBTe`5WbZRc}$xF82s`H+L`haraoydGAoaE4VXM zR{cwx6K=(P*rE6;6b6}_W*%huU~yP4TdG<%n2VTKn_e65Fb&QtY=tkdH*yA?;xZ@O zX|SY5;97gbJ0i!T4-Hp^;^F`?Rk|WAk@Fj;87~-j8k-Y?e`GG{N~x1nl>BVDm`_~B zJnn07AM=9awT4L(qKlgn9Td6Av}cWz^BesY-IMBCGG7T=m>#2`72&ii&(!KAXzzBZ z2`H%Yf^2R<$=Vnt-$-T)Eu`-60-~}#>;{Fr$*H&_v?#baP}Bd1w~J?(yNY{}`<17u z|8!_{I7!NFt?v9M;ZTw}%d(W=S;uA1ms8JCB)cPZb<+L#Mb5sCKdq}x|C9bUl!<`< zhd%i}y48$E8Cx=pnI$tTWnRkc>iO)81j}iuku?UVblTY0a@;n@aV74f^R{b9{QP*; z^|!Nj+-iH8HEjN5TrUk0sA|Lew986TXk%a!=iyAE`*h!F-ylDF`QSh$P3@pR2seax zok=$`K~9r*8FQMdo9ZxGKC9`Xu@AEwOUgZ@Z&6rA;E9w9U(trDWtFm_hQU>VLIGEx zc3?DT;)39?&_rc{`m1(MN243Lgi0kMRF)RX$BgGqKQm>zhSg@RmdydcOW%z%m2)83e*oA z4h#)`hd)4XwV&1i%r*y4FAhZLCpwrnKmvb+9Wah;tgLR(eo$rY&R*1{vxMAyW&1z1P&l$Qz z_Ub=s-P9zdZg8Hzs?Y7M==;fM^6&FE3A_qi3T`9I{a2l*uZXw}Nzy-Z$oP+Gg1MyS zoTZWVmbH|vldZJvxwWx%jAglbhv|^&cVp3`v|`~a zS;^0I^?J%G#iNW=i-O`640nJTxxmng9OOH30aIocQ3G6{M(aeL-x}VG1q`Y$-G|%o zzHU&HEddYNNNqh0*QHHgs=-L)@Rw<)$8oe+$?VK>^poDB_RAs8AO=@Cd}I6aBEy z^jbDkB~JpW$OV$Mk~6{pwlRdR$!lhVbrOGuACm9{S6ur~?`fO17=aFhKd&Yp8=Eo^$X`~cc|8kV&bKwP? z#bY8pYz6fn1hx}%Fw#Nh3cz3KN_Fjxevi9ZeeO*~@q;*0dLy}H1Jkqrr2fkRFQC78 zi9F>tpL#UR!!e-P_vn+>0YPX2^3xLpw+_9CGR%-^OHH~0)O|XAiTQN(&I=c*FWb=v zFUD%>4CZo)c<86kI|$yzU+A8nFypo$ynwg*1v1dh^hy#yADY62XcBHi7qmN{{!X|W z=u8wX&NpCgiL3%IUEi``pnuVcGeu5<0Zs$Ozrkd}k<`d{LEmzM@~j36Dg{>3mo=Ch z?p#(dscEdbXYh#4!e@gYb#*_PBMJ0WY&QNTId~15T`_M1ur*l+?uiB_{JCW^=;o57l&NKDv%sKuQ zmQy+!@uAEQyc!OMtAM*a1tp&VBemog9@rt&v8~BG-wF=Zilul`xSS;~46| z;pk@X!ABjy-PlCl*@|A@T)zJx=)oWK_iDo}`2=1O5Fdz}#Ic|N%fxX^H}$cOHPF1r z;A?eZuV)1-Ov4#!E{JAf_)<}_-emj~Lhzyn^7%`K8-jA0=rr~Rujov_ZWV~P8@#?4 zJcx=QruD$>hjY3<1$*^^)YbpeJ^vZZeE=Ma_=p9hv>Dj@U3&d!KEC8uKKS}|;plc^ z^-Y7x*#Pd}yDz$DBMjch;N$;;g%$J8enRJQ5B%1~eAXyf^La4)wXjaVBCFmHy1NNp z?_t*Z1k{{=3No{qFN(TYUiwa&$GU$b{UgnkvNI9#9&><=(hH^lS3~Xek~MOjdB!ui zZ$-f$-+|av0KvKsgYFEy)Vut5B~dXp-K8p@vn=euheXQ8%#u4x|Dra`xP#$-%$L=0 z?{QI)<>fv#rBgW~ydeBP7&D7NK^KB#&1K^A$8Zg>?b={!C16!epbwbNCte5wzl>i? zg(dV`NAUaasIhOc$5zvUtd3ImKQN?C;G#dkI!S{e`!_%T0~LN4Uet79>H z`WJd8=fV8$;PqAkR`E>Y-U-mD)?CGXrdWLjv#&@sI|l!^!l0qW`Mne@#?o*XM{yq9 zLOqa3bd9-5zknCAoUc8cEl;VhrtzHb!eaUZ=0-l^(SDv^?9O#!;#GG#b=h!K?8x7o z6z;+&NC7Y3#a^!>HInj4|KghVBO3Rb)MOqe{VpQPRiYvFo&6C7*SjCn1B0;dL(997d$5-k{hIstEb@vO&FA=++jQrXiNd{5 zP_zJ3Yswz`nWq*+HP8|~Zz$+&%&o65okj`%QOx=EGrtP5AAf_7TY#tDmCtrkH~@2E zB)#Zi#DLr&@j*0{fAbk*UEvkiZl^9Z+NE&nctj#l< zPLB!oDE~4U)U`Qhd@fE&qU#y8&oG4l7TvebG z(h!dJX0GEdRZ+}ax*a{BJH&w8?B~uzf!3&+`*O82=&1a`74PRg2k>kw(0f}+AK)z5 z@_umdokWGs_?0!o`K~p8`XLgG{K^iy#7d9xL<+(2IVp_9v7;cn_71(`Bxx$_!GqE| zX`?hsN|MfrRXFKaFpYbo7-hPA2T+wp@Y`nNRx%gn!dtpaed!AI<@Gf9s;gNMxj<0v zbAHHl@e09x4HEBu<+|j z`BDbCOLxdaerNB*bAF9tjW6Whc7&lFQzpbFwj5&p#^#I_02}BIH+lh2?Fd@+zv%wF zf;C)#te_0PClh(ROo~1ZSK}vKG_#|(c!0`lD_KQMM{u6^-OZ~#g^wR&zZc}x{f|?y z7rXEb=X`8-n}JpO2!F6|_`ldLKFPDYNoKn$Qk_p5+{a?)Twh_zuQ;1?4!SPSC&qKrXUbn+Pk@s%5Cp)kkW3t%ANH{1)Bt zI*`iNVpnm4*h=~uC!49J*5*;>_2!VdKYjep*59p(wvctN)oLwcDQNC&%46~ye=~yR z$(Lb(P7*fKX>Uze&Jk%8Zl?FwG%$~`>Sk4hWwl6KjO(9IeWp5K279z+dTC}nS0%Q1 zBb|r}czFtahkZwiN!+t(^aaB(x~~{=@CpLwX3XhgsMwvE z@^z^$jp7X=c6nZTJ}a#>dYY}#>U41XQeTcB`%yr`ijwa%0rPE-m*X5zxYO_^GQ+8K zMjo-g(jsMvGu!C`htTA;qpRrx8+^z01?a@Zrr%s7N|q;I57J*<%1)6ZBVcY8*UxI% zwYlnNP|E*>9tLZ|Y%LSW7pMXnn%uZWfw+rl1m8oL?inkrf1Y>AE; zaXp={olRZWT-JCY{pIRlyBvA!<*e<^`AoCOhjSQb$sMHqLNGcdQZ-zcismP^ zm(o2nGI%L4HBceYBw)hpZ%3#AT;^&@qEbQGtPD_BYR7e3qyXB7h~ctuMw~6R!ZGSM z<0s>P#=nfmj8}{)rpc!BrgYP5+>LGVvl+dwxBu-b*{H7EpI6!;~&cM`eMsNx7iJ(XTy; zQ{Fjh#k+9PtCNw9pig&z&dd%@*xQ`VCx|C=;kuO}8$3bIkOvi4`RFO)My|*^&iqHb z_GnmX)zs5URmB}z8;Ut}wF~YGw8Ud>Zy+@|JXj($G-OnIs+063(WPQPV-53W^GhmR zi?y1)Nu1BM8_wmM#I(eOBq3>E;@62A6Hdi1a(Uz4+k03O%sq`GaiIL4v_gD}O1Q8d zueJ$Y3(WJ6@Llj0^&avJ_I&MG=c(@P=G)Rl(fXk_BBCUcfv2L>@IlC#tUj0!tywLUU!a-K?4v z(?6z~rXt2ok}R%68N3s3*JWV+%kXCXQM*D%V43orQdUV-UWYb@`h_}%c7<9i1yvX9 z;1BvzD##|B%%R9&RB*EmZRv2kF%H-Dy|Gw9xDyh8)_rmxQ^?S!o4ER&;n_O3&LVC zMOq}CmQF}3a8!IOju(@}bL6kNgqNs;D#NSxQ)OI)AG;Am=DOAwx4t8=IvawD|AL+( z8y&>oLxVyop_9P_!Ct}g!6U)9!Lp%Rp+2EIA$MpCo^SiK(cy#9lj1{T5^S<0TQl3Q zw!3zxvsQf8#3zaG6Ne^BiAuueghmN#*;93#(;Ulf-&p=Nevqb$@2H@DN2xk3+*#|R zJPckBy!E&8KlOF>{q61R{T9c{oxaEZzQLEF;%WtYW9{@$`mFH&NIdQqjX@(98S9%a znzEYbno}(&ELN+@ddOn5)MhSyYg27gE|b}mZd^>y;b*Cf_?3{;&>9S?e&j&d7;dDG z(5`WvigFbOLPPv%wa}T+523E1nIR?gm$FNpN^j+;UIUgrJ^cvz70xjqabP$i<^%~X zB_EVO$T^Hnjpac?7s!QVx3pVoFNxA7F-o307wu===tg$%$Z)zooSy7B&Jay4sm8XYmAz;J4l28qB)I4!iR{zZ z(d#rDy{adXtin2JxltgCcx*mN*Jpq=WLx046F12@%~=!Y*YVD|&Xvv{&V6x59i!|o zt%;Tvrb5P^@@sO#`cf0|yg>*1IRuK79SkX_(jxRA*gaS@*pm1(C{#^J#r<)xHlO%a z8usu>ZNGjyTolFi5TO}3)d0yLFOidteT};~ul5_iH9nNrgAF)%YLAIs9bmY%m2!(# zp@?B7s;OmU5sSl#;h8YFe+RaApQUncCed&<&K}weW^b5O#wdo)i--#vRL5BsGc#1o3!L24QdcV?SdNJvsUn3_RNw%FEKGi=kL|TAw5_|cVW7HpWsLyAS;jV!(_J}&FKeLx& zQ%Sp1Np#@seFLK_JN3Ym$Y&~?GvvR8BCkPU9@AT%LoAl`N7`}i48NS{U5@D{I@!HI zH zVJ~>X9;%gZxW3nDKVws^R+71GL)+Vo+UNlhM-u+#1bBl#i488!DNws{oB-bNRM<*& z$;-xbUjGC;Jm%863ZC*9u6BT43WskKr+5MM6DxFG?;KvkY4(v=`3|&GgL&5#&f8z? z!q#9NH_+n7cxWnDI#bw2PCA>@M-w{0Fc74^(yTAe&W&MQcyN&ID1HMj)t3A*6DDbQ zDgYO!)N6QAyW#p4Kxgae=A&ZY`(&AUuG4 ze|h+RxCNDLGwQ~m73?jGbV{7ZeX56sAyIUJ)*J#Q>4ML^Bs>H?8N=$$OV{;R(2By* z7qGi}P)}xwJfpWd4n(d<*iX;q44s?F@Q{O+$Hlb2g*_kO`%P&3#D;D8wS!agxq2r*~V}9VqS+ckHZ>q zF&8WB6S+nvD94=mLW=?F%@2}L|J*eJFQHNipExf>qRn z9CZJeiq-}KQ4vmQN4~mHm3N{F{FWzk9?fzVR#5`TL(H+@0AI&gL-(lI_kj5hoa!o!Pk0zc!WQ4Q2$I6CF;QJPibsX9@bU;2W}wWlf`z-qkB*IoEuFTSkZ z+8}!?$-Azx|6Y=zUFW|!$eY)LZp;KpUO*N-nLn)%)=|Z^;qE$!auGC&Ik|VQVNU&z z`!f+n;B>q|hJ$5|!Dp=w@9W`xE_07e==Eb$7EAHp6n2M!PjsKPc$zC&Oa(WC>v_PH zR0U0)2qv(Lb@&6HzttBk?=c9)&ph$dypqIgn!z7qb55^wCDXa*llZro{ya9j&qCeZ zj1^an-}~~3MpMr(-=U!p&2BOx=Ep3hrkxml~{5b5wy< z7sI(+)SqrD^q5QHzAuPT9DiGf=d_!rUW%*SjYnN3c+VF0P(LuKPSmc|`P?z}dpkZy z32*{4D5=a2xJ^aBot@hg#_~u$`Fzlx5&XFaf76j(z)-lXwb_}mDF>(Egzf@CeHl?< zB^TyB+k$fqzc4E;a>k z1+U+iC_e(vbQWYL&W!8Pn%OVf!r7>TXxR%;nK^LwkHkZ&!MV^m; zE5cJ%So?Qap|^>2IY5!mu_O0@|NRDTae_4(n*$Y_MVp7$sm1F27EgxW?4jbU)7Vt6 z?5u$zykj|@(J+{i+d*6IfVcc>Nc@8Hl>k|*i3YC;y{S&<#m2CU`>>lQgAR2jzbQd? z{~oKO8NLKD4}|r+`+D9zHn({WNJV$hpXJo}i@3u-!AqS&tk})xjm_(4qGIy{kF(?4>})3v2G9BVn3Ls6khT6`0nK=)N32^P z&te95cpuMlF6jCyP`sIZPjODSPM|^W*?F;PX?NMpSLr^z;uBp56JNpW)-ucya2Ww@ zo=?Z*Rip;K5ruj6BGIOxm^WZkuZ?bx%mTZA7CFWHZQ@gma}+v_E$Eny<&2z1pQ15Jy$Fgh ziOT&XXKD*joOn>PY;a{Ca*vyH`uN$k0xRe`@A(5UXdi2EJol;t(XKFj=xD@6XUQ6U zPgar&TA2w#8(aH-!cq4{DsqJ-_}(Tw+wZx`#q7S=jGh^+fttkVqWoT)Q+X>G&{?qI zn18`J!wDSda?u6+hut}xRali>(E(g{@|S1cnxAnHH*53xDiJkg{CW1WFJ{rp$iSuW zD|SF-7`0W2I0txQr+A_Z=_+(!g~j&oM6RL%C#eJz<1)^7p zsWSP<*9_5@sHSAPXAyG3$JE06`HAP?smc8OYhud>(9O4?xUV8bz@`clW&Q?zO^&<* zO`C(7sxzFT6Y$nAp{1&!T~=p-f0RajeTMGqC}phjN^xn|_1@9;Vw$|j^un~wl!MIe zuKA=jpW|iReP^6&uG8#%9CssbbzDkZy1l5)ZN6kYCG`^*k=Ll4(_nlNyM9*@L&o6C zKwPkSutxBHU|L{*pi;0@=$7li1+we<6Gdb1V#?sxgp9wZ!n{QjJ_Hl9bT|#_Q z*Sxrj_7j$Ern<(?az|+o{$>yL!)hg^LC6QIcUW+0!0a#J9p|2&8OWUBx$o;8oS|r{ zk(wb9zU$BMiug&>Efih_;bf_n-nQ}fY>vzJfwnQ0W~TD;1<;0OboHAU3Px{-2kRZR z;a}!k6j25!Be(}8;QhukuVJ2^GZKl+B*MPs#BUA@Aye>)|4KvUX7YO~;ca|flD0_I zrTyY7w2*aBmJT7})ui9tk>`;Ewao`qa~;DIP`P&rhx8`YJkO|i4(i#-6_ZdlXGe#) z1b6dV!BK<$-(D=(}7EYDS@c} ztlt&*KF~cln{_`)n-a+_?XeVhu1fqi`BsuKQH_6@urRr6)>1i6=E%%8JY{rZvTL1V zvn|8iU%nkJr(F#U^Y(H-&Me~gxUHUL?$Vj%(+_-}|G9Yj2KQe7dF848J~GR2M%-wG zcM~VY&rMjEFeSd8>$o%3RUy7k!kC1z@k`@wS${M37KBKD?I>LAlY#BNuRIqr>ZM;x zYoFdQqhn^0dx`s#d#$IFuTn5UYZvV$9Wq(1HElNg8T$&yy0|&cUaniNF0RJTtBy|W zcbj#)d8hH7*f+XO>l}LPuk4%PmAsog4Lx(*hck0!reuaPXM39Zg8o*))uCqU*Lr4T zleo>4YM_KB8{@)hAsw0PtfeS`WqSkeE=Q`fW0 zbJP3O|99{!MOHPXEOmAn{pZLQp_3eMikhmJP3DuP!KNLkasOvJX%yrgxKGx==U@a+ zb98u?9P%eQ;ch(mgDw%(9E=k^zbyn829*k-Yh2M>^SZ`6Mr{heZrFXAUcOQ$8WZ5R?SpE{>|`S z@2V^boc8tdCVS>*zDci=miDRhr+c3&q!q~+=|1Fr>kkC;s@L?R(R<=+<2=iH+eP~Y z$Dz0-&PlFO@tYEEC7g^u=q%|WsM;mzhN z=|3NM8#UY_KDGd{MI?K4GC$-5oBv+2yMRP35 zveA{xUdvL!G*lWM&8Ft}?M>hGG48{Q4+TH}k`eUS{AT}c?-NgJZ&IMAc0*Wc?qF{b zSISx1RV|@L@}blm*%PyGOD&LeKdz`XXe=y$5;q8Kqkn2Gg8MuR(l>qT_wnM#o1d&{ zfwX_p_hr;^XZ3Xrb-+V3#q`uN*7m)9h$9@AE53bVx#a9weou}h_K*KAE@(Ys`b9co z7#v=vjP>_+=S}y2D)(vWr!Jr0fBr6QeA@A}1?i(RmwNjIhilWLig?S|&2r6Fl!)w! zubcQhu}R|n_&3gTju*C%))Z?U^K?1CP>KX=vcJ3cpl6_`kSFNA>E7;s+HSj$>FKzzMZLO&T+fEmjbDg8|GXI)w7<+(LQ_oEYIU-#$z}rAJM?rNXV-oPE-Z=O<_Bhqqw`t8H`4{JXj z`8+hEfye287Obd!6J0GiEpr@&oV_p4)||`j%#gks5~S# zGM+K()<^cZxO#Di9QW+M+uB&|mWQT|#xn9x;x6WyUev2-U#ovATj3#J4V4K^2!4fj zA&&l!qBPWg(!0T;i=sRFG136kbH8v+?1UrFFgoaMg*Vi89@Lk$^a<+wP@&LOdWn?- zuYGO2ZudXL@@<)yJrjccqaQ8*BxI&0Wbc!8OLG3i_K7Ex|4nseSF`0wU7KihzP7eD zjTcKrIxDxl&(pTO7vHRY)#dfsceb=Up7p^JTAT2&2pwtRvb@34-Z9O!Gr^tsFlkY8 zG`Vh;P09BXhq?Z+E14clf17bL)8Ogn-R1K#8SGO?(l&={ z3ImN7tqIOU3Hg#MW_go5G%0^VZ)X?#dP_}HBYCkX2t)D6-Jo?)LV>NmW}ecSqtn-? z?MZu|KE++kpGT=1P7-R!t4xI~l68o6HdCj{+Y;f(=7Em9yGfT%0?}?a?aCHP<9}od~|!-8fpV zD4i0PP~Bd~Ev^=od=A6x==I2<@H+h>ejf%cCyqOJv=^Y!t8o(P1RDOI`jhrn%LZz- zQSTQ0nvbDDF=RfTd9gJuf4JU^th{b>I1=tJc53apenDqFS7 z^z9l)t{Ga%tt@W)ZfAx|b!~P{b$MLtoiDDTdS9p&bxKP{Jel}Yi%6K=)Na;=JMYCmStjOpT|sID(j z4~1q1^?)_lJ~)v%a*{cN3JYj3l{vHihP-duuv zdQ~nUx0lL`Yv9GU*C(opN^T~L{R`9Pa&AbE-Sb=zSJ=9I3AcCjJ8)^!si(@xt9HG!l9JW*$QWy zo)Snr5dXn7jjYU+*fgQItAM?r>6sx}pAuZ@osxMqt=^~kAGG&nKHmF0$z3`4T0bU? zG(IwCvkJB{_Jt0~St_O5JYB@&O>Jta^ir+=*sd)o6Jnz!meE##}!jG3f zWlevb`NZ2luspO{J+AvA%>}dk%Je^LqN9RSil3h_CGkvR?!;$*{($ZdVMIzO~KkmR@cmU#zz$9P4b%l~H}UfHXSh$IT5q%p=vrs*i1 zH^CCInQNH}8Xrh4#lD8YkuADY>#m#&b_pDGrl(6#NonZbb1xhzzqwRW+m+p{S`3zH+g4yT%J6h?>zlI58VmwX_@`pCw#rt zIB|mQkA&)}qqA?w(UEiIZMGxX4rY6jtyH!?sU4C}#&32EwJeZL21%>t@17C;uIUCena$GHf4cW^+s79lw|~0zxl?-o%oUz%K77YQy1FZT%rHdWW>FoT6Ut^e zk~%1B&(u4~UniL2-djo=KMUQXvzarNs@+z83Ox>ZeQ!KBGkxjP(sHF$Oh1u1%~x3| z7SY6BW~=>VTpyQ>6QreUit~J2fw*IiQjQ7sW42A!=jKYrE5f<(S>>xhes5uSm(1&# z&)i+TDgK**8KE~yD&7HY!%HI>(bDik6QoblZF#-1y6HzexX#ME#i2~BXv-NAP>Z2m zY^{~i5;ajPrL92=SX}SJY|l*G2>w-ftG}uDR2%w2Bw@%Z0NK(mOCE1wj_>CdYv%GdDnK|6cV>Ycj(#F zhrz0W*WPyS&*=}-4`v?nHBtTv_dx{^66Wz)H_K;CGpyI`v*N;W=i_!b&f$7=&t|aY zv3!v48_MaqL$7?9o;~g??n3S%Zom7SXRNoNFVX)gus|887mLmp=1Gah_ojT-9(DsW zrS?1bfn7{?bhgbj50Ezs4WrhuNo%c~3HA*9x}$*OnEi=ut+k@%jcJv!nXJG}t8aJ~85Az2zgG7t$EZ232VVz! zhf)=v@=>+x8^TqiPYf-^@6iQzmTMvr-wCrmg6~@eL*r=8NM+97Hrn^njFSIc2bPx}DpsQ3}_c2|+OR`zvP3-xvfd8*)uOjk5t zTemZ_m)q#qLc{g$zM!ZChKL$2!^i z&SJIPH03t7kwyqFqZi=9J}|(pOl+!6{hp!w;ahhK-wiK{lmxx*0PFn` z(@{=>*EXP&bq#cPJcytJTx|(6=H7EYJV!Ay8Q$4?d=fl3QXGi%00FIOcn_w&6vcWK z{1kpe$C(tq&djfTFtetj+iS-uu#;Jk`{A;+fhRBy$DOM1m2%TRiv!0D(M?+dvTtKL z!2IYl!zgC<4c-AoIk{-)bknZ+|n?%Z;`d)p0cvoZ+9L)uh z(czBzUTqq_FBy6fcxYQV1AnIvG{Mk@iC3+da=DC|Of6tUk3-Ml6}I5l(?jeid}Ejb zhtdPH=PpX9x-i*pM`lMmfwxqnOE`$`WIdi;%)u?0xx^wGh-V^4fp`{H_E{kSPG1G2 zq#pgrX&}>C;PS>MxunB|`<+<=d$n9dp0e5@Z7@CN$!I0><4*CJ&sPnM^BAn+MzDQ` z3$@U9O=MP8L*Xu%+ifuU8FX4Iz#whPHUCf8N|$K{J8Cd`&XQ3#Yd7X%)}DS`Wx7pA z;2N2P9EKv%p?r?DFeYQjZ!h|qOJNun013C?oRAYH{Ak#`Ynf^?nU2gnP~9rg9Fbb# zaUes3wN&^My_gKxL4Sy+)dc*-HsJ17nOU#5^=IMi29wm!*xdBVxKDm8-IvgA)8Rgg zesPttxN#iM_$b}|nu68PB+^q4sB=^=sF)LPERjCub~wj*z^9bxYr|XNpqQ5~_;#rs zo%8%StnN4d4uj--v8m7(Pk}9Ts(RD=y$V0D3cBtNAiUqBG%dniUKwdZFTABt8E&K- z&!yfl-)qrTt-%_(kA9$#pwJsH0B5!$+KI~WhmN!F*1-b3!%V)KdLH~|#xjTbsa}9t zrX4^+3!v&K2Wu3K1|8_TbZh>h|M7-66FPyXr|-qyQ5-JHeJM1M2=O-H}{hxuo^aB@RyceGvVeeJPEEp7VY-} zaLQ_End0%CJO>Y_IiAF)qC*WWgdE}pv5eFJ#`M1^p$4G$*v;=-nYox9PxIAum0WZw z=HcDn5Xa_n(KAG~)~x;ryao~W{$gAx7UAMNgueYes+2wzTf*LN3EqC6e*H~e zwW7FQ@bSK9;2RX6r&5LPVQfalNqS*F(c#=o7rGEp;RN;NQyjEP(cgIq8~-%>vMP>h zr|EW|gIDyJzVLp!$=~28{+8~?VZ4P-vVYdnzdslb!Mtq2Gimt6VXPY31q+Pl?L0#> zuJZzUdYJ5G1P%e;!oxi)tinsC5m>_&K6^cSxO><;P3V+(P$Fz1PW{Ty|40{6qtk1{ z&!!Arx`p&_uA&>dha=B8?n-`oH7WER{qQYB9K8HQl(qCXlhFFCLHE**dnEDk)GCX!Uu!xLZE-(ZLiD>1Pfw$cnTBUx zeS9&;6NUTJ9ofO7?SDG-K0PNBhNL)QBipJW4yu&_GN5dhLxV|JvdJ$)^JG_ z3PZl|9(L3D9LxTl&dz#_9^^m14zdeY)2}=SkMt_`HSaKpv%w zH=56~jc)(Xyw+Z#^3U{*_tA~W2j_e_5xOp2{~UZzCO-0C5yNt#=Q+x!?@a%12W!p5 zRVLC)>_T49f_`!xqF|6+HJkW;kWZGIK4&c$0KK^%llh)vxF9h-1+S(*U>|ts*yksn zHH8nqRk#8(NX z%FN|#4x7)wsu|6mJIMZS&pmF1@~JfZG7oIOU-QFQu#>K3d+tL593bY5z<6pbUKVRG>As1Sh)UZn9)J_|R6NL9>yIaBeQNF` z`n^l((i_lmT;L2i$bK({R_AB9BHt5p!_?_UvYVU4+kd$Bwe)AFP`z)1lV~FU*n@B3 zcsOw>oFlRBY=}SqK<7LI#>Ek?av}~rM_|w8=P4ehpB=~9aFhFgpI-c9K51)K`hM14 zd!B4nV!~Y*BEz^_{dna!FkwEx#Zt&za-ltpdEM`0eeWm!ZR07gMK3ZJ*2yH!HVX*XFsGfXDJHuQZ>WCZ^iC z4WH%?XXG7v@^|U{zXD0ROV2*Qr+7e&Eyvy)4mYM9KbeJjEF0l+tmL`<#rMW|V;49f zE^rO|h<{hO#=rSGOxORy7ye9LetIx-fjYCdr*LMkM_JY#y~9A1pV{F=)a3lmDddLV zREG8ShWk9}%M~2uE`1A@^^A2skTp6I{>N1A!3KKi)!9)s;FgpHYk5NMMT3U5|BU={J&p9!G+3nx(on^&r zaM13etP(iAn-U!iC@xO(`K#mn@gG_3X&n4Mv-%zE*iP*8#q9K}%)2N@?)sT^G80W# zdDfE+t;b$ifX`9S1z;7N%!mde0ny z+(ZZ&UGWvpfY^+Y3q+e($XQ6J0e5H#`n)T+F8zh3@i;lxBzQPF8PYTIqz5o6cCeEAa9$^{Y8P|A zVhn|JPK`x)UJi%ZwuC>8LLpWW-OJPPZd4!R=;n<;r!)lBZ4bJw&BOV_|H4N-z^^&D zjCN&a_&Ye#vC5$=S;I3BzzsZuUNG_&N6`V6|a{7=lU%4q9U2b8g!g_SwH{5-e}8uFrr^M z$|vp!zakNxWjyP70=dUrGQchT%VYQ)8T`JV=y`zk@&TTP!hSeKl)TI;@{%LAVjWKA zte(s%{g#Za8f#(*Uwzo64TP7I>PHLwB;vIaj8GxriLSM$2x5`&yyxPZBM4zbmAp7-uX++V`^Sc7#p z>PwzKmZ{qZ9&u+O$jocAt$zn9GgUM2z%yC$%_r!U2 z5gPDc^^+*do>75k=srBJoZI`-r*O zVGZ383QHB_U*%fz*XXgsVpPgyJcmZ1x2d^lJPL>dR8QPcvN ztmmn?Ur+tQxtT%suo^zlGWJ!>&-50|sh8w||L}@6U}8RuEQJHL3f9mKI5?wt{kyOd z+7hwSIL%h^+=}sJs_%OmCUyb&%Cm+AtF5(~?BK-o&Aph??6*5nx)*hR8(6xjU`S`!IXlsI{=;fJ#cG>`M`v?R z;~nsT3c(7zLJZx&l)0g({Jq4N>u|INlN)_T19F3t>Kt{!Untg^qWqS_hj16`srSVz z?5G~%bpHh(=l`@5s0Mz-VJ}?b8@-cdo z`-PTKp9vD#P_Kj&-+b8Qt*Bk6pp0G2JobFB#j=WP(Lpa1W{NGPhSEpjO7tQbi7VV6 zK9OSh4jcgQr^q?9c;~`@M{MNe^>7Dg7 zt-dx+y`r>I2H_7<4e!8qFueXi?N(p+8(JD4npzkYLu*j624VwiAy=n_I<5z{-KGIJ z&hIgIv-Y%hu=q?RjVq+=Vk7c_FtxNV{Jr*3`9^IBm*cY`w-AXq^;aO3MYSHxPIaQW z+lU$|HkW~uP6&x<(yy?&I|vi0{Z0xyP+^}optaQF^=jez%mw~Nn9bzCG2!ldHQfQP5GV7fv9aW3u$0r0O$)K5L}BWWbo7nJBjx|q)*SJ5rq;Y`~KX0s8u zf=;;YpNGp{H~K5PxC#n|^5{E_oXC&JFZOaTYa41(#Xo{mXcBgS|LqXwhz-a{4>B1E zw?mkjbGWK}!hEJ0Fbf!EXfZ0QzllvRh*g*MuHoEN^ux#vDuNVMfvLLC(3mRsA^CO# z&iK9T*B7uHXY*Y5fF%~kwPQ5Sn{Qc9_236ddWxP6->8dB%2>rbhoj0QWf$77d7$bm zVSlw%-h^Is@m} z(_{INbPw+2d10pESfsXIRXw6yR3GTs$qR%?VXa2!W8mA+e(gGY2D}@7O3#Z9kr*j&O zG1Q2@0;?*=^pWo5*)#FgUO~iquFL2;YLLg3rxI91&Q^=s;52@mhd38gsr|BZMxI8= zPz@iO(>xyo*wF_ZYj0AyjKE)S8v8p7IA1RPJQ$w}>u$Mv2}jVVdP;qx*p-s_cBKU$ z!Dp%vUMLJPW;bUT72!Q@RSyj(EKA}$WD&Dej_+ps$&_UBo6p%g;(Yh5y_+?!*(>Lf z&I%2gWt=U%kUN?c_xcjBTB=4?=@DgGut{KdFiB(Xg=mz^NKXvw(Q(X=oG=WM<{L-g z0{NF*S}Ket;+5D2FOyZmFVS+Wt{t#0Y8ZM&`-J;xPnBiLd3Cei2S=U_;KISMK(DiW zq=uosxB?`txwHml^cYbU=i*8zp+tL%vh^i6&sx+n@1mdaa_NJU#k_DKGM9NE3ihx9 zjy*-Ih1Y0Zc;nN_`(9J$1UX{{MUNYbi?yU@IB)$$#Oq6zJeB=hL>w+U#QZ{-tnNQp z=j*_=ax-V_CD=}7vZERJ=+D!)^Q2aWdxEJ~L2L2}3~_FB8}C{Nr{M4K^5`k_gHL>m z_qhW4u{AP|lkIAx5@%Tc=(n);5{S`X>p$Y9Svx$P&-tA`2>Utx-^QAen! zU~Lv<64M-Ii&{#b8Yv-8Ga8s&azmQPDYD#H$l*?Sn>;Sb=d|0#o41+QTaz87<5oLX z+I}@}k{<~rm|vP2y$zeXc6cK_`c(BtH4hUZMrnQ2tV*{~Z6yx2)Hdcp+!I$BW--gC zWONe!!ndXz;Q5D**(4kN${I{8o`?(Q05OAnVH615b+luj@tX~3%~9SZQ5Q+kv&5ON z!h7`O@W@DC!*ubPJv4K=0bZnM&BMj6#14~H0F-{9=z`fdu)uLhyf|LTuizt+#y#yYnYBl)Gy?_KfsI3 z%JW;#nz_k+K1pmXgu6j8c*bkUkXn<=%%n0Jh~DUHGQnYRd+Wd_p3UvFz zOqTUEywZ1|c3Rj%trH4|!=ED!iCJT#v%rE6M2<%L3cKMYd_YgU1uU!-nEqo?k@Cr7 zqzI>Hf3lZ94IYr%*Ft+?I2`_QC_ZfDr}cPFelVV?)Z7m0#2cVEds#QTc#qLM-%eoP z#p%5W_#Q-~mx$FcaT}c{l^y;8U1eK3p~vYfdqY2mMuxnh((3Qp z6&M!j`cS!PuvE6+8j027NX7_cIIC8*ON#u@F6d? z(RqB+2Sv_3I^{Wd6BLmiabjy!a5ssXVWF`2yHH$ci9>8nFy6IPh1Ef?7Eu=t1AE)V z`&?k1MB%nnL&tg0@B#E@AJzL*>X-+S(R5SpgZzvjx+F(_3s)!h??Kr(n_iAe&VCd( z;-Xq_BwdT?E1y)ls79@(_Es&aR>d#yXFRl*gl;JvwI%wqa8;&bRzVrFi|lo@*is&5 zu4Z55Jc{SwV)j`}Te5ALU5C3UC%EEI#+9)hHkFfuVydJ|yHJWVXEK_hFT^)68E=Ks z#LTK_xKAqE6sI~7f7&4^Vwcg4wSZ}RqVqwW3WG1TXa3JU@u{T9n^A;HOj`Rv3X4y~ z;?ge3DCd)xNR#M6Rxos=n^G1uIHsPcADtZ88U91J>N~U<+G=JPRn!wXx0~R~nXXpV zwrKCQB6@F>fJJeqFHGLC0PhwrxxpCFupo0j|DbP`0{XrbbYm2iatc}8UK9^KjF+V4 z?E9j`f??uwIh(1A>4IE?$@4ep-k@I+_tV37NQuIhNE_zTICv&`h&prWkNk)`;WxN_ zWJC%QyZeC54W-t56P?0yt|qw2`7eWk*9HCE0t&o1dM$D`dIrRrxUAtXv5q`|J2^^RA)b+2TPizByNbl;a9wrmwRN#Ju}-yp?}&4*ajuIiV~@9_ z({CP0$2N<6MV=vNmu?$|NA8lVKi2|O6jStpoLLW;6E}#dcipvD>aXfLROFg=LubAS z=kyk447^1nTlCA6r0G(M94D`q(s}Q@WS+g`{>C(82$zwvazMNzBnY<*ezNV7!Xm@U zXopDFaDIKhmZDw7-~4yx{{6~qr$lv@QdqGmGnEvy)F-tbO!_JGWe(79`Xe;e+0eUu zMx8XCobWLn-gKdrl+XCJsUez?Vn(a6v9YY_5ghS$=HsT>#**?gJY2@3A}NgnK}c9; zFu)PCQSDx#PAJ1?pU)JXK3wHGybF5MtvHX9PA|@XFFC)R=hvKS^E3|o>!|lugT%&A zTqkqIGdT-qed$cNi5X)+f!5%Ix0l-Zy8a$VzXf^*uHa|r^=?#;l1<)MR})1`py+6W z-{>6miTWKg(za_2H8Kjc@&nx-ShDaNd(w`Rq*Anu%V zg7XrYieMdNNwJQ%nH&Y;>c`Dm7U6e$_xB2a`J5Rd|`He)>`Wg!;>Ok z8D=vGt0CJ@)L5E{?hO z`PL@pA;x)fBl*6RN6IVq5>_!AVt;tM_MehnX`%e9B&c_kZptsAXs~B6IangN8J~lW zN_Vvx+=T38$1BL%2S+C3l=BX3Vz*EO$E*R;EvcBiMSg>4#9eus+)18^D(Z|pPFC(^*b6{%)*e+Jys94$~>-p#5chX#B(Yz45%9MgB$_EItA) z9*V1^LuiV-eJikxyl@ci>MF?K2>mPliq;k0*%Dmo`|A(%Abz0v;BZciMaoDm5c>b< z%bX+E{0SCMHSqNhJVBfA7zFb!74f($T~B>Zg8qP)5Qf1@gEqdGJD%KV?Cvw*JZ z+Pd(ut8pR09f}lpXp!Px+})kx#a&z6-Q6GV?(XhdJjs>g|9AOEMqgVV4Y}v+y=2b0 z=F+d?L-7eeiXG}RwIlvheVO!l%!wJGX9ai4#H@H6b?XwUz8d(%Z4nnrr`TLI(b>s$ z#J)H+MtV6UT1l3LWw-G&B>y z7}|miwlTZmadU+!vPar(?H|0qGm(dLXzjFtXz3oJ0nSa-m=8POlGw8X#GyH#FbeL^ z$=$F%*tW>MbgVrvI~cXuehC$mlUZLZ}5Noxz%CNzkuEj2djF__q#$yYmdYJ z2%HCs;uD$xPScI;hz+U2F5p4414pdTmZ81;(|+)Q2;Ny1Sjrak7+I)qKEOK80>kV< z_b?V*y%tDWI{L}=_&A(pv)OZ_h55#Ofx}i=6gYdB@b7AFFt2hxmcbO{i0c)Fe$t;x0UVEQ&iy&w>Ir8#_XtmVZyxV)kLbyaWASKLKAc>3dro*> zxhuL>vNzy?d>T*5HF8b)5Izp0#JRlB&LD)J$$C}Tu`>WISa)*t7dAoKs3tqIHPB?Q zpC1>q={Q#vK~2yR*MY*QM%Kb)H3qvp%zR+9c#Mhp<1o1csJm1!;VSeJJK&Bh|5Pq_ z0AbvWx?&o4;{_OcM!c95zH71}Q?<6>*=|Lz_A9ZiG0LTF%)X96?@;jPgmo?$gjr}V z(tskr5UVj^I}T6LT6F2-*@JOYdItwP0*)*a%w;azjf?nq3^m0Fc>bGwpYpZ_F4+oI zlt7OfgFpT)rZ#`aF)lkb?h<;R5^x{4;W+N$Cv=c%^eFgqvQ?3OX*RslGqAd1@B_zD z!Of)JDFh#Q7{2T#JynQ3yUSbO&D&o?4joV3dyw<)w*MpVTGZm{K{r$V^f<_hUvnMK zNM1P50_2&8U=gL^l*&@4nrv07jq8SsTKfYC)F4jfM!CK+O(}t^>rqr#rd$l4*PUdt zvCM%^!!f*$@(sr4Bh|nQTtnC5@0MM<4-(QBzSCxUd@+oQ1DvQkCt85#FHJZ6J6+K+ z>bA-D(w|)Kzc6UgbX&QpMo;kBV(qDLKcP=sn4i|mGphk|F_B$tIjmP~-Mel+X7@`J zzn+;Izb&jmJJ_PLoYn%I&d_%Gy5yY|WS*IvdKE2QVZKt)5e?uREPq<{sIO&Mzp+AP`!0A*V{@$a!Y&M(Ka@Q7H__&*<+uBa z)7VB+Na`-0151J4~shjq=G&wa`&fSOu!h+12$4sdlP?0>BP zc%RMS8wbI)G-D3*hxwm%8{YK_SV&FWNGe+sQ6GlXT2(nGY52akz#zMTqTB`Ny2aE% zfQWq)CU_N^lM(p+7M7m#i5J4^M2nBXeiW1)OU2ai2baM_rwg0tO0L+ta8-}14>bbLYcQCQ69)AQu^`@F!aM#I=a56(!jR9;5U{F#WKRda1@DS+ypH%&9+Y4^v*ri5tv^4EXy=Y=x?6*AvR21*^!XTFKW^Tu{64REnZ% zu;A8Sf=E1ouk8qyloy<@1Zd+_K4mNF%L~FN+hhmUFy>);N%zF=!XVVWed&2Ck(bB9 zT@Is<_K<-F!)`z2`y7GmqD!X&FzpRs^ocyzf}$G)*#V>1oG1MS%sX0WM}6=c?i>y1 z%{rph+6to5nn;!tEOQx5dINA#f%^UiT-{xMCZCnZ)~!{1hrfvo6<~QP2>rRWnc(@F zg3kod=5`lm!wTNBhQR)n5Hm_X6zoybRa7GiI+X#axF_I=o=++#-m?1=C5l-??Q_CC zu@{O3r_@VK6l!w+{GoF9E6)66`S4W&;@ViR=z`du%ADvFhAvXe)bJK!wJ?wCjK)KC;j$nYQN5W+ECP!L7UFO zMs~H23V%ya#rnbv{F)}RVRVIUiyh@Way_xLJ)P<62b{(i_DS(S6hzmA%=Q!*#M!s3FyWOb;0Iq<8YkqpE1RQ@Gv*z;E-1wi zsjZj_jc`*c=GXkZdBs9R=dU0O?_hS9Q>%P4z4l1qI*8yLVW~9{p5Zs%`cBbD^w|oB z7;Ss3tT=?ewc8RCI!OPD@2rrrD&A_$J1B)8)>CnZ)gHI-Dqs?=a3%c!w{jImdoDHN zZF`qknLBty@LMT3uV1$^i@hZuJXUe*Bn*53E1g(UN)a-0X9uFyj0MMSfFA3ja0gT= z3(>8BI334uQC!3n!WYwLo9JEVpbA_L+I${7sH!m1lIbRP2rprx-xB$sn9uaOMi$!? z)5yO_Q+Vp@%?|W(bM1Z7Vf+BTTmPEbj3fGaGne=nb>R&0g|HZOG_&=9Tl^>YeT{IB zs^c?CvKBBuAI!E^Kk)>6c7_X^*m-x@tVLEChKqeSF${#jMT9KFZcompy@fjJUty>; zQfy+k0h@kmR<~XXQ`qP8yU+^7NJuxt&UYavTi8V@jhI^aOnvf~fT9CWyiwH0xKop5 zpR$vr4SL|h(qi#<`#O{VZOktANa?=vSs5Tj+C7Ztnx=oZWa&>Zyfnf#^O(`y(%2;+ z;Y!dD&0JA5l|@iW4MS5gLwE+VyPp0(J$vjP2rJ}g@(ppZ9bq-(#&s2c$duP${;$g^BI$ zJyd!pgt?r6GQtyZ-ScGg!s077EzGiSnvbnN(Xsz(-83uNcW}4MDYYh}<`F&`@3eA8 zJ$s9oL%Im7ddGfa)CDp7pZUsuEM1iIOBbwwvBOM-k84x*zSR@=!64-~r`xBgD|*2# z=Ct0LSF9t#L@AT}1ZBk%5TBWFrt5GG3a~?9E~uq}>knb$guP|6;5A5#DGT`m}B+wTJSl-O}H}6sn|As92v^H|=k5 z>ACs28_SiXFuOh)pq`-ZJB0}8ytGc-WMwutSYL!4a7KH{)2GnTSZq0Qi@VWp9<>jX z?>^X(;^&{vNYCL!@`D<=sPHbK_)sv$kE|8f8wk-Ux4zMMA-@U z2jMolrFYgNqpxY%PthOzLgs!?Hp@e$^qSnkl#E?fs;MZ-BypPcpWYCURhfy_&M0W6 zGo3V9++};sB>lT_+O8>ml?ysME z1r@IZ8n#dH%6}-Sq(N3@HY~o=o0-1|<>dvAJkFBJ9$}Wb2&}WBF~M#o?^Bk^mBnKA zM`MpZ(JU;qkoMyyGtr(()tmq?{D-s>Hsv#Vvl1`_0rM8qAqE?Z>&i!z4V>Nwx~fN6 z<-yoWeSatLsT63eB@IvaKe3y5Ng{ zmOUq9g~HY>^Qd9!=%K_}O1Sb~blN?r1V38YL_w-Ug|d&lz6|6)FBJw`YRx5du=j|_ zf7z?hG`wMZN?!W_lNT?|+g1@FpEO2Z1D{&VP76Znhg&I31sf0F;S*+ZaMrAkf9NZ#rj*B#%kfQG$L6QHdI4M> zdh(r1vyt&XoK!wrX~ki5m@cuC-H9rw1gG9>XB9Synb@t-QS5BhXLH>@MmICJP!}BG zCXD!II@htr2Ysbg0~gw(|Ky2gDAZh6LCbM_zE_V zw6G(@S2)WxB#X93Az20XY@6^0TA)v69%~7=WUbVO_w#@np&&W8844~p+=+^^djOpE z2WrzW%V+(~q-uoRT>M1EQX2*E|LpO?bDXITQJu$f;<}*s`a;)NfvJnWBGV;SYij*z z_9605KcS<9p^kaRm}`EriVFompsMj~Tag2uW_IhWy%r_geDntgsa!75MV%(E=Mx9< zE79U#u)__-6p(>?F!wu!4DiLj$t`$NkI-Bkuq%s=&^o;m%2|z!q2^w@25$EwamssQ z)il!KOESe?BJBm|FD7MXma@HZ&|Dxil!i%L#rwix@?$FI7wW-^7DHEEkhgr#YCvC< zLo7_6P>znZJ*WGfm`l1skF=0(^QMI>7}e4OKEp|B_oi%18c4P^M3u-nh0ahhTN@p$ zCg6(a!K4a^*X=0wuj4^wHx$duX{CO425S#B?S9aUV${eV$=bJxxCWhD2K1!MnNT~* zZL4DEq{gdl*JZNnn^XW7*D=CL>rWK5AqTw{oc)(lI&lsp#nSITUm|yHwQg> zHL;6WgWU3&oV1RfuqR0F3i}*L?HTKhogU@s3jsYGY+gxNv0sGJY!vJ$y%5%;%c+b) zv4(X|xI`xOi)F!(EoS^0S_|y|36fY$XaVMt8C_h+=e8oJ>IIc-A=su->_vgK4J>2@o?f#Rh&Xqs(27;G~Qj{ z>3f1q9_74`pl+@Tj=g|hqZwFTJU#*M(FG)1ZH0l-3i+s1i}#kyZqrOwOZyUUJ~ege z1K!1Mu$)9X#D2m8y0>42NXuX&={Phv3Fy9OQaQgtiyP$WPGTZpFu8dbmGBSLd6}6v z=#7GO9yf6x-12EE+h}Q=7%7}%he906;VJY({m}4DrjM6F$?`Lgb%>ic1_UDv^;=H5 z=^-E{9nceXLMPAzl)4H3+<^Z^ba+x1K$oZ)+{;pZHIhI=Av@A6ck7pjTSI6@kMY9#0}OZvS*{cB{XKh>C(`-XW&$u4#j6VpBokOr zPu`f>yLXo_#0nS=K}I538xoWDuO=1p4Ia=$%_| z*3O~UXO{_ji<7V=#ldFM2zl(#gl`ShFE6ZLgzWGWabPd&>^xQ`bAdUBPJ97;!x2=P z>v=0F#t|@~->LF!^yjPSs2*55!2&vgy}YL*d_eX1(JJ-RJMsWJ{V6=DgP?I8Im5F- zb5b$Aa7C&jzn4FwQ7_7t;`eMn{h#AE=VwP_$8lJiWO)ZltIA4VWj8AI59rzJF^L#2 zWoDnzGiJ}C#CFUAr{|td;-0!VMW?89bJ|bfyqAIbgn?o_#|NVc_{^_%Xc}fKnw%H# z){8m2Z|FQin_IqvtTrV&&*U8@;#26Q2g$@&QEp~DnK!3>+&WLsvkw)EhSDSr6Sjxp zJvV|RW+PJ+XG$wS8tc$@ufM_DYJ=p&^It{UK`SeFx*C5UN*|Gh{;erhw2Jn{XFdl* z3z(5Sf6@8~^O#^}VzRn2%H;w~K8CYrJ`LRLJ2R1Z^~F5Pmigu2{dGa{&w`b9pyF8o zr?3EauLD2H48HR>vOs$7^Hw~MCo)xd2u=4JQ2HDoS!0=ONC!3^4SsM~IqEp)$nIF8 z{Hx4&^mPfI51!WE7T(#O7Vf*Qo35*_6F38Qz-1>ACUu*mx+4k>xj87q944}RFxlFd zSx_3HZxbC!fggiH5RqocGQXQjA)pu)bt;tz=4}^nzn(8yaxxdLI2ZncM76J zONZC-W%{kEAc2!Waj)|}6UDvEJGKM4O-q#~gW=W%SIP(amlj#B4;CY>hr@ zFX(+~)Lz{B8oEHn^9&GkMl}c6#DHk~^d8EE}>ToHZTI zm6mc7^u;JEKm<1t>(;_3JYe>sf^~>KYK;B{H;apUe|BWgqbJ{rW~(=w%|@A9h`c|* zW;dhjne+2)?BiC>=W96};$b4q4`%CTgL_!0YPRAAyNmyw3PxE3R;mO~@F-EEH#guv zVowrX&jOxyQM#&drjl>->~DiXhw2@b4mmxy=__dJR1{!!sYYgSI~wsOLo&1k2GQjpc-2V(MiUm|8v1_rLh5kD6h`xenZkd_<)X3O15??T4JRZ0Pfbz-nDV zwe^yDlFC1y_fWI7Srtq$hndU_Mk(XIens~(t5=Q_njJO5PW>GE)0=uPw(V3v&9MbP zfY1Ep)2S)RZa3@}VuU=Aoyk)jzdIC11IJY7NcS1G^hGWf;YkZ=a6MfPx)GgOx zutvg&OyF!CA@=uxT{+AR{gb?tnP*v#$%%tZ>#l*Luk^Eye1VF4Df55#P+7akq_vry z83rb_3wCK3DCJ&|=SAeN=iHx=Pwjf%MJcAQzB0po2An;_tfdAsjv@CN{B#Ocz*o!39mxPDFoB7gATzj0cmWim^04s4O-*izW0qKiX$#F1zoX7} z;s{WZdL!iK{gHX2FlLQ*^Aia4OuJxWGH`>2^7SXX?Jo1YYQdw0qH7Hpk4I#NsYI1S z@bKx{rLrkFEyX9AD>)>?}^|YC&JcDf-h^wP0*QFRGHH|MZf(S zW%hHrj?nzlYW~#TWS6V--C1zNf6Wa%OvG#pid2I4_=#G22fRaf7_O$&p50J7Pov)M z!U=jq&2SzJ@F6qoIy~nI^0vb7`im&|H<@ZSIjjTsI5!+?HtOQfypIGsJM~%LpYutj zZ8yCAb`Z2{oUD^D#fw26SMa}E*#mI`W@IF1umbP(J2!8SIfP9l8BEJ~Yb@a2DCpOo zvCV8F`^;vbI6i|%%T3S+%iLf$4M#CgwI1jniyuKTXX z{JpQM20o*+onAKcu5heE!}b~7j3NJp3S%hD(i2#a%HkR3a4!>w#_{PJ!C*C|_WKK- zw-mf}d-MxKVA1PvmcVGKW>UjDwIHA7w9Yf#*or8)9u-C$r+F&6>iOKF!%WdDoc3vC zsr7UA%SX<9@Ali9xO ze72Hw?GLzLJIQPVK*O&aBaLE46w0(XbW81x5yll`AQJ~D(22{YP#=bSnKV*vf;-Mv zHum;%_F(>R9Q8s4*K*f8m*qO=8p&3q<+xeZb>?z9oOd0)@n7+?H872QP#T3>L=oOn zIdLcN=mpQKCFd&>o!}|n%nKs;O3ocb=JWoG)@xsQOOu-`65~$1aW#RvM4 z(3Ie7p1^7*ReMv{rRJnoWt&8KBHn#`!Vlm{vw-R^7pjxbDCn9q!Q=qhe8N`-W3y45 z+WiQfWHNQ<)CJ3=lt%tj+3sZjts#U#*)zSPNyWM&;k;1tU1Vf=aNsqb#1Pgz8L)|Z@Dh0`E#D!!5PKafR^ zQ0MPP5pa#OGmKN7m2BRFbAgL49d=1nG^b!0o`I8{=e->O`+W*Fn}^z8;?I1}>BvEE zSx2l3E^r71aZ9T8NKvhdmYGhA>V~3=&9nUrk5J?`D?AQoXsY` zqVYb7BirxnY8<3FwEOBkwS;y9t#ugv$TFsH!>A(~fqrs6*$O>ezAaZp$tgLSv8ON5 z*~Phr9oR#-IqCQ-HJjE?<5`r~aarl1q*Lz5?eLcA2g2W#PQ4NN+08Qw(k&k3tkncD zd`?te%o9Hksy*v>|v1%B};JOjRB7 zBxuH+c|i4?*y4*-OqY*0g8Wtd5rNW`#YHH$bdum|G!!>^s0BLP@Kff?tF$FAbpiN4b?y; zA2Dl~4qZihrjQOw+vRdf1D-&DjhDIEO^`v}j#{j=oK3Q+4FjN7y08+Ln{BA>>NDAK z80IN7`F5PUI*6LFF+JZ{eyYtx#D}nH(~Qd03qSN7s8g?Mtu&K;Si4Xl?o&_j*CaJs zT^rmV9Hnm8GuYMTEUxG7Ij(BTS(qY89O7v0T@&GsY#V;v)7uqvEORt-j&^;*18uT$ zK)k>tx{Rj80q)yU_(CP~mfoNZ`qhEvT4p99%^_9`CPg-)_3|)1c8bW{8HIwx$=giz zwT*5v65jeKu`-J|3tb!YOw@c#F{0!Oxg33gg9~j5f6KYv$sP28Xw8KGh>&i=72gvx zNF^i_r;&;<;k$?`_fUr{r7tQ4(wGCxtUZdg35JX}+AdVcql{GMLHfTkV35bmcFbS2 zVms3gZjwx;{R1Xs0QW;dLBx(f;VkuITOwm!^Z_9^n@MD|>tx34@auEydlZKzT zJWsj+(XRq^{8>7=Ih>c{Ftq=O&(O;>kybEAl}m2M7P{ALPAjLhSJo@nm4hgo%geo_ zk4!fDVMJh4LHZvO)lzJM}d1>Al|`3Lhcvy`UFHs;74q7GYul0$?cm_q)jL%z95 zwG!k#%%ox(#-_9=lmtN|CwFHdef$KR3UX6VWB|hn@K3#n!zn26rjzT=Q6FYO)wP|R zxrZ8jH}7yfPiF<)-yO?AEi#B|qAJyd#(7FjuVXOt8HfJruy_pRT4>{O4>}JKWcoP# zXknPtXcV(kQ1WF%l{bxeu!GoGjV|yEDwNr<6OsISfAK50cNIQuS1{UBR0`?PxQxOF zXCZ3GP;a*tcat!3?p4r|S9Ij((S839Wza(q#ZknkIn*wX=odbNCoSQgZYOfg$dmVa2u)i3&1Flu_zIBy0x;G?t#}1zsaFrOND8-vN%3NBkc+=sUjq96sSv zTww;%qr9cg&P`V6LvAi@dpIlAL9A66F(;GD_4pn6$QTo;8NK#&>ZnWf`XA^X<`G2~ zaq?DkH)81GpW>fT6&2}cD#rpu&kxo>`pCD!2Qj-`M)816WpUh9Mxl7LVX05aFJzB$ z4Og%j={41PEpq<`y0rAxCu*Hi<_+eVj~cn@aC%X5tRem-fVcYTd7aEOJfUMrgL_I> z)W6fw5-w+IN%brgkC6N8x+=Ig)4sOn08}4H*qE8%HMm?&bV)Q5Fc|)_LRQQtR zVmh*m^o@5|iGAA^Dz{T~+aa8)Gr6QVUEe|UcKhijPtZG$W&%qh-&xkb)T}>XUS2U@ zSd;(aI2m*+49H9nniROK4mj2P0c&6`jb0pB2VQxPoc?u8H*QoB1wHK&F?2 zSytk!6`#C6TEFU4vDL{ZlX(szPbQ8xE0Oa`an@&Z-ctDPA+Mlccy@n-EuXKo>>2CLCo2c5mWQu={JV*k=P-!>8hVWiWZ`RY z@FnMwmd3Paw z#@~FV>U1QhK*wgH0qezG&&>=>Dk{D*s4TiNtC$te#JGcuK2fhu=3=Vc&Bqu;p01pN@^zGLW+ z(^>cJsp0{g>+<7=I1D^H5nq;-@_2cyyc741cI-~64!5ehR z3F?_4AY%8>VeaSsC84S8Mz$D&li46impZXQa5|nYx#US~8Fits=}bKQL`7$Cw;OVQ z(pUzWW*b=jVdj!EfYV&$%%!FaJ;*%u6@2yXpzwAh zU|4(T#X9i@OHoxNfD!H{*VeTI)Je^#^)i4(Y$Mu+_R6QB2R=l#{uPIVW!#G}5c$$@ zW<9A}L*MZbQTP;H`AxhQrolFEq(5j%)nsuJFB6Rh!<~I2rX}F$71}oR58q`MT=X7# z%WM3eMEc3x{0wp6U2ab2Sbnz97Ol|UzpnHd0V>+#{CtbKVIiMXh%L@jAu!Oj+?IuW zziIT20TiOui69}qz{@$$$sOO015G71ZViQ`$+!(P~7Owf3u#LycJw_?5mliB{tV*%hEC@W*|jY1V7<_Z*tT2x6%~Iyxj7r6`=WMOBMzhrFUwa} z(Lq$1O|EgFkGaIBz5!}=8J1)hpKvn0)kbE&#t~f>;r`N!*cISR)gXR_;_`3Yr$0Ea z2f<%`#@`^V)AicyEIvmi-T*w}i_so^-F6tcba*V*MWy~#`Rv%{%!QZja@QVLd)HCt z2S-tQSC`Tm)x}tGB2{I)-GiAYIpZH(UftgY-gUK-W*Dt zEl31hMHYO9>r`3ZPH30kNZ!l=IBXA|EJe9Xb-+s^#C234x8SLsb9X*~sk zn_vfh>9&fX6zC44@`T^F3(Pk^oct5?$0gXGRtv;+6#e2I@VXYvW0%6wZv9UW$^mp- z{{yS|470k0JlqRx@FSS~PMH1mpa34N)p7KHRN7!%q2d zNk76FT#3_22K|+Goe9HmW<}@l6HPSoFz;BENK%S=u_dz}4m!}Cyzko5NEBCsTvz^A zJ}K++J9!D-C?8PB-@>`LHEsyAnEglvFRr5FN%R4a^h`e&&a~rHQ;bMO+Lkat+KPA1CPsHCJ0|>PjFhGr{^-!L{l}spf=lswssMkR)_47-SROykrpu2 zKZtPizz^K^N+#Qj5L2^&6ME_5>yiOuiDiq4e+!Avu|%s|)MhV<>2tZmXXzRnaVq}g z6C1`Wa?U((sktbZ<})F=59f*9Y#Cj}=GYowR>Qg5pQx2vaFQOGdBFZ_Q1@jQF5y*O z+_B3kvCnvxcX-%=uu5Try_-E{Jtx_?zQdW$(O-VUbb6TG(43~fW9NC_;H2QAU|Myi zT3MT?*ETm>bE(eC!h?JkTC=;bmpFteuqjdpCR#QsXO(a4R-fw_&d&QORQ8X!^UcH( zYym81?$?KCW7LB|Utn2aXy9>RWw51MPRobd?jQ4;Re)~pt=Lw&AVuN~(h2{w2J}6- z&^ISAvwILHms$9M{v)+wF6J8A&OB(=V$i7VV`62#na$h*wwj&k^4`WtBOY`tq-8kF zExpXF+-`dK*5U?qqY|!1Cz&dWMZ2|(NYD|KtP}jrGqmF+shRiT>*K*&w!74V8gZPM z7A|i*(fbNV-k z&XMJ8@U0sljScB&#(~u@G)gcVo|cYGW?noN&#HBXZj3UiP2f`9WKk11e`#qTJ`p57akIcOGav?$LD?1(golV^LJt^L9VfC5*`|8!;!peE; zd2+eaxDuTK_(c*pZ|kflMi-yvRRIPbu6bn0vt6fKX*7Bb|m;w9%|mf zLS}BsOsO~qBHXdjqTgioR#@Z zhvAGn;jXZbIP-vt)WheCph60xI(h@sA5Esr#a*lmAN|@gi5v~7BKlG}KcEBkF6tC?7Qk6Y6w=7=NoLio9BdW<=gxrgrJGbvmt?&!eIm*(!E zdz5Fcr;A7ORCAAVm3HpM4Z%boUf$fTGv%%J4E_;Z9jv4l)e7sS@q;g6r6-F;fCgtJ zFQdx?cT5ihrcx7Rf}IRGl0{Izb>#L;VS1!K4A5G+yL^R7n6e_=p^Zxjyha_S#lLAI z^xus#MBq$z9ia*pMJnp&>`F7Gtx{0A#H468DOL=i(+Ialn!lM>d8WJRB*(+D;9 zW)Gr&cZ0U=r$X)kVpRwK>G?S7Y&L2$)mzEvX>4THz97E8BM>BUw%5cfPPHJts|O5qiPui zL)M*a8BZ1(ES_TGa4r+LjpWO?6HdgR@D{U$WAMFOAWL$hH0&o*n#hf63`SKBo@)+y zs3X0oi)o$R@Kt_xi`QqmZWr}jPCm&YzT%0g>v2MxYu2FZ-$o?8Os89s37n7MiF574 zaG4FwA9(E*=F|UzibCZl%?qY#SyfRRwIqKP=Up#Ci}9EnnBF`^erk+^-&v+m+reNq zfR!vt$6gcGZzYt)Ro*X&~dD(OlAcgC7oVA6c ze+_+&7N;#z`>5lXv!1O@&|et+EYMT&p}f+eI*Yi|Fl8I%{py+Gx$3^>+KKPZ54o{) z5&SL6dTTu8?w!`=X@|L)J)jVm<1@UIDmuU!HbD!%5M%zM>mA55Tge2OlUiUTlRYow zE{fCho~Y3aCesDtC*h7`vv&lKBg=~?9Bkq zNeWEc2fo)496t{0Rq^?_h?l~0{e)4;>LA3+UmS6sD`9;i??(SGHedXn_=WLW+^N`# zF*TzKM|}0}a8GkqRVqu3g|XHLqd#6clhripb9S_R36{aNz|H5(q6ut#J&XrkqWXp1 zrl-{_S}J3eIlvw(E|#Y_3cF^zCwuaFn|c**b5C`5D%V0sapf&psDY5%&SEtJ&HluqqV_WI&B(nWK^(Sux0RE@R;h>!_2bwHZexY zOC8w+h1y5&9o|ZNPg%F>+~{bfe3xc13zXR|%(jpFMqQls_H+MI^Lg(WUN)NUgClaY zopgj=kKI0V^&)6|N~2Ej3qzy_ayB^c3NF8EmivdhxTl(@p{Kg%xjT=$kSn9}wK7*O zE!`wD=uFrDPrsqXYwgu5!TQ0EY=2pKx7oY1MXRrt3eFEq3%m@B zz<;bWxnw{0q&atHYH&_4otjfSrf)Jo*=?jVit0S=j`D8yF84-y<2{dDjhy>&=lcXR zQUcAf4Swmhc7dJwQMx4>8}v@>c$4)bS`W7UE>vr&?b#i+T05oZVS?bXa8jD2xSiQh z8x(bm?hfb;8#pQ`N98umvyGPz;xtqcb$>_YEv|h#rT^%8$_T^2#U`4ajnaBIZG`Gk z_p-h8Td=h{Kx;*P{{s&MKkuOq{mvQmD)~A8gAJ$gh-my#&xSkb2)y%3;!|)~->py5 z|J2`WXV^8oReM0}{sVr(XQ<{I>m2;hK2XYY;2{aj%Go`PtV`W zaH^{6#rU^9oZyG}OO;RybD9V0cg(ipKaQo|Ls5g`8>POPwn_Tc>7S>Q(&kAN;txd! zBRt`?y!+kHoa>dTQfr~N)yinCO$wm*Sem2EPuD|B)w3;hi*IT#dH- zC74?jc*ZLDi5y}}COp>Cr|br2t^sFXfDU2=+O?3M{2^v{3xKvo*~wt|Q#rv+$qlE` z53GQv9i)H3uj!t;Ks~O;hP>3YW&C>)D%nN2x|CN3vtc?fPEG@fo4Lqpx!GMY2COIx zO6jKH+qvwT)@<{z5#Wgr*Kj(-Bdx1iS~Y@~gV};T0?Yi*eT{uaikULjcfnsqU2IGj zOSl5z17jB^yh-yZ-OTh`)AvkwFO8b`E>(s2gRzriYDZ6vY!d#fXTRgE=(QedYXiUd z#wQ<0l9D$kZ%OIuJMD{QyXtfQ!NBccS1rLPVvQC~OVN(k&hGBTo*v$eVf}HfSrnEk zERQ$IUC9;em@RF#uNj%NXMsxoj=ny=Vr)xn6l|mB(7NLal%TCtJK>wtL|d_~;xs$LdQ8oG3IyY@v5+|sCmmaF z(6_T7-2Gvj6`_%MTS}4-DNYomgHg8am9B_CKnoM`BD_JL?`H;Rh4Gg@SUbT9eZp3$ z(&|z*Sxu!yP(Suo!_^zXZNXK+UBTT!g`aS>`c5s3V^c{oTTSzx)r~2K`FQ9Il3I)V z?aSs8{e(K6ZP#j`Bb#cEt9@__9gHW?3wA#{{iE5myCJD%Qihb&f$zE`&36YQGsn+L z{E%i!+CS1(PO~EMR;mH1-HX9Ct0|No11urn423a$db~aDxA=Z>H~8|7>frxpKlW+9|kFQO~V+q{y+7-+pUmG^S~_*>YDhxIS1}&58$3arn`WIMeLG zt9WtnpTKtici%+c4PSr%kigsERP7JroB1mke>D8m0y0TXD~b(kb@gl9o1t`o{k10A zZ^TuH_F4U(noeyH{5$ZEf3j~(%Btjo$pezB_|^qKn?AX_H&b-E_zel`6Jrw>q{z4$&pg+EiYRrnmzn43Uq2WXQ4G#CceCed zY0%-nm^>t@WKy-{0=`*+=GuMZEY6EysZ3gUm|%M@4OSXD)4Dr*vUwYM4|p!%6Q!{+ z%E>hKXc(SBOq1OJaVbR2*xF=o5xV_BFx(x{fwe%BypQd29VC}B%F)&N+!=7LaL#os zSGMEO)tPC5zrb@R;4gTdDW73_DQ$q-3&*OVxJ8A5DPPvlX!+Ecfx7jEIMm+{ISgs?&JX}Fa0;w%G8Sg8PoM_T3xk; z+FZ+E46rhZhCJ8V+5O2~!~KTsl+W;mS;i)|ekf#{DIetAa#86cGhu?&l%6Ub`jP~& zistBz%P}S2TPlq!&S$* zVB6Brd9~LkYZFTvpYC7a|IfcAFe=zUy{|@4r!U22Y#itN zwW_M8)OPB$;Hf~0-{tS=n~+jBrF+V%l*;U7t{Z5l)-W0i=afdA75)yEGd;`$_|>Q(97Bguj6Qm%t2l zneow{&a>PueV`&A0gvLhn}8UmNTcK;%0qC+NJlwXicyk;i^(B$Z%e5XcdS@GzhS4`^qM%T_tKy4^(P8A3U+0vzBO`69?8Q>;9@%Ixsi zE!|`5lvj?FXENQA0)L;`J_kydR~Uo)>Hml>nmiRvP8+5(i=#z-g<@c%*qNShGqb9z zkgd~bcv_%u2YMAU>Mt2c9lRP$2`;DB%@dgB7ya#h8GQbf7b&+>E~ku4*`6}SS25VcC@Jo6 zJjD&{h2sT24ROwI-agS+;^Pv#B&3Re9epSwf0*k2%k`_PuzQKOVnl4zg-A#EFjp^W zlvNBJ$PjA#wrDoy*~6^*=4u=}gT@wf8{NzVqC!zV{SvyFVkjT7;UaenC&jk%4N%oS zp!R=@m&6pfjShHr?_jTRb@cti!O9df+}v*HdTG76HePj59c0ul=xOZ+VjPN?D_1R2ty}a2k|%Grj8r`#d|6YdCAUJg!NO=5ktbwEd8J z#AIf5FmA*n4(4`80#o%&RSL)d|a+jt9${keGKyPgO0WjD$3osu`dF79i!*g zAF@AmyEa`bttG3vnp!JK4KxV{^j6$}V_*nJgE`5m7Xz;YX@kGAL-TWBPaqK387!#j z`U-PDw=Wkn^2d!y=6IYYvO90Ni+L}2obG*&Yq&pslyf^jxZPpb!k&5`y30HL%*-#9 z=Q`fH!o9V_eh;haz2$D;`UNM!XUYsE6VWk^JQm%OQ+g~egXJhLe6VJi!;MwAe}C6M z8aZGePvD?Z((Yz0!Dl2NeY(h$R2=Nab2fn7rP9A`W~6VNiw3PWE>BaK0f@k(vXfcZ z%7V{81(?(7b~&pnJi;GnW25+$3b+!z znLPzbI0Gaz?|p;0^Z76xGx1ou3tLwPO+yU0QePBj3EZUFsN$NS2ijz3wKu@REyCTY z5!$32_Hd@Z8yNZYLb%Ou!=Esp){<_zBGLQ3|EsTv?{&(hlyKiz-*;aRzaua^@F7qo zxGva;TI~v3jc0>af3|w!>he^YCMU@Slp;zEWxvwJ@rSd7%W(B^?{=?o&v#FAPj%m9 zmv(ATqUVi!i~GJiwP&#Bx#zs64tXrjo!3wlX|nE(IpXG{!g)AZt~kT!uq)-sCVfclLol`O_YqRU^nQcb3ZKLZw7 z5DaKFYO*XS9%`XrsRuip#ME9i|PoRXPIi+mCqj7B3D{mX%NJerZpi zH4)9hYj)!P!@hzuOhevAF?*Tsei#;T4c%)YuiiDs7PXqzoE}86 zHrs~aWphNNJc-EGPs!_ucjk1Zbsu);aWmv*CNR-48P4V#?CNVczK|1K2^7dL(6quZpe^WzvM<_F z!8GE{v8XCa8w0_F+cE7m8*J(gGYnm+F_Mi2aJnJ(@d`@5lPHSw!0M+()1ol_7l$e! z3D1i~QYm8baAtm{kOwxw_bq_;DUas*vlPyaOhj!qS*l6IdWE*~8mv%XlwReCUag7U z_nENz_ophX4m?~dZkgL`ug7Y4)C$_a+C(igXk6c*D_9};F_<4NYfGTGzm!k%)%WG} zKV@rpX8+cdAIWD@rUWYKGx5P$VLj1xb)q)i`k{RHzKUoUQPi8y=}`Vr`nV2xUx!zW zNE@ERd&WK1eb-$OoMX8Axbvk_g&dcPIqY(%UY{zhWV`_EBW4oVNqx&jb=wH;$6sQinQ3UmIinRj#YE=#hfcSxXO;F7<$U-NDCE%)vB9q_gGJxY0#97XM1$5&Z>WM;${ zYm3!RpQH`7B&R#_x3~;(jiZ)^jqnZ&{}{C(wnALd*q71OqobqqMESzIcq_QO(ZwY> zpDHb+$#73;r3hIQD_V2a3;v-0s5;-YU>%apVfrDe?5Q}y_EV>7^^LP2rX#@oYQgwS zB+3q8e@DP*VVxA#F~zRPEzzvj1>dS>elknj`NbrVt1j|Zk!eoi+zz3cv_Q_Jyut~* zg*2RAc?f*tQqd*XB(p72PRp~UxkT)OsLh6S!tOH#yU_=I`h~LWSIcs-kS`eWIhYTweB}vzO$qb_V@! zpmX53dfB*yUNE;&L>(6J1oj4A1rxRIczKUt4r&T9Yp!7!3&7S^GY{C%d|+i4Q^V|U z5G&aEQK|0cdA7h+Ftj(Z3#VxXs+sRNTa3err!(4*0rU@(V1i3ZXK~1@D&3OjI8Hk+ z($$qwBAHA2A}x_^`FF*wOoBUZCUwN`;469QAq;*7)R%WbL}sA>Okv8eHae*Ts5$ce z%v1TS2{3GHVWf`2rT=AI(v!5HT1H)iyZq+BZ-IzFX3l4GK2s0>YTy2pVkxhaUnh^F z`fBNW=ey*K_T^7GmC`jZM?Y^%@LV&^KDurQa;vaAF{|TxM)wSR>fYuZ5!EPuaDpCx zFXmj-vZ$WXr=yod9gWx=CVESFez@YCtCgo{Z2y&ah^4KW+6YjAdGg7;1+GVkveq zWkCm;883u0LQZrj15qb6AO|?72NF$?`7T{hy;zUWXIl zEHp0bIG3N;{JoDk{r}Lk_^B9nfUFJ;x$`mIupZ>;1+&v{;Hm3^^{Qx!*TB9N)eqoR zd=(7SA3Ou5u{T(m{`#0dx4#};+j#!Iz_&8xd-9;5hd zJKQrKX|MHqLMN9m;+L4A(N?(Wspx$ZkuO$C^&>6Oxc9ME)PeBnUdcP# z+u7U8bHY{5c}D3g?YHk6ad0=H8I2OKpZTxWEO;o;IOtM;hi}-0>-2U|)mUw$no6Cb zPSS?!MPUy%QQ;DAhV}6rho%<(;lfE!HJf`H`o4oMxB9yU}JNA zr&5lmJY{G6-INt6VJUV}Zup_}&B$>*df@j?o9Im$a95Cs&ck$}vl#9(kL< zfi#auh?|&DEFmJUL3CC&By5gv8NW6rQ{>99OktP9c8AUMZgih=%8u0XGhqn6E0Q_b zN=2O)Wqs3n1dj%u2779ibX&`%T~-}hb1jcnSe*h7{zz>IPu7o__&jJ-+oH-ohlW7` zuc;!`5ZyR+xKYlhA`&kbO5xHq0*BwBaz6Pl&fFV%yr480?RiG|s@zc-s$7s?!**_$ zF3STPb)1Idic%U~@m$<$S9303GXrpyS@ueBX(!PrdQhT|LG5ibn=u+q#z{DmEyfZ< z0K;!b1Yl1SsO&Q}3cuq`c!Td%HwNzpss_IJeg574U)UUn+`A($PNU&0yGnl{1f%>e1xgTALUz4>0 z>apOc;6K4O!RLVtfdZgag?y{1>JR(M`lh6KQu-vfNZy@%HF;5Tm*i~8hmwvYl}@P< z=&RG4S++h^&7>vRva5S!VoZ(5Y3|!fSLgBY3UT)n&ZJ5eTRm!ER6=Zr_+s%xVva;M zkI*CJsHc&=BmCa^?!C@g3ZzNW^xCl--r2(8SwYEf|Z4q7Swi8fj-PW@XTSRBM6 zZ6F6Va4)qpdkUUt3GfV`c$=5trlN74DQ9nlS#4&gw$7PXaIwouw9Ls|fQQ_(2#tMX zoKRbH2AAO8TbE5$Nz!;F%vsL4+ObKgz>`hyeD8eh40A?1rYqx_>UgCLB&ODq7f2VF z@j3&So+$o?yZIb6b~8{e9yc?gb*N`<;C6q4_0NkcaxptjHfx=f!pOT`I^Curk`On`^t)f5Gmunl< z%K8vtt!quhpHch5Q(SqRvS)nM{8a5zFH1;@eGokKx*Eco2ljUo3DY*jNj~a-Ip64^~&p>RZtLb=Ajd zFSUinB5N5gCdteeKQgYG>Fu#hUtEyhN@b+#Xwr&{{iILwJ$iVRKV>>T`)QbWJmDDV z$mA%=q}?#bdB+dOJx6WFFl93{WYgqA${aW*m;9J*jLE_v{D4QHPOS!i)tqhMC#!^2(k{}vs%)+qtcz-#DhQ&u73t!-Q>I%4L zhA)Y(822NtW!$9LV9b)3xR@EywWE7PSB_2`%jP0{mR)Bx6P0;w-Q?1 zbwZ5&9i2{db#8D5XSK6d8{hs$+O%NxK+V9!U@9%3y};@Io%)xyQh#KWHIKpOT+*8u zg~3Tqq1TIMetiup^pd=%c+`&n5=S$Wck-eiyp3~f6RP$6N*;W>CxJgyb@X)9#f2_e zIm*BPsr;tYV?$^Ky!KV5O)^Ra=s!2&>RnRmDLz0oUKTa?GI+WlD8WIX{moc(?Sw0kf$_ODb_ zFX`tC5pNUWSOTof{fJZJakKzU>=I0s-xqt!#T<8>e%B@UQO_CghOqMCiQx;vMuk-f zpB_;;@^!hyPlz&zQkl{DD#{1uaPk0ouj1&M_3q_3&bfVD7t= zaEEDx0rZ#ig#6T>e(KLHsF}vFHT-w!x>O7uXdHFtWH=mA9?hoFSUTfYV5vRWsO)Ac zu0Q(nj8X>t{@>!4T@kf>Ximjorgk{${k~vOxVticdz~4uYk1SMp-oSx-vaYn%)X3? z>Z9Ps;2tW#+USk+V6R|m@_4V{gJ4_rw(8QnAb{)DHR@fpI9qLQX(~4{=I0i^vf3{& zai{bg<}et7o@i<62-C$#xiGnPkK?2>hkLrG47(TVd!D)*dd7J#g!KuR!*hpK^d9ld z;a5Mo`@1dI05*^wbu?7+GZP*Q+A><62(vYb8G(%88=|n9zGn&@N&v0FH}>q@2c1=5 zlFty)9-x_6!INIbM8+mmaNR(ip5OyA4qutj2J24DkaP!&Is^(j6OZ2Uc?hZ6C%dK+CrA=L&N8|QEA|4a9h8C1oG6W0U~w+isX!|8~71vOVe zA&b~cGUPlsB^fBUla=w-X}0_JZ;lbp2F{s|753$}Jhqay-nP}Yxi-HwtyNNrQ;8o< zJh={5aY}g_T`9ie1UC|`UO?GB0uKU37>>5%81Ce6%$2Z#=LpNqv#>sX857Z>1Ul?S zg5|`Z1Z~EXo=qJ+90!nHD4<*8iqH?9>_;>#;Y94UVORC%EVAK&m6DGi)YCK^8q-SS zaO<3c_w6h3Jo>z!Vm;>dRMf9!r6f4$ONm?4flCE^wAS%=I_kj`u*n_Bq&|SLn$z!_rhv52m~XG^seA z6oRxy428A-AB=(4=vXSEk^2N5+#N2>S^9{zL(@iN6<`;+0S-P2CiZ&Peid-R(kOi_%+9{CK$aaBB0pP*)IE#0M0K_Rf&TA+b$YI;4{_n)$^>yU{>nol|V zo`9r((@wD4Z3K~=4)1;lk=i|Q*OICa=Ib5*Ebx~nzR{fJFT6k4jrYJy61|&23`%>f zo=@)a?)>gBcN+HucLqtcD8#~`SsyWJmYX#u~vfI|lzSN%4{uk!eSIZB}Ai6B2wC+?sSl;2r zJ6dXpxAH1#ESq5K78R$Wq)Lx}T5q0INnEpQlBsXSrD_jYcN{$0Huy5W!DU8+#Tm?I)&y#ga(nAVkV8C2$@t9)`&adtU3gyQ!ZH3EBv+o^)X8F z74~;!*PQr|g|m$qptiagj?@b^C5-*OR2g#Mxai<@l8v%(dU6u|)gUXgp;6n3|IG(< zu3cGi9(L-LxaLx~B#Is+2cw{TOGp#adpwce3Z6yNkQH zd#>B>wtJ4aE4k+-E6H)dO+&HbmZ{w;MfyOqBXtM6 zxmuq|w3me)W~W{gcOuw3RPi3*5u1;?=VbUytM&ax4Lgk`{iCFBf@W z9DB@t-%s*`fpjL_Le`bRTg@}kox`1!+|pgoJ{dMJXMI}`P z{mGV;(*46XT#v&QPBG7-Tj*e}RaykMh?p3eE&NmP;2>*Av+%`H{uHTFgh#Fl9~eF{ zd|i0P@I9ezgGV~b*&Zv?lx5cb_9Bikwz2XVqchKRgLcDo;Pka#h}R#0KtF{GwN0;z zN_-uh<6rt|Sg42a_PK1-)jET$T-53aF8mJih#&Pguvm285w*A8i5;k;F-xBdd-pyr z1e3L|S|rSbHE^7!2(#&e@zT5o8+x`@+?XTGl-h&7UlhxOhz~J77+2}dJ^)qnXJQgB zQKg7e@;RPhTe#6)a}Cw%_GsW|ppG4Ye@Sg&v~-%f;Z0Nt-Em^BNtNR)Rr!qF~gUCRoc>D<1iS` z%70*9KYS~|ID4>OzIy-CEp#?4=Mml_9(Qu}GVYee|rmB}FE_KsX-=kVa=8o(g;SSFf)-ZUo z{Xa{x*i#Iarz@>(vy>-7YITZxYO>(XtDQ3~;vKW0-ox+n=Ja*;3tCh3P!04L?r4F& z)NDeO{ndP?HG;+LQn#DurMGf!>8deZ-Qqj#%csgWV3pB$dRC* z_Ai#hLLGfB)oR12LaleHantwBbvU6%!mFe!p6+UGxJzZVn*Kv{JFM;V`9*ytZmrc( z1J|ZP^nngwt;NsA34I)W6e?O`tTEO)mb9Wvw`(iZC)x!#x;Z7AbV%53G{7O#r{}>@ z@D$t?yVOvgZE3DlQ?goC!NJzy%|BK4*^b&f5g)v?{FY8p#qWc2RRK#{%K*89R6tBi z&zaoB@zrsDc!J-=U!ytN=~`w7qk`TIjoE)jG`VV^uhC%Ea$%yWX{g|b5(|$o>yrtO z5nBkIaG~9;4bUD?1)s(|E3AE`A7Lb1innmq2HnK!aUsu8L8r&tkQG_+Z$CEOnAj+_x$Hlk(N zjF1&U)g7;_QI@|#8RL%n%QxLO)gOi)C5PJElRnA$_rjm1iId&!{S}DrzN?4*uYD(c zVQP}r*<38%1H@Ub6=UvVvSrFBhoNv=^zLtL%hx4rBA4lONidn3WRoj9y_uvtSJq^OamaGH)OqQljY zCSh7+npEjiuTR+~a%5P^5HV<*{jM@rmhdQFEz~w`aBjUs+rNFoJlkDPFuE4*%>I1z z_o!gx(er7a)d`wld@)B!S?R34$>y|K;Lnv((pooKW688Xfi4bKG%&FImJ4DtbE$qD z6<=|<=}Xb$*E7y)kI_23^qp24<4ts!JZ&k=>e9F^^cB1CbwepV=k`|k8HHiQd=ej# zH8#M3b30m+>L|SH8fCN*{-N}u>OHsou`@Gmi#j5OVR|e_#;VYT!)e;x)*s`_;#q}4VUPk8^=R=on^DFWysaY z=v1>)wulr^#SI8mBGyJ`iQE@HDRfG3Ct|Y-N@e*w&uW$M#Hgyh@rC02wbWn8U)sAi zS#Yg#U2qTgtycGILyMTBlpIPX5bO>e_0#j1o8vW`%uy9zINd+LT1vHW_!vNM!}V}{GT^Xs z0R>P>9HSQDA<>w6$tm{OK4u-9CEVt3xM-m`DLmCz8C~IEoQHdopudMVH-OkL8iiw+ zv5JcMJCr1!*#lGfUVEyOOHTO5Yi-~vlaw#2(v$@j>mLz78LQpWs$Sj9z*nZLa!E zwdu*mGP)m`U>yM`)%$#;p{Cr3??s{c?n!rwfVbdII(MB#gD{@jfgL~lOy(TDJxUT8 zreZ9*tGY&NR`MAfX13_Pa4xDx-v^wV

    C}73IHV_hZC$?AUdcefS7Jq^HbtaSeVO zd#IMJp+b9-I{9|`?cRsGQXHf$6S$TamFxv@?+X7+1J)3*+*c24ZQ%!xB~Bfo%>&~~ z1CBM6{>rOZTU&?*_xdmR*24WB*1hk;TI#DL`H@`40#efEF@>x^YD{YwAY5mQ!&ix{AKHCeWzSknkvuaxnf~+ z8>gsAmXOVV#`8A063Uce9=G=bRMdXH=H4WC1@|KNc<(CpfbmV}DQ*c?ko2v}-=Q0j%!LoqcK-9to+J}p6?#(D}e#&AA&)^@-r z%d6f|!}Z=UdD4M^l>xJP!2Z*O9rTKMT{wk``=I!Zj4THIiQAmZD7C`#J_Fqm{V-eF z;HxwYQ!1G>jZR7=}nAeHki_HOeW^eo4npsYuBMyF6^zQjxc<}e$K$g zI16qTa$16&Ys1P!{fc@QVGsKloG*BK$i=YikyBR0;asPIIN8bR?(d66Y1po8I`A_H;@T)+_ra_#w z^~~3}bS$vQRux~pVYYA9Jl1~7Z9LPCS*j^Fl`v~+Iv9Vl^pSUp8R%Nq0wriQ(6-I6 zGFdtLQ?$+v{2%>U;ik^w^~UR8P`)}*ybMz(!@oIfbjN#a2sMZr#$V3Rd}ah)V8j3U zleD7l6G}(=Fv|=k4HGnQ{C;T3GglkQpW z%kD^TA$5XTR(=WkNN-A`v@k_k7Bn(EPoyK_MCjPykwN2vJVBR&bA;{RRD` z;_ITF(~IF^Q%CPYN8?*+as8{AMeak!a9inPsVYB}a>!o!mSqu&#o4m(lmR}f zMqB0bI2C-u2QQ;6%3a{|<1>sdHbk9F-m;ilMVP)_&FO#TEdh6{tZy2%oVWCUKB*Pb zo}rP;g{JN~Zl^%m++fxBKCX@lH|9^DD@Be0R>u%ND5@=(>kX$iH|z+32y zG@NdJP3S^55DWG-n!8L5KQ!?Z@n)E z+|v)9x1Nhs;3iXF@%r;fwq>zUR*C?a%)B8KuQ_?*(sqpTmD$6^$yw0vMtbQOIwAw{Mlo%Zn}V zi66JvHrk5W_E`H`TUaYwTaty1vo5nXv#wDFSPIK&r0VGKn!-W-$;tkjidakTc)5Dr zKNG|(*>A$QorNN3G>V`GsDa9%XdJHArjM2nWd51<9gL$VF0E&ctZ*q+^Mud`PsprP z2%XXyBF{2-$D||^w95^o-r^}x(R$P&pU`cl5dKn6aqIQy6Va(|4fJ?L#~N_ZJ5TR~ zLGZw8knJvot!UFff`lbAOOI1=D-GlAC`j*H*4HwW#`Kk>`j!(Prf~N6hN#vg*seF# zNB-xaWi3IUVsR38s-^tR{5m?j5Bg#PP%7~HdmvH-bvM>!kz*(QF!}b@geQY z+5QOye;qn>WYa#VTX8XYp=L!TUJA#Z@;K{1)_Q6#a;jQXR4(cBaISj;Hh$4~#~IiW zN1rY*Y@3K1sib|S@6Q5QNXO~-beDX*g*2L4WfrQwG1Pmf@Vf!!L}~i0T|hbZmUaFP zti2bEW;@aRN8I~jLEKvq;{^IrNnEp&%Ebq?%R_kbUh=`IoDc8mDexOU#&s%%xo}XN z1LMyNmfsWC@fc$Nv)ogR9*jSEVAOvYKiD_MgW}XSWKM~)xEpf%qJA!ERA%N4q%!y) zJAXbgO!^3Wx;f}-6l&ifOGfJ%y5&Yvm;a5KoSH%CJuk4DhEH6=qN~|#nzr$$b6y7+w zbeo=_UDtNNHN2yp(t6?b;YKOf5I>3yc(j~htwn=3eAfrCt9^p~Top&wHG&9F?<#I1 zkAz_2h|a`4{qZO(FFhs}r1u^yo$5I3wS`qSM=UGuD1Cg#1ycytI&0CHa={%@bg-X?@#HO#b_nTW6Z z5e-Os8TezddJtcS8sU0zt$@D6D9uQhGy55Sysq-mu``Kr@_|L|WY*lLE*nn|#B1DN zJhcyj=Qe>{`l5h?`9dWIh1$OH%bTxH_y6_yc!&7{Y!{11{gcdep^%=$kp%)#s z@G3I`9usZJHe1lYx1(eeZlYRvNY?nmxJk@J=P{6+&)jQXX2BAwEceX>YKyJ$j)eOaZm zLQV8f?WkOyV-!B2qFSd{Kqs{su5V^@2s0$cSj-L?EQXm2$QT+J31)uY<*6A%?)9IM zNl3|a7zW3hUg~%w-V@t$k`9D_9!#&V8Pxl~fS+F#3?UOrn9BeL@y9TyLjs)idD)IFsn2A@#i6#wUFvOofd`DbS}1T=_*R zSqqFH_Rddm2@G78Uz;WI3<{D;;HKpiTJj9$Rhf<^rjQjK`s8W%a zjz^!Qz!5DZ=HPCK>VOMruSTq67_;J2g9)PaK4I?LzuswdUQ6g2&|& zm(h*x3*HHh*@L^#2|LO7kJE5JN{Zt6S6##Fsu4=N5qN=4#SLyX9eUS_t%R9|(>yN} z64!7N_=u}Jn>O)3DO##0Ok?$gFoSmqWpJ?%6)NLl-Ok)AjHF7nUp!}KBtz;-=id6# zNzTJQMDkztmF!>R#Dk1kAF#?%sHYEslZ=GTFq9MJIX$Kmn8Ew$s?ZVVx2@qWY7n?|#OQ6sXxOFEbl~$E!E`B%VAWJdMf}6KP4CcBVt$wcn?Yami*Zz_@4~%s znI(y>w~4tpYmUK59Vc$U5jZVZGLUl;9uPTUcVeaI|JWLDjVdsVuH*Cff}J%j9F3Bo zn4?gs6~!U2I4+D)oUym)PP>PdnU5Vb8dtYSsU9)<7$JONhCJUGPEL8eOO#J{Fpx%WsOGZEK?W!$ue$lTOxJLBR#G zFr1y-PS1<$=6jUWOHtDF7W=V(zJ>|f(#Sv$#5vMldTM?OL~po9b;I?dmXOEjrkh3( zYQ2X<8{GvJ1qB_g%ojopatTG8hw7)guwEDVC9>k{NDh*c;EA?%V{m~o%!(3gcSY)LraUFB8p z5Rsl`F5e|f-;JYz2)6P<2*dj{i535fZnSr)#sxap#);q9)mnocAL2wb;JJ1sSz$LMn7x-Czn<1uZya z%2HanjZ{-`u|iLA@~smp6Kj>B4zSyN%$`_*%KkPwSr*m{n|DB>uH#bslhNC1tml0E zADLT8_NBD+T&yB)2T{4DZ>HDoHK8@U{a)fd#x07j_Qj+Q^g-Og*-;c7KmqZw@R?Y7 z2Rn4AuuYsS-4&wPql>YZ{D+RG9-X_lkoOK@6fDvmiQbTU4{f1tnzx1H;%)rH`rpG&xo=j?kNeFvK@j6Ql9o@`OMm$-qfzAoL-e~X2rD?$r%GJWer z_Lg~cL7iZX)5hV=^hCHCK>18Vi(m{AS=F0RnWWP@XdR3Z(jM8%UU3e;<-dBMnvhb= zM?9Ak{7TnPn!DKp>zsI3@?#Bd5X48bOV> z0ke1$>v6sC5#>Yzor3?Owd`zGM>UX6a)VoZ1}EQTj>4rN8urdlBAH?AF2~SbKOpys z;Y_(l=k~AQ9u>)GqN%OEg*&5ALC|@zDH4F1QR9(N$xz{=uwG1m6HZqa}1& z&5B#BO;q8hRAS_234d^3J55YklvDW!t49zfoBhdD+T(CwlBbQwgE%dY_R*ZevF0Ho z82|3f>^5V;h&K=)wt!oc21L3bF4`tBPam?2QpDmZ#69K?_vhTM9%XVK;fcS@MLP)#aL_tHl&*5c^L^M_h@ z13KtR+W_vn^rWMPZC%`I-8{36+_|(3~!=i=R5kKU|oR|yn>$**zywGSObfNS2QscSM zfad}`6FqsHR#uM{f5=&+5AgfCny<9s#y0U1yUqzPnWIK~eT{jAZq4sGpHES3+@+7A zLi5giAPj-wcZp}4gIsV07^q2~(249jF;YhH3DI|MGNz4Y#()>9n9i(+e|jIY2(zq& z*q2^#b3 z?lyj#`-!0%3DeDM=tJKbZ-nz=8zI5SMt{M!;Eq4dW#R|cn{FP*M{WIejK$KBa{0#fHg*en~10r6^T!i!QKIxZm5k+||vnIHIX7Lu!s4|#hFaFX`oWi+G zho(7ZJWLINo~S**1h3HhI#{Sq2b~Sf=|P+p!`Wkc8EcFeW`A)nPwq3$oHq?8I*GLG z8!=#&gYdE~7AyY(~PvvK?!1 zxj2Xjsj;!noJ|(die2RwW0H}xbua5c$eJJ(1>-2DOH-Uen@ z&Y&}_;aJ##)9LJ6j#v6=zM&^@W+KDJfj$Ies~4D?`*`=PTumIBx+20JDx@huLG!>i zoFN=QI}l8+Kbck8T)e>Z?Q3L0XOaf}!z?oAdSrmH;PoHb8A}uO4CdS~%N>3I3qOZ? ze?03wiG6)IO6o5-LDIJuq~Hv!P#t|_H9Y1d*rIVfo7KV@p8Iz2^)Q}63E?Z)!EVO? zg0T!Xa6>qfo6Y9NZ9R-S%>Vv%!?B_`3xi2npBZ16opLwMt?|YzvhhoJ1}ewY2yAN=bOL8vW7jBRpk7n((Bun}Zm)j{eA*F+OOm0kY`e?NwEW2N~OH^yr0xJ7wB zQLqf7Sr?D!3G^L=yB82 zL%)6w#!)dk9Gk{tqZm0#0rt@WaPm&U2oCtPf5)A@HF;7d{E9~5`E-+fuMCLkaag=B z=q2%i81+9^z&5;bo1>3E4?>j8Zh4x!YD#AQ6|H6;oLO%X@hYGo1&C0O@?5U6YtO?~ ze>bt$SP;1a#6FjJ0y$X^#Tc0{a00vII&hdLlZ*8+74`9EBFI2j(I$8vujH%yLR!4# zJ96^QA-8_R-ya~SXD{a`&v?QgxfYYVJ;I0{;K|k`a;?J&+=^Oi;C>{$`G1%i6}pDF z`rqJ0xk2x@*TjZX(2JB{$4`fbq9adb6MIrX(;U!=9w*=3OK!0Yq(tONI@lk#vKx=( zd7t7ZCpZ&s@vLIm7gNA~m_^O6p?R8fC;^`JND!1v#4-2S{ar>W;vy&UTSi8$2boY| z97!Kj7kAT#VJEvqCdSWgv?cEB0H)I3OlH56ajJOEc=rRLX$tG560hBb9ywn*6EZMz zNobV%6Q6Zwz8qne)M97a$aB8V+zkWy$po`=IBwMar7}2gr2;K$Dd)oF`~rL8Tlf`^ z@u}`iHWm!GEWofo1)8)DY;YQi+#|n&6kCuP#^FkMji*r`E_4er|58K} zow=&-)J7%~^Q1Gcf-uUqm%E44AqBaZkLYGPGhh=dF)$Zaz})RbZG0fROMTXZ!04yu-;AaYIK~i8Xv(^m*AHFTMu-s zd#*px?PSD{^~129=1?((yQMoo?F!@R+Y^-e5)88;eANYx`atbCRgS58q8@N7%}j1m z6i+IxE@eyO$bAus6yN(Abk?`~;|LYI1j?uUYv+b4m zQj8bBp+ELg(=Lgd$U9@aXV+86xMU!cyo4)<(7iic)eM$`mNv{tD2E5l%ph7EfR zH_?TBu16&^MQu+tyQrEGM)qfaL3N*+o@b$IW8u-2Cqr0mHU-P8BCVBg(iOcFcFYcD&*9f#aOxkt4`i(P?*{U{qS#kJw7t?6%Dy{NJe&j=FbCP- z1}eargihvJa;bE<3SFh95Q*REL%NtRXAE4hrx!B{ZK*c3f@4{StFBJ zJ~IZ#;L*hKllgpH0{8Hu~&}bKp)D36sT(%I7a$<8Sfac*A%+q;7MR z)8GRAxkEWICNhdo^gr}Qs=-aH-*4Pf)mfF(W`dQY-|AeBXGBugb{46t-nLk7yPIk}q zbP@YtTx2)BLS*$E?&cQeOgp%8o!I5Z;-@g3ise>fvFyBFpu?|&C@T^sUN*AWvS4;~ zdBqw;ft{${b)j405IzS{_h`Xx*^rsHnrm7@yca_RIF`-}6{-4`;u?~u@l54insTzW zfCpEM7~%%n(M|YXPsG2XHlI1!S6i{;Z^FUw0du-Nb-`*xZJALw{Sex)TaTsoQyWE9 zW6MK&^)^-dDPxrLN)793>kaEY>udD+E2-+wv96;(`6cUiYi;XxrJN!v$6(n^wRD8z zHcBob-$kXTNWbwt=>_t0ojCg)9Kt5%H_rdzXrnq%%jNhZZaBgIFpJYF8@ot-s`A4b z*)fdmJkI3>jOanC*5^2(7wWVBd3hhFj(tME3-Y*(j>{W}F}4yX-2=}I%#+td8z=S8 zdI3~^!-%tb@HvmH^dP!K#OK5sZKxt1Voxni-ZX+e^&WM+8RAg-fz*-nhUW!}s zavF^LU>=;F64@o2QZ=|u6w#8nF#>E`fOF`fDq4~;If#GLYtXN6yy7CVhk!XBj~2Li_T>EdG5B&8G;6jKo^CCamP%cEmak;Q%PLOPa0e{) zEkQ6z+tF1fo%~uF$|?7P5gkT8zm5?tNY?a%wN;L}_lD~qL}X`ha+$CU8gP#6*4uD~ zOolVIhEsh zqH4Q`NPifdpp!7~9vh=U4(?LrGpT^hK+E)(IQ4+J%e*giM<2CLIBllqoKT6qs(?L} z1@-Soc2^cn;$w2pBE$jH$XWZKOUVxo5)6j?hQEsiT@C`({0fps#hAbSK^!eJSGS41 zgiHEWvNRAd5Wj)wVVg2~`>4H7Cab$mK6HwXiV65bJ`ysDWx*+vjCTUhji{y?I7>BR zoE7X)n}`Oxi*rDY5gUU!@54XBjY3AH;va-V=U9-}LS{ZQqwt$GKbjqJKDgdpPTw40 z2Q7(LI}zb8WA-mJ=JIrTDZdwlxX~6-vdDgi(g5VC*0_%A2C*d73YNEoQ+)W1AL3hAU~5h@fMQ1 z@1zRi(r1EmDnbb(S?f-%Co>wkj;MRS>EDHL$qQ!n*=#O;M-_6?oDZgyXa-B8&3CNo zvqFTtP_$546!1GrA+3XX=+nPYE8EPev)f!i1$8Tx?83qXBO|+$12kuWaYx81y`&f9 zH1ic#{#JOV-!xl;emoP7akf1NA#sX5IJ5lhPO*YbwBV6)9Gyx7Do+Q92O~f^(-{)A z>ZL+O-AQH?B0P{b36)u4y~TpO&PAaJ*mHVn*Xzw#Ar|)d8!ZF)Qm%kUhThgt#QAiG z-A6TS70)lD(MSwtA|sR?vpXIn2# zlT511O*pMLnF;vp6ejP=1gh!=BW~ zm^X3wdacJW=lAPRsM7Yr12Eb+0e|_KPIp(aGZoij!c7?Og+Y0?g0=UcO=5y^53j>m za~yo@aA}}0TC7gr*&?S;lCev6h%|GJH`tJkfH@uIZc}X>qe(0)OME*#^A!n9*6EYkm>OnjyFk zSJ%c1s+38zn-8>T<24-Y+|&z#^e)19X(m{CJMEG=24?hiP^3Twr80X$SMfEwQ$C{U z+Higkf$U`GTxc&IW>0%X6jPO|*m&x#4~6pb0|8|@I7daHAJvA|<}a8?nc)$PBPWjq zqc+T(umSv_M>mbK!VBWPX!x;2Y2-h1gil7iHcn^>Q?j3#ReuBbs*hQXzv*KvheuJ+ z94_4l`+lP@r6wJ?j=RErW0;UaYR>uyhJ~At`o|TjGL7_u#w9el!_8vspjqg%VWF0H z+8Cg_Ot*Z%e5+P5-bmHOeP%XeC$n}o6|*bSI^(-mUTP=Rffd$FIw!2w3kfBqeMC-e z$(>fgX()nkSXnb(8;J@mvy{O+sPBO%3Pd+aut}%IndVXTwy{b6PkKsxJwy0q z?jTk_N!F4Xwo9Z?-e{;LfiWyM=ipJA*&KxKr;qNU^V?Ur6I)5to4GPQ;ae2L8&)_(E3Fw+NHvDfEXCSyMgClQ^HB)P}Ru)D=>3 zw#)*jYNR`edQ;-B5GGF3D;e{_jjiHmy`VN!47ZGAJgw}(qokGcG$8^ngOqS!74e4d zGD76hjA4IfP7&5aD319Sst;ShR{si-RLmEM<&55Z=QY*5+iEfKiLxFvKy#zHl?GG+kJ1%n+{_)3kKLEor5>UEfOu56utk_&5#fDYc$=#&7XIj?geg)NRDyw%;0emz!`d!9pXQkJh}Q42v)ngEs)Ok}l9mc>JEJ3g zV7{{NPnEM7oAp#^c;dux{kCyeIxWN*dxc+8395^EITh2f=au42+(4(qPn^6mb={W2 zSN(^ni$AGieC8dURIx>K9(`33L4+cyGKHeS87wU_7aN<{UD`>V#US?0RiHq#gxB=R zZG|s=P9kdyynw7!lco{zZX|Mg@sDfbVr4hwl&T9CE{NUlzPXUyXFuvdLA(WH?;s46 zHTa_j`px_`;)Dos8C9ZvXcb0?Yrt;f=z0GgK3{5cEq=8J%w+wQagOWPj2WELABhIL z@am7r#;?GqjyGb>iee^M8ayeZtaw{kXo+QXesM(?13;-(`$`A%)R3Hx44 zPQQv&j8^iU%+!33P@NH&%VOCt=6igzHKOsyMJ8-X!%mY+Q z@({yiV5e(Cy=D(}+C^q@<_L8k_-I9q1fKN}s%+bgP}am>5VqpfEP~-$RR{GR0JShd7nUFe|5rMSLOjAje%oG@73((`9Ng5!7t*@r|n(K~!|N)eYOeLst8>huuw*)Nw?UxWD{#0u zNUk@I>pVm)U>e^`%XbUH0eEY?fva_k97}?qn3;^LE@wbn^FK!E1@nD7h~hdTsAkNC zl$@Wp;Vs`FI|(E5tR$qShBFB~q8~M!R(u@>>gHmEPBOpJFkctp10KZ;Oym@}!T0a- zB)0QzmAT3~Tth>m*JbqlY{uR726>$)guuz^0~hcPQGPmh%zk8^2G^UKm^q4TAj)A* zJ_J3yMI{LaHCMTV8spo4Tp~CB=Vdhp?&lkLZwK@Ebjd5W{#5)47^;WXXA{ zaIEBdw@^=_3Q7$nD|_J}UiAuGfxOJr=Ztqs#=Z}~RghOWYhE|Psl}9}zfMDbdnaxw zcZ?$DW*FD|j8G8yrd*r-U;Q*AdWadEqJzl{8o`TW|DYS&Jh&o@V0Iie?or!l&pT%$ z-n4@5EGHJlZN+TpQZuoOxEglLXt3KNpbUm!ax%~66xu=6Eswc@_k2sPtim||pf|y>s0eD- z_i!@)k^z??Q;PwE>dt>3)77W~*;#dvt2N+eqc}fPfgd^HUQFk&ZhTkP?qc3 zRQzt=s5QZzU-0iRoF`+zqZ?{fPwz^#Z4p%)a|xzxSiJKi*);-dX-%T5fZ%n9*`fSzvQG+XnRuUKlbl zG%S2%cx3pUuo+?H!p?;D3+)wpBxG3dOlOGwkuulPNlp|Wz-^d>PuCl@n*XHlHHu)( zm(@3$*WKqi>IuUAcAoFMe~20db~{4((qWr;yKF_0LOY8eNqes^?JizV}&Bnlwjm`_T8sm*DH+?}nGmTs0Ti z=qcu$P3TX@yBw$`##nMGPn8N*hqZz=!rImP-kOE^vfnz@YOz|Cd6wVu38}vrEPSV9 zZ7%%`y-l0bl_;-26VE3r{k^)wNeaBSvuAB zR7X>#P4zj&pr{6s=_4kD6%L&hT*7(7=CfRsRQMQO=|r^8JKdAgz1>yDRVe9e;@YI= zNp)Rb*Q(?m?)skA-gN%EYAMhK8&%VW(tSBb8EkE7n`p1-nCQ6VxbLXq(Coz=PweIG zBW#VVrIj1#@*DxdgqhW7Qj) zOaE%5=VZ1?ALInfWaXVw-+JDfh_>Q9;~itIX?>uiRT_as?9hce3YMCr#Vsu?+f*Y z=ZzGiGNj0wazx52DUL*iMa~Rg6t*<0Vm07W#PFTqoZ5%MRl zOSt#<+~1}N&l2_~?oOKF8j^g)-PPO9e^8rid@(zUUa78Sv~t^e*fzj^$PpdXHE31P zgP=J)6S^q~LDtfB0!}hfawsqFV)_8n2I#@Q~=ei4ZYa$GX zk0@>2)J|hriJyFaUpM?41HO0re7Aiq824UORTXMRwW&Syp>EuSn5QrudrHeeun5kf z%*}5pP51Q!xS?N^;-sA@a)XFZ4^gGhB%A=h>q`t@PLIS>ol)YjZ5)4;#&Cp@K0f5Lw^NHj!4^B#gHpYlg+!@ReuvS z#x~Cs*UE(6fBMGT{*?I}k#Hj+I&oZLde;v3Cf`792rT~$QYlMS>pR;J`$+pa`vr&J z86A8ncyRCmG<|^{ZOelGIHR09?N_WnEGMNr!f}0+I^XBUk#$eM*kXh7G9@S%?5A~U5xxUrLZl(J(J_F7blTS^d`LHlj$?i!Bft&z~lAQW_P(x2hwc5`@Z)0X>?cPIRoG5htW+P zBR-a?Kz^a3`c>|1nQa+hnT(r8O?v7+lLzBHc?kZ}N;r@YIiFsD>kb8rd92-4_t4p- zl<$eRzV|uvDzE1+-HWc##WBC{fv*>&W6_VKpN){U0xvMzRmJ1rVJwbklfJR(!zMmW^FkCD!R*L8VcbIzD+teUU% zkX)f9YlWzhc=VA(4b9*Tbr(8=m0qOtd`3&WWrGr8jkNx!=me+L zw$^e(sw9p!$Ld?ut^V)+1e_c4dOIY`Nv{*;BveaS^SAV$7r$Hle*e4GpIHe{lI|tP zde5o_%!QIuS#P~%%kNn0yc4t~_*n3>pbyS4XEO&4a_ymXv-xPBX>VtbwvV#)w>p#s zaUl*6@yhB|EyCym`|S;Uyed+C zdAMaP9dj;NhuJFGQ#)kmGUrL>G-pcZO-CokQpQKLuCTO`3rM*|a!kFc=I8Vu@AJ`z zc(G@``%khvIfuIh{UkcMXShS?jkLjA!S~ho+^?$n7@zT=5trcvM{{bulJ}!ludB4j z881*FU4_rqE%@H^VeD?E+jS9tbKf=Z3hy9qb8ims5zj79oM#e#iNAR5(>P|;b`4E> zlvpXTOk!+er=*>(-tH6LqUvu@l)g%P$Ial_uvHN;QK6}BrumWXWAw@BC+W_o&6oCh zn$M}DQw@vi5&kUXb5M|Tt8JrYsi=ay_12F1dwbKnLtGydyC-H%Y?k2qb0pb={v`zmrNPO-{^~m?QCWqMo!N zxuBq6TV`*X(;=iDF-kKUcZMM7diN`!O| zE*7-fk3HIx#B_-wpK*yzlYB{el25y3U!<1GOeO8JG_cihJ_)%Q(J$qn z)WgzFOIIV>mOg*_)zKBx#iSXTYEgY+iS3^9Fe<8E2vo%*11Xb35xzx z?%?F%NkZa*zp;OM$D6-H;#b5E_*3d{SVG#wO-VhI>v>A}GN_-my&&m^I8@$id85Qz z72JxqIsA^F4xgifbCh$cv$NCXnC@ueh__R(x4pM!w2r~QRIvohpq*&j>KbMB*Es6w zM3xr%sO4}UN*)Ecm}KlOBm*aNl2TWYNc|*L}sj=iIH6)41{{bxKT;ST%8XVuqyc zNt0YBlEXYjy(4{F{jb$1s)V|6!R!f2w2s=qA{=)=!YRxqo02Y_pr4$H0<$z@@;`V* z!G=}OtEE#z{8fDQy~RDV-FK1;;2q&|HKZ@+K~GtqMV+h_GSY&fw^9^)V`skLvmsf- zj)l7;9!Ew;Rf@_KwL3CXq)yBg6uLCHM3CUTVef78!j|if$9`SyCP-Y}0ZIdk$iJdrWV#Cn$eP$$8Oy+vJH-0=n)iU^TO0 zhE^oM9*3GJ8@*MVXrbC;^%UO1<>;ulnf_M~e0?x+-s~Oi%|vY52?xcy_y8>T)%6!3 z4=at%(+j$F1dpF`xWayu6xqtVIOO+qvnnQ?fZ>u39mGp;k#@YF70;&ttK}2u%}0GV z4!lp{(v{Lox*AL9@DFPhsNSzN+JSxC67GoSr0H@EOOj=|QUG7b0Wg5(mwUpXT63|APVe3XG)%jJzFiF!t&1sT?c?e~18`*+JE_3V1_# zDjF7e);WZXTzgsg0Q-fHpo8_{KRps7c%RY#?s*zt1xfGW4Q#^OwE^F9?E)s}Vx1qtQ#_OL0 zG0Q0Cqx(q^Paqye!XBkl{djh2zwb`Dlb6%g*MaEN@w-y5jC)P=b>0#2A!swUN!IxsR7 z=w4WZQOO|%9NFigt?Ug3P*D6v&2S1BWh}L^fU02_9Gy8}(j#%-F3%l~B}<(S^4tgH zb_h9ZH*^K%{&DytQ5|1lq*lYk9}Ppg7B%+@FpCPoU&%^U|11bkJ8+%+uryNR2~Z8^ zraho-m%)2-ftz|?@i3?wQnS6wdp|a8 z@XvFA+9ez5;XbVb|F{b-d6lbK4qqUk@~(*IgJ`~lTd|Z89R}_<7ZmUy?2tRuTXZA9 z(8|I+2BR-fjT9h}jlf)IgM5Af`Od|x>Hx?42=nbeD;!u3WZTaiJOyKY1~^PNuB;&U z8w{H}FY|FJSNt3d`Zk^ipTJ*}n0HgD9t;3&8^(WoQ)64f-z&_`=I|@yz@GQwbudF5 zh!0*Pw0&vC)M)rz{OwO(=_Y!Y3;e}QK4KWJ{@@7#O_;)ny#Eio0;D zdJNAfnrD*@Mpa8@N>4Dd*36EHV9o16_Mf7`ElPEL6;FCTYkW4VCKgq~Kv3=O%)Vye z>5W-64OmCT*}EF^{)7294bLis`JNfX{Rhm63*7NlcsYAu8_i%I#lpVp4$7RHtE#~J zb^~qQ$4YyH^1{hDj$noF6}G}LInEu#GRCnerq{BgY+zUF!3kK89V&`3`om~n2Aw~_ z)o$ZyZ2PBZ*-YhR3s3qYXTT#~)y>$3GLC_@KZJi9vp?3MUQ&+F-n`=Ye|u#P7+*iR ziiaS<5113Hxz>5ikNpzXU?HlWr~1sExpcrt;fvJd?E2=$A{T&DnA(vM$SF*qav zwMJl^LwJrNU*~2w?#T5I=8DHL9$QgUoMA1zBD)H#tOwxa%ecdyj812MuOd1liJHV) zX38a=P#jvQz+P|_=H7ZLPcwO5*SM=QJi`n8RWziI6w6iFglx=fJ9qA5r6e%|5B`mupBTW!6)mHVw25n7NWY61=1YB6 zbP(u#Dmb4(>`{@-rN*rD;q21A`54ByE#zZA_Y+0UuP3K!J@%W!>=~!H#_i1L!OWsM zoNYFqb4Kd-FL|Hc#8iPjVG?}M5imu2QuFA-Ro3CP7gEzk3C%kdCGM&NBd8r~xHCVQ z!(2Pcb>?Cn^k!`=VXV(GDi2wM9u#FK*bf_X71j92&76HtY;&1Ay!$WCSxG%;D(}^s zmDiCwtH$}#kozmmnytZ|tYi<{`ETdgM=j(X=SUFqA|ubZ3gerXJ+=+2^CXdBJo+^Q zp4&%wxu5vIP%$gof)G0DT!TwFl{wyswN-$-l_+pmH>p zySdI49${WyW=^=7-{05^?d;~koQvV?2>~A4AVzu~9P6^|nZ>x4KxYms)x{^=*HPxw zHukI!)M6tU-8%gC92lVciC?#Ky;J$SwyeSqj6xSaIx$AQIg6Y9o4XyU@)hS*8~t19 z&3SswnfXQ7e~WSU^!%Z5@ z&pt4h#<3T6rveqtSodNs{Nj}(Ilt1fv)tspJFq7QQ=yKBI~U6P_hxnn*1>jmxE+ky zY{u>cAFp6;W@TqS%&7aQD?cQbUdGdI#u(+{zKb$X(lJ|J@smjL7+2dCmyK&E)`H1d z{zsQT!ze>sVBnD#!zk8fr`gYYl_#3`!_UgW1G|eB>i-;_19)6(7ljXI&LFIf6e&{v zwcWH$-QF6hZ8L4^TO)O9B;~Db+qSK6=J0=~Po5&4^I>D{z1IGhewC|Owg0d>Q+TGb z-0-Z_Qk(O;M)Hw4>~C9Yw(0P|`+0T4@GAMO-29yJR3zhBfdCZFz0~Kna~{uNG1lQL zLpV{W~xn-?1BSIU!rPdzY}MZgx5pAC-aAkd?1(z$#7W zG*97-I;b7*8ZX{fyVK4~vwINbWcNgOS*zA6VM|(Gwm?WICPP z<03q{t8}3RuoFf3+J&59g+0oEb-XqgKy_mZDei#2an|q`>~c&zL@n* zrsDsUCl26QYhW>d@q731^j(O%Ch|RFcy;SJS4()B!hF6lG43&PjX2JdFMcYG)oV+1 zy@{#jA~odZALiL(@X{SQWw+?q&(9s{!ACA}7BbTpGXeV?N_3H(yZ!}MVm<7Eg(d6D z32KeKe~FK~!Y=+n%c(85GJ>_ZjuqI<*`12FKh0};$jSPN1+GtpJw42YXkJ}g>~uKY zyQQe!SHeI?8!Y;%g)D6|FhJ1=#6qtoBIk#ASTLGWr1giQ*%v zXfG0PqZ#GTq}>z{jI2zQ%E=U-f>gF^qMQ|g>g7PXG|{0Uk7+FSfZF}OSJVl*0G67u{Cl0=n}x8*bD27v zLdk4F&s;QrD$oUxfTrI|7-VPRt9*gCa*Nz`-d~PDD{9}v;15KR{SSi0vH@1&NV?EJ zp!BEecIHIAK_U5xzDi%fpV#^e=F|l1d%!?$YPa<$bg2fIv(XH>hzd|)rV*`|O4)KN z>l}TZl54Q*foll-FvRa{=5h^mKJmfT!eGkmR^Di%tH3UZ`?`FI)lvh zKjR{dQI*X8E*UI<4N_b_Lfs~pozH8Tu%@AI5>-AB~%9OX`L5c zr`@4`|M*?>tLs0&zYe(YcfUBl?0)^+JzPVa!OjTykY^oHN>2M_`5+vUO5$+tMSE^h zV|q13qp_YJ)xrGw4{erKPRpz{(>_p5U&GGTqlcj(ihzg61$z>aZ6Yr#Pe)U#_?#KN zkK~(F4EHbxaSJ?%ip<0)z+A+-@)wv5fvCGoN0BXo*mp3t^%J*rBKuU2Sl>ylcNW=Z zE$YZ7y3^<3WO%8n__3x=W{c&9uh$j!#w5O`7#xPyWI~7O27N`Ia*Y^r3AuJ{Ix2Eg z?K%XjYA3w7Fqlgw?8Z2FaB20=tmzi57qh$~m@?HLEzZ}T;vSdhBQ>{Zsx1do@1}+` z_rO$BS2&~ z!?uqjZXb;8EQX(dg^w9cTwu}dmyU=!Eq=hjM)ra4G6c_Xl}y)xQo&FSk%&ML3t+%AK?7R3#%3CHaRC$~6?+!e@v$DsWw*YFww3uQy4?k=IRY~p*BHpi<0XkvjCR&9QrCtK_9m3tQG0@S|(aTZO@lq+Ow6({fo_UbDX-(xs81MelI54GM zWKb8#bK>zaQ|O|b!s?bn31=X>9V=Mpd)&wwu=5IVMnb5ECm4mzaF{)fV8Bcx(wk48 zcWv(RSu>1Ce%IfeXeW7(jhuc3u~Q}1IU2spc2K2Huy5C5YkfFHJf3SQanCdIe1ray z6#6gE;N8X(HPj|@?M%$S3f7lSk3@W^biXiH)T)_^&padTgkcUCW_;Q4ceRsTy;c!qt)Ld~nHdI&1EG&oK>Rv-gs> zl$xOW>FK#O7r|FwTJY!Tu>Hq}5>)FV2CVuxIr0G_g=k{ATyPkEW4EhtAHwCmzp9@w-F&hAcq1nQ%)@Hv0!qLG8yAIbQqO|Xdf zz#~3K-$;Iazg9sLwX+Ij`WtECbqP?)48U$SwvzW1D$ou3PeO$6O|Prmkx=iN)}8O>84!4^Cv+WpPiHNk3z5qY4k$()DY;EdJbhTR4$ z3Pn%%HeGBv$an9fhT+8m>>_eGLgxMs1SORC>IG}RiyZGI?76~NiX2pk&Jnk1#5l7w`6+iP8k#E z2gw5d{}cxEL;V^Y-{By)#q}UCc0+r|{E6Y3pv_Ucs6|wr`9;^g*SsgGt6x`BwcJ!E zs>9=4V-{r|!5y)`yqx)Q@2Cjwwx?6RDFvPB=y+=8igUhnba3Q$6nFgVILpMj->6t` zW3I>q6sAwh$K@9C68Ztd(eND%TiA}e^m6eBR`EXTk_{^xg)a|avQ|~z`jL+sAPW0A zy-C*&;Y8$lh;Cw2U-+})Z|pdLTy-n^vD2#@6n z3X8}2`vvU931)GkuLmIF@8Lx~f>}BR?0W?!X^T4~Fs+b0vL6{h9L@Wea0sY&K@` zo<})hoI`Q`bVNJ8Dm|DOHk~SP8>~eMM>}PTeV47Jt+{P9ddAu5H~Nao*$Ei&=g=3f z1MY4Ro9<(k+YwRM<9<4+;x!}VE6Qp&A&zek;E{Win7l;ZHurY;Ntl(SxrPQeG*l)*6FJ22TY4BN4WxgSJvCx+q>B7*z4Py+DkGeqPH!b&BN^Z8=UqT^8e&h)Hhzz2a9qw=jcFkh3MKcQ4)nv=Km!M^;D?BKQe@GlPi znm*M$AQGj}8p;A*UYh^qXRWhfYyGeVE@F#Fa-A#S8h+U3!u-j{-^DoZHnQ9l@XOo0 z-Nf?5fNl&U3TcbAo(pQR1Vp6^IPYL`lDS|n4Tw~J5ZPU$lC%RvurqN-aUwg^_+hjj zFfMT77ol~K-*`rCagyGgj>9^{A@k@jo5x#!{$HC(hL!nD46*(-UDmhK2h`3i!p}`H z*Mp{1=7w$;uS;v?lkz3m&NS+oXdo0pm5UFT%+Sa&VsNbBdaC zs{X*cO)zrs(b{+?k#1lgGdYGFYYW*|EW1(`ZHociiR-Mv9aiB9)?7EUkj>7=Ld*gK zIgPE}L`Ldk=Dx+6xyfJCll7MdO&ZRf*h~hTf{pvY-S|di86aE+_o~6?`jgL=2L18y zy)QwaQpneRwtqbv7ulicfbouaDtOLMax-HlbrQ1XXgpI#cHyHXl`OR zUey97sC;2ZGSJD8oz8cW%(n@(%*9k%=Kn=WOzcQEYMcwn+k0@z>k~a60XMn=fgGVy^Ez5lMm&C4%B*j64gYarW%79dpR`NVnLD%p;j8lZpMIUr-55q zkgCdi7(+*}`91mkT(E=V)PZYozj9c!VQ3q~v4x4IRvAt*^AltbrHywew9haj$kFqf z%Z#GtQKCMNe$AL?&LG?D%Xz8*){}rDQXOL2!ub5A)=9GhIMGeK_gWa02ZTd-y%dxv zKY0d_!wPRxKQRWAC zCSQMvHcdmUeir&Q)}kHM2Bt(f)qqSOR|R3DPQl*#;cwFlOIgQv)&Me-QPwx~94xA! z16WguxO*QMelA#_E%B^xL7B#boNNYXi6>TRLbfe|rhS52(+qTJxX=V`#_=F~v#FPs z1)(cTgt8BA&SlWcIUrZ%;AOeUeHLNOcANXL_22OTA;MT|lG&0x>KU2OZfm^I6ML7Q zr&owK=sC6;a^{z#F*AV)1Ra@caFgzUFruELD7Uo2ik{#uT;euY1aB&7wPQDy5pA|L zC*rLOTg$n>DCYAN7mQ>0^JIKRTG(045H<%2rN!*9E~=soJR8-$a;VMCVMagDD@u(x3{~e+ z!UeGp+M2)2jp#62;8)*FCQS+Gr&5C)A~XckF9#Q?F`TmuMD!w0mPg3OD$X?H&>QSa zk5wVC-04_^Zrq{~>_<`J@EJZnGB{mNFpn}!aw|oR;3cS^K$hy|>@$;0xM)UM>BWuo z<@xcwrKK)nEpB*WAyTR*`Y^?gCARO4`5Rw*>P_^UK2(1H(1sFV5f|{{{A2zW77~pOwoZ~$JQgO1$G|+lSi?j&y-oM=9DVuiYe1oI zn||DvmZ0RHh{qa$#a=g;zz7^coc7DCDL%C>>Cr}v^;MW>Of@GG{T?+l@T#_PH?|w) zs2TJ|8*h+N-1O;)^qPJkvl^Qf(4ClTy}#(zGg(a~Q4Awr z@ZsDyc#=ol!CRm?zUdIhxdr*q#@s2~Gs_dLHlTjn!SbLqaog(5S&I`+!1exVEkN^i zs96v6`GPpmIA&&}Q~0?xm9FRr6tnY^p=5(!x|w_0KnOG%!lQ3vH6pitEOrvkpdv6D z{(lX8cOmN3#d+%WSi%7)R@cMNSN7G0=~%iZB#QOn=$Ay1c9+mwmj##H493nXd|6W| zK+I1zdjaO#Ff=cYiec7W(<{2<_u!b5#C&o)I)`OGo@iA!hQmA$6Z@LsMs2A&y#TGT z=5JA+ABc7CY;`w_fp~N>_rTN~0(Uzdm_L;bc$kBI&qS3Z6!!U8R2H7#NA0Ml&JnXQ z6WLE3L^hR(A1y)EEkf)d<+E-Y8<~EQftchFx?49<-kpW^hChlARiy85rb?oHS`t?O zJ}XG1T2ED?x+GzR{>77}8bhs#(t4{KvCJJQyAWzj6&}Dibox{=iH|Rs3ymPDDzU#$ z%V8tCR6^WN_V>t2gqPiip2gePzu{yo*HIYg0Ja~9#dPi9T+A+LxsPlEq6 z1;0)(lj*4_ik~WAwxg>v-a3KzE=#Yk_k2foBAC~Bfv?1u zX~iGrDuJo=tV<*|Y=*g?xodgIz48hPl7O9_3y*R&o~)JiR_rP)F%~gbp`(};@A-{L zs)O0V+9~>2&6*hn;%S+U*v%(mC31~mvqhB%eNe5s6%)ocQ$HP=Z zl_fhBu-+&upTG{zFpG#jg|CMEds2+UXCxc(LM!tb zEaE}JdgF_^K?)`V2@|@TN66hXp&v1i-Jgvu%xU>1-$a26ej^4SB=&?cjjEMZLu`W{ z$26kwH7K6%BPN+Ih6r=a0b(+-wl6zvDeQ-3?>iAAv5ck7jQp;%d^`p#)t9GSV|J4i zYoYl;Xh;9VzdCb7r2h$F<{RwCT&C{+HW!JRt-5eu55hCOhkbBzQr;7R+{SnO!cPtt zqD2?@MJjyRy4E6tGe^XC*1T?=L&ah%>Jrt>MW|dPljmJU0e2V-@R6vMjIl0Jy9)w^ zNY4oiW3I?evpvzPM9euFCi!Kvt92S%eb9iND{i;^(5v}JJ`AE-UK%5|MC<>yH9&q# zhjBw{v2Da8VxB6TY2qj0s`%RKWM&cv5Mz}`YbO~c@7qLh9jvY9O{NH3G2dew@(3z( zCc6+F{!83_l)L|tis}Zq^qtT~SVm2K20AZ6f`R9J$1QJ!ib`ecsTfQoca8a%C&)GX z)5}_ssAvH-#K+i~+1!593`gmwp_D=NO$7_U|9F|>;2^Sy=Jgb#hVzRUGb?H!C&1aJ zSqDr_WS)zemWcfapXo>)DZ#u6H&_b|bd=wRjk8sG-=L=@e^O^Z7(a zXYk$ysQK@J0lA&i*pcY{EFXK$Ii80)iAvOc6F>S=IOwZb@Y8}&PacZ}xhft&@xD0K zhIYbPV%Dyr#7u=c)B@{Ky)~>#tZHqTyJ2M1bCiPQNvG3Ikkv6vZ}F+U_n}W`Jr#o0uf>qgRvh zZz1;UMf5nF&rUPTqCI}eT5D+5C@~r|yDnDYI%w@(@|RE)YnTrUzS0*}rzO%xYpFSm z&(_9jd?cRzMKu10e5xpYmP@!LKaB3;Q?zp485Ni*kcYj0j5eo3oaOT5Vgfq;Ifq*Wj~@V-4-r zH11J*>IuU@IC9d@iy{fRaSs@GuPPoN%?k;wR)KovlUGcuqTu#{|~hq*x9 zDQpH=IcbG)+MKY6I&jV!@wv0~<~XtNTZo>YL^>0# z+hju`x!nTpqYs~t1-pNZ?I_B0m0LVddVEt>5P*V0Rq-va%!laG$4S5c4pjPcVp|;I z|Ii-4OT>5&zW00HilZ$=hZQm5CM(u#f*Qtsx=>n!LzLlb!r0r#W(;Zsh1sWy;1cc0 z-gXnacNga1Te4dv=quK#k1oRpR3=aBizWOYiscqHfQ_JG)yOIakRkl$Mm}fn`;w7< zFk{guct>wUK7Q^&Fq`bCgmoqcT0+Of8~pWFV!S1&3Y4Imst-OqoM&Fat!j_|iN`Cn zr%UM;6Fm~~b3M>%|4!F-4RIQlBNjh3opW%Orw2p#sWpI?e559knaJ%dwfzC)Kr6^t z@1m=em%2?7@$Mx2^+@L3WudB*TkK0_pTInzJN*2uXm)K!9is~wWEJ+UAz!tLpS2ln zjILOLIsAlGpm9!6{q|r1zL=;xc4!OM?KUyuc5Z7&{6|ZnEk7f|S2?0ipMz)11Rn4V z4`|VWH3j4#FEOM~*}@5e{Tw7~7^uxikk)GSk1>%ML^%)mXdaZ$Yja9NnaxxQ^kNwn z?G{$3C$ZoIR;4AqUY|ha+EYL2!2hday;l&8XXTV-Cs%(4NB0=s%bz$Yk+08;ZpH&F zokVBWDzf3U`2EuSc8!%%h+j3jV*Icp5#(8Kz^E>EGI!6_(b0N<0LbSdPv%@9X-+0c&E%3s7aw_`qL5f33Tk6^_vqin-e~e9asexnU_2#5safT7ON5+rD4R;d#HO7KY?6E zv5INYw4O{|qaEvdi6`nnjMExKU@Vo=;^;7i(Sfp(&#j;XdM&Y3C>FUqcFyFzlhmnnp=6y;NG7B);N= zofj9%g8jICCz_rs?K7xx9JZ%d>cFpG4%#)6DWL1@5A8XW>`EE?D0w0JYptcOM8y8m z5^)cwV=uVXXzDH7(P``k0=Gy%rybSWp*S#7E2CAU;(bzUq2JJ_pipoXF33#7pJ;eK zUM7tG!?a*C?Z|yEpjtM98vQk?q+CZ{EU%@4zm)$S0?RLgg4!)Q&`Z)opPhPOf1e5& zJvS!(L?@}Awm<>l34I`2sIW!Rqkjv`HIBb?G5a>g@S_Ibn%ZkkYQ_)HXkJFW=#ZI_ zSosE)|1+p^C*CTf#nF$`y@|8kft3w}`8}H+l0)b-#Rj+$JJPM zy?Q|1K~=82`oJ3ihbA1hPqHr5*q2}`j{eTpDGj75>vS}q$ z%Ugwt>kDsr?-NgP?<((2^`WNF$MRK;R!6I)wAy+bqdjrrT3<#@T;GkX?JoKTUU4*7 z*;lDCZ0d*dE&04WN#2Gg>sh&hZKutEhm;-WX<6qL@OD4Y&G_cZ81J>tYKv2L(V zGl;P*YXM)cjNFQt9c<>FwlA@eWaCO*CFGY2rV0 z<+>VWj7Qcq+Z|_rzczlEU6s(@-sbG=ZtPz(V0OT)09Rm-Kr0|yKqvo({&)TQ_%(GO zaz1n{cN}-Fc2#lpQjSRntWQQJ^nzSu45KWsQC=^o_VYA^EjuDLi|4T?$urP1E44=I zlhneVwx0K?#ZrHz1bQxb>r(AnNvF+fCSeD`BWY}WMjh|AbW@%uUjx;wf%2U$J(ts? z)}9&8ZUQxPMd@UpWDBz8v}Lj7vBjeAzQp!Uz9~%>3o!X%0xSFpU62WM0loA*a_%PsTVEUj|fl>b5{nGhuVg7AZcQ>c0 zlyc;8ws5vlR!FJld2s(eW+HxKxOq)m=lznpEahDCyyRgiBU2lCx_f4)c1)?8+#oq^ zO25?1odWyc!@+N54o3d{BB&=+5}@ymVC6rxT$pHLOtc z6BhU&_2jKoAHtc&Qqw3&Ww{NLLKiYKeLc4)h~Ak{?UFi29ZuKGd+#{!L2rL-+k0)a zF@RWb5{i#)jV1ac^Pqgrxzf*aKXBZ#^|akos<@N=Uj`HmDCu9z|CxW`fGYm$-P2ra zU0>YA{oHQ5GeQxSPRa%)!d^qFWG$uMu@9tRfUw+j=|$9Uo|&F1p5Li0nChH^)yeCX zJzrD0rc{Ti8%KZ19Pc7;c{NIF&3WF5Lb=K`=>Fyiw6mH}S&x9T*Pgd_%$u;=X4!Wr z9#o6dD&tVO9r|}#Ya8W%$_M*6dp_9KmRv#}ilysBcS$F4Gx7dyy5g>**U^+o#Z9!f z+Avsn_n3Vi;{EEW;EnVS@fK0Psv&g4bwfjDmv_H9Sx+>Av9jIizH5us+s{nErcz_S20v&gQgVwe_)eq+Yj*8mnlVBj=VcNsI7m&%}798`Pt(B%Dm`A9FcX zp^RwYHlmX91_k+E@bg#DE7+1g#m`zvl*%jUI^BvFv}7%i|K~;_rXGLJbKAz#8FE_H z;0Vq|?c%X_g*VO{!VYHFI_NXa>%ta(OS0LST6<+NRr$xSn134gY-Omen!U4giC@!z zUI9D&CI9yRJNy&;qTLN))Ms?p_bcx=&6V4++rGozUFoe%vrQ+Xdjd|8#H0r1(HecU zHz+!Y-VdJFp4r}|YH6*X`qC4edM3F}N`6n6T1cx+$Iw!Bg8Hv|TvgTO+8F)5(Skf+ zHPK}($p0O&kMu_BLj8Dzt*-r+J;^@Ho@nc5Yiqk;YiR#&&w{$oMEe2yT0T<$sR^Fk z3#$|%?jpu2W)?KI>XY>lxU9plHRsh%Y5`_kXYwxatnmcG(QS_xI0l2VB^o1TnSlL5 zTdhYLQAArEiTpmmN1$^}$}JZ_Ct;HP6cxX#_HFi5dwb=W@?5#A+)&P;IFm!!Z4bBK zw9T?Lu%*bY;hw7089GvL3nA-Y$L-2a&&e11O{(kvaB~jv+m5Mc)hp^fbsLH?q3Ds^ z_GaR}1=>UA6O_~vRZ(lnJmu@!U3N1LChiSpyie5%YbM>40a}83f%~^k%>`d}BwSxZ z&q^M$j?8{4S$Z2QR;uV&?S{VZs-qmVEwQh6y8M3zqz!BuaKrzB|3&|oejD6r+{@j% zUueK?|6}fO=V4_8)@rko*Zy6qLqF9h^sgI;SFOW_uBmhg6jAqiyL+d2EpKo2C%OY; zQ!l2-sX^!o`JuznOl_i$SC4YKa%c;+g8ES7E~w5=I(TM~VP&Grx+;9QP-&^;Cy$qR z6Sq}hKDQ>7mJiChT+>#YnHGoW!aO29W+uUZ^y)VNRU9iW7rI*YQIKv%?^6mhiqA52 zV2m#4Ew!b@Q*G6|YOMB=PMscF4eW9*eLJ(k+tXQ@Y*eBqy_^Z2lj(@92>x1Lx+^`B zQ*Gnzd+ooOfRanm@SYi!`AQAP6h}wLapkK0vaK3gC3Tp{yhBcbpPP+I#>J?S?nT*d zI&q9*PBq@@BdO(Q(06Dz)k5lgZ)tBXZ$FeS4tmFX1H4C3;h68a?n&{CLknY@`h)r3 zkJPleM-L+VYeGKW6tzThUOh%|Lj{|aPQ=qiC_H=>&1Q#0RS6eGee2$~vA7u*qJ2TEuE znm^6nR0NXXD|pFWbJMxBn<=_;i0J5VhDF4u=!zEKXaZL)Q>=c4xBNU5m=Df{e^ z_9|F|r-VgcxTlf!^R}vgmFTY!sh$?MQft}|a*~yL=4I@;h#hT957<+BG#cx-wHDZ- z?P>?Lq*_~zFf|(OWZ~MhPoFxk17Az9g418awIsS9PgB}N;ExWon=X?fTu6O+j6jg zs^mHospsC&FKcDBbf~r1nKWNie?mub6dvmPqB7Gv z#xZRqk_uXN`qAgmw=#)tgGFG_TZrWa^qx0UHx7V7(3E`i2a0y%sDWOj*I+cLo{Ma^ z0a(FvSXKs|P{Cy5eq_YOsC6e%JZ}YJ6iV0RGce8nQQ2Az)9(+}h#PRsE>hzi4W2Yf zXi2qDkot)Ch%N_$8FZrO;x+7oRoL=hbXCsd0`XEHFT=12REqHa(mdm$OiR*{@PEy%=b zJ~kD^)`vzkp}%9FFa_V0UyhXTNZ;u{A0uv%a@dC3>nTH+i{U2Ld4NCD?bQ^ALv!qO zu5(^-oTHPZfYMeOsAN%|*kkPz=%(_w*R{2mk4e4gol50(?58uvH_2!a`REO^61S!Y zp2gs#P0}OvA^5vGc>LztJ9?R~s-IP_`bte9@2g7GeO9fZenfAkGTtMO-merjfRBct z@YF@StX0z2Gd<+3{)e|>bnP|aT-TuU?-CeB7NW-sMEzr69^~bQSD_blDR+D%k;)Az zRdO)#H&NQlN22Ms=nbR2j+6sGcpdI{NqGK0={*iaYoZxw=1{uXvXNihrMqq?otb0l z@tQ=1r4qAM9?>r6p+S8jzk> zphz`{k}7iwUxTw(r{>fSwCp7o>ot37P&?QT8k398rvUOwZXU?MH`vNCbngFV4ncw; zV3o>){ynC9u>^5&Mt*B3{gq4TrQArj_IMclC%{^!V$HsR=)9$-R1hZdTfq-6(F`?+ zA?!{cu`L}wqs95W*A?#fA|}ee6CLO))T7^UFMXb?!6_=UO3%n1=E<{}PjgLL%lxxF za#`CJ+ji7H^4ng^FXa3Dc`bjGfAjZkd8QmB$I`u0LMqR6)FM(Gob@$es8hhY($Le@ zmmHuS9Pw}T!Grp9n>)}M*bkfF`?Hh|qpdu{5<0j1)ZFM zQgd2BcV1=?s4J|nZ)eBR$v1|M#hM@{cj&vl3jZX@C}j@f^~|C_xg_&C-qEA=59{nT z%hOX-leeNgOCwIoRjUraC!XKdo0?TOn5h;Ttog*Myq<8_Lw53-kMyXRd|V*nI?j%C zrZ?F)?eHtu*;&x$II!bbu(xt(l)j~ZZ4*0ro$e*Ux(C|O90V-Eu+yC}lkUpPoXPF5 zV79VWYsr5#vn{vn3E19#tYJJSEfEB+6U1_1a(~pTZU|U^g=`y{U? zHG(bRI;DuO+LNv3k^}#$o`Ai z6g_0w!7o43ADl`BDH|5}DeE#4J2Zxyb$~Mz!xMCd>!#ugjE#LaXr@cSBH-X=jpI*+R zoUvM*hSvP+2bVP;_)~jSJ}S}Y7J`M)>CGL<8Ci&RIB9;QvvCbfo-5`7&VvX3-dk)? zB7G>&d8U`_#7}f61oom3aiUIV?`pnM;-{8mJ?iosUUTYnvnoDoB^dlDsyglHk~{&r zpNwrf${LElcK`oiB~eg1$e$FW|cd8U!R zehrY~64Wp2ac>eh+dHVw#L}O6p3WQ*OPiL@*Pz;Qm{kepq&DH^PJ;P3fE$02Gy0JF zVF0(O6!!g!m`H!L%#;@uPM@7~@m-XpqEZ{waDSojF;9#F3A#Xs%K;+vx?%;o=R#1n zI87bl2FfoM`VFtCGtA(3bfaUZ6wg$NOh^EYzra%;;I#kX964e3pTg=aWT$--RgSO< zKHVFiT}tKiK4xuADklB;IlcLJI6J~D{!8@997wvSW zJ>@4~0WmK{bg>ls(Ti$Z2o;2Za1`d@*JpEry780ISEqiyfR*UTPx7&H|3?kRH|H@L zOshP#%rJaPYrd~7)rZ3DPZ1E5_jtq|JjVrUPG5P=7qEK&;b9W-lJq`M-T2IVXL&or ztGG)=Ckf*e%ujc5hCSF2FCR(dbf2aB^#>m6GC%PE>)`uykhf)={dMeaSMV9%sjP@C zoW%ZhB+e_%UixwO(@_sg4|mo;3&hSn%f|UD!Jjgmf)3Q(Z1R7>=cH&Hs7}{a6t{ zRv{OdUwZD=3(jK_clOm^rmb(zQ3B8B#+4ig#A22Cxt{8Nj$Mu7tV2h zYM1R;*+xeIH9NhRr&HL))9h6&KH&`axf6R8f%hB8 zZU6As>XpHQr$vo@AG;6=!scRq->@tD>Eh4DPaIC1e45vjgpVwY1#QMFtir2X&)vOC zZ)pSalSg1zzKA6epV*1#h@%SctGjjL?yuzU!JP4RR7T%&f^^QtOwL&kVL$uxncrQQ zbH0dET^NSPb?&%8)Y_Wq7bYFFHR%Sti5l1$x_*YD$g~rbr7--_RO0QY;tx=#cvxi3 zQ1bbPvd?flt(TtMSa$F<9xt4m<3%+;hY*d`O2Env<2Br8b>_1E5;iLWJLqSg~+fmha=$h)ri zl1jZ#YiKPIcvW!B8gL04iXW&fKBB8urdwws(cT5>$M@-JP9z>%%uO!HtNq5BX;{z2 z+>*^^M@}VuH{|gf&06@ZX>{B4GiOoDt;>A)Y}6=Ysh>sBv)7PpzXhLpfRgDRSfACo zZS(kP$GCq*sPPrS0`%n8F2Gm!<4pWyKUFgqyR(HAc?-+vp*exC_QwOH<7baFdvg*z zrpH)H|E`

    Hello, world!

    -

    This is a test of the text-to-speech API.

    - - """ - - # Set up the mock for asynchronous calls - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_async_post: - mock_async_post.return_value = mock_response - model = "vertex_ai/test" - - try: - response = await litellm.aspeech( - input=ssml, - model=model, - voice={ - "languageCode": "en-UK", - "name": "en-UK-Studio-O", - }, - audioConfig={ - "audioEncoding": "LINEAR22", - "speakingRate": "10", - }, - ) - except litellm.APIConnectionError as e: - if "Your default credentials were not found" in str(e): - pytest.skip("skipping test, credentials not found") - - # Assert asynchronous call - mock_async_post.assert_called_once() - _, kwargs = mock_async_post.call_args - print("call args", kwargs) - - assert kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" - - assert "x-goog-user-project" in kwargs["headers"] - assert kwargs["headers"]["Authorization"] is not None - - assert kwargs["json"] == { - "input": {"ssml": ssml}, - "voice": {"languageCode": "en-UK", "name": "en-UK-Studio-O"}, - "audioConfig": {"audioEncoding": "LINEAR22", "speakingRate": "10"}, - } - - - - @pytest.mark.asyncio @pytest.mark.flaky(retries=3, delay=1) async def test_azure_ava_tts_async(): diff --git a/tests/audio_tests/test_whisper.py b/tests/audio_tests/test_whisper.py index 5380a57c871..99b084be1d3 100644 --- a/tests/audio_tests/test_whisper.py +++ b/tests/audio_tests/test_whisper.py @@ -15,29 +15,21 @@ from dotenv import load_dotenv from openai import AsyncOpenAI import litellm -from litellm.integrations.custom_logger import CustomLogger # Get the current directory of the file being run pwd = os.path.dirname(os.path.realpath(__file__)) print(pwd) file_path = os.path.join(pwd, "gettysburg.wav") -file2_path = os.path.join(pwd, "eagle.wav") with open(file_path, "rb") as _f: _GETTYSBURG_BYTES = _f.read() -with open(file2_path, "rb") as _f: - _EAGLE_BYTES = _f.read() def _audio_file(): return ("gettysburg.wav", _GETTYSBURG_BYTES, "audio/wav") -def _audio_file2(): - return ("eagle.wav", _EAGLE_BYTES, "audio/wav") - - load_dotenv() from litellm import Router @@ -75,104 +67,3 @@ async def test_transcription_azure_whisper(response_format, timestamp_granularit response_format=response_format, timestamp_granularities=timestamp_granularities, ) - - -@pytest.mark.asyncio() -async def test_transcription_caching(): - import litellm - from litellm.caching.caching import Cache - - litellm.set_verbose = True - litellm.cache = Cache() - - # make raw llm api call - - response_1 = await litellm.atranscription( - model="whisper-1", - file=_audio_file(), - ) - - await asyncio.sleep(5) - - # cache hit - - response_2 = await litellm.atranscription( - model="whisper-1", - file=_audio_file(), - ) - - print("response_1", response_1) - print("response_2", response_2) - print("response2 hidden params", response_2._hidden_params) - assert response_2._hidden_params["cache_hit"] is True - - # cache miss - - response_3 = await litellm.atranscription( - model="whisper-1", - file=_audio_file2(), - ) - print("response_3", response_3) - print("response3 hidden params", response_3._hidden_params) - assert response_3._hidden_params.get("cache_hit") is not True - assert response_3.text != response_2.text - - litellm.cache = None - - -@pytest.mark.asyncio -async def test_whisper_log_pre_call(): - from litellm.litellm_core_utils.litellm_logging import Logging - from datetime import datetime - from unittest.mock import patch, MagicMock - - custom_logger = CustomLogger() - - litellm.callbacks = [custom_logger] - - with patch.object(custom_logger, "log_pre_api_call") as mock_log_pre_call: - await litellm.atranscription( - model="whisper-1", - file=_audio_file(), - ) - mock_log_pre_call.assert_called_once() - - -@pytest.mark.asyncio -async def test_gpt_4o_transcribe_model_mapping(): - """Test that GPT-4o transcription models are correctly mapped and not hardcoded to whisper-1""" - - # Test GPT-4o mini transcribe - response = await litellm.atranscription( - model="openai/gpt-4o-mini-transcribe", - file=_audio_file(), - response_format="json", - ) - - # Check that the response contains the correct model in hidden params - assert response._hidden_params is not None - assert response._hidden_params["model"] == "gpt-4o-mini-transcribe" - assert response._hidden_params["custom_llm_provider"] == "openai" - assert response.text is not None - - # Test GPT-4o transcribe - response2 = await litellm.atranscription( - model="openai/gpt-4o-transcribe", file=_audio_file(), response_format="json" - ) - - # Check that the response contains the correct model in hidden params - assert response2._hidden_params is not None - assert response2._hidden_params["model"] == "gpt-4o-transcribe" - assert response2._hidden_params["custom_llm_provider"] == "openai" - assert response2.text is not None - - # Test traditional whisper-1 still works - response3 = await litellm.atranscription( - model="openai/whisper-1", file=_audio_file(), response_format="json" - ) - - # Check that the response contains the correct model in hidden params - assert response3._hidden_params is not None - assert response3._hidden_params["model"] == "whisper-1" - assert response3._hidden_params["custom_llm_provider"] == "openai" - assert response3.text is not None diff --git a/tests/batches_tests/test_batch_rate_limits.py b/tests/batches_tests/test_batch_rate_limits.py deleted file mode 100644 index 87d35e11c74..00000000000 --- a/tests/batches_tests/test_batch_rate_limits.py +++ /dev/null @@ -1,896 +0,0 @@ -""" -Integration Tests for Batch Rate Limits -""" - -import asyncio -import json -import os - -import pytest -from fastapi import HTTPException - - -import litellm -from litellm import DualCache -from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.hooks.batch_rate_limiter import ( - BatchFileUsage, - PROXY_BatchRateLimiter, -) -from litellm.proxy.hooks.parallel_request_limiter_v3 import ( - PROXY_MaxParallelRequestsHandler_v3, -) -from litellm.proxy.utils import InternalUsageCache - - -def _build_batch_limiter() -> PROXY_BatchRateLimiter: - internal_usage_cache = InternalUsageCache(dual_cache=DualCache()) - return PROXY_BatchRateLimiter( - internal_usage_cache=internal_usage_cache, - parallel_request_limiter=PROXY_MaxParallelRequestsHandler_v3( - internal_usage_cache=internal_usage_cache - ), - ) - - -def get_expected_batch_file_usage(file_path: str) -> tuple[int, int]: - """ - Helper function to calculate expected request count and token count from a batch JSONL file. - - Returns: - tuple[int, int]: (expected_request_count, expected_total_tokens) - """ - with open(file_path, "r") as f: - file_contents = [json.loads(line) for line in f if line.strip()] - - expected_request_count = len(file_contents) - expected_total_tokens = 0 - - for item in file_contents: - body = item.get("body", {}) - model = body.get("model", "") - messages = body.get("messages", []) - if messages: - item_tokens = litellm.token_counter(model=model, messages=messages) - expected_total_tokens += item_tokens - - return expected_request_count, expected_total_tokens - - -def _write_batch_file(tmp_path, file_name: str, content: str) -> str: - path = tmp_path / file_name - path.write_text(content) - return str(path) - - -@pytest.mark.asyncio() -@pytest.mark.skipif( - os.environ.get("OPENAI_API_KEY") is None, - reason="OPENAI_API_KEY not set - skipping integration test", -) -async def test_batch_rate_limits(): - """ - Integration test for batch rate limits with real OpenAI API calls. - Tests the full flow: file creation -> token counting -> cleanup - """ - litellm.turn_on_debug() - CUSTOM_LLM_PROVIDER = "openai" - BATCH_LIMITER = _build_batch_limiter() - - file_name = "openai_batch_completions.jsonl" - _current_dir = os.path.dirname(os.path.abspath(__file__)) - file_path = os.path.join(_current_dir, file_name) - - # Create file on OpenAI - print(f"Creating file from {file_path}") - file_obj = await litellm.acreate_file( - file=open(file_path, "rb"), - purpose="batch", - custom_llm_provider=CUSTOM_LLM_PROVIDER, - ) - print(f"Response from creating file: {file_obj}") - - assert file_obj.id is not None, "File ID should not be None" - - # Give API a moment to process the file - await asyncio.sleep(1) - - # Count requests and token usage in input file - tracked_batch_file_usage: BatchFileUsage = ( - await BATCH_LIMITER.count_input_file_usage( - file_id=file_obj.id, - custom_llm_provider=CUSTOM_LLM_PROVIDER, - ) - ) - print(f"Actual total tokens: {tracked_batch_file_usage.total_tokens}") - print(f"Actual request count: {tracked_batch_file_usage.request_count}") - - # Calculate expected values by reading the JSONL file - expected_request_count, expected_total_tokens = get_expected_batch_file_usage( - file_path=file_path - ) - - print(f"Expected request count: {expected_request_count}") - print(f"Expected total tokens: {expected_total_tokens}") - - # Verify token counting results - assert ( - tracked_batch_file_usage.request_count == expected_request_count - ), f"Expected {expected_request_count} requests, got {tracked_batch_file_usage.request_count}" - assert ( - tracked_batch_file_usage.total_tokens == expected_total_tokens - ), f"Expected {expected_total_tokens} total_tokens, got {tracked_batch_file_usage.total_tokens}" - - -@pytest.mark.asyncio() -async def test_batch_rate_limit_single_file(tmp_path): - """ - Test batch rate limiting with a single file. - - Key has TPM = 200 - - File with < 200 tokens: should go through - - File with > 200 tokens: should hit rate limit - """ - CUSTOM_LLM_PROVIDER = "openai" - - # Setup: Create internal usage cache and rate limiter - dual_cache = DualCache() - internal_usage_cache = InternalUsageCache(dual_cache=dual_cache) - rate_limiter = PROXY_MaxParallelRequestsHandler_v3( - internal_usage_cache=internal_usage_cache - ) - - # Setup: Get batch rate limiter - batch_limiter = rate_limiter._get_batch_rate_limiter() - assert batch_limiter is not None, "Batch rate limiter should be available" - - # Setup: Create user API key with TPM = 200 - user_api_key_dict = UserAPIKeyAuth( - api_key="test-key-123", - tpm_limit=200, - rpm_limit=10, - ) - - # Test 1: File with < 200 tokens should go through - print("\n=== Test 1: File under 200 tokens ===") - - # Create a small batch file with ~150 tokens - small_batch_content = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}]}} -{"custom_id": "request-2", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hi"}]}} -{"custom_id": "request-3", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hey"}]}}""" - - small_file_path = _write_batch_file( - tmp_path, "small-batch-rate-limit.jsonl", small_batch_content - ) - - try: - # Upload file to OpenAI - with open(small_file_path, "rb") as batch_file: - file_obj_small = await litellm.acreate_file( - file=batch_file, - purpose="batch", - custom_llm_provider=CUSTOM_LLM_PROVIDER, - ) - print(f"Created small file: {file_obj_small.id}") - await asyncio.sleep(1) # Give API time to process - - data_under_limit = { - "model": "gpt-3.5-turbo", - "input_file_id": file_obj_small.id, - "custom_llm_provider": CUSTOM_LLM_PROVIDER, - } - - # Should not raise an exception - result = await batch_limiter.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=dual_cache, - data=data_under_limit, - call_type="acreate_batch", - ) - print(f"✓ File with ~150 tokens passed (under limit of 200)") - print(f" Actual tokens: {result.get('_batch_token_count')}") - except HTTPException as e: - pytest.fail(f"Should not have hit rate limit with small file: {e.detail}") - - # Test 2: File with > 200 tokens should hit rate limit - print("\n=== Test 2: File over 200 tokens ===") - - # Reset cache for clean test - dual_cache = DualCache() - internal_usage_cache = InternalUsageCache(dual_cache=dual_cache) - rate_limiter = PROXY_MaxParallelRequestsHandler_v3( - internal_usage_cache=internal_usage_cache - ) - batch_limiter = rate_limiter._get_batch_rate_limiter() - - # Create a larger batch file with ~10000+ tokens (100x larger to ensure it exceeds 200 token limit) - base_message = ( - "This is a longer message that will consume more tokens from the rate limit. " - * 100 - ) - - # Build JSONL content with json.dumps to avoid f-string nesting issues - import json as json_lib - - requests = [] - for i in range(1, 4): - request_obj = { - "custom_id": f"request-{i}", - "method": "POST", - "url": "/v1/chat/completions", - "body": { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": base_message}], - }, - } - requests.append(json_lib.dumps(request_obj)) - - large_batch_content = "\n".join(requests) - - large_file_path = _write_batch_file( - tmp_path, "large-batch-rate-limit.jsonl", large_batch_content - ) - - # Upload file to OpenAI - with open(large_file_path, "rb") as batch_file: - file_obj_large = await litellm.acreate_file( - file=batch_file, - purpose="batch", - custom_llm_provider=CUSTOM_LLM_PROVIDER, - ) - print(f"Created large file: {file_obj_large.id}") - await asyncio.sleep(1) # Give API time to process - - data_over_limit = { - "model": "gpt-3.5-turbo", - "input_file_id": file_obj_large.id, - "custom_llm_provider": CUSTOM_LLM_PROVIDER, - } - - # Should raise HTTPException with 429 status - with pytest.raises(HTTPException) as exc_info: - await batch_limiter.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=dual_cache, - data=data_over_limit, - call_type="acreate_batch", - ) - - assert exc_info.value.status_code == 429, "Should return 429 status code" - assert ( - "tokens" in exc_info.value.detail.lower() - ), "Error message should mention tokens" - print(f"✓ File with 250+ tokens correctly rejected (over limit of 200)") - print(f" Error: {exc_info.value.detail}") - - -@pytest.mark.asyncio() -async def test_batch_rate_limit_multiple_requests(tmp_path): - """ - Test batch rate limiting with multiple requests. - - Key has TPM = 200 - - Request 1: file with ~100 tokens (should go through, 100/200 used) - - Request 2: file with ~105 tokens (should hit limit, 100+105=205 > 200) - """ - CUSTOM_LLM_PROVIDER = "openai" - - # Setup: Create internal usage cache and rate limiter - dual_cache = DualCache() - internal_usage_cache = InternalUsageCache(dual_cache=dual_cache) - rate_limiter = PROXY_MaxParallelRequestsHandler_v3( - internal_usage_cache=internal_usage_cache - ) - - # Setup: Get batch rate limiter - batch_limiter = rate_limiter._get_batch_rate_limiter() - assert batch_limiter is not None, "Batch rate limiter should be available" - - # Setup: Create user API key with TPM = 200 - user_api_key_dict = UserAPIKeyAuth( - api_key="test-key-456", - tpm_limit=200, - rpm_limit=10, - ) - - # Request 1: File with ~100 tokens - print("\n=== Request 1: File with ~100 tokens ===") - - # Create file with ~100 tokens - import json as json_lib - - message_1 = "This message has some content to reach about 100 tokens total. " * 4 - requests_1 = [] - for i in range(1, 3): - request_obj = { - "custom_id": f"request-{i}", - "method": "POST", - "url": "/v1/chat/completions", - "body": { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": message_1}], - }, - } - requests_1.append(json_lib.dumps(request_obj)) - - batch_content_1 = "\n".join(requests_1) - - file_path_1 = _write_batch_file( - tmp_path, "batch-rate-limit-request-1.jsonl", batch_content_1 - ) - - try: - # Upload file to OpenAI - with open(file_path_1, "rb") as batch_file: - file_obj_1 = await litellm.acreate_file( - file=batch_file, - purpose="batch", - custom_llm_provider=CUSTOM_LLM_PROVIDER, - ) - print(f"Created file 1: {file_obj_1.id}") - await asyncio.sleep(1) # Give API time to process - - data_request1 = { - "model": "gpt-3.5-turbo", - "input_file_id": file_obj_1.id, - "custom_llm_provider": CUSTOM_LLM_PROVIDER, - } - - # Should not raise an exception - result1 = await batch_limiter.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=dual_cache, - data=data_request1, - call_type="acreate_batch", - ) - tokens_used_1 = result1.get("_batch_token_count", 0) - print( - f"✓ Request 1 with {tokens_used_1} tokens passed ({tokens_used_1}/200 used)" - ) - except HTTPException as e: - pytest.fail(f"Request 1 should not have hit rate limit: {e.detail}") - - # Request 2: File with ~105+ tokens (total would exceed 200) - print("\n=== Request 2: File with ~105 tokens (should hit limit) ===") - - # Create file with ~105+ tokens - message_2 = ( - "This is another message with more content to exceed the remaining limit. " * 11 - ) - requests_2 = [] - for i in range(1, 3): - request_obj = { - "custom_id": f"request-{i}", - "method": "POST", - "url": "/v1/chat/completions", - "body": { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": message_2}], - }, - } - requests_2.append(json_lib.dumps(request_obj)) - - batch_content_2 = "\n".join(requests_2) - - file_path_2 = _write_batch_file( - tmp_path, "batch-rate-limit-request-2.jsonl", batch_content_2 - ) - - # Upload file to OpenAI - with open(file_path_2, "rb") as batch_file: - file_obj_2 = await litellm.acreate_file( - file=batch_file, - purpose="batch", - custom_llm_provider=CUSTOM_LLM_PROVIDER, - ) - print(f"Created file 2: {file_obj_2.id}") - await asyncio.sleep(1) # Give API time to process - - data_request2 = { - "model": "gpt-3.5-turbo", - "input_file_id": file_obj_2.id, - "custom_llm_provider": CUSTOM_LLM_PROVIDER, - } - - # Should raise HTTPException with 429 status - with pytest.raises(HTTPException) as exc_info: - await batch_limiter.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=dual_cache, - data=data_request2, - call_type="acreate_batch", - ) - - assert exc_info.value.status_code == 429, "Should return 429 status code" - assert ( - "tokens" in exc_info.value.detail.lower() - ), "Error message should mention tokens" - print(f"✓ Request 2 correctly rejected") - print(f" Error: {exc_info.value.detail}") - - -@pytest.mark.asyncio() -@pytest.mark.skipif( - os.environ.get("OPENAI_API_KEY") is None, - reason="OPENAI_API_KEY not set - skipping integration test", -) -async def test_batch_rate_limiter_with_managed_files(tmp_path): - """ - Test for GEN-2166: Verify batch rate limiter can read user files when managed files are enabled. - - This test ensures that: - 1. The batch rate limiter passes user_api_key_dict to afile_content() - 2. The managed files hook can verify file ownership correctly - 3. Rate limiting is enforced (not silently bypassed) - 4. No 403 Permission Denied errors occur for files owned by the user - """ - from unittest.mock import AsyncMock, MagicMock, patch - - CUSTOM_LLM_PROVIDER = "openai" - - # Setup: Create internal usage cache and rate limiter - dual_cache = DualCache() - internal_usage_cache = InternalUsageCache(dual_cache=dual_cache) - rate_limiter = PROXY_MaxParallelRequestsHandler_v3( - internal_usage_cache=internal_usage_cache - ) - - # Setup: Get batch rate limiter - batch_limiter = rate_limiter._get_batch_rate_limiter() - assert batch_limiter is not None, "Batch rate limiter should be available" - - # Setup: Create user API key with TPM = 500, RPM = 10 - test_user_id = "test-user-abc123" - user_api_key_dict = UserAPIKeyAuth( - api_key="test-key-managed-files", - user_id=test_user_id, - tpm_limit=500, - rpm_limit=10, - ) - - print(f"\n=== Testing Batch Rate Limiter with Managed Files ===") - print(f"User ID: {test_user_id}") - - # Create a batch file with ~200 tokens - import json as json_lib - - message = "This is a test message for batch rate limiting with managed files. " * 5 - requests = [] - for i in range(1, 4): - request_obj = { - "custom_id": f"request-{i}", - "method": "POST", - "url": "/v1/chat/completions", - "body": { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": message}], - }, - } - requests.append(json_lib.dumps(request_obj)) - - batch_content = "\n".join(requests) - - file_path = _write_batch_file( - tmp_path, "managed-files-batch-rate-limit.jsonl", batch_content - ) - - try: - # Step 1: Upload file to OpenAI (simulating user upload) - print("\n1. Uploading batch input file...") - with open(file_path, "rb") as batch_file: - file_obj = await litellm.acreate_file( - file=batch_file, - purpose="batch", - custom_llm_provider=CUSTOM_LLM_PROVIDER, - ) - print(f" ✓ File uploaded: {file_obj.id}") - await asyncio.sleep(1) # Give API time to process - - # Step 2: Mock managed files hook to simulate file ownership check - # In a real scenario, the managed files hook would check if the user owns the file - # For this test, we'll verify that user_api_key_dict is passed correctly - print("\n2. Testing rate limiter file access with user context...") - - # Track if user_api_key_dict was passed to afile_content - original_afile_content = litellm.afile_content - user_context_passed = {"value": False} - - async def mock_afile_content(*args, **kwargs): - # Check if user_api_key_dict was passed - if ( - "user_api_key_dict" in kwargs - and kwargs["user_api_key_dict"] is not None - ): - user_context_passed["value"] = True - print(f" ✓ user_api_key_dict passed to afile_content") - print(f" User ID: {kwargs['user_api_key_dict'].user_id}") - else: - print(f" ✗ user_api_key_dict NOT passed to afile_content (BUG!)") - - # Call original function - return await original_afile_content(*args, **kwargs) - - # Patch afile_content to track the call - with patch("litellm.afile_content", side_effect=mock_afile_content): - data = { - "model": "gpt-3.5-turbo", - "input_file_id": file_obj.id, - "custom_llm_provider": CUSTOM_LLM_PROVIDER, - } - - # Step 3: Submit batch and verify rate limiting works - print("\n3. Submitting batch with rate limiting...") - result = await batch_limiter.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=dual_cache, - data=data, - call_type="acreate_batch", - ) - - tokens_used = result.get("_batch_token_count", 0) - requests_count = result.get("_batch_request_count", 0) - print(f" ✓ Batch submitted successfully") - print(f" Tokens counted: {tokens_used}") - print(f" Requests counted: {requests_count}") - print( - f" Rate limit usage: {tokens_used}/500 TPM, {requests_count}/10 RPM" - ) - - # Step 4: Verify user context was passed - print("\n4. Verifying fix for GEN-2166...") - assert user_context_passed["value"], ( - "FAILED: user_api_key_dict was not passed to afile_content(). " - "This means the bug GEN-2166 is not fixed!" - ) - print(" ✓ Fix verified: user_api_key_dict is correctly passed") - - # Step 5: Verify rate limiting is actually enforced (not bypassed) - print("\n5. Verifying rate limiting is enforced...") - assert tokens_used > 0, "Token count should be greater than 0" - assert requests_count > 0, "Request count should be greater than 0" - print(" ✓ Rate limiting is active (not silently bypassed)") - - print("\n=== Test Passed: GEN-2166 Fix Verified ===") - print("✓ Batch rate limiter can access user files") - print("✓ User context is correctly passed") - print("✓ Rate limiting is enforced") - print("✓ No silent failures") - - except HTTPException as e: - if e.status_code == 403: - pytest.fail( - f"FAILED: Got 403 Permission Denied error. " - f"This indicates the bug GEN-2166 is not fixed. " - f"Error: {e.detail}" - ) - else: - raise - except Exception as e: - pytest.fail(f"Unexpected error: {str(e)}") - - -@pytest.mark.asyncio() -async def test_batch_rate_limiter_without_user_context(tmp_path): - """ - Test that verifies the bug scenario from GEN-2166. - - When user_api_key_dict is NOT passed to count_input_file_usage(), - the function should still work for non-managed files, but would fail - for managed files (which is the bug we fixed). - - This test documents the expected behavior with and without user context. - """ - CUSTOM_LLM_PROVIDER = "openai" - - # Setup - BATCH_LIMITER = _build_batch_limiter() - - # Create a simple batch file - batch_content = """{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Hello"}]}}""" - - file_path = _write_batch_file( - tmp_path, "without-user-context-batch-rate-limit.jsonl", batch_content - ) - - # Upload file - with open(file_path, "rb") as batch_file: - file_obj = await litellm.acreate_file( - file=batch_file, - purpose="batch", - custom_llm_provider=CUSTOM_LLM_PROVIDER, - ) - await asyncio.sleep(1) - - # Test 1: Without user context (old behavior - would fail with managed files) - print("\n=== Test 1: count_input_file_usage WITHOUT user context ===") - try: - usage_without_context = await BATCH_LIMITER.count_input_file_usage( - file_id=file_obj.id, - custom_llm_provider=CUSTOM_LLM_PROVIDER, - user_api_key_dict=None, # Explicitly passing None - ) - print( - f"✓ Works for non-managed files (tokens: {usage_without_context.total_tokens})" - ) - print(" Note: Would fail with 403 for managed files (GEN-2166 bug)") - except Exception as e: - print(f"✗ Failed: {str(e)}") - - # Test 2: With user context (new behavior - works with managed files) - print("\n=== Test 2: count_input_file_usage WITH user context ===") - user_api_key_dict = UserAPIKeyAuth( - api_key="test-key", - user_id="test-user-123", - ) - - usage_with_context = await BATCH_LIMITER.count_input_file_usage( - file_id=file_obj.id, - custom_llm_provider=CUSTOM_LLM_PROVIDER, - user_api_key_dict=user_api_key_dict, # Passing user context - ) - print(f"✓ Works with user context (tokens: {usage_with_context.total_tokens})") - print(" Note: This fixes GEN-2166 for managed files") - - # Verify both return the same results - assert usage_with_context.total_tokens == usage_without_context.total_tokens - assert usage_with_context.request_count == usage_without_context.request_count - print("\n✓ Both methods return identical results for non-managed files") - - -@pytest.mark.asyncio() -async def test_batch_rate_limiter_managed_files_regression(): - """ - Regression test for GEN-2166: Batch Rate Limiter Cannot Access User Files - - This test ensures that the batch rate limiter can properly access managed files - by verifying that: - 1. Managed files are detected correctly (base64 encoded unified file IDs) - 2. The _fetch_managed_file_content method uses the managed files hook - 3. User context (user_api_key_dict) is properly passed through - 4. No 403 errors occur when accessing files owned by the user - 5. The fix doesn't break non-managed file access - - This is a unit test that doesn't require external API calls. - """ - from unittest.mock import AsyncMock, MagicMock, patch - from litellm.llms.base_llm.files.transformation import BaseFileEndpoints - from litellm.types.llms.openai import HttpxBinaryResponseContent - import httpx - - print("\n=== Regression Test: GEN-2166 Batch Rate Limiter Managed Files ===") - - # Setup: Create batch rate limiter - dual_cache = DualCache() - internal_usage_cache = InternalUsageCache(dual_cache=dual_cache) - rate_limiter = PROXY_MaxParallelRequestsHandler_v3( - internal_usage_cache=internal_usage_cache - ) - batch_limiter = rate_limiter._get_batch_rate_limiter() - assert batch_limiter is not None - - # Setup: Create user API key dict - user_api_key_dict = UserAPIKeyAuth( - api_key="test-key-regression", - user_id="test-user-regression", - tpm_limit=1000, - rpm_limit=10, - ) - - # Setup: Create mock file content (batch input file) - batch_content = b'{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", "body": {"model": "gpt-3.5-turbo", "messages": [{"role": "user", "content": "Test message for regression"}]}}' - - # Mock managed file ID (base64 encoded unified file ID format) - managed_file_id = "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxyZWdyZXNzaW9uLXRlc3QtZmlsZQ==" - - # Test 1: Verify managed file detection - print("\n1. Verifying managed file detection...") - from litellm.proxy.openai_files_endpoints.common_utils import ( - is_base64_encoded_unified_file_id, - ) - - is_managed = is_base64_encoded_unified_file_id(managed_file_id) - assert is_managed, "Managed file should be detected correctly" - print(" ✓ Managed file detected") - - # Test 2: Verify _fetch_managed_file_content uses managed files hook - print("\n2. Verifying managed files hook integration...") - - # Create mock managed files hook - class MockManagedFiles(BaseFileEndpoints): - def __init__(self): - self._afile_content_called = False - self._last_call_args = None - - async def acreate_file(self, *args, **kwargs): - pass - - async def afile_content(self, *args, **kwargs): - self._afile_content_called = True - self._last_call_args = kwargs - # Return mock file content - mock_response = httpx.Response( - status_code=200, - content=batch_content, - headers={"content-type": "application/octet-stream"}, - ) - return HttpxBinaryResponseContent(response=mock_response) - - async def afile_delete(self, *args, **kwargs): - pass - - async def afile_list(self, *args, **kwargs): - pass - - async def afile_retrieve(self, *args, **kwargs): - pass - - mock_managed_files = MockManagedFiles() - mock_llm_router = MagicMock() - mock_proxy_logging_obj = MagicMock() - mock_proxy_logging_obj.get_proxy_hook.return_value = mock_managed_files - - # Patch proxy_server imports - with patch.dict( - "sys.modules", - { - "litellm.proxy.proxy_server": MagicMock( - llm_router=mock_llm_router, - proxy_logging_obj=mock_proxy_logging_obj, - ) - }, - ): - # Call _fetch_managed_file_content - result = await batch_limiter._fetch_managed_file_content( - file_id=managed_file_id, - user_api_key_dict=user_api_key_dict, - ) - - # Verify managed files hook was called - assert ( - mock_managed_files._afile_content_called - ), "REGRESSION: managed_files_obj.afile_content was not called! Bug GEN-2166 has returned." - - # Verify user context was passed - assert ( - mock_managed_files._last_call_args is not None - ), "REGRESSION: No arguments passed to afile_content" - assert ( - "file_id" in mock_managed_files._last_call_args - ), "REGRESSION: file_id not passed to managed files hook" - assert ( - mock_managed_files._last_call_args["file_id"] == managed_file_id - ), "REGRESSION: Incorrect file_id passed" - assert ( - "llm_router" in mock_managed_files._last_call_args - ), "REGRESSION: llm_router not passed to managed files hook" - - print(" ✓ Managed files hook called correctly") - print(" ✓ User context passed correctly") - - # Test 3: Verify count_input_file_usage uses managed files path - print("\n3. Verifying count_input_file_usage integration...") - - with patch.object(batch_limiter, "_fetch_managed_file_content") as mock_fetch: - mock_response = httpx.Response( - status_code=200, - content=batch_content, - headers={"content-type": "application/octet-stream"}, - ) - mock_fetch.return_value = HttpxBinaryResponseContent(response=mock_response) - - # Call count_input_file_usage with managed file - usage = await batch_limiter.count_input_file_usage( - file_id=managed_file_id, - custom_llm_provider="openai", - user_api_key_dict=user_api_key_dict, - ) - - # Verify _fetch_managed_file_content was called - assert ( - mock_fetch.called - ), "REGRESSION: _fetch_managed_file_content not called for managed files! Bug GEN-2166 has returned." - - # Verify correct parameters were passed - call_kwargs = mock_fetch.call_args.kwargs - assert ( - call_kwargs["file_id"] == managed_file_id - ), "REGRESSION: Incorrect file_id passed to _fetch_managed_file_content" - assert ( - call_kwargs["user_api_key_dict"] == user_api_key_dict - ), "REGRESSION: user_api_key_dict not passed! Bug GEN-2166 has returned." - - # Verify usage was calculated - assert usage.total_tokens > 0, "Token count should be greater than 0" - assert usage.request_count == 1, "Request count should be 1" - - print(" ✓ Managed file path used") - print(f" ✓ Token count: {usage.total_tokens}") - print(f" ✓ Request count: {usage.request_count}") - - # Test 4: Verify non-managed files still work - print("\n4. Verifying non-managed files still work...") - - non_managed_file_id = "file-abc123" # Standard OpenAI file ID - - with patch("litellm.afile_content") as mock_afile_content: - mock_response = httpx.Response( - status_code=200, - content=batch_content, - headers={"content-type": "application/octet-stream"}, - ) - mock_afile_content.return_value = HttpxBinaryResponseContent( - response=mock_response - ) - - # Call count_input_file_usage with non-managed file - usage = await batch_limiter.count_input_file_usage( - file_id=non_managed_file_id, - custom_llm_provider="openai", - user_api_key_dict=user_api_key_dict, - ) - - # Verify litellm.afile_content was called - assert ( - mock_afile_content.called - ), "REGRESSION: litellm.afile_content not called for non-managed files" - - print(" ✓ Standard file path used") - print(f" ✓ Token count: {usage.total_tokens}") - - # Test 5: Verify the fix prevents 403 errors - print("\n5. Verifying 403 error prevention...") - - # Simulate the bug scenario: managed files hook not being used - with patch.object(batch_limiter, "_fetch_managed_file_content") as mock_fetch: - # If this is NOT called for managed files, the bug has returned - mock_fetch.side_effect = Exception("Should not be called if bug exists") - - # This should call _fetch_managed_file_content - try: - with patch("litellm.afile_content") as mock_afile_content: - # If litellm.afile_content is called for managed files, bug exists - mock_afile_content.side_effect = Exception( - "Error code: 403 - User does not have access to the file" - ) - - # Reset mock_fetch to return valid content - mock_response = httpx.Response( - status_code=200, - content=batch_content, - headers={"content-type": "application/octet-stream"}, - ) - mock_fetch.side_effect = None - mock_fetch.return_value = HttpxBinaryResponseContent( - response=mock_response - ) - - # This should use _fetch_managed_file_content, not litellm.afile_content - usage = await batch_limiter.count_input_file_usage( - file_id=managed_file_id, - custom_llm_provider="openai", - user_api_key_dict=user_api_key_dict, - ) - - # Verify managed files path was used (not standard path that causes 403) - assert ( - mock_fetch.called - ), "REGRESSION: Managed files path not used! This would cause 403 errors." - assert ( - not mock_afile_content.called - ), "REGRESSION: Standard path used for managed files! This causes 403 errors." - - print(" ✓ 403 error prevention verified") - - except Exception as e: - if "403" in str(e): - pytest.fail( - f"REGRESSION: 403 error occurred! Bug GEN-2166 has returned. Error: {str(e)}" - ) - raise - - print("\n=== Regression Test Passed ===") - print("✓ Bug GEN-2166 is fixed and protected against regression") - print("✓ Managed files are properly accessed via managed files hook") - print("✓ User context is correctly passed through") - print("✓ No 403 errors occur") - print("✓ Non-managed files still work correctly\n") diff --git a/tests/batches_tests/test_openai_batches_and_files.py b/tests/batches_tests/test_openai_batches_and_files.py deleted file mode 100644 index 6341fe2f91c..00000000000 --- a/tests/batches_tests/test_openai_batches_and_files.py +++ /dev/null @@ -1,580 +0,0 @@ -# What is this? -## Unit Tests for OpenAI Batches API -import asyncio -import json -import os -import tempfile -from dotenv import load_dotenv - -load_dotenv() - -import logging -import time - -import pytest -from typing import Optional -import litellm -from litellm._logging import verbose_logger -import openai - -verbose_logger.setLevel(logging.DEBUG) - -from litellm.integrations.custom_logger import CustomLogger -from litellm.types.utils import StandardLoggingPayload -import socket -import httpx -from unittest.mock import patch, MagicMock, AsyncMock - - -def _can_resolve_openai(): - """Check if api.openai.com is reachable (DNS resolves).""" - try: - socket.getaddrinfo("api.openai.com", 443, socket.AF_UNSPEC, socket.SOCK_STREAM) - return True - except socket.gaierror: - return False - - -skip_if_no_openai_network = pytest.mark.skipif( - not _can_resolve_openai(), - reason="Cannot resolve api.openai.com - skipping integration test due to DNS issues", -) - - -async def _wait_for_standard_logging_object( - custom_logger: "TestCustomLogger", timeout: float = 15.0 -) -> StandardLoggingPayload: - from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER - - deadline = time.monotonic() + timeout - while time.monotonic() < deadline: - await GLOBAL_LOGGING_WORKER.flush() - if custom_logger.standard_logging_object is not None: - return custom_logger.standard_logging_object - await asyncio.sleep(0.25) - assert custom_logger.standard_logging_object is not None - return custom_logger.standard_logging_object - - -def load_vertex_ai_credentials(): - # Define the path to the vertex_key.json file - print("loading vertex ai credentials") - os.environ["GCS_FLUSH_INTERVAL"] = "1" - filepath = os.path.dirname(os.path.abspath(__file__)) - vertex_key_path = filepath + "/vertex_key.json" - - # Read the existing content of the file or create an empty dictionary - try: - with open(vertex_key_path, "r") as file: - # Read the file content - print("Read vertexai file path") - content = file.read() - - # If the file is empty or not valid JSON, create an empty dictionary - if not content or not content.strip(): - service_account_key_data = {} - else: - # Attempt to load the existing JSON content - file.seek(0) - service_account_key_data = json.load(file) - except FileNotFoundError: - # If the file doesn't exist, create an empty dictionary - service_account_key_data = {} - - # Update the service_account_key_data with environment variables - private_key_id = os.environ.get("VERTEX_AI_PRIVATE_KEY_ID", "") - private_key = os.environ.get("VERTEX_AI_PRIVATE_KEY", "") - private_key = private_key.replace("\\n", "\n") - service_account_key_data["private_key_id"] = private_key_id - service_account_key_data["private_key"] = private_key - - # Create a temporary file - with tempfile.NamedTemporaryFile(mode="w+", delete=False) as temp_file: - # Write the updated content to the temporary files - json.dump(service_account_key_data, temp_file, indent=2) - - # Export the temporary file as GOOGLE_APPLICATION_CREDENTIALS - os.environ["GCS_PATH_SERVICE_ACCOUNT"] = os.path.abspath(temp_file.name) - os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = os.path.abspath(temp_file.name) - print("created gcs path service account=", os.environ["GCS_PATH_SERVICE_ACCOUNT"]) - - -async def cancel_batch_unless_already_terminal(batch_id: str, provider: str) -> None: - try: - cancel_batch_response = await litellm.acancel_batch(batch_id=batch_id, custom_llm_provider=provider) - except openai.ConflictError as e: - if "Cannot cancel a batch with status 'completed'" in str(e): - print(f"Batch already completed, cannot cancel: {e}") - return - if "Cannot cancel a batch with status 'failed'" not in str(e): - raise - failed_batch = await litellm.aretrieve_batch(batch_id=batch_id, custom_llm_provider=provider) - print(f"Batch failed before cancel, errors={failed_batch.errors}") - failure_codes = {err.code for err in (failed_batch.errors.data if failed_batch.errors else None) or []} - assert failure_codes == {"token_limit_exceeded"}, ( - f"batch failed for a reason other than the org's enqueued token limit: {failed_batch.errors}" - ) - return - print("cancel_batch_response=", cancel_batch_response) - - -class TestCustomLogger(CustomLogger): - def __init__(self): - super().__init__() - self.standard_logging_object: Optional[StandardLoggingPayload] = None - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - print( - "Success event logged with kwargs=", - kwargs, - "and response_obj=", - response_obj, - ) - self.standard_logging_object = kwargs["standard_logging_object"] - - -def cleanup_azure_files(): - """ - Delete all files for Azure - helper for when we run out of Azure Files Quota - """ - azure_files = litellm.file_list( - custom_llm_provider="azure", - api_key=os.getenv("AZURE_FT_API_KEY"), - api_base=os.getenv("AZURE_FT_API_BASE"), - ) - print("azure_files=", azure_files) - for _file in azure_files: - print("deleting file=", _file) - delete_file_response = litellm.file_delete( - file_id=_file.id, - custom_llm_provider="azure", - api_key=os.getenv("AZURE_FT_API_KEY"), - api_base=os.getenv("AZURE_FT_API_BASE"), - ) - print("delete_file_response=", delete_file_response) - assert delete_file_response.id == _file.id - - -def cleanup_azure_ft_models(): - """ - Test CLEANUP: Delete all existing fine tuning jobs for Azure - """ - try: - from openai import AzureOpenAI - import requests - - client = AzureOpenAI( - api_key=os.getenv("AZURE_AI_API_KEY"), - azure_endpoint=os.getenv("AZURE_AI_API_BASE"), - api_version=os.getenv("AZURE_AI_API_VERSION"), - ) - - _list_ft_jobs = client.fine_tuning.jobs.list() - print("_list_ft_jobs=", _list_ft_jobs) - - # delete all ft jobs make post request to this - # Delete all fine-tuning jobs - for job in _list_ft_jobs: - try: - endpoint = os.getenv("AZURE_FT_API_BASE").rstrip("/") - url = f"{endpoint}/openai/fine_tuning/jobs/{job.id}?api-version=2024-10-21" - print("url=", url) - - headers = { - "api-key": os.getenv("AZURE_FT_API_KEY"), - "Content-Type": "application/json", - } - - response = requests.delete(url, headers=headers) - print(f"Deleting job {job.id}: Status {response.status_code}") - if response.status_code != 204: - print(f"Error deleting job {job.id}: {response.text}") - - except Exception as e: - print(f"Error deleting job {job.id}: {str(e)}") - except Exception as e: - print(f"Error on cleanup_azure_ft_models: {str(e)}") - - -@pytest.mark.parametrize("provider", ["openai"]) -@pytest.mark.asyncio() -@skip_if_no_openai_network -async def test_async_create_batch(provider, tmp_path): - """ - 1. Create File for Batch completion - 2. Create Batch Request - 3. Retrieve the specific batch - """ - litellm.turn_on_debug() - print("Testing async create batch") - litellm.logging_callback_manager._reset_all_callbacks() - - file_name = "openai_batch_completions.jsonl" - _current_dir = os.path.dirname(os.path.abspath(__file__)) - file_path = os.path.join(_current_dir, file_name) - with open(file_path, "rb") as batch_file: - file_obj = await litellm.acreate_file( - file=batch_file, - purpose="batch", - custom_llm_provider=provider, - ) - print("Response from creating file=", file_obj) - - await asyncio.sleep(10) - batch_input_file_id = file_obj.id - assert ( - batch_input_file_id is not None - ), "Failed to create file, expected a non null file_id but got {batch_input_file_id}" - - extra_metadata_field = { - "user_api_key_alias": "special_api_key_alias", - "user_api_key_team_alias": "special_team_alias", - } - custom_logger = TestCustomLogger() - litellm.callbacks = [custom_logger, "datadog"] - create_batch_response = await litellm.acreate_batch( - completion_window="24h", - endpoint="/v1/chat/completions", - input_file_id=batch_input_file_id, - custom_llm_provider=provider, - metadata={"key1": "value1", "key2": "value2"}, - # litellm specific param - used for logging metadata on logging callback - litellm_metadata=extra_metadata_field, - ) - - print("response from litellm.create_batch=", create_batch_response) - - assert ( - create_batch_response.id is not None - ), f"Failed to create batch, expected a non null batch_id but got {create_batch_response.id}" - assert ( - create_batch_response.endpoint == "/v1/chat/completions" - or create_batch_response.endpoint == "/chat/completions" - ), f"Failed to create batch, expected endpoint to be /v1/chat/completions but got {create_batch_response.endpoint}" - assert ( - create_batch_response.input_file_id == batch_input_file_id - ), f"Failed to create batch, expected input_file_id to be {batch_input_file_id} but got {create_batch_response.input_file_id}" - - # Assert that the create batch event is logged on CustomLogger - standard_logging_object = await _wait_for_standard_logging_object(custom_logger) - print( - "standard_logging_object=", - json.dumps(standard_logging_object, indent=4, default=str), - ) - assert ( - standard_logging_object["metadata"]["user_api_key_alias"] - == extra_metadata_field["user_api_key_alias"] - ) - assert ( - standard_logging_object["metadata"]["user_api_key_team_alias"] - == extra_metadata_field["user_api_key_team_alias"] - ) - - retrieved_batch = await litellm.aretrieve_batch( - batch_id=create_batch_response.id, custom_llm_provider=provider - ) - print("retrieved batch=", retrieved_batch) - # just assert that we retrieved a non None batch - - assert retrieved_batch.id == create_batch_response.id - - # list all batches - list_batches = await litellm.alist_batches(custom_llm_provider=provider, limit=2) - print("list_batches=", list_batches) - - # try to get file content for our original file - - file_content = await litellm.afile_content( - file_id=batch_input_file_id, custom_llm_provider=provider - ) - - print("file content = ", file_content) - - # file obj - file_obj = await litellm.afile_retrieve( - file_id=batch_input_file_id, custom_llm_provider=provider - ) - print("file obj = ", file_obj) - assert file_obj.id == batch_input_file_id - - # delete file - delete_file_response = await litellm.afile_delete( - file_id=batch_input_file_id, custom_llm_provider=provider - ) - - print("delete file response = ", delete_file_response) - - assert delete_file_response.id == batch_input_file_id - - all_files_list = await litellm.afile_list( - custom_llm_provider=provider, - ) - - print("all_files_list = ", all_files_list) - - result_file_path = tmp_path / "batch_job_results_furniture.jsonl" - result_file_path.write_bytes(file_content.content) - - await cancel_batch_unless_already_terminal(batch_id=create_batch_response.id, provider=provider) - - -mock_file_response = { - "kind": "storage#object", - "id": "litellm-local/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/5f7b99ad-9203-4430-98bf-3b45451af4cb/1739598666670574", - "selfLink": "https://www.googleapis.com/storage/v1/b/litellm-local/o/litellm-vertex-files%2Fpublishers%2Fgoogle%2Fmodels%2Fgemini-1.5-flash-001%2F5f7b99ad-9203-4430-98bf-3b45451af4cb", - "mediaLink": "https://storage.googleapis.com/download/storage/v1/b/litellm-local/o/litellm-vertex-files%2Fpublishers%2Fgoogle%2Fmodels%2Fgemini-1.5-flash-001%2F5f7b99ad-9203-4430-98bf-3b45451af4cb?generation=1739598666670574&alt=media", - "name": "litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/5f7b99ad-9203-4430-98bf-3b45451af4cb", - "bucket": "litellm-local", - "generation": "1739598666670574", - "metageneration": "1", - "contentType": "application/json", - "storageClass": "STANDARD", - "size": "416", - "md5Hash": "hbBNj7C8KJ7oVH+JmyRM6A==", - "crc32c": "oDmiUA==", - "etag": "CO7D0IT+xIsDEAE=", - "timeCreated": "2025-02-15T05:51:06.741Z", - "updated": "2025-02-15T05:51:06.741Z", - "timeStorageClassUpdated": "2025-02-15T05:51:06.741Z", - "timeFinalized": "2025-02-15T05:51:06.741Z", -} - -mock_vertex_batch_response = { - "name": "projects/123456789/locations/us-central1/batchPredictionJobs/test-batch-id-456", - "displayName": "litellm_batch_job", - "model": "projects/123456789/locations/us-central1/models/gemini-1.5-flash-001", - "modelVersionId": "v1", - "inputConfig": { - "gcsSource": { - "uris": [ - "gs://litellm-local/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/5f7b99ad-9203-4430-98bf-3b45451af4cb" - ] - } - }, - "outputConfig": { - "gcsDestination": {"outputUriPrefix": "gs://litellm-local/batch-outputs/"} - }, - "dedicatedResources": { - "machineSpec": { - "machineType": "n1-standard-4", - "acceleratorType": "NVIDIA_TESLA_T4", - "acceleratorCount": 1, - }, - "startingReplicaCount": 1, - "maxReplicaCount": 1, - }, - "state": "JOB_STATE_RUNNING", - "createTime": "2025-02-15T05:51:06.741Z", - "startTime": "2025-02-15T05:51:07.741Z", - "updateTime": "2025-02-15T05:51:08.741Z", - "labels": {"key1": "value1", "key2": "value2"}, - "completionStats": {"successfulCount": 0, "failedCount": 0, "remainingCount": 100}, -} - -mock_vertex_list_response = { - "batchPredictionJobs": [ - mock_vertex_batch_response, - { - **mock_vertex_batch_response, - "name": "projects/123456789/locations/us-central1/batchPredictionJobs/test-batch-id-789", - "state": "JOB_STATE_SUCCEEDED", - }, - ], - "nextPageToken": "", -} - - -@pytest.mark.asyncio -async def test_avertex_batch_prediction(monkeypatch): - monkeypatch.setenv("GCS_BUCKET_NAME", "litellm-local") - monkeypatch.setenv("VERTEXAI_PROJECT", "mock-project") - monkeypatch.setenv("VERTEXAI_LOCATION", "us-central1") - - # Mock Google auth so the test doesn't need real credentials - mock_creds = MagicMock() - mock_creds.token = "mock-token" - mock_creds.valid = True - mock_creds.expiry = None - monkeypatch.setattr( - "google.auth.default", - lambda *args, **kwargs: (mock_creds, "mock-project"), - ) - - from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler - - # Configure mock response object - mock_response = MagicMock() - mock_response.raise_for_status.return_value = None - - async def mock_side_effect(*args, **kwargs): - print("args", args, "kwargs", kwargs) - url = kwargs.get("url", "") - if "files" in url: - mock_response.json.return_value = mock_file_response - elif "batch" in url: - mock_response.json.return_value = mock_vertex_batch_response - mock_response.status_code = 200 - return mock_response - - # Batch jsonl creation now stages the body to a temp file and issues a single - # uploadType=media POST against the raw httpx.AsyncClient (client.client) inside - # _astage_and_upload_media, not AsyncHTTPHandler.post. Patch that raw POST so the - # real staging/upload + response transform run while the GCS object response is - # mocked; AsyncHTTPHandler.post still handles the batch-prediction call. - with ( - patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - side_effect=mock_side_effect, - ), - patch.object( - httpx.AsyncClient, - "post", - new_callable=AsyncMock, - return_value=httpx.Response( - 200, - json=mock_file_response, - request=httpx.Request("POST", "https://storage.googleapis.com/upload"), - ), - ) as mock_gcs_upload, - ): - litellm.set_verbose = True - litellm.turn_on_debug() - file_name = "vertex_batch_completions.jsonl" - _current_dir = os.path.dirname(os.path.abspath(__file__)) - file_path = os.path.join(_current_dir, file_name) - - # Create file - file_obj = await litellm.acreate_file( - file=open(file_path, "rb"), - purpose="batch", - custom_llm_provider="vertex_ai", - ) - print("Response from creating file=", file_obj) - - assert ( - file_obj.id - == "gs://litellm-local/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/5f7b99ad-9203-4430-98bf-3b45451af4cb" - ) - - mock_gcs_upload.assert_awaited_once() - upload_url = str(mock_gcs_upload.call_args.args[0]) - assert "uploadType=media" in upload_url - assert "/b/litellm-local/o" in upload_url - assert ( - mock_gcs_upload.call_args.kwargs["headers"]["Content-Type"] - == "application/json" - ) - - # Create batch - create_batch_response = await litellm.acreate_batch( - completion_window="24h", - endpoint="/v1/chat/completions", - input_file_id=file_obj.id, - custom_llm_provider="vertex_ai", - metadata={"key1": "value1", "key2": "value2"}, - ) - print("create_batch_response=", create_batch_response) - - assert create_batch_response.id == "test-batch-id-456" - assert ( - create_batch_response.input_file_id - == "gs://litellm-local/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/5f7b99ad-9203-4430-98bf-3b45451af4cb" - ) - - # Mock the retrieve batch response - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.get" - ) as mock_get: - mock_get_response = MagicMock() - mock_get_response.json.return_value = mock_vertex_batch_response - mock_get_response.status_code = 200 - mock_get_response.is_redirect = False - mock_get_response.raise_for_status.return_value = None - mock_get_response.is_redirect = False - mock_get.return_value = mock_get_response - - retrieved_batch = await litellm.aretrieve_batch( - batch_id=create_batch_response.id, - custom_llm_provider="vertex_ai", - ) - print("retrieved_batch=", retrieved_batch) - - assert retrieved_batch.id == "test-batch-id-456" - - -@pytest.mark.asyncio - - -@pytest.mark.asyncio - - -@pytest.mark.asyncio -@skip_if_no_openai_network -async def test_delete_batch_output_file(): - """ - Test that deleting a batch output file works correctly. - - This test verifies the fix for: - - When a batch is retrieved and has an output_file_id, the file object is properly stored - - The output file can be deleted without validation errors - - The file_object is fetched and stored with proper metadata instead of None - """ - litellm.turn_on_debug() - print("Testing delete batch output file") - - file_name = "openai_batch_completions.jsonl" - _current_dir = os.path.dirname(os.path.abspath(__file__)) - file_path = os.path.join(_current_dir, file_name) - - # Create file for batch - file_obj = await litellm.acreate_file( - file=open(file_path, "rb"), - purpose="batch", - custom_llm_provider="openai", - ) - print("Response from creating file=", file_obj) - batch_input_file_id = file_obj.id - - # Create batch - create_batch_response = await litellm.acreate_batch( - completion_window="24h", - endpoint="/v1/chat/completions", - input_file_id=batch_input_file_id, - custom_llm_provider="openai", - ) - print("Batch created with ID=", create_batch_response.id) - - # Retrieve batch to get output_file_id - retrieved_batch = await litellm.aretrieve_batch( - batch_id=create_batch_response.id, custom_llm_provider="openai" - ) - print("Retrieved batch=", retrieved_batch) - - # If batch has completed and has output file, test deleting it - if retrieved_batch.output_file_id: - print(f"Testing deletion of output file: {retrieved_batch.output_file_id}") - - # This is the key test - deleting the output file should work - # without validation errors (file_object should not be None) - delete_output_file_response = await litellm.afile_delete( - file_id=retrieved_batch.output_file_id, custom_llm_provider="openai" - ) - - print("Delete output file response=", delete_output_file_response) - assert delete_output_file_response.id == retrieved_batch.output_file_id - assert delete_output_file_response.deleted is True or hasattr( - delete_output_file_response, "id" - ) - print("✓ Successfully deleted batch output file") - else: - print( - "⚠ Batch has not completed yet or no output file available, skipping output file deletion test" - ) - - # Clean up - delete the input file - delete_input_file_response = await litellm.afile_delete( - file_id=batch_input_file_id, custom_llm_provider="openai" - ) - print("Delete input file response=", delete_input_file_response) - assert delete_input_file_response.id == batch_input_file_id - print("✓ Successfully deleted batch input file") diff --git a/tests/e2e/quota_management/spend_tracking/SPEND_TRACKING_COVERAGE_MATRIX.md b/tests/e2e/quota_management/spend_tracking/SPEND_TRACKING_COVERAGE_MATRIX.md index d9380098891..e3f3bf009dc 100644 --- a/tests/e2e/quota_management/spend_tracking/SPEND_TRACKING_COVERAGE_MATRIX.md +++ b/tests/e2e/quota_management/spend_tracking/SPEND_TRACKING_COVERAGE_MATRIX.md @@ -43,7 +43,7 @@ proxy + SpendLogs rows. Status: `covered` / `partial` / `gap`. | Tag | `test_update_daily_tag_spend.py` | partial | yes (`test_tag_spend_matches_sum_of_tagged_logs`) | | End-user | `test_proxy_update_spend.py` | covered | yes | | Spend == sum(logs) consistency | none | gap | yes (key + tag aggregate == sum of rows) | -| Concurrent increments (one key, parallel writers) | `tests/spend_tracking_tests/test_spend_accuracy_tests.py` (burst) | partial | yes (`test_burst_of_concurrent_calls_loses_no_spend`) | +| Concurrent increments (one key, parallel writers) | `tests/integration/spend/test_spend_rollup_accuracy.py`, `tests/integration/spend/test_chaos_burst_spend_once.py` (burst) | partial | yes (`test_burst_of_concurrent_calls_loses_no_spend`) | ## Spend read endpoints (verification surface) diff --git a/tests/guardrails_tests/test_bedrock_guardrails.py b/tests/guardrails_tests/test_bedrock_guardrails.py deleted file mode 100644 index aeeb34f4e8d..00000000000 --- a/tests/guardrails_tests/test_bedrock_guardrails.py +++ /dev/null @@ -1,100 +0,0 @@ -import io, asyncio -import pytest - -import litellm -from litellm.proxy.guardrails.guardrail_hooks.bedrock_guardrails import ( - BedrockGuardrail, - _redact_pii_matches, -) -from litellm.proxy._types import UserAPIKeyAuth -from unittest.mock import MagicMock, AsyncMock, patch - - -@pytest.mark.asyncio -async def test_bedrock_guardrails_pii_masking(): - # Create proper mock objects - mock_user_api_key_dict = UserAPIKeyAuth() - - guardrail = BedrockGuardrail( - guardrailIdentifier="wf0hkdb5x07f", - guardrailVersion="DRAFT", - ) - - request_data = { - "model": "gpt-5.5", - "messages": [ - {"role": "user", "content": "Hello, my phone number is +1 412 555 1212"}, - {"role": "assistant", "content": "Hello, how can I help you today?"}, - {"role": "user", "content": "I need to cancel my order"}, - { - "role": "user", - "content": "ok, my credit card number is 1234-5678-9012-3456", - }, - ], - } - - response = await guardrail.async_moderation_hook( - data=request_data, - user_api_key_dict=mock_user_api_key_dict, - call_type="completion", - ) - print("response after moderation hook", response) - - if response: # Only assert if response is not None - assert response["messages"][0]["content"] == "Hello, my phone number is {PHONE}" - assert response["messages"][1]["content"] == "Hello, how can I help you today?" - assert response["messages"][2]["content"] == "I need to cancel my order" - assert ( - response["messages"][3]["content"] - == "ok, my credit card number is {CREDIT_DEBIT_CARD_NUMBER}" - ) - - -@pytest.mark.asyncio -async def test_bedrock_guardrails_pii_masking_content_list(): - # Create proper mock objects - mock_user_api_key_dict = UserAPIKeyAuth() - - guardrail = BedrockGuardrail( - guardrailIdentifier="wf0hkdb5x07f", - guardrailVersion="DRAFT", - ) - - request_data = { - "model": "gpt-5.5", - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": "Hello, my phone number is +1 412 555 1212", - }, - {"type": "text", "text": "what time is it?"}, - ], - }, - {"role": "assistant", "content": "Hello, how can I help you today?"}, - {"role": "user", "content": "who is the president of the united states?"}, - ], - } - - response = await guardrail.async_moderation_hook( - data=request_data, - user_api_key_dict=mock_user_api_key_dict, - call_type="completion", - ) - print(response) - - if response: # Only assert if response is not None - # Verify that the list content is properly masked - assert isinstance(response["messages"][0]["content"], list) - assert ( - response["messages"][0]["content"][0]["text"] - == "Hello, my phone number is {PHONE}" - ) - assert response["messages"][0]["content"][1]["text"] == "what time is it?" - assert response["messages"][1]["content"] == "Hello, how can I help you today?" - assert ( - response["messages"][2]["content"] - == "who is the president of the united states?" - ) diff --git a/tests/guardrails_tests/test_presidio_pii.py b/tests/guardrails_tests/test_presidio_pii.py deleted file mode 100644 index 595cc95fa9f..00000000000 --- a/tests/guardrails_tests/test_presidio_pii.py +++ /dev/null @@ -1,211 +0,0 @@ -import os -import pytest -from litellm import mock_completion -from unittest.mock import patch - -import litellm -from litellm.proxy.guardrails.guardrail_hooks.presidio import ( - OPTIONAL_PresidioPIIMasking, - PresidioPerRequestConfig, -) -from litellm.types.guardrails import PiiEntityType, PiiAction -from litellm.proxy._types import UserAPIKeyAuth -from litellm.caching.caching import DualCache -from litellm.exceptions import BlockedPiiEntityError - - -@pytest.mark.asyncio -async def test_presidio_with_blocked_entities(): - """Test for Presidio guardrail with blocked entities - requires actual Presidio API""" - # Setup the guardrail with specific entities config - BLOCK for credit card - litellm.turn_on_debug() - pii_entities_config = { - PiiEntityType.CREDIT_CARD: PiiAction.BLOCK, # This entity should cause a block - PiiEntityType.EMAIL_ADDRESS: PiiAction.MASK, # This entity should be masked - } - - presidio_guardrail = OPTIONAL_PresidioPIIMasking( - pii_entities_config=pii_entities_config, - presidio_analyzer_api_base=os.environ.get("PRESIDIO_ANALYZER_API_BASE"), - presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE"), - ) - - # Test text with blocked PII type - test_text = ( - "My credit card number is 4111-1111-1111-1111 and my email is test@example.com" - ) - - # Verify the analyze request configuration - analyze_request = presidio_guardrail._get_presidio_analyze_request_payload( - text=test_text, presidio_config=None, request_data={} - ) - - # Verify entities were passed correctly - assert "entities" in analyze_request - assert set(analyze_request["entities"]) == set(pii_entities_config.keys()) - - # Test that BlockedPiiEntityError is raised when check_pii is called - with pytest.raises(BlockedPiiEntityError) as excinfo: - await presidio_guardrail.check_pii( - text=test_text, output_parse_pii=True, presidio_config=None, request_data={} - ) - - # Verify the error contains the correct entity type - assert excinfo.value.entity_type == PiiEntityType.CREDIT_CARD - assert excinfo.value.guardrail_name == presidio_guardrail.guardrail_name - - -@pytest.mark.asyncio -async def test_presidio_pre_call_hook_with_blocked_entities(): - """Test for Presidio guardrail pre-call hook with blocked entities on a chat completion request""" - # Setup the guardrail with specific entities config - pii_entities_config = { - PiiEntityType.CREDIT_CARD: PiiAction.BLOCK, # This entity should cause a block - PiiEntityType.EMAIL_ADDRESS: PiiAction.MASK, # This entity should be masked - } - - presidio_guardrail = OPTIONAL_PresidioPIIMasking( - pii_entities_config=pii_entities_config, - presidio_analyzer_api_base=os.environ.get("PRESIDIO_ANALYZER_API_BASE"), - presidio_anonymizer_api_base=os.environ.get("PRESIDIO_ANONYMIZER_API_BASE"), - ) - - # Create a sample chat completion request with PII data - data = { - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - { - "role": "user", - "content": "My credit card is 4111-1111-1111-1111 and my email is test@example.com.", - }, - ], - "model": "gpt-5-mini", - } - - # Mock objects needed for the pre-call hook - user_api_key_dict = UserAPIKeyAuth(api_key="test_key") - cache = DualCache() - - # Call the pre-call hook and expect BlockedPiiEntityError - with pytest.raises(BlockedPiiEntityError) as excinfo: - await presidio_guardrail.async_pre_call_hook( - user_api_key_dict=user_api_key_dict, - cache=cache, - data=data, - call_type="completion", - ) - - print(f"got error: {excinfo}") - - # Verify the error contains the correct entity type - assert excinfo.value.entity_type == PiiEntityType.CREDIT_CARD - assert excinfo.value.guardrail_name == presidio_guardrail.guardrail_name - - - - - - -# asyncio.run(test_output_parsing()) - - -### UNIT TESTS FOR PRESIDIO PII MASKING ### - -input_a_anonymizer_results = { - "text": "hello world, my name is . My number is: ", - "items": [ - { - "start": 48, - "end": 62, - "entity_type": "PHONE_NUMBER", - "text": "", - "operator": "replace", - }, - { - "start": 24, - "end": 32, - "entity_type": "PERSON", - "text": "", - "operator": "replace", - }, - ], -} - -input_b_anonymizer_results = { - "text": "My name is , who are you? Say my name in your response", - "items": [ - { - "start": 11, - "end": 19, - "entity_type": "PERSON", - "text": "", - "operator": "replace", - } - ], -} - - -# Test if PII masking works with input A - - -# Test if PII masking works with input B (also test if the response != A's response) - - - - -@pytest.mark.asyncio -@patch.dict( - os.environ, - { - "PRESIDIO_ANALYZER_API_BASE": "http://localhost:5002", - "PRESIDIO_ANONYMIZER_API_BASE": "http://localhost:5001", - }, -) -async def test_presidio_pii_masking_logging_output_only_logged_response_guardrails_config(): - from typing import Dict, List, Optional - - import litellm - from litellm.proxy.guardrails.init_guardrails import initialize_guardrails - from litellm.types.guardrails import ( - GuardrailItemSpec, - GuardrailEventHooks, - ) - - litellm.set_verbose = True - # Environment variables are now patched via the decorator instead of setting them directly - - guardrails_config: List[Dict[str, GuardrailItemSpec]] = [ - { - "pii_masking": { - "callbacks": ["presidio"], - "default_on": True, - "logging_only": True, - } - } - ] - litellm_settings = {"guardrails": guardrails_config} - - assert len(litellm.guardrail_name_config_map) == 0 - initialize_guardrails( - guardrails_config=guardrails_config, - premium_user=True, - config_file_path="", - litellm_settings=litellm_settings, - ) - - assert len(litellm.guardrail_name_config_map) == 1 - - pii_masking_obj: Optional[OPTIONAL_PresidioPIIMasking] = None - for callback in litellm.callbacks: - print(f"CALLBACK: {callback}") - if isinstance(callback, OPTIONAL_PresidioPIIMasking): - pii_masking_obj = callback - - assert pii_masking_obj is not None - - assert hasattr(pii_masking_obj, "logging_only") - assert pii_masking_obj.event_hook == GuardrailEventHooks.logging_only - - assert pii_masking_obj.should_run_guardrail( - data={}, event_type=GuardrailEventHooks.logging_only - ) diff --git a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py b/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py deleted file mode 100644 index e7d526beeb8..00000000000 --- a/tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py +++ /dev/null @@ -1,112 +0,0 @@ -import logging -import traceback - -from dotenv import load_dotenv -from openai.types.image import Image - - -from litellm.llms.bedrock.image_generation.amazon_nova_canvas_transformation import ( - AmazonNovaCanvasConfig, -) - -logging.basicConfig(level=logging.DEBUG) -load_dotenv() -import asyncio - -import pytest -from litellm.llms.bedrock.image_generation.cost_calculator import cost_calculator -from litellm.types.utils import ImageResponse, ImageObject - -import litellm -from litellm.llms.bedrock.image_generation.amazon_stability3_transformation import ( - AmazonStability3Config, -) -from litellm.llms.bedrock.image_generation.amazon_stability1_transformation import ( - AmazonStabilityConfig, -) -from litellm.types.llms.bedrock import ( - AmazonStability3TextToImageRequest, - AmazonStability3TextToImageResponse, -) -from unittest.mock import MagicMock, patch -from litellm.llms.bedrock.image_generation.image_handler import ( - BedrockImageGeneration, - BedrockImagePreparedRequest, -) -from litellm.llms.bedrock.common_utils import BedrockError - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -# Test cases for issue #14373 - Bedrock Application Inference Profiles with Nova Canvas - - - - - - - - -def test_amazon_nova_canvas_image_gen(): - """Test Amazon Nova Canvas image generation with cost tracking.""" - from litellm import image_generation - - model_id = "bedrock/amazon.nova-canvas-v1:0" - - response = litellm.image_generation( - model=model_id, - prompt="A serene mountain landscape at sunset with a lake reflection", - aws_region_name="us-east-1", - ) - - print(f"response cost: {response._hidden_params['response_cost']}") - - assert response._hidden_params["response_cost"] > 0 diff --git a/tests/image_gen_tests/test_image_edits.py b/tests/image_gen_tests/test_image_edits.py index fdcbf6fcd8b..eca22196287 100644 --- a/tests/image_gen_tests/test_image_edits.py +++ b/tests/image_gen_tests/test_image_edits.py @@ -193,182 +193,3 @@ async def test_openai_image_edit_litellm_router(): f.write(image_bytes) except litellm.ContentPolicyViolationError as e: pass - - -@pytest.mark.flaky(retries=3, delay=2) -@pytest.mark.asyncio -async def test_openai_image_edit_with_bytesio(): - """Test image editing using BytesIO objects instead of file readers""" - from litellm import aimage_edit, image_edit - - litellm.turn_on_debug() - try: - prompt = """ - Create a studio ghibli style image that combines all the reference images. Make sure the person looks like a CTO. - """ - - # Get images as BytesIO objects - bytesio_images = get_test_images_as_bytesio() - - result = await aimage_edit( - prompt=prompt, - model="gpt-image-1", - image=bytesio_images, - ) - print("result from image edit with BytesIO", result) - - # Validate the response meets expected schema - ImageResponse.model_validate(result) - - if isinstance(result, ImageResponse) and result.data: - image_base64 = result.data[0].b64_json - if image_base64: - image_bytes = base64.b64decode(image_base64) - - # Save the image to a file - with open("test_image_edit_bytesio.png", "wb") as f: - f.write(image_bytes) - except litellm.ContentPolicyViolationError as e: - pass - - - - - - -@pytest.mark.asyncio -async def test_azure_image_edit_cost_tracking(): - """Test Azure image edit cost tracking with custom logger""" - from litellm import aimage_edit, image_edit - - test_custom_logger = TestCustomLogger() - litellm.logging_callback_manager._reset_all_callbacks() - litellm.callbacks = [test_custom_logger] - - # Mock response for Azure image edit with usage data for cost tracking - mock_response = { - "created": 1589478378, - "data": [ - { - "b64_json": "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" - } - ], - "usage": { - "total_tokens": 1100, - "input_tokens": 100, - "input_tokens_details": {"image_tokens": 50, "text_tokens": 50}, - "output_tokens": 1000, - }, - } - - class MockResponse: - def __init__(self, json_data, status_code): - self._json_data = json_data - self.status_code = status_code - self.text = json.dumps(json_data) - self.headers = {} - - def json(self): - return self._json_data - - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_post: - # Configure the mock to return our response - mock_post.return_value = MockResponse(mock_response, 200) - - litellm.turn_on_debug() - - prompt = """ - Create a studio ghibli style image that combines all the reference images. Make sure the person looks like a CTO. - """ - - # Set up test environment variables - - result = await aimage_edit( - prompt=prompt, - model="azure/CUSTOM_AZURE_DEPLOYMENT_NAME", - base_model="azure/gpt-image-1", - image=_make_test_images(), - ) - - # Verify the request was made correctly - mock_post.assert_called_once() - - # Validate the response meets expected schema - ImageResponse.model_validate(result) - - if isinstance(result, ImageResponse) and result.data: - image_base64 = result.data[0].b64_json - if image_base64: - image_bytes = base64.b64decode(image_base64) - - # Save the image to a file - with open("test_image_edit.png", "wb") as f: - f.write(image_bytes) - - await asyncio.sleep(5) - print( - "standard logging payload", - json.dumps( - test_custom_logger.standard_logging_payload, indent=4, default=str - ), - ) - - # check model - assert ( - test_custom_logger.standard_logging_payload["model"] - == "CUSTOM_AZURE_DEPLOYMENT_NAME" - ) - assert ( - test_custom_logger.standard_logging_payload["custom_llm_provider"] - == "azure" - ) - - # check response_cost - assert test_custom_logger.standard_logging_payload["response_cost"] is not None - assert test_custom_logger.standard_logging_payload["response_cost"] > 0 - - - - - - -@pytest.mark.flaky(retries=3, delay=2) -@pytest.mark.asyncio -async def test_multiple_image_edit_with_different_formats(): - """Test multiple images editing with different file formats and types""" - from litellm import aimage_edit - - litellm.turn_on_debug() - - try: - prompt = "Create a cohesive artistic style across all images" - - mixed_images = [ - _make_single_test_image(), - get_test_images_as_bytesio()[1], - ] - - result = await aimage_edit( - prompt=prompt, - model="gpt-image-1", - image=mixed_images, - ) - - print("Mixed format images result:", result) - ImageResponse.model_validate(result) - - assert result is not None - assert result.data is not None - assert len(result.data) > 0 - - # Save result if available - if result.data and result.data[0].b64_json: - image_bytes = base64.b64decode(result.data[0].b64_json) - with open("test_multiple_image_edit_mixed.png", "wb") as f: - f.write(image_bytes) - - except litellm.ContentPolicyViolationError as e: - pytest.skip(f"Content policy violation: {e}") diff --git a/tests/integration/messages_endpoint/providers/anthropic/test_passthrough_migration_wire.py b/tests/integration/messages_endpoint/providers/anthropic/test_passthrough_migration_wire.py new file mode 100644 index 00000000000..834e10e32b8 --- /dev/null +++ b/tests/integration/messages_endpoint/providers/anthropic/test_passthrough_migration_wire.py @@ -0,0 +1,584 @@ +import json +import uuid +from collections.abc import Callable, Generator, Mapping +from contextlib import contextmanager +from hashlib import sha256 +from pathlib import Path +from typing import Final + +from integration._support.client import Gateway, JsonValue, eventually +from integration._support.database import read_rows +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, Wire, wire_server + +_MODEL: Final = "claude-sonnet-4-5-20250929" +_KEY: Final = "synthetic-anthropic-key" + +_PROXY_CONFIG: Final = ( + "model_list: []\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + " disable_spend_logs: false\n" + " proxy_batch_write_at: 1\n" +) + + +def _message(message_id: str, model: str = _MODEL) -> dict[str, JsonValue]: + return { + "id": message_id, + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": "hello test"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 11, "output_tokens": 7}, + } + + +def _thinking_message(message_id: str) -> dict[str, JsonValue]: + return { + **_message(message_id, "claude-haiku-4-5-20251001"), + "content": [ + {"type": "thinking", "thinking": "pondering the joke", "signature": "sig1"}, + {"type": "text", "text": "hello thinking"}, + ], + "usage": {"input_tokens": 11, "output_tokens": 30}, + } + + +def _sse(event: str, payload: Mapping[str, JsonValue]) -> bytes: + return f"event: {event}\ndata: {json.dumps(payload)}\n\n".encode() + + +def _stream_chunks(message_id: str) -> tuple[bytes, ...]: + return ( + _sse( + "message_start", + { + "type": "message_start", + "message": { + "id": message_id, + "type": "message", + "role": "assistant", + "model": _MODEL, + "content": [], + "usage": {"input_tokens": 11, "output_tokens": 1}, + }, + }, + ), + _sse( + "content_block_start", + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + ), + _sse( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello stream"}}, + ), + _sse("content_block_stop", {"type": "content_block_stop", "index": 0}), + _sse( + "message_delta", + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 7}}, + ), + _sse("message_stop", {"type": "message_stop"}), + ) + + +def _bad_request_reply() -> Reply: + return Reply( + status=400, + body=json.dumps( + {"type": "error", "error": {"type": "invalid_request_error", "message": "messages must be objects"}} + ).encode(), + ) + + +def _messages_are_objects(body: Mapping[str, JsonValue]) -> bool: + messages: Final = body.get("messages") + return isinstance(messages, list) and all(isinstance(message, dict) for message in messages) + + +_SPEND_COLUMNS: Final = ( + "SELECT request_id, status, call_type, prompt_tokens, completion_tokens, total_tokens, spend, request_tags, " + 'end_user, api_base, custom_llm_provider, model, cache_hit, ("startTime" <= "endTime") AS times_ordered ' + 'FROM "LiteLLM_SpendLogs" ' +) + + +def _spend_row(request_id: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows(_SPEND_COLUMNS + "WHERE request_id=%s", (request_id,)), + lambda values: len(values) == 1, + seconds=90, + ) + return rows[0] + + +def _key_spend_row(key: str) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + _SPEND_COLUMNS + "WHERE api_key=%s AND call_type=%s", + (sha256(key.encode()).hexdigest(), "pass_through_endpoint"), + ), + lambda values: len(values) == 1, + seconds=90, + ) + return rows[0] + + +_MODEL_LIST_PROBE: Final = ("GET", "/v1/models") + + +def _is_model_list_probe(request: Request) -> bool: + return (request.method, request.target) == _MODEL_LIST_PROBE + + +@contextmanager +def _upstream(respond: Callable[[Request], Reply]) -> Generator[Wire]: + with wire_server( + lambda request: ( + Reply(body=b'{"object":"list","data":[]}') if _is_model_list_probe(request) else respond(request) + ) + ) as wire: + yield wire + + +def _provider_calls(wire: Wire) -> tuple[Request, ...]: + return tuple(request for request in wire.drain() if not _is_model_list_probe(request)) + + +def _tags(row: Mapping[str, JsonValue]) -> list[JsonValue]: + raw: Final = row["request_tags"] + tags: Final = json.loads(raw) if isinstance(raw, str) else raw + assert isinstance(tags, list), row + return [tag for tag in tags if not (isinstance(tag, str) and tag.startswith("User-Agent: "))] + + +def _assert_usage_row(row: Mapping[str, JsonValue], call_type: str, tags: list[str]) -> None: + assert row["status"] == "success", row + assert row["call_type"] == call_type, row + assert row["prompt_tokens"] == 11, row + assert row["completion_tokens"] == 7, row + assert row["total_tokens"] == 18, row + spend: Final = row["spend"] + assert isinstance(spend, (int, float)) and spend > 0, row + assert _tags(row) == tags, row + assert row["custom_llm_provider"] == "anthropic", row + assert str(row["cache_hit"]).lower() != "true", row + assert row["times_ordered"] is True, row + + +def _stream_text(gateway: Gateway, path: str, body: Mapping[str, JsonValue], key: str | None = None) -> str: + with gateway.client.stream( + "POST", + f"{gateway.client.base_url}{path}", + json=body, + headers={"Authorization": f"Bearer {key or gateway.key}"}, + ) as stream: + assert stream.status_code == 200, stream.read() + return "".join(stream.iter_text()) + + +def _owned_config(tmp_path: Path, text: str) -> Path: + config: Final = tmp_path / "proxy_config.yaml" + config.write_text(text) + return config + + +def test_passthrough_basic_completion_spend_row_v1_messages(gateway: Gateway) -> None: + marker: Final = "pt-basic-" + uuid.uuid4().hex + prompt: Final = f"say hello {marker}" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == _KEY + body: Final = json.loads(request.body) + assert body["model"] == _MODEL + assert body["messages"] == [{"role": "user", "content": prompt}] + assert "litellm_metadata" not in body + return Reply(body=json.dumps(_message(f"msg_{marker}")).encode()) + + with _upstream(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_KEY) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 100, + "messages": [{"role": "user", "content": prompt}], + "litellm_metadata": {"tags": [f"{marker}-1", f"{marker}-2"], "user": f"end-user-{marker}"}, + }, + ) + assert response.status_code == 200, response.text + assert response.json()["id"] == f"msg_{marker}" + assert len(_provider_calls(wire)) == 1 + row: Final = _spend_row(f"msg_{marker}") + _assert_usage_row(row, "anthropic_messages", [f"{marker}-1", f"{marker}-2"]) + assert row["end_user"] == f"end-user-{marker}", row + + +def test_passthrough_streaming_spend_row_v1_messages(gateway: Gateway) -> None: + marker: Final = "pt-stream-" + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + assert request.target == "/v1/messages", request.target + body: Final = json.loads(request.body) + assert body["model"] == _MODEL + assert body["stream"] is True + return Reply(content_type="text/event-stream", chunks=_stream_chunks(f"msg_{marker}")) + + with _upstream(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_KEY) + text: Final = _stream_text( + gateway, + "/v1/messages", + { + "model": model, + "max_tokens": 100, + "stream": True, + "messages": [{"role": "user", "content": f"say hello {marker}"}], + "litellm_metadata": {"tags": [f"{marker}-1", f"{marker}-2"], "user": f"end-user-{marker}"}, + }, + ) + assert "hello stream" in text + row: Final = _spend_row(f"msg_{marker}") + _assert_usage_row(row, "anthropic_messages", [f"{marker}-1", f"{marker}-2"]) + assert row["end_user"] == f"end-user-{marker}", row + + +def test_passthrough_wildcard_model_strips_provider_prefix(gateway: Gateway) -> None: + marker: Final = "pt-wildcard-" + uuid.uuid4().hex + + def respond(request: Request) -> Reply: + body: Final = json.loads(request.body) + assert body["model"] == "claude-haiku-4-5-20251001" + return Reply(body=json.dumps(_message(f"msg_{marker}", "claude-haiku-4-5-20251001")).encode()) + + with _upstream(respond) as wire, gateway.scenario() as scenario: + created: Final = gateway.post( + "/model/new", + { + "model_name": "anthropic/*", + "litellm_params": {"model": "anthropic/*", "api_base": wire.url, "api_key": _KEY}, + }, + ) + model_info: Final = created["model_info"] + assert isinstance(model_info, dict), created + identity: Final = model_info["id"] + assert isinstance(identity, str), created + scenario.cleanups.callback(scenario.delete_model, identity) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": "anthropic/claude-haiku-4-5-20251001", + "max_tokens": 100, + "messages": [{"role": "user", "content": f"hello wildcard {marker}"}], + }, + ) + assert response.status_code == 200, response.text + assert response.json()["content"][0]["text"] == "hello test" + assert len(_provider_calls(wire)) == 1 + + +def test_passthrough_thinking_block_round_trips_v1_messages(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + body: Final = json.loads(request.body) + assert body["model"] == "claude-haiku-4-5-20251001" + assert body["thinking"] == {"type": "enabled", "budget_tokens": 16000} + assert body["max_tokens"] == 20000 + return Reply(body=json.dumps(_thinking_message("msg_" + uuid.uuid4().hex)).encode()) + + with _upstream(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model="anthropic/claude-haiku-4-5-20251001", api_base=wire.url, api_key=_KEY) + response: Final = gateway.request( + "POST", + "/v1/messages", + { + "model": model, + "max_tokens": 20000, + "thinking": {"type": "enabled", "budget_tokens": 16000}, + "messages": [{"role": "user", "content": "Just pinging with thinking enabled"}], + }, + ) + assert response.status_code == 200, response.text + content: Final = response.json()["content"] + assert content[0]["type"] == "thinking" + assert content[0]["thinking"] == "pondering the joke" + + +def test_passthrough_bad_request_returns_400_v1_messages(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + body: Final = json.loads(request.body) + assert not _messages_are_objects(body), body + return _bad_request_reply() + + with _upstream(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"anthropic/{_MODEL}", api_base=wire.url, api_key=_KEY) + responses: Final = tuple( + gateway.request( + "POST", + "/v1/messages", + {"model": model, "max_tokens": 10, "stream": stream, "messages": ["hi"]}, + ) + for stream in (False, True) + ) + assert [response.status_code for response in responses] == [400, 400], [r.text for r in responses] + + +def test_native_anthropic_route_completion_stream_thinking_and_bad_request(gateway: Gateway, tmp_path: Path) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/v1/messages", request.target + assert request.headers["x-api-key"] == _KEY + body: Final = json.loads(request.body) + if not _messages_are_objects(body): + return _bad_request_reply() + if body.get("stream") is True: + return Reply(content_type="text/event-stream", chunks=_stream_chunks("msg_" + uuid.uuid4().hex)) + if body.get("thinking") is not None: + return Reply(body=json.dumps(_thinking_message("msg_" + uuid.uuid4().hex)).encode()) + return Reply(body=json.dumps(_message("msg_" + uuid.uuid4().hex)).encode()) + + with ( + _upstream(respond) as wire, + owned_proxy( + gateway, + tmp_path, + {"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _KEY}, + config=_owned_config(tmp_path, _PROXY_CONFIG), + ) as candidate, + ): + completion: Final = candidate.request( + "POST", + "/anthropic/v1/messages", + {"model": _MODEL, "max_tokens": 100, "messages": [{"role": "user", "content": "say hello native"}]}, + ) + assert completion.status_code == 200, completion.text + assert completion.json()["content"][0]["text"] == "hello test" + thinking: Final = candidate.request( + "POST", + "/anthropic/v1/messages", + { + "model": "claude-haiku-4-5-20251001", + "max_tokens": 20000, + "thinking": {"type": "enabled", "budget_tokens": 16000}, + "messages": [{"role": "user", "content": "ping"}], + }, + ) + assert thinking.status_code == 200, thinking.text + assert thinking.json()["content"][0]["type"] == "thinking" + assert thinking.json()["content"][0]["thinking"] == "pondering the joke" + bad: Final = tuple( + candidate.request( + "POST", + "/anthropic/v1/messages", + {"model": _MODEL, "max_tokens": 10, "stream": stream, "messages": ["hi"]}, + ) + for stream in (False, True) + ) + assert [response.status_code for response in bad] == [400, 400], [r.text for r in bad] + text: Final = _stream_text( + candidate, + "/anthropic/v1/messages", + { + "model": _MODEL, + "max_tokens": 100, + "stream": True, + "messages": [{"role": "user", "content": "hello native stream"}], + }, + ) + assert "hello stream" in text + + +def test_native_passthrough_spend_rows_record_usage_tags_and_spend(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "pt-native-" + uuid.uuid4().hex + completion_id: Final = f"msg_{marker}_completion" + stream_id: Final = f"msg_{marker}_stream" + + def respond(request: Request) -> Reply: + body: Final = json.loads(request.body) + assert "litellm_metadata" not in body + if body.get("stream") is True: + return Reply(content_type="text/event-stream", chunks=_stream_chunks(stream_id)) + return Reply(body=json.dumps(_message(completion_id)).encode()) + + with ( + _upstream(respond) as wire, + owned_proxy( + gateway, + tmp_path, + {"ANTHROPIC_API_BASE": wire.url, "ANTHROPIC_API_KEY": _KEY}, + config=_owned_config(tmp_path, _PROXY_CONFIG), + ) as candidate, + candidate.scenario() as scenario, + ): + completion_key: Final = scenario.key() + stream_key: Final = scenario.key() + response: Final = candidate.request( + "POST", + "/anthropic/v1/messages", + { + "model": _MODEL, + "max_tokens": 10, + "messages": [{"role": "user", "content": "Say 'hello test' and nothing else"}], + "litellm_metadata": {"tags": [f"{marker}-1", f"{marker}-2"]}, + }, + key=completion_key, + ) + assert response.status_code == 200, response.text + assert response.json()["id"] == completion_id + text: Final = _stream_text( + candidate, + "/anthropic/v1/messages", + { + "model": _MODEL, + "max_tokens": 10, + "stream": True, + "messages": [{"role": "user", "content": "Say 'hello stream test' and nothing else"}], + "litellm_metadata": {"tags": [f"{marker}-s1", f"{marker}-s2"], "user": f"end-user-{marker}"}, + }, + key=stream_key, + ) + assert "hello stream" in text + completion_row: Final = _key_spend_row(completion_key) + stream_row: Final = _key_spend_row(stream_key) + assert completion_row["request_id"] == completion_id, completion_row + assert stream_row["request_id"] == stream_id, stream_row + _assert_usage_row(completion_row, "pass_through_endpoint", [f"{marker}-1", f"{marker}-2"]) + assert completion_row["api_base"] == f"{wire.url}/v1/messages", completion_row + assert "claude" in str(completion_row["model"]), completion_row + _assert_usage_row(stream_row, "pass_through_endpoint", [f"{marker}-s1", f"{marker}-s2"]) + assert stream_row["end_user"] == f"end-user-{marker}", stream_row + + +def _openai_responses_stream() -> tuple[bytes, ...]: + response: Final[dict[str, JsonValue]] = { + "id": "resp_pt1", + "object": "response", + "created_at": 1700000000, + "model": "gpt-4o-mini", + "status": "completed", + "output": [ + { + "type": "message", + "role": "assistant", + "content": [{"type": "output_text", "text": "hi from openai"}], + } + ], + "usage": {"input_tokens": 12, "output_tokens": 8, "total_tokens": 20}, + } + return ( + _sse( + "response.created", + {"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}, + ), + _sse( + "response.output_text.delta", + { + "type": "response.output_text.delta", + "item_id": "msg_pto", + "output_index": 0, + "content_index": 0, + "delta": "hi from openai", + }, + ), + _sse("response.completed", {"type": "response.completed", "response": response}), + ) + + +def _openai_chat_stream() -> tuple[bytes, ...]: + chunk: Final[dict[str, JsonValue]] = { + "id": "chatcmpl-pt1", + "object": "chat.completion.chunk", + "created": 1700000000, + "model": "gpt-4o", + } + return ( + f"data: {json.dumps({**chunk, 'choices': [{'index': 0, 'delta': {'role': 'assistant', 'content': 'Hi'}}]})}\n\n".encode(), + f"data: {json.dumps({**chunk, 'choices': [{'index': 0, 'delta': {}, 'finish_reason': 'stop'}]})}\n\n".encode(), + f"data: {json.dumps({**chunk, 'choices': [], 'usage': {'prompt_tokens': 12, 'completion_tokens': 8, 'total_tokens': 20}})}\n\n".encode(), + b"data: [DONE]\n\n", + ) + + +def _delta_usages(text: str) -> list[Mapping[str, JsonValue]]: + events: Final = [json.loads(line[len("data: ") :]) for line in text.splitlines() if line.startswith("data: ")] + return [event["usage"] for event in events if event.get("type") == "message_delta" and "usage" in event] + + +def _cost_config(wire_url: str) -> str: + return ( + "model_list:\n" + " - model_name: amsg\n" + " litellm_params:\n" + f" model: anthropic/{_MODEL}\n" + f" api_base: {wire_url}\n" + f" api_key: {_KEY}\n" + " - model_name: omsg\n" + " litellm_params:\n" + " model: openai/gpt-4o-mini\n" + f" api_base: {wire_url}\n" + " api_key: synthetic-openai-key\n" + "litellm_settings:\n" + " include_cost_in_streaming_usage: true\n" + "general_settings:\n" + " master_key: os.environ/LITELLM_MASTER_KEY\n" + " database_url: os.environ/DATABASE_URL\n" + " store_model_in_db: true\n" + " disable_spend_logs: false\n" + " proxy_batch_write_at: 1\n" + ) + + +def _assert_cost_in_every_delta(gateway: Gateway, model: str) -> None: + text: Final = _stream_text( + gateway, + "/v1/messages", + {"model": model, "max_tokens": 20, "stream": True, "messages": [{"role": "user", "content": "Say 'Hi'"}]}, + ) + usages: Final = _delta_usages(text) + assert usages, (model, text) + costs: Final = [usage.get("cost") for usage in usages] + assert all(isinstance(cost, (int, float)) and cost > 0 for cost in costs), (model, text) + + +def test_streaming_cost_injected_into_usage_for_anthropic_and_openai_responses( + gateway: Gateway, tmp_path: Path +) -> None: + def respond(request: Request) -> Reply: + if request.target.endswith("/responses"): + return Reply(content_type="text/event-stream", chunks=_openai_responses_stream()) + assert request.target == "/v1/messages", request.target + return Reply(content_type="text/event-stream", chunks=_stream_chunks("msg_" + uuid.uuid4().hex)) + + with ( + _upstream(respond) as wire, + owned_proxy(gateway, tmp_path, {}, config=_owned_config(tmp_path, _cost_config(wire.url))) as candidate, + ): + _assert_cost_in_every_delta(candidate, "amsg") + _assert_cost_in_every_delta(candidate, "omsg") + targets: Final = [request.target for request in _provider_calls(wire)] + assert targets == ["/v1/messages", "/responses"], targets + + +def test_streaming_cost_injected_into_usage_for_openai_chat_completions_bridge( + gateway: Gateway, tmp_path: Path +) -> None: + def respond(request: Request) -> Reply: + assert request.target.endswith("/chat/completions"), request.target + assert json.loads(request.body)["stream"] is True + return Reply(content_type="text/event-stream", chunks=_openai_chat_stream()) + + with ( + _upstream(respond) as wire, + owned_proxy( + gateway, + tmp_path, + {"LITELLM_USE_CHAT_COMPLETIONS_URL_FOR_ANTHROPIC_MESSAGES": "true"}, + config=_owned_config(tmp_path, _cost_config(wire.url)), + ) as candidate, + ): + _assert_cost_in_every_delta(candidate, "omsg") + assert len(_provider_calls(wire)) == 1 diff --git a/tests/integration/providers/test_ocr_router_wire.py b/tests/integration/providers/test_ocr_router_wire.py new file mode 100644 index 00000000000..cc6455aa5f1 --- /dev/null +++ b/tests/integration/providers/test_ocr_router_wire.py @@ -0,0 +1,69 @@ +import json +from typing import Final + +import pytest + +from integration._support.client import Gateway, eventually, string_value +from integration._support.database import read_rows +from integration._support.wire import Reply, Request, wire_server + +_DOCUMENT_URL: Final = "https://example.com/doc.pdf" +_OCR_COST_PER_PAGE: Final = 0.0125 +_MISTRAL_OCR_BODY: Final = json.dumps( + { + "model": "mistral-ocr-latest", + "pages": [{"index": 0, "markdown": "Test PDF File"}], + "usage_info": {"pages_processed": 1, "doc_size_bytes": 1024}, + } +).encode() + + +def _mistral_ocr_peer(request: Request) -> Reply: + return Reply(body=_MISTRAL_OCR_BODY) + + +def test_router_aocr_routes_to_mistral_and_logs_spend(gateway: Gateway) -> None: + with wire_server(_mistral_ocr_peer) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="mistral/mistral-ocr-latest", + api_base=wire.url, + api_key="fake-mistral-key", + ocr_cost_per_page=_OCR_COST_PER_PAGE, + ) + response: Final = gateway.request( + "POST", "/v1/ocr", {"model": model, "document": {"type": "document_url", "document_url": _DOCUMENT_URL}} + ) + assert response.status_code == 200, response.text + upstream: Final = wire.drain() + assert len(upstream) == 1, upstream + assert (upstream[0].method, upstream[0].target) == ("POST", "/v1/ocr"), upstream[0] + sent: Final = json.loads(upstream[0].body) + assert sent["model"] == "mistral-ocr-latest", sent + assert sent["document"]["type"] == "document_url", sent + assert sent["document"]["document_url"] == _DOCUMENT_URL, sent + payload: Final = response.json() + assert payload["object"] == "ocr", payload + assert payload["model"] == model, payload + assert [page["index"] for page in payload["pages"]] == [0], payload + assert payload["pages"][0]["markdown"] == "Test PDF File", payload + assert payload["usage_info"]["pages_processed"] == len(payload["pages"]), payload + assert float(response.headers["x-litellm-response-cost"]) == pytest.approx(_OCR_COST_PER_PAGE), response.headers + + request_id: Final = string_value(response.headers["x-litellm-call-id"]) + rows: Final = eventually( + lambda: read_rows( + "SELECT status, call_type, custom_llm_provider, model, model_group, spend, prompt_tokens, " + 'completion_tokens, total_tokens FROM "LiteLLM_SpendLogs" WHERE request_id=%s', + (request_id,), + ), + lambda values: len(values) == 1, + seconds=70, + ) + row: Final = rows[0] + assert row["status"] == "success", row + assert row["call_type"] == "aocr", row + assert row["custom_llm_provider"] == "mistral", row + assert row["model"] == "mistral/mistral-ocr-latest", row + assert row["model_group"] == model, row + assert float(row["spend"]) == pytest.approx(_OCR_COST_PER_PAGE), row + assert (row["prompt_tokens"], row["completion_tokens"], row["total_tokens"]) == (0, 0, 0), row diff --git a/tests/integration/providers/test_openai_passthrough_files_wire.py b/tests/integration/providers/test_openai_passthrough_files_wire.py new file mode 100644 index 00000000000..bf62db8aec6 --- /dev/null +++ b/tests/integration/providers/test_openai_passthrough_files_wire.py @@ -0,0 +1,49 @@ +import uuid +from pathlib import Path +from typing import Final + +from integration._support.client import Gateway +from integration._support.process import owned_proxy +from integration._support.wire import Reply, Request, wire_server + +_UPSTREAM_KEY: Final = "synthetic-openai-key" + + +def test_openai_passthrough_file_upload_and_delete(gateway: Gateway, tmp_path: Path) -> None: + marker: Final = "openai-file-" + uuid.uuid4().hex + file_id: Final = f"file-{marker}" + + def respond(request: Request) -> Reply: + assert request.headers["authorization"] == f"Bearer {_UPSTREAM_KEY}", request.headers + if request.method == "POST" and request.target == "/files": + assert request.headers["content-type"].startswith("multipart/form-data"), request.headers + assert b'name="purpose"\r\n\r\nassistants\r\n' in request.body, request.body[:400] + assert b'filename="notes.txt"' in request.body and marker.encode() in request.body, request.body[:400] + return Reply( + body=( + b'{"id": "' + file_id.encode() + b'", "object": "file", "bytes": 12, ' + b'"created_at": 1700000000, "purpose": "assistants", "filename": "notes.txt"}' + ), + ) + if request.method == "DELETE" and request.target == f"/files/{file_id}": + return Reply(body=b'{"id": "' + file_id.encode() + b'", "object": "file", "deleted": true}') + return Reply(status=404) + + with wire_server(respond) as wire: + with owned_proxy( + gateway, + tmp_path, + {"OPENAI_API_BASE": wire.url, "OPENAI_API_KEY": _UPSTREAM_KEY}, + ) as candidate: + upload: Final = candidate.request_multipart( + "/openai/files", + {"purpose": "assistants"}, + {"file": ("notes.txt", f"contents {marker}".encode(), "text/plain")}, + ) + assert upload.status_code == 200, upload.text + assert upload.json()["id"] == file_id + delete: Final = candidate.request("DELETE", f"/openai/files/{file_id}") + assert delete.status_code == 200, delete.text + assert delete.json()["deleted"] is True + forwarded: Final = tuple((request.method, request.target) for request in wire.drain()) + assert forwarded == (("POST", "/files"), ("DELETE", f"/files/{file_id}")), forwarded diff --git a/tests/integration/providers/test_responses_error_status_wire.py b/tests/integration/providers/test_responses_error_status_wire.py new file mode 100644 index 00000000000..01372c8ae7a --- /dev/null +++ b/tests/integration/providers/test_responses_error_status_wire.py @@ -0,0 +1,71 @@ +import json +from typing import Final + +from integration._support.client import Gateway +from integration._support.wire import Reply, Request, wire_server + + +def test_unknown_model_provider_404_surfaces_to_client_as_404(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/responses", request.target + body: Final = json.loads(request.body) + assert body["model"] == "non-existent-model" + return Reply( + status=404, + body=json.dumps( + {"error": {"message": "model not found", "type": "invalid_request_error", "code": "404"}} + ).encode(), + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/responses/non-existent-model", api_base=wire.url, api_key="synthetic-openai-key" + ) + response: Final = gateway.request("POST", "/v1/responses", {"model": model, "input": "say hi"}) + assert response.status_code == 404, response.text + assert len(wire.drain()) == 1 + + +def test_provider_400_for_bad_temperature_surfaces_to_client_as_400(gateway: Gateway) -> None: + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == "/responses", request.target + body: Final = json.loads(request.body) + assert body["model"] == "gpt-4o" + assert body["temperature"] == 2000 + return Reply( + status=400, + body=json.dumps( + {"error": {"message": "temperature out of range", "type": "invalid_request_error", "code": "400"}} + ).encode(), + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/responses/gpt-4o", api_base=wire.url, api_key="synthetic-openai-key" + ) + response: Final = gateway.request( + "POST", "/v1/responses", {"model": model, "input": "say hi", "temperature": 2000} + ) + assert response.status_code == 400, response.text + assert len(wire.drain()) == 1 + + +def test_cancel_invalid_response_id_surfaces_error_status(gateway: Gateway) -> None: + response_id: Final = "invalid_response_id_12345" + + def respond(request: Request) -> Reply: + assert request.method == "POST" and request.target == f"/responses/{response_id}/cancel", request.target + return Reply( + status=404, + body=json.dumps( + {"error": {"message": "No such response", "type": "invalid_request_error", "code": "404"}} + ).encode(), + ) + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = scenario.model( + model="openai/responses/gpt-4o", api_base=wire.url, api_key="synthetic-openai-key" + ) + response: Final = gateway.request("POST", f"/v1/responses/{response_id}/cancel", {"model": model}) + assert response.status_code == 404, response.text + assert len(wire.drain()) == 1 diff --git a/tests/ocr_tests/test_ocr_matrix.py b/tests/ocr_tests/test_ocr_matrix.py index 13cffbbc9a1..3f8c9b402fc 100644 --- a/tests/ocr_tests/test_ocr_matrix.py +++ b/tests/ocr_tests/test_ocr_matrix.py @@ -24,7 +24,6 @@ from typing import Final, Literal import pytest import litellm -from litellm import Router from litellm.integrations.custom_logger import CustomLogger from litellm.llms.base_llm.ocr.transformation import OCRResponse @@ -301,17 +300,3 @@ async def test_ocr(case: Case, monkeypatch: pytest.MonkeyPatch, logger: Recordin _assert_logged(await logger.wait_for_call(), response, case.provider.model, response.model, case.call) -async def test_router_aocr(monkeypatch: pytest.MonkeyPatch, logger: RecordingLogger) -> None: - case: Final = Case(MISTRAL, MISTRAL_KEY, "explicit", PDF_BY_URL, "async") - router: Final = Router( - model_list=[ - { - "model_name": "ocr-alias", - "litellm_params": {"model": MISTRAL.model, **case.bind_credentials(monkeypatch)}, - } - ] - ) - response: Final = await router.aocr(model="ocr-alias", document=PDF_BY_URL.build()) # pyright: ignore[reportUnknownMemberType, reportUnknownVariableType] # Router.aocr is untyped - assert isinstance(response, OCRResponse) - _assert_ocr_response(response, MISTRAL.model, PDF_TEXT) - _assert_logged(await logger.wait_for_call(), response, MISTRAL.model, MISTRAL.model, case.call) diff --git a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py b/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py deleted file mode 100644 index 260656600ec..00000000000 --- a/tests/openai_endpoints_tests/test_e2e_openai_responses_api.py +++ /dev/null @@ -1,115 +0,0 @@ -import os -import time -from collections.abc import Iterator -from typing import Final - -import httpx -import pytest -from openai import APIStatusError, BadRequestError, NotFoundError, OpenAI, Stream -from openai.types.responses import ResponseStreamEvent - -BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS: Final = 90 - - -def generate_key(): - """Generate a key for testing""" - url = "http://0.0.0.0:4000/key/generate" - headers = { - "Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", - "Content-Type": "application/json", - } - data = {} - - response = httpx.post(url, headers=headers, json=data) - if response.status_code != 200: - raise Exception(f"Key generation failed with status: {response.status_code}") - return response.json()["key"] - - -def get_test_client(): - """Create OpenAI client with generated key""" - key = generate_key() - return OpenAI(api_key=key, base_url="http://0.0.0.0:4000") - - -def validate_response(response): - """ - Validate basic response structure from OpenAI responses API - """ - assert response is not None - assert hasattr(response, "choices") - assert len(response.choices) > 0 - assert hasattr(response.choices[0], "message") - assert hasattr(response.choices[0].message, "content") - assert isinstance(response.choices[0].message.content, str) - assert hasattr(response, "id") - assert isinstance(response.id, str) - assert hasattr(response, "model") - assert isinstance(response.model, str) - assert hasattr(response, "created") - assert isinstance(response.created, int) - assert hasattr(response, "usage") - assert hasattr(response.usage, "prompt_tokens") - assert hasattr(response.usage, "completion_tokens") - assert hasattr(response.usage, "total_tokens") - - -def validate_stream_chunk(chunk): - """ - Validate streaming chunk structure from OpenAI responses API - """ - assert chunk is not None - assert hasattr(chunk, "choices") - assert len(chunk.choices) > 0 - assert hasattr(chunk.choices[0], "delta") - - # Some chunks might not have content in the delta - if ( - hasattr(chunk.choices[0].delta, "content") - and chunk.choices[0].delta.content is not None - ): - assert isinstance(chunk.choices[0].delta.content, str) - - assert hasattr(chunk, "id") - assert isinstance(chunk.id, str) - assert hasattr(chunk, "model") - assert isinstance(chunk.model, str) - assert hasattr(chunk, "created") - assert isinstance(chunk.created, int) - - -def test_model_not_found_error(): - client = get_test_client() - with pytest.raises(NotFoundError): - client.responses.create(model="non-existent-model", input="This should fail") - - -def test_bad_request_bad_param_error(): - client = get_test_client() - with pytest.raises(BadRequestError): - # Out-of-range temperature on a non-reasoning model, so drop_params forwards it - client.responses.create( - model="gpt-4.1", input="This should fail", temperature=2000 - ) - - -def admitted_response_id(chunk: ResponseStreamEvent) -> str | None: - response: Final = getattr(chunk, "response", None) - return None if response is None else response.id - - -def events_until_admission(stream: Stream[ResponseStreamEvent], started: float) -> Iterator[ResponseStreamEvent]: - for chunk in stream: - print("stream chunk=", chunk) - yield chunk - if admitted_response_id(chunk) is not None: - return - if time.monotonic() - started > BACKGROUND_STREAM_ADMISSION_DEADLINE_SECONDS: - return - - -def test_cancel_invalid_response_id(): - client = get_test_client() - with pytest.raises(APIStatusError): - # Try to cancel a non-existent response ID - client.responses.cancel("invalid_response_id_12345") diff --git a/tests/openai_endpoints_tests/test_openai_batches_endpoint.py b/tests/openai_endpoints_tests/test_openai_batches_endpoint.py deleted file mode 100644 index f488095aa12..00000000000 --- a/tests/openai_endpoints_tests/test_openai_batches_endpoint.py +++ /dev/null @@ -1,463 +0,0 @@ -# What this tests ? -## Tests /batches endpoints -import os -import pytest -import asyncio -import aiohttp, openai -from openai import OpenAI, AsyncOpenAI -from typing import Optional, List, Union -from test_openai_files_endpoints import upload_file, delete_file -import sys -import time - - -BASE_URL = "http://localhost:4000" # Replace with your actual base URL -API_KEY = os.environ["LITELLM_MASTER_KEY"] # Replace with your actual API key - - -client = OpenAI(base_url=BASE_URL, api_key=API_KEY) - - -def create_batch_oai_sdk(filepath: str, custom_llm_provider: str) -> str: - batch_input_file = client.files.create( - file=open(filepath, "rb"), - purpose="batch", - extra_headers={"custom-llm-provider": custom_llm_provider}, - ) - batch_input_file_id = batch_input_file.id - - print("waiting for file to be processed......") - time.sleep(5) - rq = client.batches.create( - input_file_id=batch_input_file_id, - endpoint="/v1/chat/completions", - completion_window="24h", - metadata={ - "description": filepath, - }, - extra_headers={"custom-llm-provider": custom_llm_provider}, - ) - - print(f"Batch submitted. ID: {rq.id}") - return rq.id - - -def await_batch_completion(batch_id: str, custom_llm_provider: str): - max_tries = 3 - tries = 0 - - while tries < max_tries: - batch = client.batches.retrieve( - batch_id, extra_headers={"custom-llm-provider": custom_llm_provider} - ) - if batch.status == "completed": - print(f"Batch {batch_id} completed.") - return batch.id - - tries += 1 - print(f"waiting for batch to complete... (attempt {tries}/{max_tries})") - time.sleep(10) - - print( - f"Reached maximum number of attempts ({max_tries}). Batch may still be processing." - ) - - -def write_content_to_file( - batch_id: str, output_path: str, custom_llm_provider: str -) -> str: - batch = client.batches.retrieve( - batch_id=batch_id, extra_headers={"custom-llm-provider": custom_llm_provider} - ) - content = client.files.content( - file_id=batch.output_file_id, - extra_headers={"custom-llm-provider": custom_llm_provider}, - ) - print("content from files.content", content.content) - content.write_to_file(output_path) - - -def read_jsonl(filepath: str): - import json - - results = [] - with open(filepath, "r") as f: - for line in f: - if line.strip(): - results.append(json.loads(line)) - - for item in results: - print(item) - custom_id = item["custom_id"] - print(custom_id) - - -def get_any_completed_batch_id_azure(): - print("AZURE getting any completed batch id") - list_of_batches = client.batches.list( - extra_headers={"custom-llm-provider": "azure"} - ) - print("list of batches", list_of_batches) - for batch in list_of_batches: - if batch.status == "completed": - return batch.id - return None - - -@pytest.mark.skip(reason="Local only test to verify if things work well") -def test_vertex_batches_endpoint(): - """ - Test VertexAI Batches Endpoint - """ - import os - - oai_client = OpenAI(api_key=API_KEY, base_url=BASE_URL) - file_name = "local_testing/vertex_batch_completions.jsonl" - _current_dir = os.path.dirname(os.path.abspath(__file__)) - file_path = os.path.join(_current_dir, file_name) - file_obj = oai_client.files.create( - file=open(file_path, "rb"), - purpose="batch", - extra_headers={"custom-llm-provider": "vertex_ai"}, - ) - print("Response from creating file=", file_obj) - - batch_input_file_id = file_obj.id - assert ( - batch_input_file_id is not None - ), f"Failed to create file, expected a non null file_id but got {batch_input_file_id}" - - create_batch_response = oai_client.batches.create( - completion_window="24h", - endpoint="/v1/chat/completions", - input_file_id=batch_input_file_id, - extra_headers={"custom-llm-provider": "vertex_ai"}, - metadata={"key1": "value1", "key2": "value2"}, - ) - print("response from create batch", create_batch_response) - pass - - -@pytest.mark.asyncio -async def test_batch_status_sync_from_provider_to_database(): - """ - Test that when batch status changes at the provider, - it gets synced to the ManagedObjectTable database. - - This tests the new refactored utility functions: - - get_batch_from_database() - - update_batch_in_database() - """ - from unittest.mock import MagicMock, AsyncMock - from litellm.proxy.openai_files_endpoints.common_utils import ( - get_batch_from_database, - update_batch_in_database, - ) - from litellm.types.utils import LiteLLMBatch - import json - - # Setup: Create mock objects - batch_id = "batch_test123" - unified_batch_id = "litellm_proxy:test_unified_batch" - - # Mock database batch object with "validating" status - mock_db_batch = MagicMock() - mock_db_batch.unified_object_id = batch_id - mock_db_batch.status = "validating" - mock_db_batch.file_object = json.dumps( - { - "id": batch_id, - "object": "batch", - "status": "validating", - "endpoint": "/v1/chat/completions", - "input_file_id": "file-test123", - "completion_window": "24h", - "created_at": 1234567890, - } - ) - - # Mock prisma client - mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock( - return_value=mock_db_batch - ) - mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() - - # Mock managed_files_obj - mock_managed_files = MagicMock() - - # Mock logger - mock_logger = MagicMock() - mock_logger.debug = MagicMock() - mock_logger.info = MagicMock() - mock_logger.warning = MagicMock() - mock_logger.error = MagicMock() - - # Test 1: Retrieve batch from database (initial state) - db_batch_object, response_batch = await get_batch_from_database( - batch_id=batch_id, - unified_batch_id=unified_batch_id, - managed_files_obj=mock_managed_files, - prisma_client=mock_prisma_client, - verbose_proxy_logger=mock_logger, - ) - - # Verify database was queried - mock_prisma_client.db.litellm_managedobjecttable.find_first.assert_called_once_with( - where={"unified_object_id": batch_id} - ) - - # Verify batch was retrieved correctly - assert db_batch_object is not None - assert response_batch is not None - assert response_batch.id == batch_id - assert response_batch.status == "validating" - - # Test 2: Simulate provider returning updated status - updated_batch_response = LiteLLMBatch( - id=batch_id, - object="batch", - status="completed", # Status changed from "validating" to "completed" - endpoint="/v1/chat/completions", - input_file_id="file-test123", - completion_window="24h", - created_at=1234567890, - output_file_id="file-output123", - ) - - # Test 3: Update database with new status from provider - await update_batch_in_database( - batch_id=batch_id, - unified_batch_id=unified_batch_id, - response=updated_batch_response, - managed_files_obj=mock_managed_files, - prisma_client=mock_prisma_client, - verbose_proxy_logger=mock_logger, - db_batch_object=db_batch_object, - operation="retrieve", - ) - - # Verify database was updated - mock_prisma_client.db.litellm_managedobjecttable.update.assert_called_once() - update_call_args = mock_prisma_client.db.litellm_managedobjecttable.update.call_args - - # Verify the update call had correct parameters - assert update_call_args.kwargs["where"]["unified_object_id"] == batch_id - assert ( - update_call_args.kwargs["data"]["status"] == "complete" - ) # "completed" normalized to "complete" - assert "file_object" in update_call_args.kwargs["data"] - assert "updated_at" in update_call_args.kwargs["data"] - # batch_processed must be set to True when batch transitions to complete - assert update_call_args.kwargs["data"]["batch_processed"] is True - - # Verify logger was called with status change message - mock_logger.info.assert_called() - log_message = mock_logger.info.call_args[0][0] % mock_logger.info.call_args[0][1:] - assert "validating" in log_message - assert "completed" in log_message - - print("✅ Test passed: Batch status synced from provider to database") - - -@pytest.mark.asyncio -async def test_batch_cancel_updates_database(): - """ - Test that canceling a batch updates the database status. - """ - from unittest.mock import MagicMock, AsyncMock - from litellm.proxy.openai_files_endpoints.common_utils import ( - update_batch_in_database, - ) - from litellm.types.utils import LiteLLMBatch - - # Setup - batch_id = "batch_cancel_test" - unified_batch_id = "litellm_proxy:cancel_test" - - # Mock cancelled batch response from provider - cancelled_batch_response = LiteLLMBatch( - id=batch_id, - object="batch", - status="cancelled", - endpoint="/v1/chat/completions", - input_file_id="file-test123", - completion_window="24h", - created_at=1234567890, - cancelled_at=1234567999, - ) - - # Mock prisma client - mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock( - return_value=None - ) - mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() - - # Mock managed_files_obj - mock_managed_files = MagicMock() - - # Mock logger - mock_logger = MagicMock() - mock_logger.info = MagicMock() - mock_logger.error = MagicMock() - - # Call update_batch_in_database for cancel operation - await update_batch_in_database( - batch_id=batch_id, - unified_batch_id=unified_batch_id, - response=cancelled_batch_response, - managed_files_obj=mock_managed_files, - prisma_client=mock_prisma_client, - verbose_proxy_logger=mock_logger, - operation="cancel", - ) - - # Verify database was updated - mock_prisma_client.db.litellm_managedobjecttable.update.assert_called_once() - update_call_args = mock_prisma_client.db.litellm_managedobjecttable.update.call_args - - # Verify the update call had correct parameters - assert update_call_args.kwargs["where"]["unified_object_id"] == batch_id - assert update_call_args.kwargs["data"]["status"] == "cancelled" - assert "file_object" in update_call_args.kwargs["data"] - - # Verify logger was called - mock_logger.info.assert_called() - log_message = mock_logger.info.call_args[0][0] % mock_logger.info.call_args[0][1:] - assert "cancel" in log_message.lower() - assert "cancelled" in log_message - - print("✅ Test passed: Batch cancel updates database") - - -@pytest.mark.asyncio -async def test_batch_terminal_state_skip_provider_call(): - """ - Test that when a batch is in a terminal state (completed, failed, cancelled, expired), - it returns immediately from database without calling the provider. - """ - from unittest.mock import MagicMock, AsyncMock - from litellm.proxy.openai_files_endpoints.common_utils import ( - get_batch_from_database, - ) - from litellm.types.utils import LiteLLMBatch - import json - - # Setup: Create mock objects for a completed batch - batch_id = "batch_completed_test" - unified_batch_id = "litellm_proxy:completed_test" - - # Mock database batch object with "completed" status - mock_db_batch = MagicMock() - mock_db_batch.unified_object_id = batch_id - mock_db_batch.status = "complete" - mock_db_batch.file_object = json.dumps( - { - "id": batch_id, - "object": "batch", - "status": "completed", - "endpoint": "/v1/chat/completions", - "input_file_id": "file-test123", - "output_file_id": "file-output123", - "completion_window": "24h", - "created_at": 1234567890, - "completed_at": 1234567999, - } - ) - - # Mock prisma client - mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock( - return_value=mock_db_batch - ) - - # Mock managed_files_obj - mock_managed_files = MagicMock() - - # Mock logger - mock_logger = MagicMock() - mock_logger.debug = MagicMock() - - # Retrieve batch from database - db_batch_object, response_batch = await get_batch_from_database( - batch_id=batch_id, - unified_batch_id=unified_batch_id, - managed_files_obj=mock_managed_files, - prisma_client=mock_prisma_client, - verbose_proxy_logger=mock_logger, - ) - - # Verify batch was retrieved - assert db_batch_object is not None - assert response_batch is not None - assert response_batch.status == "completed" - - # In the actual endpoint, when status is in terminal states, - # it should return immediately without calling the provider - # This test verifies the database retrieval works correctly - assert response_batch.status in ["completed", "failed", "cancelled", "expired"] - - print("✅ Test passed: Terminal state batch retrieved from database") - - -@pytest.mark.asyncio -async def test_batch_no_status_change_skip_update(): - """ - Test that when batch status hasn't changed, database update is skipped. - """ - from unittest.mock import MagicMock, AsyncMock - from litellm.proxy.openai_files_endpoints.common_utils import ( - update_batch_in_database, - ) - from litellm.types.utils import LiteLLMBatch - - # Setup - batch_id = "batch_no_change_test" - unified_batch_id = "litellm_proxy:no_change_test" - - # Mock database batch object with "validating" status - mock_db_batch = MagicMock() - mock_db_batch.status = "validating" - - # Mock batch response from provider with same status - batch_response = LiteLLMBatch( - id=batch_id, - object="batch", - status="validating", # Same status as in database - endpoint="/v1/chat/completions", - input_file_id="file-test123", - completion_window="24h", - created_at=1234567890, - ) - - # Mock prisma client - mock_prisma_client = MagicMock() - mock_prisma_client.db.litellm_managedobjecttable.update = AsyncMock() - - # Mock managed_files_obj - mock_managed_files = MagicMock() - - # Mock logger - mock_logger = MagicMock() - mock_logger.info = MagicMock() - - # Call update_batch_in_database - await update_batch_in_database( - batch_id=batch_id, - unified_batch_id=unified_batch_id, - response=batch_response, - managed_files_obj=mock_managed_files, - prisma_client=mock_prisma_client, - verbose_proxy_logger=mock_logger, - db_batch_object=mock_db_batch, - operation="retrieve", - ) - - # Verify database update was NOT called (status hasn't changed) - mock_prisma_client.db.litellm_managedobjecttable.update.assert_not_called() - - # Verify logger info was NOT called (no status change to log) - mock_logger.info.assert_not_called() - - print("✅ Test passed: Database update skipped when status unchanged") diff --git a/tests/openai_endpoints_tests/test_openai_files_endpoints.py b/tests/openai_endpoints_tests/test_openai_files_endpoints.py deleted file mode 100644 index 9398a0d1c53..00000000000 --- a/tests/openai_endpoints_tests/test_openai_files_endpoints.py +++ /dev/null @@ -1,113 +0,0 @@ -import os -# What this tests ? -## Tests /chat/completions by generating a key and then making a chat completions request -import pytest -import asyncio -import aiohttp, openai -from openai import OpenAI, AsyncOpenAI -from typing import Optional, List, Union - - -BASE_URL = "http://localhost:4000" # Replace with your actual base URL -API_KEY = os.environ["LITELLM_MASTER_KEY"] # Replace with your actual API key - - -@pytest.mark.asyncio -async def test_file_operations(): - openai_client = AsyncOpenAI(api_key=API_KEY, base_url=BASE_URL) - file_content = b'{"prompt": "Hello", "completion": "Hi"}' - uploaded_file = await openai_client.files.create( - purpose="fine-tune", - file=file_content, - ) - list_files = await openai_client.files.list() - print("list_files=", list_files) - - get_file = await openai_client.files.retrieve(file_id=uploaded_file.id) - print("get_file=", get_file) - - get_file_content = await openai_client.files.content(file_id=uploaded_file.id) - print("get_file_content=", get_file_content.content) - response = get_file_content.response - - assert get_file_content.content == file_content - assert response.status_code == 200 - assert response.headers.get("content-type") == "application/octet-stream" - assert response.headers.get("content-length") is not None - assert int(response.headers["content-length"]) == len(get_file_content.content) - assert response.headers.get("content-disposition") is not None - assert uploaded_file.filename in response.headers["content-disposition"] - assert response.headers.get("x-request-id") is not None - # try get_file_content.write_to_file - get_file_content.write_to_file("get_file_content.jsonl") - - delete_file = await openai_client.files.delete(file_id=uploaded_file.id) - print("delete_file=", delete_file) - - -async def upload_file(session, purpose="fine-tune"): - url = f"{BASE_URL}/v1/files" - headers = {"Authorization": f"Bearer {API_KEY}"} - data = aiohttp.FormData() - data.add_field("purpose", purpose) - data.add_field( - "file", b'{"prompt": "Hello", "completion": "Hi"}', filename="mydata.jsonl" - ) - - async with session.post(url, headers=headers, data=data) as response: - assert response.status == 200 - result = await response.json() - assert "id" in result - print(f"File upload successful. File ID: {result['id']}") - return result["id"] - - -async def list_files(session): - url = f"{BASE_URL}/v1/files" - headers = {"Authorization": f"Bearer {API_KEY}"} - - async with session.get(url, headers=headers) as response: - assert response.status == 200 - result = await response.json() - assert "data" in result - print("List files successful") - - -async def get_file(session, file_id): - url = f"{BASE_URL}/v1/files/{file_id}" - headers = {"Authorization": f"Bearer {API_KEY}"} - - async with session.get(url, headers=headers) as response: - assert response.status == 200 - result = await response.json() - assert result["id"] == file_id - assert result["object"] == "file" - assert "bytes" in result - assert "created_at" in result - assert "filename" in result - assert result["purpose"] == "fine-tune" - print(f"Get file successful for file ID: {file_id}") - - -async def get_file_content(session, file_id): - url = f"{BASE_URL}/v1/files/{file_id}/content" - headers = {"Authorization": f"Bearer {API_KEY}"} - - async with session.get(url, headers=headers) as response: - assert response.status == 200 - content = await response.text() - print("content from /files/{file_id}/content=", content) - assert content # Check if content is not empty - print(f"Get file content successful for file ID: {file_id}") - - -async def delete_file(session, file_id): - url = f"{BASE_URL}/v1/files/{file_id}" - headers = {"Authorization": f"Bearer {API_KEY}"} - - async with session.delete(url, headers=headers) as response: - assert response.status == 200 - result = await response.json() - assert "deleted" in result - assert result["id"] == file_id - print(f"Delete file successful for file ID: {file_id}") diff --git a/tests/pass_through_tests/base_anthropic_messages_test.py b/tests/pass_through_tests/base_anthropic_messages_test.py index 95f709e3880..710dfa453cf 100644 --- a/tests/pass_through_tests/base_anthropic_messages_test.py +++ b/tests/pass_through_tests/base_anthropic_messages_test.py @@ -13,61 +13,8 @@ class BaseAnthropicMessagesTest(ABC): def get_client(self): return anthropic.Anthropic() - def test_anthropic_basic_completion(self): - print("making basic completion request to anthropic passthrough") - client = self.get_client() - response = client.messages.create( - model="claude-sonnet-4-5-20250929", - max_tokens=1024, - messages=[{"role": "user", "content": "Say 'hello test' and nothing else"}], - extra_body={ - "litellm_metadata": { - "tags": ["test-tag-1", "test-tag-2"], - } - }, - ) - print(response) - def test_anthropic_streaming(self): - print("making streaming request to anthropic passthrough") - collected_output = [] - client = self.get_client() - with client.messages.stream( - max_tokens=10, - messages=[ - {"role": "user", "content": "Say 'hello stream test' and nothing else"} - ], - model="claude-sonnet-4-5-20250929", - extra_body={ - "litellm_metadata": { - "tags": ["test-tag-stream-1", "test-tag-stream-2"], - } - }, - ) as stream: - for text in stream.text_stream: - collected_output.append(text) - full_response = "".join(collected_output) - print(full_response) - - def test_anthropic_messages_with_thinking(self): - print("making request to anthropic passthrough with thinking") - client = self.get_client() - response = client.messages.create( - model="claude-haiku-4-5-20251001", - max_tokens=20000, - thinking={"type": "enabled", "budget_tokens": 16000}, - messages=[ - {"role": "user", "content": "Just pinging with thinking enabled"} - ], - ) - - print(response) - - # Verify the first content block is a thinking block - response_thinking = response.content[0].thinking - assert response_thinking is not None - assert len(response_thinking) > 0 def test_anthropic_streaming_with_thinking(self): print("making streaming request to anthropic passthrough with thinking enabled") @@ -105,41 +52,4 @@ class BaseAnthropicMessagesTest(ABC): assert len(collected_response) > 0 assert len(full_response) > 0 - def test_bad_request_error_handling_streaming(self): - print("making request to anthropic passthrough with bad request") - try: - client = self.get_client() - response = client.messages.create( - model="claude-sonnet-4-5-20250929", - max_tokens=10, - stream=True, - messages=["hi"], - ) - print(response) - assert pytest.fail("Expected BadRequestError") - except anthropic.BadRequestError as e: - print("Got BadRequestError from anthropic, e=", e) - print(e.__cause__) - print(e.status_code) - print(e.response) - except Exception as e: - pytest.fail(f"Got unexpected exception: {e}") - def test_bad_request_error_handling_non_streaming(self): - print("making request to anthropic passthrough with bad request") - try: - client = self.get_client() - response = client.messages.create( - model="claude-sonnet-4-5-20250929", - max_tokens=10, - messages=["hi"], - ) - print(response) - assert pytest.fail("Expected BadRequestError") - except anthropic.BadRequestError as e: - print("Got BadRequestError from anthropic, e=", e) - print(e.__cause__) - print(e.status_code) - print(e.response) - except Exception as e: - pytest.fail(f"Got unexpected exception: {e}") diff --git a/tests/pass_through_tests/test_anthropic_passthrough.py b/tests/pass_through_tests/test_anthropic_passthrough.py deleted file mode 100644 index 5d6ddb1fbd0..00000000000 --- a/tests/pass_through_tests/test_anthropic_passthrough.py +++ /dev/null @@ -1,472 +0,0 @@ -""" -This test ensures that the proxy can passthrough anthropic requests -""" - -import os -import pytest -import anthropic -import aiohttp -import asyncio -import json - - -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=2) -async def test_anthropic_basic_completion_with_headers(): - print("making basic completion request to anthropic passthrough with aiohttp") - - headers = { - "Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", - "Content-Type": "application/json", - "Anthropic-Version": "2023-06-01", - } - - payload = { - "model": "claude-sonnet-4-5-20250929", - "max_tokens": 10, - "messages": [{"role": "user", "content": "Say 'hello test' and nothing else"}], - "litellm_metadata": { - "tags": ["test-tag-1", "test-tag-2"], - }, - } - - async with aiohttp.ClientSession() as session: - async with session.post( - "http://0.0.0.0:4000/anthropic/v1/messages", json=payload, headers=headers - ) as response: - response_text = await response.text() - print(f"Response text: {response_text}") - - response_json = await response.json() - response_headers = response.headers - print( - "non-streaming response", - json.dumps(response_json, indent=4, default=str), - ) - reported_usage = response_json.get("usage", None) - # fix null checks for reported_usage - anthropic_api_input_tokens = ( - reported_usage.get("input_tokens", None) if reported_usage else None - ) - anthropic_api_output_tokens = ( - reported_usage.get("output_tokens", None) if reported_usage else None - ) - anthropic_message_id = response_json.get("id") - - print(f"Anthropic message ID: {anthropic_message_id}") - - # Wait for spend to be logged - await asyncio.sleep(15) - - # Check spend logs for this specific request with retry logic - spend_data = None - max_retries = 2 - for attempt in range(max_retries): - print(f"Attempt {attempt + 1}/{max_retries} to check spend logs") - - async with session.get( - f"http://0.0.0.0:4000/spend/logs?request_id={anthropic_message_id}", - headers={"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}"}, - ) as spend_response: - print("text spend response") - print(f"Spend response: {spend_response}") - spend_data = await spend_response.json() - print(f"Spend data: {spend_data}") - - # Check if spend data exists and has entries - if spend_data and len(spend_data) > 0: - print("Spend logs found!") - break - else: - print("Spend logs not found yet...") - if ( - attempt < max_retries - 1 - ): # Don't wait after the last attempt - print("Waiting 10 seconds before retry...") - await asyncio.sleep(10) - - if not isinstance(spend_data, list): - print(f"Spend endpoint answered with an error response: {spend_data}") - print("Skipping spend assertions (spend logs unreachable in CI)") - return - - assert spend_data, ( - f"GET /spend/logs?request_id={anthropic_message_id} found no row for the id " - "the caller received" - ) - - log_entry = spend_data[0] - - # Basic existence checks - assert isinstance(log_entry, dict), "Log entry should be a dictionary" - - # Request metadata assertions - assert ( - log_entry["request_id"] == anthropic_message_id - ), "Request ID should be the message id the caller received" - assert ( - log_entry["call_type"] == "pass_through_endpoint" - ), "Call type should be pass_through_endpoint" - assert ( - log_entry["api_base"] == "https://api.anthropic.com/v1/messages" - ), "API base should be Anthropic's endpoint" - - # Token and spend assertions - assert log_entry["spend"] > 0, "Spend value should not be None" - assert isinstance( - log_entry["spend"], (int, float) - ), "Spend should be a number" - assert log_entry["total_tokens"] > 0, "Should have some tokens" - assert ( - log_entry["prompt_tokens"] == anthropic_api_input_tokens - ), f"Should have prompt tokens matching anthropic api. Expected {anthropic_api_input_tokens} but got {log_entry['prompt_tokens']}" - assert ( - log_entry["completion_tokens"] == anthropic_api_output_tokens - ), f"Should have completion tokens matching anthropic api. Expected {anthropic_api_output_tokens} but got {log_entry['completion_tokens']}" - assert ( - log_entry["total_tokens"] - == log_entry["prompt_tokens"] + log_entry["completion_tokens"] - ), "Total tokens should equal prompt + completion" - - # Time assertions - assert all( - key in log_entry - for key in ["startTime", "endTime", "completionStartTime"] - ), "Should have all time fields" - assert ( - log_entry["startTime"] < log_entry["endTime"] - ), "Start time should be before end time" - - # Metadata assertions - assert str(log_entry["cache_hit"]).lower() != "true", "Cache should be off" - assert log_entry["request_tags"] == [ - "test-tag-1", - "test-tag-2", - ], "Tags should match input" - assert ( - "user_api_key" in log_entry["metadata"] - ), "Should have user API key in metadata" - - assert "claude" in log_entry["model"] - assert log_entry["custom_llm_provider"] == "anthropic" - - -@pytest.mark.asyncio -async def test_anthropic_streaming_with_headers(): - print("making streaming request to anthropic passthrough with aiohttp") - - headers = { - "Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", - "Content-Type": "application/json", - "Anthropic-Version": "2023-06-01", - } - - payload = { - "model": "claude-sonnet-4-5-20250929", - "max_tokens": 10, - "messages": [ - {"role": "user", "content": "Say 'hello stream test' and nothing else"} - ], - "stream": True, - "litellm_metadata": { - "tags": ["test-tag-stream-1", "test-tag-stream-2"], - "user": "test-user-1", - }, - } - - async with aiohttp.ClientSession() as session: - async with session.post( - "http://0.0.0.0:4000/anthropic/v1/messages", json=payload, headers=headers - ) as response: - print("response status") - print(response.status) - assert response.status == 200, "Response should be successful" - response_headers = response.headers - print(f"Response headers: {response_headers}") - - collected_output = [] - async for line in response.content: - if line: - text = line.decode("utf-8").strip() - if text.startswith("data: "): - collected_output.append(text[6:]) # Remove 'data: ' prefix - - print("Collected output:", "".join(collected_output)) - anthropic_api_usage_chunks = [] - anthropic_message_id = None - for chunk in collected_output: - chunk_json = json.loads(chunk) - if chunk_json.get("type") == "message_start": - anthropic_message_id = chunk_json.get("message", {}).get("id") - if "usage" in chunk_json: - anthropic_api_usage_chunks.append(chunk_json["usage"]) - elif "message" in chunk_json and "usage" in chunk_json["message"]: - anthropic_api_usage_chunks.append(chunk_json["message"]["usage"]) - - print(f"Anthropic message ID: {anthropic_message_id}") - - print( - "anthropic_api_usage_chunks", - json.dumps(anthropic_api_usage_chunks, indent=4, default=str), - ) - - print("anthropic_api_usage_chunks: ", anthropic_api_usage_chunks) - # Get the most recent value of input tokens (iterate backwards to find last non-zero value) - anthropic_api_input_tokens = 0 - for usage in reversed(anthropic_api_usage_chunks): - if usage.get("input_tokens", 0) > 0: - anthropic_api_input_tokens = usage.get("input_tokens", 0) - break - anthropic_api_output_tokens = 0 - for usage in reversed(anthropic_api_usage_chunks): - if usage.get("output_tokens", 0) > 0: - anthropic_api_output_tokens = usage.get("output_tokens", 0) - break - - print("anthropic_api_input_tokens", anthropic_api_input_tokens) - print("anthropic_api_output_tokens", anthropic_api_output_tokens) - - # Wait for spend to be logged - await asyncio.sleep(20) - - # Check spend logs for this specific request with retry logic - spend_data = None - max_retries = 2 - for attempt in range(max_retries): - print(f"Attempt {attempt + 1}/{max_retries} to check spend logs") - - async with session.get( - f"http://0.0.0.0:4000/spend/logs?request_id={anthropic_message_id}", - headers={"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}"}, - ) as spend_response: - spend_data = await spend_response.json() - print(f"Spend data: {spend_data}") - - # Check if spend data exists and has entries - if spend_data and len(spend_data) > 0: - print("Spend logs found!") - break - else: - print("Spend logs not found yet...") - if ( - attempt < max_retries - 1 - ): # Don't wait after the last attempt - print("Waiting 10 seconds before retry...") - await asyncio.sleep(10) - - if not isinstance(spend_data, list): - print(f"Spend endpoint answered with an error response: {spend_data}") - print("Skipping spend assertions (spend logs unreachable in CI)") - return - - assert spend_data, ( - f"GET /spend/logs?request_id={anthropic_message_id} found no row for the id " - "the caller received" - ) - - log_entry = spend_data[0] - - # Basic existence checks - assert isinstance(log_entry, dict), "Log entry should be a dictionary" - - # Request metadata assertions - assert ( - log_entry["request_id"] == anthropic_message_id - ), "Request ID should be the message id the caller received" - assert ( - log_entry["call_type"] == "pass_through_endpoint" - ), "Call type should be pass_through_endpoint" - # assert ( - # log_entry["api_base"] == "https://api.anthropic.com/v1/messages" - # ), "API base should be Anthropic's endpoint" - - # Token and spend assertions - assert log_entry["spend"] > 0, "Spend value should not be None" - assert isinstance( - log_entry["spend"], (int, float) - ), "Spend should be a number" - assert log_entry["total_tokens"] > 0, "Should have some tokens" - assert ( - log_entry["prompt_tokens"] == anthropic_api_input_tokens - ), f"Should have prompt tokens matching anthropic api. Expected {anthropic_api_input_tokens} but got {log_entry['prompt_tokens']}" - assert ( - log_entry["completion_tokens"] == anthropic_api_output_tokens - ), f"Should have completion tokens matching anthropic api. Expected {anthropic_api_output_tokens} but got {log_entry['completion_tokens']}" - assert ( - log_entry["total_tokens"] - == log_entry["prompt_tokens"] + log_entry["completion_tokens"] - ), "Total tokens should equal prompt + completion" - - # Time assertions - assert all( - key in log_entry - for key in ["startTime", "endTime", "completionStartTime"] - ), "Should have all time fields" - assert ( - log_entry["startTime"] < log_entry["endTime"] - ), "Start time should be before end time" - - # Metadata assertions - assert str(log_entry["cache_hit"]).lower() != "true", "Cache should be off" - assert log_entry["request_tags"] == [ - "test-tag-stream-1", - "test-tag-stream-2", - ], "Tags should match input" - assert ( - "user_api_key" in log_entry["metadata"] - ), "Should have user API key in metadata" - - assert "claude" in log_entry["model"] - - assert log_entry["end_user"] == "test-user-1" - assert log_entry["custom_llm_provider"] == "anthropic" - - -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=2) -async def test_anthropic_messages_streaming_cost_injection(): - """ - Test that cost is injected into message_delta usage for Anthropic Messages API streaming - """ - print("Testing cost injection in Anthropic Messages API streaming response") - - headers = { - "Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", - "Content-Type": "application/json", - "anthropic-version": "2023-06-01", - } - - payload = { - "model": "claude-haiku-4-5-20251001", - "max_tokens": 10, - "stream": True, - "messages": [{"role": "user", "content": "Say 'Hi'"}], - } - - async with aiohttp.ClientSession() as session: - async with session.post( - "http://0.0.0.0:4000/v1/messages", - json=payload, - headers=headers, - ) as response: - assert response.status == 200 - - # Collect all SSE events. - # Split each chunk by newlines to handle both: - # - Anthropic direct path: chunks arrive as individual lines - # - OpenAI/Responses API path: chunks are full multi-line SSE events - events = [] - async for chunk in response.content: - chunk_str = chunk.decode("utf-8") - for line in chunk_str.split("\n"): - line = line.strip() - if line.startswith("data: "): - try: - data = json.loads(line[6:]) # Remove 'data: ' prefix - events.append(data) - except json.JSONDecodeError: - continue - - # Find message_delta event with usage - message_delta_events = [ - event - for event in events - if event.get("type") == "message_delta" and "usage" in event - ] - - assert ( - len(message_delta_events) > 0 - ), "No message_delta events with usage found" - - # Check that cost is included in usage - for event in message_delta_events: - usage = event.get("usage", {}) - assert "cost" in usage, f"Cost not found in usage: {usage}" - assert isinstance( - usage["cost"], (int, float) - ), f"Cost should be numeric: {usage['cost']}" - assert ( - usage["cost"] >= 0 - ), f"Cost should be non-negative: {usage['cost']}" - - print(f"Found message_delta with cost: {usage}") - - print( - f"Test passed: Found {len(message_delta_events)} message_delta events with cost" - ) - - -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=2) -async def test_anthropic_messages_openai_model_streaming_cost_injection(): - """ - Test that cost is injected into message_delta usage for OpenAI model via Anthropic Messages API - """ - print("Testing cost injection in Anthropic Messages API with OpenAI model") - - headers = { - "Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", - "Content-Type": "application/json", - "anthropic-version": "2023-06-01", - } - - payload = { - "model": "openai/gpt-4o", - "max_tokens": 20, - "stream": True, - "messages": [{"role": "user", "content": "Say 'Hi'"}], - } - - async with aiohttp.ClientSession() as session: - async with session.post( - "http://0.0.0.0:4000/v1/messages", - json=payload, - headers=headers, - ) as response: - assert response.status == 200 - - # Collect all SSE events. - # Split each chunk by newlines to handle both: - # - Direct API paths: chunks arrive as individual lines - # - OpenAI/Responses API path: AnthropicResponsesStreamWrapper yields - # full multi-line SSE events as single bytes objects, so a naive - # startswith('data: ') check on the whole chunk misses them. - events = [] - async for chunk in response.content: - chunk_str = chunk.decode("utf-8") - for line in chunk_str.split("\n"): - line = line.strip() - if line.startswith("data: "): - try: - data = json.loads(line[6:]) # Remove 'data: ' prefix - events.append(data) - except json.JSONDecodeError: - continue - - # Find message_delta event with usage - message_delta_events = [ - event - for event in events - if event.get("type") == "message_delta" and "usage" in event - ] - - assert ( - len(message_delta_events) > 0 - ), "No message_delta events with usage found" - - # Check that cost is included in usage - for event in message_delta_events: - usage = event.get("usage", {}) - assert "cost" in usage, f"Cost not found in usage: {usage}" - assert isinstance( - usage["cost"], (int, float) - ), f"Cost should be numeric: {usage['cost']}" - assert ( - usage["cost"] >= 0 - ), f"Cost should be non-negative: {usage['cost']}" - - print(f"Found message_delta with cost: {usage}") - - print( - f"Test passed: Found {len(message_delta_events)} message_delta events with cost" - ) diff --git a/tests/pass_through_tests/test_anthropic_passthrough_basic.py b/tests/pass_through_tests/test_anthropic_passthrough_basic.py index c7e9fea867c..4ef9887bc89 100644 --- a/tests/pass_through_tests/test_anthropic_passthrough_basic.py +++ b/tests/pass_through_tests/test_anthropic_passthrough_basic.py @@ -19,11 +19,3 @@ class TestAnthropicMessagesEndpoint(BaseAnthropicMessagesTest): api_key=os.environ["LITELLM_MASTER_KEY"], ) - def test_anthropic_messages_to_wildcard_model(self): - client = self.get_client() - response = client.messages.create( - model="anthropic/claude-haiku-4-5-20251001", - messages=[{"role": "user", "content": "Hello, world!"}], - max_tokens=100, - ) - print(response) diff --git a/tests/pass_through_tests/test_assembly_ai.py b/tests/pass_through_tests/test_assembly_ai.py deleted file mode 100644 index 09999bc2bed..00000000000 --- a/tests/pass_through_tests/test_assembly_ai.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -This test ensures that the proxy can passthrough requests to assemblyai -""" - -import os -import time - -import pytest -import httpx -import aiohttp -import asyncio - -TEST_MASTER_KEY = os.environ["LITELLM_MASTER_KEY"] -TEST_BASE_URL = "http://0.0.0.0:4000/assemblyai" - - -def _transcribe_and_verify(virtual_key: str, base_url: str): - file_url = "https://assembly.ai/wildfires.mp3" - headers = { - "Authorization": f"Bearer {virtual_key}", - "Content-Type": "application/json", - } - create_payload = { - "audio_url": file_url, - "speech_models": ["universal-2"], - } - - create_response = httpx.post( - url=f"{base_url}/v2/transcript", - headers=headers, - json=create_payload, - timeout=60.0, - ) - if create_response.status_code != 200: - pytest.fail( - "Failed to create transcript request: " - f"status={create_response.status_code}, body={create_response.text}" - ) - - transcript = create_response.json() - transcript_id = transcript.get("id") - if not transcript_id: - pytest.fail("Failed to get transcript id") - - for _ in range(60): - poll_response = httpx.get( - url=f"{base_url}/v2/transcript/{transcript_id}", - headers=headers, - timeout=30.0, - ) - if poll_response.status_code != 200: - pytest.fail( - "Failed to poll transcript status: " - f"status={poll_response.status_code}, body={poll_response.text}" - ) - transcript = poll_response.json() - if transcript.get("status") in ("completed", "error"): - break - time.sleep(1) - - httpx.delete( - url=f"{base_url}/v2/transcript/{transcript_id}", - headers=headers, - timeout=30.0, - ) - - if transcript.get("status") == "error": - pytest.fail(f"Failed to transcribe file error: {transcript.get('error')}") - - print(transcript.get("text")) - - -def test_assemblyai_basic_transcribe(): - print("making basic transcribe request to assemblyai passthrough") - _transcribe_and_verify(TEST_MASTER_KEY, TEST_BASE_URL) - - -async def generate_key(calling_key: str) -> str: - """Helper function to generate a new API key""" - url = "http://0.0.0.0:4000/key/generate" - headers = { - "Authorization": f"Bearer {calling_key}", - "Content-Type": "application/json", - } - - async with aiohttp.ClientSession() as session: - async with session.post(url, headers=headers, json={}) as response: - if response.status == 200: - data = await response.json() - return data.get("key") - raise Exception(f"Failed to generate key: {response.status}") - - -@pytest.mark.asyncio -async def test_assemblyai_transcribe_with_non_admin_key(): - non_admin_key = await generate_key(TEST_MASTER_KEY) - print(f"Generated non-admin key: {non_admin_key}") - - request_start_time = time.time() - _transcribe_and_verify(non_admin_key, TEST_BASE_URL) - request_end_time = time.time() - print(f"Request took {request_end_time - request_start_time} seconds") diff --git a/tests/pass_through_tests/test_hosted_vllm_passthrough.py b/tests/pass_through_tests/test_hosted_vllm_passthrough.py deleted file mode 100644 index 272b4e1bb00..00000000000 --- a/tests/pass_through_tests/test_hosted_vllm_passthrough.py +++ /dev/null @@ -1,71 +0,0 @@ -import asyncio -from unittest.mock import AsyncMock, patch - -import httpx -import pytest - -from litellm.passthrough.main import allm_passthrough_route -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -from litellm.utils import ProviderConfigManager -from litellm.types.utils import LlmProviders -from litellm.llms.vllm.passthrough.transformation import ( - VLLMPassthroughConfig, -) - - -def test_get_provider_passthrough_config_for_hosted_vllm_returns_vllm_config(): - # When requesting passthrough config for HOSTED_VLLM - cfg = ProviderConfigManager.get_provider_passthrough_config( - model="hosted_vllm/my-deployment", - provider=LlmProviders.HOSTED_VLLM, - ) - - # Then we should get a VLLMPassthroughConfig instance - assert isinstance(cfg, VLLMPassthroughConfig) - - -@pytest.mark.asyncio -async def test_allm_passthrough_route_with_hosted_vllm_model_does_not_raise(): - # Given a hosted_vllm model and an async http client - client = AsyncHTTPHandler() - - # Mock the provider resolution to ensure we use hosted_vllm and provide api_base - with patch( - "litellm.passthrough.main.get_llm_provider", - return_value=( - "my-deployment", # normalized model name - "hosted_vllm", # provider - "fake-api-key", # api key (not required for vllm) - "http://localhost:8090", # api base - ), - ): - # Mock the underlying AsyncClient.send to avoid real network I/O - fake_request = httpx.Request( - method="POST", url="http://localhost:8090/v1/chat/completions" - ) - fake_response = httpx.Response( - status_code=200, - content=b'{\n "ok": true\n}', - request=fake_request, - headers={"content-type": "application/json"}, - ) - - with patch.object( - client.client, "send", new=AsyncMock(return_value=fake_response) - ): - # When calling the async passthrough route with a hosted_vllm/* model - response = await allm_passthrough_route( - method="POST", - endpoint="v1/chat/completions", - model="hosted_vllm/my-deployment", - api_base="http://localhost:8090", - json={ - "model": "anything", # will be replaced internally with normalized model - "messages": [{"role": "user", "content": "Hello"}], - }, - client=client, - ) - - # Then it should not raise and return a successful httpx.Response - assert isinstance(response, httpx.Response) - assert response.status_code == 200 diff --git a/tests/pass_through_tests/test_openai_assistants_passthrough.py b/tests/pass_through_tests/test_openai_assistants_passthrough.py deleted file mode 100644 index 4da84ce5ca3..00000000000 --- a/tests/pass_through_tests/test_openai_assistants_passthrough.py +++ /dev/null @@ -1,23 +0,0 @@ -import os -import openai -import tempfile - - -client = openai.OpenAI(base_url="http://0.0.0.0:4000/openai", api_key=os.environ["LITELLM_MASTER_KEY"]) - - -def test_pass_through_file_operations(): - with tempfile.NamedTemporaryFile( - mode="w+", suffix=".txt", delete=False - ) as temp_file: - temp_file.write("This is a test file for the OpenAI Assistants API.") - temp_file.flush() - - file = client.files.create( - file=open(temp_file.name, "rb"), - purpose="assistants", - ) - print("file created", file) - - delete_file = client.files.delete(file.id) - print("file deleted", delete_file) diff --git a/tests/pass_through_tests/test_vertex_ai.py b/tests/pass_through_tests/test_vertex_ai.py index c1de9ae777d..834373df650 100644 --- a/tests/pass_through_tests/test_vertex_ai.py +++ b/tests/pass_through_tests/test_vertex_ai.py @@ -82,93 +82,6 @@ def get_tracked_spend() -> float: return sum(float(row.get("spend") or 0.0) for row in rows) -VERTEX_PROJECT = "litellm-ci-cd" -VERTEX_MODEL = "gemini-3.1-flash-lite" -VERTEX_GENERATE_CONTENT_URL = ( - f"{LITE_LLM_ENDPOINT}/vertex_ai/v1/projects/{VERTEX_PROJECT}" - f"/locations/global/publishers/google/models/{VERTEX_MODEL}:generateContent" -) - - -def _vertex_access_token() -> str: - import google.auth - import google.auth.transport.requests - - credentials, _ = google.auth.default( - scopes=["https://www.googleapis.com/auth/cloud-platform"] - ) - credentials.refresh(google.auth.transport.requests.Request()) - return credentials.token - - -def _spend_log_for_request(call_id: str) -> dict | None: - response = requests.get( - f"{LITE_LLM_ENDPOINT}/spend/logs?request_id={call_id}", - headers={"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}"}, - timeout=30, - ) - if response.status_code != 200: - return None - rows = response.json() - return rows[0] if rows else None - - -def _is_vertex_quota_error(response: requests.Response) -> bool: - return response.status_code == 429 or "RESOURCE_EXHAUSTED" in response.text - - -@pytest.mark.asyncio() -async def test_basic_vertex_ai_pass_through_with_spendlog(): - load_vertex_ai_credentials() - access_token = _vertex_access_token() - - # Drive the pass-through over HTTP instead of the vertexai SDK: the SDK intermittently - # routes generateContent to the public Vertex endpoint rather than the proxy override, - # so the call never reaches LiteLLM and no spend is logged. A direct request always - # hits the proxy. Spend logging then runs on a best-effort background worker that can - # drop a single event, so retry a few billed calls and assert that one specific call's - # spend log lands. Failing every attempt still fails hard, which is the signal we want - # if cost tracking is broken. - max_attempts = 3 - poll_seconds = 60 - poll_interval = 5 - - for attempt in range(1, max_attempts + 1): - response = requests.post( - VERTEX_GENERATE_CONTENT_URL, - headers={ - "Authorization": f"Bearer {access_token}", - "Content-Type": "application/json", - }, - json={"contents": [{"role": "user", "parts": [{"text": "hi"}]}]}, - timeout=60, - ) - if _is_vertex_quota_error(response): - pytest.skip("Vertex AI quota exhausted") - assert ( - response.status_code == 200 - ), f"vertex pass-through call failed: {response.status_code} {response.text}" - - call_id = response.headers.get("x-litellm-call-id") - assert call_id, "proxy response missing x-litellm-call-id header" - - for _ in range(poll_seconds // poll_interval): - await asyncio.sleep(poll_interval) - row = _spend_log_for_request(call_id) - if row is not None and float(row.get("spend") or 0) > 0: - assert "gemini" in row["model"], f"unexpected model in spend log: {row}" - assert ( - row["custom_llm_provider"] == "vertex_ai" - ), f"unexpected provider in spend log: {row}" - return - - print(f"attempt {attempt}: spend log for call {call_id} not found yet, re-billing") - - pytest.fail( - f"Vertex pass-through spend never recorded after {max_attempts} billed calls" - ) - - @pytest.mark.asyncio() @pytest.mark.skip(reason="skip flaky test - vertex pass through streaming is flaky") async def test_basic_vertex_ai_pass_through_streaming_with_spendlog(): diff --git a/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py b/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py index a7f04466d14..41b351631f6 100644 --- a/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py +++ b/tests/pass_through_unit_tests/base_anthropic_messages_prompt_caching_test.py @@ -202,50 +202,6 @@ class BaseAnthropicMessagesPromptCachingTest(ABC): f"but got {cache_read}. Full usage: {usage}" ) - @pytest.mark.asyncio - async def test_prompt_caching_with_system_message(self): - """ - E2E test: Prompt caching with system message should work. - """ - _skip_live_prompt_caching_test() - litellm.turn_on_debug() - - messages = [ - { - "role": "user", - "content": "What are the key terms?", - }, - ] - - system = [ - { - "type": "text", - "text": LARGE_DOCUMENT_FOR_CACHING, - "cache_control": {"type": "ephemeral"}, - }, - ] - - response = await litellm.anthropic.messages.acreate( - model=self.get_model(), - messages=messages, - system=system, - max_tokens=100, - ) - - print(f"Response: {json.dumps(response, indent=2, default=str)}") - - usage = response.get("usage", {}) - cache_creation = usage.get("cache_creation_input_tokens", 0) - cache_read = usage.get("cache_read_input_tokens", 0) - - print(f"cache_creation_input_tokens: {cache_creation}") - print(f"cache_read_input_tokens: {cache_read}") - - assert cache_creation > 0 or cache_read > 0, ( - f"Expected cache tokens > 0 for system message caching, " - f"but got cache_creation={cache_creation}, cache_read={cache_read}" - ) - def _parse_sse_chunks(self, chunk: bytes) -> list: """ Parse SSE format chunks and return list of JSON objects. @@ -432,94 +388,3 @@ class BaseAnthropicMessagesPromptCachingTest(ABC): f"Expected cache_read_input_tokens > 0 on second streaming call, " f"but got {cache_read}" ) - - @pytest.mark.asyncio - async def test_prompt_caching_message_start_indicates_caching_support(self): - """ - E2E test: message_start event should contain cache fields to indicate caching support. - - This validates that the message_start event includes cache_creation_input_tokens - and cache_read_input_tokens fields (even if initialized to 0) so that clients - like Claude Code can detect that prompt caching is supported. - - This test specifically addresses the issue where Bedrock converse API streaming - didn't include cache fields in message_start, causing clients to think caching - wasn't supported. - """ - _skip_live_prompt_caching_test() - litellm.turn_on_debug() - - messages = self.get_messages_with_cache_control() - - response = await litellm.anthropic.messages.acreate( - model=self.get_model(), - messages=messages, - max_tokens=100, - stream=True, - ) - - # Look for message_start event and validate it has cache fields - message_start_found = False - message_start_has_cache_creation_field = False - message_start_has_cache_read_field = False - - async for chunk in response: - # Handle SSE format chunks (bytes) - if isinstance(chunk, bytes): - json_chunks = self._parse_sse_chunks(chunk) - for json_data in json_chunks: - if json_data.get("type") == "message_start": - message_start_found = True - message = json_data.get("message", {}) - usage = message.get("usage", {}) - - print( - f"message_start usage: {json.dumps(usage, indent=2, default=str)}" - ) - - # Check that cache fields are present (even if 0) - if "cache_creation_input_tokens" in usage: - message_start_has_cache_creation_field = True - if "cache_read_input_tokens" in usage: - message_start_has_cache_read_field = True - - # Break after first message_start - break - elif isinstance(chunk, dict): - if chunk.get("type") == "message_start": - message_start_found = True - message = chunk.get("message", {}) - usage = message.get("usage", {}) - - print( - f"message_start usage: {json.dumps(usage, indent=2, default=str)}" - ) - - # Check that cache fields are present (even if 0) - if "cache_creation_input_tokens" in usage: - message_start_has_cache_creation_field = True - if "cache_read_input_tokens" in usage: - message_start_has_cache_read_field = True - - # Break after first message_start - break - - # Break if we found message_start - if message_start_found: - break - - # Validate that message_start was found - assert ( - message_start_found - ), "Expected to find message_start event in streaming response" - - # Validate that cache fields are present in message_start - assert message_start_has_cache_creation_field, ( - "Expected cache_creation_input_tokens field in message_start event. " - "This field should be present (even if 0) to indicate caching support to clients." - ) - - assert message_start_has_cache_read_field, ( - "Expected cache_read_input_tokens field in message_start event. " - "This field should be present (even if 0) to indicate caching support to clients." - ) diff --git a/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py b/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py index 9e706f99316..966048e609e 100644 --- a/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py +++ b/tests/pass_through_unit_tests/base_anthropic_messages_tool_search_test.py @@ -147,48 +147,6 @@ class BaseAnthropicMessagesToolSearchTest(ABC): content = response.get("content", []) assert len(content) > 0, "Response should have content" - @pytest.mark.asyncio - async def test_tool_search_discovers_tool(self): - """ - E2E test: Tool search should discover and use a deferred tool. - - This validates that when the user asks about weather, the model - discovers the get_weather tool via tool search and attempts to use it. - """ - litellm.turn_on_debug() - - tools = self.get_tools_with_tool_search() - messages = [ - { - "role": "user", - "content": "I need to know the current weather in New York City. Please use the appropriate tool.", - } - ] - - response = await litellm.anthropic.messages.acreate( - model=self.get_model(), - messages=messages, - tools=tools, - max_tokens=1024, - extra_headers=self.get_extra_headers(), - ) - - print(f"Response: {json.dumps(response, indent=2, default=str)}") - - content = response.get("content", []) - - # Check if the model used tool_use (either tool_search or get_weather) - tool_uses = [block for block in content if block.get("type") == "tool_use"] - - print(f"Tool uses: {json.dumps(tool_uses, indent=2, default=str)}") - - # The model should attempt to use tools when asked about weather - # It might use tool_search first, or directly use get_weather if discovered - if response.get("stop_reason") == "tool_use": - assert ( - len(tool_uses) > 0 - ), "Expected tool_use blocks when stop_reason is tool_use" - @pytest.mark.asyncio @pytest.mark.flaky(retries=3, delay=5) async def test_tool_search_streaming(self): @@ -234,39 +192,3 @@ class BaseAnthropicMessagesToolSearchTest(ABC): # Should have message_start message_starts = [c for c in chunks if c.get("type") == "message_start"] assert len(message_starts) > 0, "Expected message_start in streaming response" - - @pytest.mark.asyncio - async def test_tool_search_with_multiple_deferred_tools(self): - """ - E2E test: Tool search should work with multiple deferred tools. - - This validates that the model can discover the appropriate tool - from a larger catalog of deferred tools. - """ - litellm.turn_on_debug() - - tools = self.get_tools_with_tool_search() - messages = [ - {"role": "user", "content": "What's the stock price of Apple (AAPL)?"} - ] - - response = await litellm.anthropic.messages.acreate( - model=self.get_model(), - messages=messages, - tools=tools, - max_tokens=1024, - extra_headers=self.get_extra_headers(), - ) - - print(f"Response: {json.dumps(response, indent=2, default=str)}") - - # Validate response - assert "content" in response, "Response should contain content" - - content = response.get("content", []) - tool_uses = [block for block in content if block.get("type") == "tool_use"] - - # If the model decides to use a tool, it should be related to stocks - if tool_uses: - tool_names = [t.get("name") for t in tool_uses] - print(f"Tools used: {tool_names}") diff --git a/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py b/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py index 16055b5a29b..858a6713f7b 100644 --- a/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py +++ b/tests/pass_through_unit_tests/base_anthropic_unified_messages_test.py @@ -62,234 +62,3 @@ class BaseAnthropicMessagesTest: assert "content" in response assert "model" in response assert response.get("role") == "assistant" - - @pytest.mark.asyncio - async def test_non_streaming_base(self): - """Base test for non-streaming requests""" - litellm.turn_on_debug() - - request_params = self.model_config - - # Set up test parameters - messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] - - # Prepare call arguments - call_args = { - "messages": messages, - "max_tokens": 100, - } - - # Add any additional config from subclass - call_args.update(request_params) - - # Call the handler - response = await litellm.anthropic.messages.acreate(**call_args) - - print(f"Non-streaming {request_params['model']} response: ", response) - - # Verify response - self._validate_response(response) - - print(f"Non-streaming response: {json.dumps(response, indent=2, default=str)}") - return response - - @pytest.mark.asyncio - async def test_response_format_consistency(self): - """ - Test that response content blocks are consistently dicts (not Pydantic objects). - - This ensures that code like response["content"][0]["type"] works - regardless of the target provider. - - Issue: https://github.com/BerriAI/litellm/issues/20342 - """ - litellm.turn_on_debug() - - request_params = self.model_config - - # Set up test parameters - messages = [{"role": "user", "content": "Say hi"}] - - # Prepare call arguments - call_args = { - "messages": messages, - "max_tokens": 100, - } - - # Add any additional config from subclass - call_args.update(request_params) - - # Call the handler - response = await litellm.anthropic.messages.acreate(**call_args) - - print( - f"Response for {request_params['model']}: {json.dumps(response, indent=2, default=str)}" - ) - - # Verify response structure - assert "content" in response, "Response should have 'content' field" - assert len(response["content"]) > 0, "Response content should not be empty" - - # Get the first content block - block = response["content"][0] - - # Check that the block is a dict, not a Pydantic object - assert isinstance(block, dict), ( - f"Content block should be a dict, but got {type(block)}. " - f"This means response format is inconsistent across providers." - ) - - # Verify we can access fields using dict syntax (not object attributes) - try: - block_type = block["type"] - print(f"✓ Successfully accessed block['type']: {block_type}") - except TypeError as e: - pytest.fail( - f"Cannot access content block using dict syntax: {e}. " - f"Block type: {type(block)}" - ) - - # Verify the block has expected structure - assert "type" in block, "Content block should have 'type' field" - if block["type"] == "text": - assert "text" in block, "Text content block should have 'text' field" - - print( - f"✓ Response format consistency test passed for {request_params['model']}" - ) - - @pytest.mark.asyncio - async def test_anthropic_messages_litellm_router_streaming_with_logging(self): - """ - Test that logging and cost tracking works for anthropic_messages with streaming request - """ - test_custom_logger = TestCustomLogger() - litellm.callbacks = [test_custom_logger] - litellm.turn_on_debug() - router = Router( - model_list=[ - { - "model_name": "claude-special-alias", - "litellm_params": {**self.model_config}, - } - ] - ) - - # Set up test parameters - messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] - - # Call the handler - response = await router.aanthropic_messages( - messages=messages, - model="claude-special-alias", - max_tokens=100, - stream=True, - ) - - response_prompt_tokens = 0 - response_completion_tokens = 0 - all_anthropic_usage_chunks = [] - buffer = "" - - async for chunk in response: - # Decode chunk if it's bytes - print("chunk=", chunk) - - # Handle SSE format chunks - if isinstance(chunk, bytes): - chunk_str = chunk.decode("utf-8") - buffer += chunk_str - # Extract the JSON data part from SSE format - for line in buffer.split("\n"): - if line.startswith("data: "): - try: - json_data = json.loads(line[6:]) # Skip the 'data: ' prefix - print( - "\n\nJSON data:", - json.dumps(json_data, indent=4, default=str), - ) - - # Extract usage information - if ( - json_data.get("type") == "message_start" - and "message" in json_data - ): - if "usage" in json_data["message"]: - usage = json_data["message"]["usage"] - all_anthropic_usage_chunks.append(usage) - print( - "USAGE BLOCK", - json.dumps(usage, indent=4, default=str), - ) - elif "usage" in json_data: - usage = json_data["usage"] - all_anthropic_usage_chunks.append(usage) - print( - "USAGE BLOCK", - json.dumps(usage, indent=4, default=str), - ) - except json.JSONDecodeError: - print(f"Failed to parse JSON from: {line[6:]}") - elif hasattr(chunk, "message"): - if chunk.message.usage: - print( - "USAGE BLOCK", - json.dumps(chunk.message.usage, indent=4, default=str), - ) - all_anthropic_usage_chunks.append(chunk.message.usage) - elif hasattr(chunk, "usage"): - print("USAGE BLOCK", json.dumps(chunk.usage, indent=4, default=str)) - all_anthropic_usage_chunks.append(chunk.usage) - - print( - "all_anthropic_usage_chunks", - json.dumps(all_anthropic_usage_chunks, indent=4, default=str), - ) - - # Extract token counts from usage data - if all_anthropic_usage_chunks: - response_prompt_tokens = max( - [usage.get("input_tokens", 0) for usage in all_anthropic_usage_chunks] - ) - response_completion_tokens = max( - [usage.get("output_tokens", 0) for usage in all_anthropic_usage_chunks] - ) - - print("input_tokens_anthropic_api", response_prompt_tokens) - print("output_tokens_anthropic_api", response_completion_tokens) - - await asyncio.sleep(4) - - print( - "logged_standard_logging_payload", - json.dumps( - test_custom_logger.logged_standard_logging_payload, - indent=4, - default=str, - ), - ) - - assert ( - test_custom_logger.logged_standard_logging_payload is not None - ), "Logging payload should not be None" - assert ( - test_custom_logger.logged_standard_logging_payload["messages"] == messages - ) - assert ( - test_custom_logger.logged_standard_logging_payload["response"] is not None - ) - assert ( - test_custom_logger.logged_standard_logging_payload["model"] - == self.expected_model_name_in_logging - ) - - # check logged usage + spend - assert test_custom_logger.logged_standard_logging_payload["response_cost"] > 0 - assert ( - test_custom_logger.logged_standard_logging_payload["prompt_tokens"] - == response_prompt_tokens - ) - assert ( - test_custom_logger.logged_standard_logging_payload["completion_tokens"] - == response_completion_tokens - ) diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py index 0f0bb8f091e..001de2f7d52 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py @@ -118,13 +118,6 @@ class TestAnthropicOpenAIAPI(BaseAnthropicMessagesTest): """ return "gpt-4.1-mini" - @pytest.mark.asyncio - async def test_anthropic_messages_litellm_router_streaming_with_logging(self): - """ - Test the anthropic_messages with streaming request - """ - pass - @pytest.mark.asyncio async def test_anthropic_messages_litellm_router_non_streaming(): @@ -163,283 +156,3 @@ async def test_anthropic_messages_litellm_router_non_streaming(): print(f"Non-streaming response: {json.dumps(response, indent=2)}") return response - -@pytest.mark.asyncio -async def test_anthropic_messages_litellm_router_routing_strategy(): - """ - Test the anthropic_messages with routing strategy + non-streaming request - """ - litellm.turn_on_debug() - router = Router( - model_list=[ - { - "model_name": "claude-special-alias", - "litellm_params": { - "model": "claude-haiku-4-5-20251001", - "api_key": os.getenv("ANTHROPIC_API_KEY"), - }, - } - ], - routing_strategy="latency-based-routing", - ) - - # Set up test parameters - messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] - - # Call the handler - response = await router.aanthropic_messages( - messages=messages, - model="claude-special-alias", - max_tokens=100, - metadata={ - "user_id": "hello", - }, - ) - - # Verify response - assert "id" in response - assert "content" in response - assert "model" in response - assert response["role"] == "assistant" - - print(f"Non-streaming response: {json.dumps(response, indent=2)}") - return response - - -@pytest.mark.asyncio -async def test_anthropic_messages_fallbacks(): - """ - E2E test the anthropic_messages fallbacks from Anthropic API to Bedrock - """ - litellm.turn_on_debug() - router = Router( - model_list=[ - { - "model_name": "anthropic/claude-opus-4-7", - "litellm_params": { - "model": "anthropic/claude-opus-4-7", - "api_key": "bad-key", - }, - }, - { - "model_name": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "litellm_params": { - "model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - }, - }, - ], - fallbacks=[ - { - "anthropic/claude-opus-4-7": [ - "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0" - ] - } - ], - ) - - # Set up test parameters - messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] - - # Call the handler - response = await router.aanthropic_messages( - messages=messages, - model="anthropic/claude-opus-4-7", - max_tokens=100, - metadata={ - "user_id": "hello", - }, - ) - - # Verify response - assert "id" in response - assert "content" in response - assert "model" in response - assert response["role"] == "assistant" - - print(f"Non-streaming response: {json.dumps(response, indent=2)}") - return response - - -class TestCustomLogger(CustomLogger): - def __init__(self): - super().__init__() - self.logged_standard_logging_payload: Optional[StandardLoggingPayload] = None - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - print("inside async_log_success_event") - self.logged_standard_logging_payload = kwargs.get("standard_logging_object") - - pass - - -@pytest.mark.asyncio -async def test_anthropic_messages_litellm_router_non_streaming_with_logging(): - """ - Test the anthropic_messages with non-streaming request - - - Ensure Cost + Usage is tracked - """ - test_custom_logger = TestCustomLogger() - litellm.callbacks = [test_custom_logger] - litellm.turn_on_debug() - MODEL_GROUP = "claude-special-alias" - router = Router( - model_list=[ - { - "model_name": MODEL_GROUP, - "litellm_params": { - "model": "claude-haiku-4-5-20251001", - "api_key": os.getenv("ANTHROPIC_API_KEY"), - }, - } - ] - ) - - # Set up test parameters - messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] - - # Call the handler - response = await router.aanthropic_messages( - messages=messages, - model=MODEL_GROUP, - max_tokens=100, - ) - - # Verify response - _validate_anthropic_response(response) - - print(f"Non-streaming response: {json.dumps(response, indent=2)}") - - await asyncio.sleep(1) - - assert ( - test_custom_logger.logged_standard_logging_payload is not None - ), "Logging payload should not be None" - print( - "tracked standard logging payload", - json.dumps( - test_custom_logger.logged_standard_logging_payload, indent=4, default=str - ), - ) - assert test_custom_logger.logged_standard_logging_payload["messages"] == messages - assert test_custom_logger.logged_standard_logging_payload["response"] is not None - assert ( - test_custom_logger.logged_standard_logging_payload["model"] - == "claude-haiku-4-5-20251001" - ) - - # check logged usage + spend - assert test_custom_logger.logged_standard_logging_payload["response_cost"] > 0 - assert ( - test_custom_logger.logged_standard_logging_payload["prompt_tokens"] - == response["usage"]["input_tokens"] - ) - assert ( - test_custom_logger.logged_standard_logging_payload["completion_tokens"] - == response["usage"]["output_tokens"] - ) - - # assert model_group - assert ( - test_custom_logger.logged_standard_logging_payload["model_group"] == MODEL_GROUP - ) - - -# @pytest.mark.asyncio -# async def test_bedrock_messages_api_header_forwarding(): -# """ -# Test that headers from kwargs (set by proxy's add_headers_to_llm_call_by_model_group) -# are correctly passed to validate_anthropic_messages_environment for Bedrock Invoke API. - -# This verifies that forward_client_headers_to_llm_api works for Bedrock Invoke API (Messages API). - -# Issue: When calling Anthropic models via the Messages API, LiteLLM makes a call to -# Bedrock's Invoke API, and custom headers were not being forwarded, even though -# they worked correctly for Chat Completions API with Bedrock's Converse API. -# """ -# from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler -# from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj -# from litellm.types.router import GenericLiteLLMParams - -# handler = BaseLLMHTTPHandler() - -# # Headers that would be set by the proxy when forward_client_headers_to_llm_api is configured -# custom_headers = { -# "X-Custom-Header": "CustomValue", -# "X-Request-ID": "req-123", -# } - -# # Mock the provider config -# mock_provider_config = MagicMock() - -# # We'll check what headers are passed to this method -# mock_provider_config.validate_anthropic_messages_environment.return_value = ( -# {"Authorization": "Bearer test"}, -# "https://bedrock-runtime.us-east-1.amazonaws.com/invoke" -# ) -# mock_provider_config.transform_anthropic_messages_request.return_value = {"model": "test"} -# mock_provider_config.get_complete_url.return_value = "https://test.com" -# mock_provider_config.sign_request.return_value = ({}, None) -# mock_provider_config.transform_anthropic_messages_response.return_value = {"id": "test"} - -# # Mock HTTP client to prevent actual network calls -# with unittest.mock.patch("litellm.llms.custom_httpx.llm_http_handler.get_async_httpx_client") as mock_get_client: -# mock_http_client = AsyncMock() -# mock_response = MagicMock() -# mock_response.status_code = 200 -# mock_response.json.return_value = {"id": "test", "content": []} -# mock_response.text = "{}" -# mock_http_client.post.return_value = mock_response -# mock_get_client.return_value = mock_http_client - -# # Mock logging object -# mock_logging_obj = MagicMock(spec=LiteLLMLoggingObj) -# mock_logging_obj.model_call_details = {} - -# # Call the handler with headers in kwargs -# try: -# await handler.async_anthropic_messages_handler( -# model="bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0", -# messages=[{"role": "user", "content": "Hello"}], -# anthropic_messages_provider_config=mock_provider_config, -# anthropic_messages_optional_request_params={"max_tokens": 100}, -# custom_llm_provider="bedrock", -# litellm_params=GenericLiteLLMParams( -# api_key="test-key", -# aws_region_name="us-east-1" -# ), -# logging_obj=mock_logging_obj, -# api_key="test-key", -# stream=False, -# kwargs={"headers": custom_headers} # Headers set by proxy -# ) -# except Exception: -# pass # Ignore errors, we're only checking if headers were passed - -# # Verify that validate_anthropic_messages_environment was called -# assert mock_provider_config.validate_anthropic_messages_environment.called - -# # Get the headers that were passed -# call_args = mock_provider_config.validate_anthropic_messages_environment.call_args -# passed_headers = call_args[1]["headers"] - -# # The custom headers from kwargs should be in the passed headers -# assert "X-Custom-Header" in passed_headers or "x-custom-header" in passed_headers -# assert "X-Request-ID" in passed_headers or "x-request-id" in passed_headers - - -def test_sync_openai_messages(): - """ - Test the anthropic_messages with sync request - """ - litellm.turn_on_debug() - response = litellm.anthropic.messages.create( - messages=[{"role": "user", "content": "Hello, can you tell me a short joke?"}], - model="openai/gpt-4.1-mini", - max_tokens=100, - ) - print("ANT response", response) - - assert response is not None - assert isinstance(response, dict) - assert response["content"][0]["text"] is not None diff --git a/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py index 451a09d30bb..aacf65ea4a3 100644 --- a/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py +++ b/tests/pass_through_unit_tests/test_bedrock_anthropic_messages_test.py @@ -14,54 +14,6 @@ from base_anthropic_unified_messages_test import BaseAnthropicMessagesTest INSTANCE_BASE_ANTHROPIC_MESSAGES_TEST = BaseAnthropicMessagesTest() -@pytest.mark.asyncio -async def test_anthropic_messages_litellm_router_bedrock(): - """ - Test the anthropic_messages with non-streaming request - """ - - litellm.turn_on_debug() - router = Router( - model_list=[ - { - "model_name": "bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "litellm_params": { - "model": "bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - }, - }, - { - "model_name": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - "litellm_params": { - "model": "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - }, - }, - ] - ) - - # Set up test parameters - messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] - - # Call 1 using bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0 - response = await router.aanthropic_messages( - messages=messages, - model="bedrock/converse/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - max_tokens=100, - ) - - # Verify response - INSTANCE_BASE_ANTHROPIC_MESSAGES_TEST._validate_response(response) - - # Call 2 using bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0 - response = await router.aanthropic_messages( - messages=messages, - model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - max_tokens=100, - ) - - # Verify response - INSTANCE_BASE_ANTHROPIC_MESSAGES_TEST._validate_response(response) - - @pytest.mark.asyncio async def test_anthropic_messages_bedrock_converse_with_thinking(): """ diff --git a/tests/search_tests/base_search_unit_tests.py b/tests/search_tests/base_search_unit_tests.py index 7028f58a1a3..8c16e790607 100644 --- a/tests/search_tests/base_search_unit_tests.py +++ b/tests/search_tests/base_search_unit_tests.py @@ -113,62 +113,3 @@ class BaseSearchTest(ABC): except Exception as e: pytest.fail(f"Search call failed: {str(e)}") - - def test_search_response_structure(self): - """ - Test that the Search response has the correct structure. - """ - litellm.set_verbose = True - search_provider = self.get_search_provider() - - response = litellm.search( - query="artificial intelligence recent news", - search_provider=search_provider, - ) - - # Validate response structure - assert hasattr(response, "results"), "Response should have 'results' attribute" - assert hasattr(response, "object"), "Response should have 'object' attribute" - - assert isinstance(response.results, list), "results should be a list" - assert len(response.results) > 0, "Should have at least one result" - assert response.object == "search", "object should be 'search'" - - # Validate first result structure - first_result = response.results[0] - assert hasattr(first_result, "title"), "Result should have 'title' attribute" - assert hasattr(first_result, "url"), "Result should have 'url' attribute" - assert hasattr( - first_result, "snippet" - ), "Result should have 'snippet' attribute" - assert isinstance(first_result.title, str), "title should be a string" - assert isinstance(first_result.url, str), "url should be a string" - assert isinstance(first_result.snippet, str), "snippet should be a string" - - print(f"\nResponse structure validated:") - print(f" - object: {response.object}") - print(f" - results: {len(response.results)}") - print(f" - first result has all required fields") - - def test_search_with_optional_params(self): - """ - Test search with optional parameters. - """ - litellm.set_verbose = True - search_provider = self.get_search_provider() - - response = litellm.search( - query="machine learning", - search_provider=search_provider, - max_results=5, - ) - - # Validate response - assert hasattr(response, "results"), "Response should have 'results' attribute" - assert isinstance(response.results, list), "results should be a list" - assert len(response.results) > 0, "Should have at least one result" - assert len(response.results) <= 5, "Should have at most 5 results as requested" - - print(f"\nSearch with optional params validated:") - print(f" - Requested max_results: 5") - print(f" - Received results: {len(response.results)}") diff --git a/tests/search_tests/test_duckduckgo_search.py b/tests/search_tests/test_duckduckgo_search.py deleted file mode 100644 index 682221326bb..00000000000 --- a/tests/search_tests/test_duckduckgo_search.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -Tests for DuckDuckGo Search API integration. -""" - -import os - -import pytest - -import litellm -from tests.search_tests.base_search_unit_tests import BaseSearchTest - - -class TestDuckDuckGoSearch(BaseSearchTest): - """ - Tests for DuckDuckGo Search functionality. - """ - - def get_search_provider(self) -> str: - """ - Return search_provider for DuckDuckGo Search. - """ - return "duckduckgo" - - @pytest.mark.asyncio - async def test_basic_search(self): - """ - Test basic search functionality with a simple query. - """ - os.environ["LITELLM_LOCAL_MODEL_COST_MAP"] = "True" - litellm.model_cost = litellm.get_model_cost_map(url="") - litellm.turn_on_debug() - search_provider = self.get_search_provider() - print("Search Provider=", search_provider) - - try: - response = await litellm.asearch( - query="india", - search_provider=search_provider, - ) - print("Search response=", response.model_dump_json(indent=4)) - - print(f"\n{'='*80}") - print(f"Response type: {type(response)}") - print( - f"Response object: {response.object if hasattr(response, 'object') else 'N/A'}" - ) - - # Check if response has expected Search format - assert hasattr( - response, "results" - ), "Response should have 'results' attribute" - assert hasattr( - response, "object" - ), "Response should have 'object' attribute" - assert ( - response.object == "search" - ), f"Expected object='search', got '{response.object}'" - - # Validate results structure - assert isinstance(response.results, list), "results should be a list" - assert len(response.results) > 0, "Should have at least one result" - - # Check first result structure - first_result = response.results[0] - assert hasattr( - first_result, "title" - ), "Result should have 'title' attribute" - assert hasattr(first_result, "url"), "Result should have 'url' attribute" - assert hasattr( - first_result, "snippet" - ), "Result should have 'snippet' attribute" - - print(f"Total results: {len(response.results)}") - print(f"First result title: {first_result.title}") - print(f"First result URL: {first_result.url}") - print(f"First result snippet: {first_result.snippet[:100]}...") - print(f"{'='*80}\n") - - assert len(first_result.title) > 0, "Title should not be empty" - assert len(first_result.url) > 0, "URL should not be empty" - assert len(first_result.snippet) > 0, "Snippet should not be empty" - - # Validate cost tracking in _hidden_params - assert hasattr( - response, "_hidden_params" - ), "Response should have '_hidden_params' attribute" - hidden_params = response._hidden_params - assert ( - "response_cost" in hidden_params - ), "_hidden_params should contain 'response_cost'" - - response_cost = hidden_params["response_cost"] - assert response_cost is not None, "response_cost should not be None" - assert isinstance( - response_cost, (int, float) - ), "response_cost should be a number" - assert response_cost == 0, "response_cost should be 0" - - print(f"Cost tracking: ${response_cost:.6f}") - - except Exception as e: - pytest.fail(f"Search call failed: {str(e)}") - - def test_search_response_structure(self): - """ - Test that the Search response has the correct structure. - """ - litellm.set_verbose = True - search_provider = self.get_search_provider() - - response = litellm.search( - query="india", - search_provider=search_provider, - ) - - # Validate response structure - assert hasattr(response, "results"), "Response should have 'results' attribute" - assert hasattr(response, "object"), "Response should have 'object' attribute" - - assert isinstance(response.results, list), "results should be a list" - assert len(response.results) > 0, "Should have at least one result" - assert response.object == "search", "object should be 'search'" - - # Validate first result structure - first_result = response.results[0] - assert hasattr(first_result, "title"), "Result should have 'title' attribute" - assert hasattr(first_result, "url"), "Result should have 'url' attribute" - assert hasattr( - first_result, "snippet" - ), "Result should have 'snippet' attribute" - assert isinstance(first_result.title, str), "title should be a string" - assert isinstance(first_result.url, str), "url should be a string" - assert isinstance(first_result.snippet, str), "snippet should be a string" - - print(f"\nResponse structure validated:") - print(f" - object: {response.object}") - print(f" - results: {len(response.results)}") - print(f" - first result has all required fields") diff --git a/tests/search_tests/test_firecrawl_search.py b/tests/search_tests/test_firecrawl_search.py deleted file mode 100644 index eec74d48e26..00000000000 --- a/tests/search_tests/test_firecrawl_search.py +++ /dev/null @@ -1,42 +0,0 @@ -from unittest.mock import Mock, patch -import litellm - - -def test_firecrawl_search_request_body(): - """ - Test that validates the Firecrawl search request body is correctly formatted. - """ - mock_response = Mock() - mock_response.status_code = 200 - mock_response.json.return_value = { - "success": True, - "data": { - "web": [ - { - "title": "Test Title", - "url": "https://example.com", - "markdown": "Test content", - } - ] - }, - } - - with patch( - "litellm.llms.custom_httpx.http_handler.HTTPHandler.post", - return_value=mock_response, - ) as mock_post: - litellm.search( - query="test query", - search_provider="firecrawl", - max_results=10, - country="US", - ) - - assert mock_post.called - call_kwargs = mock_post.call_args.kwargs - request_body = call_kwargs.get("json") - - assert request_body is not None - assert request_body["query"] == "test query" - assert request_body["limit"] == 10 - assert request_body["country"] == "US" diff --git a/tests/spend_tracking_tests/test_ocr_spend_tracking.py b/tests/spend_tracking_tests/test_ocr_spend_tracking.py deleted file mode 100644 index 3ce77c56361..00000000000 --- a/tests/spend_tracking_tests/test_ocr_spend_tracking.py +++ /dev/null @@ -1,296 +0,0 @@ -""" -Unit tests for OCR spend tracking in get_logging_payload. - -This test file verifies that OCR/AOCR calls correctly extract usage_info -and populate the spend logs payload with pages_processed instead of token counts. -""" - -import pytest -from datetime import datetime, timezone -from unittest.mock import Mock -from pydantic import BaseModel -from typing import Optional - -from litellm.proxy.spend_tracking.spend_tracking_utils import ( - get_logging_payload, - _extract_usage_for_ocr_call, -) - - -class MockUsageInfo(BaseModel): - """Mock Pydantic model for OCR usage_info""" - - pages_processed: int - doc_size_bytes: Optional[int] = None - - -class MockOCRResponse(BaseModel): - """Mock Pydantic model for OCR response""" - - id: str - object: str - model: str - usage_info: MockUsageInfo - - -class TestExtractUsageForOCRCall: - """Test the _extract_usage_for_ocr_call helper method""" - - def test_extract_usage_from_dict(self): - """Test extracting usage from dict response""" - response_obj_dict = {"usage_info": {"pages_processed": 5}} - - usage = _extract_usage_for_ocr_call(response_obj_dict, response_obj_dict) - - assert usage["prompt_tokens"] == 0 - assert usage["completion_tokens"] == 0 - assert usage["total_tokens"] == 0 - assert usage["pages_processed"] == 5 - - def test_extract_usage_from_pydantic_model(self): - """Test extracting usage from Pydantic model response""" - usage_info = MockUsageInfo(pages_processed=10, doc_size_bytes=1024) - response_obj = MockOCRResponse( - id="ocr-123", object="ocr", model="test-ocr-model", usage_info=usage_info - ) - response_obj_dict = response_obj.model_dump() - - usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict) - - assert usage["prompt_tokens"] == 0 - assert usage["completion_tokens"] == 0 - assert usage["total_tokens"] == 0 - assert usage["pages_processed"] == 10 - - def test_extract_usage_with_object_attributes(self): - """Test extracting usage from object with __dict__""" - - class SimpleUsageInfo: - def __init__(self, pages_processed): - self.pages_processed = pages_processed - - class SimpleOCRResponse: - def __init__(self): - self.usage_info = SimpleUsageInfo(pages_processed=3) - - response_obj = SimpleOCRResponse() - response_obj_dict = {} - - usage = _extract_usage_for_ocr_call(response_obj, response_obj_dict) - - assert usage.get("prompt_tokens") == 0 - assert usage.get("completion_tokens") == 0 - assert usage.get("total_tokens") == 0 - assert usage.get("pages_processed") == 3 - - def test_extract_usage_missing_usage_info(self): - """Test handling missing usage_info""" - response_obj_dict = {} - - usage = _extract_usage_for_ocr_call(response_obj_dict, response_obj_dict) - - assert usage == {} - - def test_extract_usage_empty_usage_info(self): - """Test handling empty usage_info""" - response_obj_dict = {"usage_info": {}} - - usage = _extract_usage_for_ocr_call(response_obj_dict, response_obj_dict) - - assert usage.get("prompt_tokens") == 0 - assert usage.get("completion_tokens") == 0 - assert usage.get("total_tokens") == 0 - assert usage.get("pages_processed") == 0 - - -class TestGetLoggingPayloadOCR: - """Test get_logging_payload with OCR call types""" - - @pytest.fixture - def mock_datetime(self): - """Fixture for consistent timestamps""" - return datetime.now(timezone.utc) - - @pytest.fixture - def base_kwargs(self): - """Fixture for base kwargs used in tests""" - return { - "model": "test-ocr-model", - "call_type": "ocr", - "litellm_params": {}, - "response_cost": 0.05, - } - - def test_ocr_call_with_dict_response(self, mock_datetime, base_kwargs): - """Test OCR call with dict response containing usage_info""" - response_obj = { - "id": "ocr-test-123", - "object": "ocr", - "model": "test-ocr-model", - "usage_info": {"pages_processed": 7, "doc_size_bytes": 2048}, - } - - payload = get_logging_payload( - kwargs=base_kwargs, - response_obj=response_obj, - start_time=mock_datetime, - end_time=mock_datetime, - ) - - assert payload["call_type"] == "ocr" - assert payload["prompt_tokens"] == 0 - assert payload["completion_tokens"] == 0 - assert payload["total_tokens"] == 0 - assert payload["spend"] == 0.05 - - # Verify pages_processed is in additional_usage_values - import json - - metadata = json.loads(payload["metadata"]) - assert "additional_usage_values" in metadata - assert metadata["additional_usage_values"]["pages_processed"] == 7 - - def test_aocr_call_with_pydantic_response(self, mock_datetime, base_kwargs): - """Test AOCR (async OCR) call with Pydantic model response""" - base_kwargs["call_type"] = "aocr" - - usage_info = MockUsageInfo(pages_processed=12) - response_obj = MockOCRResponse( - id="aocr-test-456", - object="ocr", - model="test-ocr-model", - usage_info=usage_info, - ) - - payload = get_logging_payload( - kwargs=base_kwargs, - response_obj=response_obj, - start_time=mock_datetime, - end_time=mock_datetime, - ) - - assert payload["call_type"] == "aocr" - assert payload["prompt_tokens"] == 0 - assert payload["completion_tokens"] == 0 - assert payload["total_tokens"] == 0 - - # Verify pages_processed is in additional_usage_values - import json - - metadata = json.loads(payload["metadata"]) - assert "additional_usage_values" in metadata - assert metadata["additional_usage_values"]["pages_processed"] == 12 - - def test_ocr_call_missing_usage_info(self, mock_datetime, base_kwargs): - """Test OCR call with missing usage_info returns empty usage""" - response_obj = { - "id": "ocr-test-789", - "object": "ocr", - "model": "test-ocr-model", - } - - payload = get_logging_payload( - kwargs=base_kwargs, - response_obj=response_obj, - start_time=mock_datetime, - end_time=mock_datetime, - ) - - assert payload["call_type"] == "ocr" - assert payload["prompt_tokens"] == 0 - assert payload["completion_tokens"] == 0 - assert payload["total_tokens"] == 0 - - def test_ocr_call_with_zero_pages(self, mock_datetime, base_kwargs): - """Test OCR call with zero pages processed""" - response_obj = { - "id": "ocr-test-000", - "object": "ocr", - "model": "test-ocr-model", - "usage_info": {"pages_processed": 0}, - } - - payload = get_logging_payload( - kwargs=base_kwargs, - response_obj=response_obj, - start_time=mock_datetime, - end_time=mock_datetime, - ) - - assert payload["call_type"] == "ocr" - assert payload["prompt_tokens"] == 0 - assert payload["completion_tokens"] == 0 - assert payload["total_tokens"] == 0 - - # Verify pages_processed is 0 - import json - - metadata = json.loads(payload["metadata"]) - assert metadata["additional_usage_values"]["pages_processed"] == 0 - - def test_non_ocr_call_uses_token_based_usage(self, mock_datetime): - """Test that non-OCR calls still use token-based usage""" - kwargs = { - "model": "gpt-5.5", - "call_type": "completion", - "litellm_params": {}, - "response_cost": 0.02, - } - - response_obj = { - "id": "completion-test-123", - "object": "chat.completion", - "model": "gpt-5.5", - "usage": { - "prompt_tokens": 50, - "completion_tokens": 100, - "total_tokens": 150, - }, - } - - payload = get_logging_payload( - kwargs=kwargs, - response_obj=response_obj, - start_time=mock_datetime, - end_time=mock_datetime, - ) - - assert payload["call_type"] == "completion" - assert payload["prompt_tokens"] == 50 - assert payload["completion_tokens"] == 100 - assert payload["total_tokens"] == 150 - - def test_ocr_with_metadata(self, mock_datetime, base_kwargs): - """Test OCR call with additional metadata""" - base_kwargs["litellm_params"] = { - "metadata": { - "user_api_key_user_id": "test-user", - "user_api_key_team_id": "test-team", - } - } - - response_obj = { - "id": "ocr-metadata-test", - "object": "ocr", - "model": "test-ocr-model", - "usage_info": {"pages_processed": 5, "doc_size_bytes": 1024}, - } - - payload = get_logging_payload( - kwargs=base_kwargs, - response_obj=response_obj, - start_time=mock_datetime, - end_time=mock_datetime, - ) - - assert payload["call_type"] == "ocr" - assert payload["user"] == "test-user" - assert payload["prompt_tokens"] == 0 - assert payload["completion_tokens"] == 0 - - # Verify pages_processed and doc_size_bytes are both in additional_usage_values - import json - - metadata = json.loads(payload["metadata"]) - assert metadata["additional_usage_values"]["pages_processed"] == 5 - assert metadata["additional_usage_values"]["doc_size_bytes"] == 1024 diff --git a/tests/test_spend_logs.py b/tests/test_spend_logs.py index 57109ea01e5..f239fbbb53f 100644 --- a/tests/test_spend_logs.py +++ b/tests/test_spend_logs.py @@ -116,7 +116,7 @@ async def generate_team(session: aiohttp.ClientSession, org_id: str) -> dict: @pytest.mark.skip( - reason="Flaky in CI: /spend/logs?request_id=... returns 500 even after a 20s wait for the spend log to be written. Spend-log accuracy is covered by tests/unit/proxy/spend_tracking/ and the proxy_spend_accuracy_tests CircleCI job." + reason="Flaky in CI: /spend/logs?request_id=... returns 500 even after a 20s wait for the spend log to be written. Spend-log accuracy is covered by tests/unit/proxy/spend_tracking/ and tests/integration/spend/." ) @pytest.mark.asyncio async def test_spend_logs_with_org_id(): diff --git a/tests/unit/batches/test_main.py b/tests/unit/batches/test_main.py index 3303c13b6a3..fb97e0e4919 100644 --- a/tests/unit/batches/test_main.py +++ b/tests/unit/batches/test_main.py @@ -34,6 +34,16 @@ import pytest import litellm import litellm.batches.main as bm +import asyncio +import datetime +import json +from collections.abc import Mapping +from typing import Final +import httpx +import respx +from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict +from litellm.integrations.custom_logger import CustomLogger # --------------------------------------------------------------------------- # @@ -986,3 +996,211 @@ async def test_batch_logging_azure_credentials_regression(): print("✓ Batch output files can be fetched with Azure credentials") print("✓ Cost and usage tracking works for Azure batches") print("✓ Backwards compatibility maintained\n") + + +_OPENAI_FILE_JSON: Final = MappingProxyType( + { + "id": "file-abc123", + "object": "file", + "purpose": "batch", + "filename": "batch.jsonl", + "bytes": 416, + "created_at": 1739598666, + "status": "processed", + } +) + + +_OPENAI_BATCH_JSON: Final = MappingProxyType( + { + "id": "batch_abc123", + "object": "batch", + "endpoint": "/v1/chat/completions", + "input_file_id": "file-abc123", + "status": "validating", + "completion_window": "24h", + "created_at": 1739598666, + } +) + + +class _KeyAliasMetadata(TypedDict): + user_api_key_alias: ReadOnly[str | None] + user_api_key_team_alias: ReadOnly[str | None] + + +class _LoggedCall(TypedDict): + call_type: ReadOnly[str] + metadata: ReadOnly[_KeyAliasMetadata] + + +_LOGGED_CALL: Final = TypeAdapter(_LoggedCall) + + +class _SuccessPayloadRecorder(CustomLogger): + def __init__(self, call_type: str) -> None: + super().__init__() + self._call_type: Final = call_type + self.logged: Final = asyncio.Event() + self.payload: _LoggedCall | None = None + + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + payload: Final = _LOGGED_CALL.validate_python(kwargs["standard_logging_object"]) + if payload["call_type"] != self._call_type: + return + self.payload = payload + self.logged.set() + + +@pytest.mark.asyncio +async def test_acreate_batch_full_crud_and_logging_metadata( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.logging_callback_manager._reset_all_callbacks() + recorder: Final = _SuccessPayloadRecorder("acreate_batch") + monkeypatch.setattr(litellm, "callbacks", [recorder]) + + upload_route: Final = respx_mock.post("https://api.openai.com/v1/files").mock( + return_value=httpx.Response(200, json=dict(_OPENAI_FILE_JSON)) + ) + create_route: Final = respx_mock.post("https://api.openai.com/v1/batches").mock( + return_value=httpx.Response(200, json=dict(_OPENAI_BATCH_JSON)) + ) + retrieve_route: Final = respx_mock.get("https://api.openai.com/v1/batches/batch_abc123").mock( + return_value=httpx.Response(200, json=dict(_OPENAI_BATCH_JSON)) + ) + list_batches_route: Final = respx_mock.get("https://api.openai.com/v1/batches").mock( + return_value=httpx.Response(200, json={"object": "list", "data": [dict(_OPENAI_BATCH_JSON)]}) + ) + respx_mock.get("https://api.openai.com/v1/files/file-abc123/content").mock( + return_value=httpx.Response(200, content=b'{"custom_id": "request-1"}\n') + ) + respx_mock.get("https://api.openai.com/v1/files/file-abc123").mock( + return_value=httpx.Response(200, json=dict(_OPENAI_FILE_JSON)) + ) + respx_mock.delete("https://api.openai.com/v1/files/file-abc123").mock( + return_value=httpx.Response(200, json={"id": "file-abc123", "object": "file", "deleted": True}) + ) + list_files_route: Final = respx_mock.get("https://api.openai.com/v1/files").mock( + return_value=httpx.Response(200, json={"object": "list", "data": [dict(_OPENAI_FILE_JSON)]}) + ) + cancel_route: Final = respx_mock.post("https://api.openai.com/v1/batches/batch_abc123/cancel").mock( + return_value=httpx.Response(200, json={**_OPENAI_BATCH_JSON, "status": "cancelling"}) + ) + + batch_file: Final = ( + "batch.jsonl", + b'{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", ' + b'"body": {"model": "gpt-4o", "messages": [{"role": "user", "content": "hi"}]}}\n', + "application/jsonl", + ) + file_obj: Final = await litellm.acreate_file( + file=batch_file, purpose="batch", custom_llm_provider="openai", api_key="fake-key" + ) + assert file_obj.id == "file-abc123" + upload_body: Final = upload_route.calls.last.request.content + assert b'name="purpose"\r\n\r\nbatch' in upload_body + assert batch_file[1] in upload_body + + extra_metadata_field: Final = { + "user_api_key_alias": "special_api_key_alias", + "user_api_key_team_alias": "special_team_alias", + } + create_batch_response: Final = await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id=file_obj.id, + custom_llm_provider="openai", + api_key="fake-key", + metadata={"key1": "value1", "key2": "value2"}, + litellm_metadata=extra_metadata_field, + ) + + assert json.loads(create_route.calls.last.request.content) == { + "completion_window": "24h", + "endpoint": "/v1/chat/completions", + "input_file_id": "file-abc123", + "metadata": {"key1": "value1", "key2": "value2"}, + } + assert create_batch_response.id == "batch_abc123" + assert create_batch_response.endpoint == "/v1/chat/completions" + assert create_batch_response.input_file_id == file_obj.id + + await asyncio.wait_for(recorder.logged.wait(), timeout=10) + assert recorder.payload is not None + standard_logging_object: Final = recorder.payload + assert standard_logging_object["metadata"]["user_api_key_alias"] == extra_metadata_field["user_api_key_alias"] + assert ( + standard_logging_object["metadata"]["user_api_key_team_alias"] + == extra_metadata_field["user_api_key_team_alias"] + ) + + retrieved_batch: Final = await litellm.aretrieve_batch( + batch_id=create_batch_response.id, custom_llm_provider="openai", api_key="fake-key" + ) + assert retrieve_route.called + assert retrieved_batch.id == create_batch_response.id + + list_batches: Final = await litellm.alist_batches(custom_llm_provider="openai", limit=2, api_key="fake-key") + assert list_batches_route.calls.last.request.url.params["limit"] == "2" + assert [batch.id for batch in list_batches.data] == ["batch_abc123"] + + file_content: Final = await litellm.afile_content( + file_id=file_obj.id, custom_llm_provider="openai", api_key="fake-key" + ) + assert file_content.content == b'{"custom_id": "request-1"}\n' + + retrieved_file: Final = await litellm.afile_retrieve( + file_id=file_obj.id, custom_llm_provider="openai", api_key="fake-key" + ) + assert retrieved_file.id == file_obj.id + + delete_file_response: Final = await litellm.afile_delete( + file_id=file_obj.id, custom_llm_provider="openai", api_key="fake-key" + ) + assert delete_file_response.id == file_obj.id + + all_files_list: Final = await litellm.afile_list(custom_llm_provider="openai", api_key="fake-key") + assert list_files_route.called + assert [file.id for file in all_files_list.data] == ["file-abc123"] + + cancel_batch_response: Final = await litellm.acancel_batch( + batch_id=create_batch_response.id, custom_llm_provider="openai", api_key="fake-key" + ) + assert cancel_route.called + assert cancel_batch_response.id == create_batch_response.id + + +@pytest.mark.asyncio +async def test_delete_batch_output_file(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + batch_with_output: Final = { + **_OPENAI_BATCH_JSON, + "status": "completed", + "output_file_id": "file-output123", + } + respx_mock.get("https://api.openai.com/v1/batches/batch_abc123").mock( + return_value=httpx.Response(200, json=batch_with_output) + ) + delete_route: Final = respx_mock.delete("https://api.openai.com/v1/files/file-output123").mock( + return_value=httpx.Response(200, json={"id": "file-output123", "object": "file", "deleted": True}) + ) + + batch: Final = await litellm.aretrieve_batch( + batch_id="batch_abc123", custom_llm_provider="openai", api_key="fake-key" + ) + assert batch.output_file_id == "file-output123" + + delete_response: Final = await litellm.afile_delete( + file_id=batch.output_file_id, custom_llm_provider="openai", api_key="fake-key" + ) + assert delete_route.call_count == 1 + assert delete_response.id == "file-output123" + assert delete_response.deleted is True diff --git a/tests/unit/files/test_main.py b/tests/unit/files/test_main.py index cb70b39f4d5..e91951fb5ba 100644 --- a/tests/unit/files/test_main.py +++ b/tests/unit/files/test_main.py @@ -1,3 +1,4 @@ +from types import MappingProxyType from typing import Final from urllib.parse import parse_qs, urlparse @@ -121,3 +122,68 @@ async def test_afile_retrieve_rejects_a_provider_file_without_its_size(): assert exc_info.value.title == "OpenAIFileObject" assert [error["loc"] for error in exc_info.value.errors()] == [("bytes",)] + + +_FILE_BODY: Final = b'{"prompt": "Hello", "completion": "Hi"}' +_FINE_TUNE_FILE_JSON: Final = MappingProxyType( + { + "id": "file-abc123", + "object": "file", + "bytes": len(_FILE_BODY), + "created_at": 1699000000, + "filename": "mydata.jsonl", + "purpose": "fine-tune", + } +) + + +@pytest.mark.asyncio +async def test_openai_file_operations_roundtrip(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + files_route: Final = respx_mock.post("https://api.openai.com/v1/files").mock( + return_value=httpx.Response(200, json=dict(_FINE_TUNE_FILE_JSON)) + ) + list_route: Final = respx_mock.get("https://api.openai.com/v1/files").mock( + return_value=httpx.Response(200, json={"object": "list", "data": [dict(_FINE_TUNE_FILE_JSON)]}) + ) + retrieve_route: Final = respx_mock.get("https://api.openai.com/v1/files/file-abc123").mock( + return_value=httpx.Response(200, json=dict(_FINE_TUNE_FILE_JSON)) + ) + content_route: Final = respx_mock.get("https://api.openai.com/v1/files/file-abc123/content").mock( + return_value=httpx.Response(200, content=_FILE_BODY) + ) + delete_route: Final = respx_mock.delete("https://api.openai.com/v1/files/file-abc123").mock( + return_value=httpx.Response(200, json={"id": "file-abc123", "object": "file", "deleted": True}) + ) + + uploaded: Final = await litellm.acreate_file( + file=("mydata.jsonl", _FILE_BODY), purpose="fine-tune", custom_llm_provider="openai", api_key="fake-key" + ) + assert files_route.call_count == 1 + upload_body: Final = files_route.calls.last.request.content + assert b'name="purpose"\r\n\r\nfine-tune' in upload_body + assert b'filename="mydata.jsonl"' in upload_body + assert _FILE_BODY in upload_body + assert uploaded.id == "file-abc123" + + listed: Final = await litellm.afile_list(custom_llm_provider="openai", api_key="fake-key") + assert list_route.call_count == 1 + assert [file.id for file in listed.data] == ["file-abc123"] + + retrieved: Final = await litellm.afile_retrieve( + file_id="file-abc123", custom_llm_provider="openai", api_key="fake-key" + ) + assert retrieve_route.call_count == 1 + assert retrieved.filename == "mydata.jsonl" + assert retrieved.purpose == "fine-tune" + + content: Final = await litellm.afile_content( + file_id="file-abc123", custom_llm_provider="openai", api_key="fake-key" + ) + assert content_route.call_count == 1 + assert content.content == _FILE_BODY + + deleted: Final = await litellm.afile_delete(file_id="file-abc123", custom_llm_provider="openai", api_key="fake-key") + assert delete_route.call_count == 1 + assert deleted.id == "file-abc123" + assert deleted.deleted is True diff --git a/tests/unit/images/test_image_edit.py b/tests/unit/images/test_image_edit.py new file mode 100644 index 00000000000..583474cba90 --- /dev/null +++ b/tests/unit/images/test_image_edit.py @@ -0,0 +1,152 @@ +import asyncio +import io +from collections.abc import Iterator, Mapping +from datetime import datetime +from typing import Final + +import httpx +import pytest +import respx +from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict, override + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.utils import ImageResponse + +_PNG_SIGNATURE: Final = b"\x89PNG\r\n\x1a\n" +_FIRST_IMAGE: Final = _PNG_SIGNATURE + b"first-reference-image" +_SECOND_IMAGE: Final = _PNG_SIGNATURE + b"second-reference-image" +_EDITED_IMAGE_B64: Final = ( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/5+hHgAHggJ/PchI7wAAAABJRU5ErkJggg==" +) +_TEXT_TOKENS: Final = 50 +_IMAGE_TOKENS: Final = 50 +_OUTPUT_TOKENS: Final = 1000 +_EDIT_RESPONSE: Final = { + "created": 1589478378, + "data": [{"b64_json": _EDITED_IMAGE_B64}], + "usage": { + "total_tokens": _TEXT_TOKENS + _IMAGE_TOKENS + _OUTPUT_TOKENS, + "input_tokens": _TEXT_TOKENS + _IMAGE_TOKENS, + "input_tokens_details": {"image_tokens": _IMAGE_TOKENS, "text_tokens": _TEXT_TOKENS}, + "output_tokens": _OUTPUT_TOKENS, + }, +} + + +class _LoggedImageEdit(TypedDict): + model: ReadOnly[str] + custom_llm_provider: ReadOnly[str] + response_cost: ReadOnly[float] + + +_LOGGED_IMAGE_EDIT: Final = TypeAdapter(_LoggedImageEdit) + + +class _SuccessLogger(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.payload: _LoggedImageEdit | None = None + self.logged: Final = asyncio.Event() + + @override + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + self.payload = _LOGGED_IMAGE_EDIT.validate_python(kwargs.get("standard_logging_object")) + self.logged.set() + + +@pytest.fixture +def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +def _multipart_image_parts(request: httpx.Request) -> tuple[bytes, ...]: + body: Final = request.read() + boundary: Final = request.headers["content-type"].split("boundary=", 1)[1].encode() + parts: Final = body.split(b"--" + boundary) + return tuple(part.split(b"\r\n\r\n", 1)[1].removesuffix(b"\r\n") for part in parts if b'name="image[]"' in part) + + +@pytest.mark.asyncio +async def test_openai_image_edit_accepts_bytesio_images(respx_mock: respx.MockRouter, httpx_transport: None) -> None: + route: Final = respx_mock.post("https://api.openai.com/v1/images/edits").mock( + return_value=httpx.Response(200, json=_EDIT_RESPONSE) + ) + + result: Final = await litellm.aimage_edit( + prompt="combine the reference images", + model="gpt-image-1", + image=[io.BytesIO(_FIRST_IMAGE), io.BytesIO(_SECOND_IMAGE)], + api_key="fake-key", + ) + + assert isinstance(result, ImageResponse) + assert result.data is not None and result.data[0].b64_json == _EDITED_IMAGE_B64 + assert route.call_count == 1 + assert _multipart_image_parts(route.calls[0].request) == (_FIRST_IMAGE, _SECOND_IMAGE) + + +@pytest.mark.asyncio +async def test_openai_image_edit_accepts_mixed_bytes_and_bytesio( + respx_mock: respx.MockRouter, httpx_transport: None +) -> None: + route: Final = respx_mock.post("https://api.openai.com/v1/images/edits").mock( + return_value=httpx.Response(200, json=_EDIT_RESPONSE) + ) + + result: Final = await litellm.aimage_edit( + prompt="Create a cohesive artistic style across all images", + model="gpt-image-1", + image=[_FIRST_IMAGE, io.BytesIO(_SECOND_IMAGE)], + api_key="fake-key", + ) + + assert isinstance(result, ImageResponse) + assert result.data is not None and len(result.data) == 1 + assert result.data[0].b64_json == _EDITED_IMAGE_B64 + assert route.call_count == 1 + assert _multipart_image_parts(route.calls[0].request) == (_FIRST_IMAGE, _SECOND_IMAGE) + + +@pytest.mark.asyncio +async def test_azure_image_edit_logs_deployment_model_and_positive_cost( + respx_mock: respx.MockRouter, httpx_transport: None, monkeypatch: pytest.MonkeyPatch +) -> None: + logger: Final = _SuccessLogger() + monkeypatch.setattr(litellm, "callbacks", [logger]) + route: Final = respx_mock.post( + url__startswith="https://fake.openai.azure.com/openai/deployments/CUSTOM_AZURE_DEPLOYMENT_NAME/images/edits" + ).mock(return_value=httpx.Response(200, json=_EDIT_RESPONSE)) + + result: Final = await litellm.aimage_edit( + prompt="combine the reference images", + model="azure/CUSTOM_AZURE_DEPLOYMENT_NAME", + base_model="azure/gpt-image-1", + image=[_FIRST_IMAGE, _SECOND_IMAGE], + api_key="fake-key", + api_base="https://fake.openai.azure.com", + api_version="2025-04-01-preview", + ) + await asyncio.wait_for(logger.logged.wait(), timeout=10) + + assert isinstance(result, ImageResponse) + assert route.call_count == 1 + payload: Final = logger.payload + assert payload is not None + assert payload["model"] == "CUSTOM_AZURE_DEPLOYMENT_NAME" + assert payload["custom_llm_provider"] == "azure" + pricing: Final = litellm.model_cost["azure/gpt-image-1"] + expected_cost: Final = ( + _TEXT_TOKENS * pricing["input_cost_per_token"] + + _IMAGE_TOKENS * pricing["input_cost_per_image_token"] + + _OUTPUT_TOKENS * pricing["output_cost_per_image_token"] + ) + assert expected_cost > 0 + assert payload["response_cost"] == pytest.approx(expected_cost) + assert result._hidden_params["response_cost"] == pytest.approx(expected_cost) # pyright: ignore[reportPrivateUsage] # cost is only surfaced on _hidden_params diff --git a/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py new file mode 100644 index 00000000000..c55fa476c06 --- /dev/null +++ b/tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py @@ -0,0 +1,619 @@ +from __future__ import annotations + +import asyncio +import base64 +import json +import struct +import uuid +from collections.abc import AsyncIterable, Mapping +from typing import Final +from zlib import crc32 + +import httpx +import pytest + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.router import Router +from litellm.types.llms.anthropic import ( + AnthropicMessagesTextParam, + AnthropicMessagesTool, + AnthropicMessagesUserMessageParam, + AnthropicToolSearchToolRegex, +) +from litellm.types.utils import StandardLoggingPayload + +_ALIAS: Final = "claude-special-alias" +_ANTHROPIC_MODEL: Final = "claude-haiku-4-5-20251001" +_BEDROCK_MODEL: Final = "bedrock/us.anthropic.claude-haiku-4-5-20251001-v1:0" +_BEDROCK_SONNET: Final = "us.anthropic.claude-sonnet-4-5-20250929-v1:0" +_OPENAI_MODEL: Final = "openai/gpt-4.1-mini" +_JOKE: Final = "why did the chicken cross the road" +_PROMPT: Final = "Hello, can you tell me a short joke?" +_ANTHROPIC_URL: Final = r".*api\.anthropic\.com/v1/messages.*" +_OPENAI_RESPONSES_URL: Final = r".*api\.openai\.com/v1/responses.*" +_BEDROCK_INVOKE_URL: Final = r".*bedrock-runtime.*/invoke$" +_BEDROCK_INVOKE_STREAM_URL: Final = r".*bedrock-runtime.*/invoke-with-response-stream$" +_BEDROCK_CONVERSE_URL: Final = r".*bedrock-runtime.*/converse$" +_BEDROCK_CONVERSE_STREAM_URL: Final = r".*bedrock-runtime.*/converse-stream$" + + +def _anthropic_body(model: str = _ANTHROPIC_MODEL, msg_id: str = "msg_1") -> Mapping[str, object]: + return { + "id": msg_id, + "type": "message", + "role": "assistant", + "model": model, + "content": [{"type": "text", "text": _JOKE}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 10, "output_tokens": 20}, + } + + +_CONVERSE_BODY: Final = { + "output": {"message": {"role": "assistant", "content": [{"text": _JOKE}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 20, "totalTokens": 30}, +} + +_OPENAI_BODY: Final = { + "id": "resp_1", + "object": "response", + "status": "completed", + "created_at": 1700000000, + "model": "gpt-4.1-mini", + "output": [ + { + "type": "message", + "id": "msg_out_1", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": _JOKE, "annotations": []}], + } + ], + "usage": {"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, +} + +_STREAM_EVENTS: Final = ( + { + "type": "message_start", + "message": { + "id": "msg_1", + "type": "message", + "role": "assistant", + "model": _ANTHROPIC_MODEL, + "content": [], + "usage": { + "input_tokens": 10, + "output_tokens": 1, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 0, + }, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": _JOKE}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 20}}, + {"type": "message_stop"}, +) + + +class _RecordingLogger(CustomLogger): + def __init__(self, messages: list[AnthropicMessagesUserMessageParam]) -> None: + super().__init__() + self.messages: Final = messages + self.payloads: tuple[StandardLoggingPayload, ...] = () + self.received: Final = asyncio.Event() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + payload: Final = kwargs.get("standard_logging_object") + if payload is not None and payload["messages"] == self.messages: + self.payloads = (*self.payloads, payload) + self.received.set() + + +def _unique_messages() -> list[AnthropicMessagesUserMessageParam]: + return [{"role": "user", "content": f"{_PROMPT} {uuid.uuid4().hex}"}] + + +def _router(model_name: str, model: str, **router_kwargs: object) -> Router: + return Router( + model_list=[{"model_name": model_name, "litellm_params": {"model": model, "api_key": "fake-key"}}], + **router_kwargs, + ) + + +def _event_frame(event_type: str, payload: Mapping[str, object]) -> bytes: + def header(name: str, value: str) -> bytes: + name_b: Final = name.encode() + value_b: Final = value.encode() + return ( + struct.pack("!B", len(name_b)) + name_b + struct.pack("!B", 7) + struct.pack("!H", len(value_b)) + value_b + ) + + payload_b: Final = json.dumps(payload).encode() + headers_b: Final = ( + header(":event-type", event_type) + + header(":content-type", "application/json") + + header(":message-type", "event") + ) + prelude: Final = struct.pack("!II", 16 + len(headers_b) + len(payload_b), len(headers_b)) + prelude_crc: Final = crc32(prelude) & 0xFFFFFFFF + message: Final = struct.pack("!I", prelude_crc) + headers_b + payload_b + return prelude + message + struct.pack("!I", crc32(message, prelude_crc) & 0xFFFFFFFF) + + +def _invoke_stream_body() -> bytes: + return b"".join( + _event_frame("chunk", {"bytes": base64.b64encode(json.dumps(event).encode()).decode()}) + for event in _STREAM_EVENTS + ) + + +def _anthropic_sse_body() -> bytes: + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in _STREAM_EVENTS).encode() + + +def _converse_stream_body() -> bytes: + return ( + _event_frame("messageStart", {"role": "assistant"}) + + _event_frame("contentBlockDelta", {"contentBlockIndex": 0, "delta": {"text": _JOKE}}) + + _event_frame("contentBlockStop", {"contentBlockIndex": 0}) + + _event_frame("messageStop", {"stopReason": "end_turn"}) + + _event_frame( + "metadata", + { + "usage": { + "inputTokens": 10, + "outputTokens": 20, + "totalTokens": 530, + "cacheReadInputTokens": 500, + "cacheWriteInputTokens": 0, + }, + "metrics": {"latencyMs": 10}, + }, + ) + ) + + +def _set_fake_aws_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AWS_ACCESS_KEY_ID", "fake") + monkeypatch.setenv("AWS_SECRET_ACCESS_KEY", "fake") + monkeypatch.setenv("AWS_REGION_NAME", "us-east-1") + + +def _sse_events(raw: str) -> tuple[Mapping[str, object], ...]: + return tuple(json.loads(line[len("data: ") :]) for line in raw.splitlines() if line.startswith("data: ")) + + +async def _stream_events(stream: object) -> tuple[Mapping[str, object], ...]: + assert isinstance(stream, AsyncIterable), type(stream) + chunks: Final = [chunk async for chunk in stream] + raw: Final = "".join(chunk.decode() for chunk in chunks if isinstance(chunk, bytes)) + dict_events: Final = tuple(chunk for chunk in chunks if isinstance(chunk, Mapping)) + return _sse_events(raw) + dict_events + + +async def _wait_for_payload(recorder: _RecordingLogger) -> None: + await asyncio.wait_for(recorder.received.wait(), timeout=30.0) + + +def _assert_anthropic_message(response: object, model: str) -> None: + assert isinstance(response, dict), type(response) + assert response["type"] == "message" + assert response["role"] == "assistant" + assert response["model"] == model + assert isinstance(response["id"], str) and response["id"] + block: Final = response["content"][0] + assert isinstance(block, dict), type(block) + assert block["type"] == "text" + assert block["text"] == _JOKE + + +@pytest.fixture(autouse=True) +def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +@pytest.mark.asyncio +async def test_router_aanthropic_messages_non_streaming_posts_anthropic_body(respx_mock): + route: Final = respx_mock.post(url__regex=_ANTHROPIC_URL).mock( + return_value=httpx.Response(200, json=_anthropic_body()) + ) + response: Final = await _router(_ALIAS, _ANTHROPIC_MODEL).aanthropic_messages( + messages=[{"role": "user", "content": _PROMPT}], + model=_ALIAS, + max_tokens=100, + ) + sent: Final = json.loads(route.calls.last.request.read()) + assert sent["model"] == _ANTHROPIC_MODEL + assert sent["max_tokens"] == 100 + assert sent["messages"] == [{"role": "user", "content": _PROMPT}] + _assert_anthropic_message(response, _ANTHROPIC_MODEL) + + +@pytest.mark.asyncio +async def test_router_aanthropic_messages_latency_routing_forwards_user_id(respx_mock): + route: Final = respx_mock.post(url__regex=_ANTHROPIC_URL).mock( + return_value=httpx.Response(200, json=_anthropic_body()) + ) + response: Final = await _router( + _ALIAS, _ANTHROPIC_MODEL, routing_strategy="latency-based-routing" + ).aanthropic_messages( + messages=[{"role": "user", "content": _PROMPT}], + model=_ALIAS, + max_tokens=100, + metadata={"user_id": "hello"}, + ) + sent: Final = json.loads(route.calls.last.request.read()) + assert sent["model"] == _ANTHROPIC_MODEL + assert sent["metadata"] == {"user_id": "hello"} + _assert_anthropic_message(response, _ANTHROPIC_MODEL) + + +@pytest.mark.asyncio +async def test_router_aanthropic_messages_falls_back_to_bedrock_after_anthropic_401(respx_mock, monkeypatch): + _set_fake_aws_env(monkeypatch) + anthropic_route: Final = respx_mock.post(url__regex=_ANTHROPIC_URL).mock( + return_value=httpx.Response( + 401, json={"type": "error", "error": {"type": "authentication_error", "message": "invalid x-api-key"}} + ) + ) + bedrock_route: Final = respx_mock.post(url__regex=_BEDROCK_INVOKE_URL).mock( + return_value=httpx.Response(200, json=_anthropic_body(model=_BEDROCK_SONNET, msg_id="msg_bedrock")) + ) + router: Final = Router( + model_list=[ + { + "model_name": "anthropic/claude-opus-4-7", + "litellm_params": {"model": "anthropic/claude-opus-4-7", "api_key": "bad-key"}, + }, + {"model_name": f"bedrock/{_BEDROCK_SONNET}", "litellm_params": {"model": f"bedrock/{_BEDROCK_SONNET}"}}, + ], + fallbacks=[{"anthropic/claude-opus-4-7": [f"bedrock/{_BEDROCK_SONNET}"]}], + ) + response: Final = await router.aanthropic_messages( + messages=[{"role": "user", "content": _PROMPT}], + model="anthropic/claude-opus-4-7", + max_tokens=100, + metadata={"user_id": "hello"}, + ) + assert anthropic_route.call_count == 1 + assert anthropic_route.calls.last.request.headers["x-api-key"] == "bad-key" + assert bedrock_route.call_count == 1 + assert "authorization" in bedrock_route.calls.last.request.headers + _assert_anthropic_message(response, _BEDROCK_SONNET) + assert response["id"] == "msg_bedrock" + + +@pytest.mark.asyncio +async def test_router_aanthropic_messages_bedrock_converse_and_invoke(respx_mock, monkeypatch): + _set_fake_aws_env(monkeypatch) + converse_route: Final = respx_mock.post(url__regex=_BEDROCK_CONVERSE_URL).mock( + return_value=httpx.Response(200, json=_CONVERSE_BODY) + ) + invoke_route: Final = respx_mock.post(url__regex=_BEDROCK_INVOKE_URL).mock( + return_value=httpx.Response(200, json=_anthropic_body(model=_BEDROCK_SONNET)) + ) + converse_model: Final = f"bedrock/converse/{_BEDROCK_SONNET}" + invoke_model: Final = f"bedrock/{_BEDROCK_SONNET}" + router: Final = Router( + model_list=[ + {"model_name": converse_model, "litellm_params": {"model": converse_model}}, + {"model_name": invoke_model, "litellm_params": {"model": invoke_model}}, + ] + ) + converse_response: Final = await router.aanthropic_messages( + messages=[{"role": "user", "content": _PROMPT}], model=converse_model, max_tokens=100 + ) + invoke_response: Final = await router.aanthropic_messages( + messages=[{"role": "user", "content": _PROMPT}], model=invoke_model, max_tokens=100 + ) + assert converse_route.call_count == 1 + assert invoke_route.call_count == 1 + converse_request: Final = converse_route.calls.last.request + invoke_request: Final = invoke_route.calls.last.request + assert "authorization" in converse_request.headers + assert "authorization" in invoke_request.headers + assert json.loads(converse_request.read())["messages"] == [{"role": "user", "content": [{"text": _PROMPT}]}] + assert json.loads(invoke_request.read())["messages"] == [{"role": "user", "content": _PROMPT}] + _assert_anthropic_message(converse_response, _BEDROCK_SONNET) + _assert_anthropic_message(invoke_response, _BEDROCK_SONNET) + + +def test_sync_openai_bridge_anthropic_messages_returns_content_blocks(respx_mock): + route: Final = respx_mock.post(url__regex=_OPENAI_RESPONSES_URL).mock( + return_value=httpx.Response(200, json=_OPENAI_BODY) + ) + response: Final = litellm.anthropic.messages.create( + messages=[{"role": "user", "content": _PROMPT}], + model=_OPENAI_MODEL, + max_tokens=100, + api_key="fake-key", + ) + sent: Final = json.loads(route.calls.last.request.read()) + assert sent["model"] == "gpt-4.1-mini" + assert isinstance(response, dict) + assert response["content"][0]["text"] == _JOKE + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "url", "body", "expected_model"), + [ + pytest.param(_ANTHROPIC_MODEL, _ANTHROPIC_URL, _anthropic_body(), _ANTHROPIC_MODEL, id="anthropic"), + pytest.param( + _BEDROCK_MODEL, + _BEDROCK_INVOKE_URL, + _anthropic_body(model="us.anthropic.claude-haiku-4-5-20251001-v1:0"), + "us.anthropic.claude-haiku-4-5-20251001-v1:0", + id="bedrock-invoke", + ), + pytest.param(_OPENAI_MODEL, _OPENAI_RESPONSES_URL, _OPENAI_BODY, "gpt-4.1-mini", id="openai-bridge"), + ], +) +async def test_acreate_non_streaming_returns_dict_content_blocks( + respx_mock, monkeypatch, model: str, url: str, body: Mapping[str, object], expected_model: str +): + _set_fake_aws_env(monkeypatch) + route: Final = respx_mock.post(url__regex=url).mock(return_value=httpx.Response(200, json=body)) + response: Final = await litellm.anthropic.messages.acreate( + messages=[{"role": "user", "content": _PROMPT}], + model=model, + max_tokens=100, + api_key="fake-key", + ) + assert route.call_count == 1 + _assert_anthropic_message(response, expected_model) + + +@pytest.mark.asyncio +async def test_router_aanthropic_messages_non_streaming_logs_usage_model_and_cost(respx_mock, monkeypatch): + messages: Final = _unique_messages() + recorder: Final = _RecordingLogger(messages) + monkeypatch.setattr(litellm, "callbacks", [recorder]) + respx_mock.post(url__regex=_ANTHROPIC_URL).mock(return_value=httpx.Response(200, json=_anthropic_body())) + response: Final = await _router(_ALIAS, _ANTHROPIC_MODEL).aanthropic_messages( + messages=messages, model=_ALIAS, max_tokens=100 + ) + await _wait_for_payload(recorder) + assert len(recorder.payloads) == 1 + payload: Final = recorder.payloads[0] + assert payload["status"] == "success" + assert payload["messages"] == messages + assert payload["response"] is not None + assert payload["model"] == _ANTHROPIC_MODEL + assert payload["model_group"] == _ALIAS + assert payload["response_cost"] > 0 + assert payload["prompt_tokens"] == response["usage"]["input_tokens"] == 10 + assert payload["completion_tokens"] == response["usage"]["output_tokens"] == 20 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "url", "content_type", "body", "expected_model"), + [ + pytest.param( + _ANTHROPIC_MODEL, + _ANTHROPIC_URL, + "text/event-stream", + _anthropic_sse_body(), + _ANTHROPIC_MODEL, + id="anthropic", + ), + pytest.param( + _BEDROCK_MODEL, + _BEDROCK_INVOKE_STREAM_URL, + "application/vnd.amazon.eventstream", + _invoke_stream_body(), + _BEDROCK_MODEL, + id="bedrock-invoke", + ), + ], +) +async def test_router_aanthropic_messages_streaming_logs_usage_model_and_cost( + respx_mock, monkeypatch, model: str, url: str, content_type: str, body: bytes, expected_model: str +): + _set_fake_aws_env(monkeypatch) + messages: Final = _unique_messages() + recorder: Final = _RecordingLogger(messages) + monkeypatch.setattr(litellm, "callbacks", [recorder]) + respx_mock.post(url__regex=url).mock( + return_value=httpx.Response(200, content=body, headers={"content-type": content_type}) + ) + stream: Final = await _router(_ALIAS, model).aanthropic_messages( + messages=messages, model=_ALIAS, max_tokens=100, stream=True + ) + events: Final = await _stream_events(stream) + usages: Final = tuple( + event["usage"] if "usage" in event else event["message"]["usage"] + for event in events + if "usage" in event or (event.get("type") == "message_start" and "usage" in event["message"]) + ) + assert usages, events + await _wait_for_payload(recorder) + assert len(recorder.payloads) == 1 + payload: Final = recorder.payloads[0] + assert payload["status"] == "success" + assert payload["messages"] == messages + assert payload["response"] is not None + assert payload["model"] == expected_model + assert payload["response_cost"] > 0 + assert payload["prompt_tokens"] == max(usage.get("input_tokens", 0) for usage in usages) == 10 + assert payload["completion_tokens"] == max(usage.get("output_tokens", 0) for usage in usages) == 20 + + +_LARGE_SYSTEM_PROMPT: Final = "This is a comprehensive legal agreement between Party A and Party B. " * 100 + + +def _cached_system() -> list[AnthropicMessagesTextParam]: + return [{"type": "text", "text": _LARGE_SYSTEM_PROMPT, "cache_control": {"type": "ephemeral"}}] + + +@pytest.mark.asyncio +async def test_bedrock_converse_system_prompt_caching_returns_cache_tokens(respx_mock, monkeypatch): + _set_fake_aws_env(monkeypatch) + route: Final = respx_mock.post(url__regex=_BEDROCK_CONVERSE_URL).mock( + return_value=httpx.Response( + 200, + json={ + **_CONVERSE_BODY, + "usage": { + "inputTokens": 10, + "outputTokens": 20, + "totalTokens": 580, + "cacheReadInputTokens": 500, + "cacheWriteInputTokens": 50, + }, + }, + ) + ) + response: Final = await litellm.anthropic.messages.acreate( + model=f"bedrock/converse/{_BEDROCK_SONNET}", + messages=[{"role": "user", "content": "What are the key terms?"}], + system=_cached_system(), + max_tokens=100, + ) + sent: Final = json.loads(route.calls.last.request.read()) + assert sent["system"] == [{"text": _LARGE_SYSTEM_PROMPT}, {"cachePoint": {"type": "default"}}] + assert isinstance(response, dict) + assert response["usage"]["cache_creation_input_tokens"] == 50 + assert response["usage"]["cache_read_input_tokens"] == 500 + + +@pytest.mark.asyncio +async def test_bedrock_invoke_system_prompt_caching_returns_cache_tokens(respx_mock, monkeypatch): + _set_fake_aws_env(monkeypatch) + route: Final = respx_mock.post(url__regex=_BEDROCK_INVOKE_URL).mock( + return_value=httpx.Response( + 200, + json={ + **_anthropic_body(model=_BEDROCK_SONNET), + "usage": { + "input_tokens": 10, + "output_tokens": 20, + "cache_creation_input_tokens": 50, + "cache_read_input_tokens": 500, + }, + }, + ) + ) + response: Final = await litellm.anthropic.messages.acreate( + model=f"bedrock/invoke/{_BEDROCK_SONNET}", + messages=[{"role": "user", "content": "What are the key terms?"}], + system=_cached_system(), + max_tokens=100, + ) + sent: Final = json.loads(route.calls.last.request.read()) + assert sent["system"] == _cached_system() + assert isinstance(response, dict) + assert response["usage"]["cache_creation_input_tokens"] == 50 + assert response["usage"]["cache_read_input_tokens"] == 500 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "url", "body"), + [ + pytest.param( + f"bedrock/converse/{_BEDROCK_SONNET}", _BEDROCK_CONVERSE_STREAM_URL, _converse_stream_body(), id="converse" + ), + pytest.param( + f"bedrock/invoke/{_BEDROCK_SONNET}", _BEDROCK_INVOKE_STREAM_URL, _invoke_stream_body(), id="invoke" + ), + ], +) +async def test_bedrock_streaming_message_start_carries_cache_usage_fields( + respx_mock, monkeypatch, model: str, url: str, body: bytes +): + _set_fake_aws_env(monkeypatch) + route: Final = respx_mock.post(url__regex=url).mock( + return_value=httpx.Response(200, content=body, headers={"content-type": "application/vnd.amazon.eventstream"}) + ) + stream: Final = await litellm.anthropic.messages.acreate( + model=model, + messages=[ + { + "role": "user", + "content": [ + {"type": "text", "text": _LARGE_SYSTEM_PROMPT, "cache_control": {"type": "ephemeral"}}, + {"type": "text", "text": "What are the payment terms in this agreement?"}, + ], + } + ], + max_tokens=100, + stream=True, + ) + events: Final = await _stream_events(stream) + assert route.call_count == 1 + message_starts: Final = [event for event in events if event.get("type") == "message_start"] + assert len(message_starts) == 1, events + usage: Final = message_starts[0]["message"]["usage"] + assert "cache_creation_input_tokens" in usage, usage + assert "cache_read_input_tokens" in usage, usage + + +def _tool_search_tools() -> list[AnthropicToolSearchToolRegex | AnthropicMessagesTool]: + def deferred(name: str, description: str, field: str) -> AnthropicMessagesTool: + return { + "name": name, + "description": description, + "input_schema": {"type": "object", "properties": {field: {"type": "string"}}, "required": [field]}, + "defer_loading": True, + } + + return [ + {"type": "tool_search_tool_regex_20251119", "name": "tool_search_tool_regex"}, + deferred("get_weather", "Get the current weather for a location", "location"), + deferred("get_stock_price", "Get the current stock price for a ticker symbol", "ticker"), + deferred("search_web", "Search the web for information", "query"), + ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("prompt", "tool_name", "tool_input"), + [ + pytest.param( + "I need to know the current weather in New York City. Please use the appropriate tool.", + "get_weather", + {"location": "New York, NY"}, + id="discovers-weather-tool", + ), + pytest.param( + "What's the stock price of Apple (AAPL)?", "get_stock_price", {"ticker": "AAPL"}, id="multiple-deferred" + ), + ], +) +async def test_tool_search_forwards_deferred_tools_and_beta_header( + respx_mock, prompt: str, tool_name: str, tool_input: Mapping[str, str] +): + route: Final = respx_mock.post(url__regex=_ANTHROPIC_URL).mock( + return_value=httpx.Response( + 200, + json={ + **_anthropic_body(model="claude-sonnet-4-5-20250929", msg_id="msg_tool"), + "content": [ + {"type": "tool_use", "id": "toolu_1", "name": tool_name, "input": tool_input}, + ], + "stop_reason": "tool_use", + }, + ) + ) + response: Final = await litellm.anthropic.messages.acreate( + model="anthropic/claude-sonnet-4-5-20250929", + messages=[{"role": "user", "content": prompt}], + tools=[dict(tool) for tool in _tool_search_tools()], + max_tokens=1024, + api_key="fake-key", + extra_headers={"anthropic-beta": "advanced-tool-use-2025-11-20"}, + ) + request: Final = route.calls.last.request + assert "advanced-tool-use-2025-11-20" in request.headers["anthropic-beta"].split(",") + assert json.loads(request.read())["tools"] == _tool_search_tools() + assert isinstance(response, dict) + assert response["stop_reason"] == "tool_use" + assert [block for block in response["content"] if block["type"] == "tool_use"] == [ + {"type": "tool_use", "id": "toolu_1", "name": tool_name, "input": tool_input} + ] diff --git a/tests/unit/llms/anthropic/pass_through/test_anthropic_native_passthrough_spend_logging.py b/tests/unit/llms/anthropic/pass_through/test_anthropic_native_passthrough_spend_logging.py new file mode 100644 index 00000000000..de2cb03bf48 --- /dev/null +++ b/tests/unit/llms/anthropic/pass_through/test_anthropic_native_passthrough_spend_logging.py @@ -0,0 +1,181 @@ +from __future__ import annotations + +import asyncio +import json +import uuid +from collections.abc import Mapping +from typing import Final +from unittest.mock import AsyncMock, MagicMock + +import httpx +import pytest +from fastapi import Request, Response + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER +from litellm.proxy._types import UserAPIKeyAuth, hash_token +from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import anthropic_proxy_route +from litellm.types.utils import StandardLoggingPayload + +_UPSTREAM: Final = "https://api.anthropic.com/v1/messages" +_MODEL: Final = "claude-sonnet-4-5-20250929" +_VIRTUAL_KEY: Final = "sk-native-passthrough" + + +class _RecordingLogger(CustomLogger): + def __init__(self, message_id: str) -> None: + super().__init__() + self.message_id: Final = message_id + self.payloads: tuple[StandardLoggingPayload, ...] = () + self.received: Final = asyncio.Event() + + async def async_log_success_event(self, kwargs, response_obj, start_time, end_time) -> None: + payload: Final = kwargs.get("standard_logging_object") + if payload is not None and payload["id"] == self.message_id: + self.payloads = (*self.payloads, payload) + self.received.set() + + +def _proxy_request(body: Mapping[str, object]) -> Request: + request: Final = MagicMock(spec=Request) + request.method = "POST" + request.url = httpx.URL("http://proxy/anthropic/v1/messages") + request.headers = {"content-type": "application/json", "anthropic-version": "2023-06-01"} + request.scope = {"path": "/anthropic/v1/messages", "type": "http", "method": "POST", "headers": []} + request.query_params = {} + request.body = AsyncMock(return_value=json.dumps(body).encode()) + request.json = AsyncMock(return_value=body) + return request + + +def _sse(events: tuple[Mapping[str, object], ...]) -> bytes: + return "".join(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n" for event in events).encode() + + +async def _wait_for_payload(recorder: _RecordingLogger) -> None: + GLOBAL_LOGGING_WORKER.start() + await asyncio.wait_for(recorder.received.wait(), timeout=30.0) + + +def _assert_spend_payload( + payload: StandardLoggingPayload, message_id: str, tags: list[str], prompt_tokens: int, completion_tokens: int +) -> None: + assert payload["id"] == message_id + assert payload["call_type"] == "pass_through_endpoint" + assert payload["status"] == "success" + assert payload["custom_llm_provider"] == "anthropic" + assert payload["model"] == _MODEL + assert payload["prompt_tokens"] == prompt_tokens + assert payload["completion_tokens"] == completion_tokens + assert payload["total_tokens"] == prompt_tokens + completion_tokens + assert payload["response_cost"] > 0 + assert payload["request_tags"] == tags + assert payload["cache_hit"] is not True + assert payload["startTime"] <= payload["endTime"] + assert payload["metadata"]["user_api_key_hash"] == hash_token(_VIRTUAL_KEY) + + +@pytest.fixture +def recorder(monkeypatch: pytest.MonkeyPatch) -> _RecordingLogger: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("ANTHROPIC_API_KEY", "synthetic-anthropic-key") + monkeypatch.delenv("ANTHROPIC_API_BASE", raising=False) + monkeypatch.delenv("ANTHROPIC_BASE_URL", raising=False) + logger: Final = _RecordingLogger(f"msg_{uuid.uuid4().hex}") + monkeypatch.setattr(litellm, "callbacks", [logger]) + monkeypatch.setattr(litellm, "_async_success_callback", [logger]) + return logger + + +@pytest.mark.asyncio +async def test_native_anthropic_passthrough_logs_usage_tags_and_spend(respx_mock, recorder: _RecordingLogger): + tags: Final = ["test-tag-1", "test-tag-2"] + route: Final = respx_mock.post(_UPSTREAM).mock( + return_value=httpx.Response( + 200, + json={ + "id": recorder.message_id, + "type": "message", + "role": "assistant", + "model": _MODEL, + "content": [{"type": "text", "text": "hello test"}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 11, "output_tokens": 7}, + }, + ) + ) + response: Final = await anthropic_proxy_route( + endpoint="v1/messages", + request=_proxy_request( + { + "model": _MODEL, + "max_tokens": 10, + "messages": [{"role": "user", "content": "Say 'hello test' and nothing else"}], + "litellm_metadata": {"tags": tags}, + } + ), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key=_VIRTUAL_KEY, token=_VIRTUAL_KEY), + ) + assert response.status_code == 200 + assert json.loads(response.body)["id"] == recorder.message_id + outbound: Final = route.calls.last.request + assert outbound.headers["x-api-key"] == "synthetic-anthropic-key" + assert json.loads(outbound.content) == { + "model": _MODEL, + "max_tokens": 10, + "messages": [{"role": "user", "content": "Say 'hello test' and nothing else"}], + } + await _wait_for_payload(recorder) + assert len(recorder.payloads) == 1 + payload: Final = recorder.payloads[0] + _assert_spend_payload(payload, recorder.message_id, tags, prompt_tokens=11, completion_tokens=7) + assert payload["api_base"] == _UPSTREAM + + +@pytest.mark.asyncio +async def test_native_anthropic_passthrough_streaming_logs_usage_tags_and_spend(respx_mock, recorder: _RecordingLogger): + tags: Final = ["test-tag-stream-1", "test-tag-stream-2"] + events: Final = ( + { + "type": "message_start", + "message": { + "id": recorder.message_id, + "type": "message", + "role": "assistant", + "model": _MODEL, + "content": [], + "usage": {"input_tokens": 11, "output_tokens": 1}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": "hello stream test"}}, + {"type": "content_block_stop", "index": 0}, + {"type": "message_delta", "delta": {"stop_reason": "end_turn"}, "usage": {"output_tokens": 7}}, + {"type": "message_stop"}, + ) + route: Final = respx_mock.post(_UPSTREAM).mock( + return_value=httpx.Response(200, content=_sse(events), headers={"content-type": "text/event-stream"}) + ) + response: Final = await anthropic_proxy_route( + endpoint="v1/messages", + request=_proxy_request( + { + "model": _MODEL, + "max_tokens": 10, + "stream": True, + "messages": [{"role": "user", "content": "Say 'hello stream test' and nothing else"}], + "litellm_metadata": {"tags": tags, "user": "test-user-1"}, + } + ), + fastapi_response=MagicMock(spec=Response), + user_api_key_dict=UserAPIKeyAuth(api_key=_VIRTUAL_KEY, token=_VIRTUAL_KEY), + ) + assert response.status_code == 200 + streamed: Final = b"".join([chunk async for chunk in response.body_iterator]) + assert b"hello stream test" in streamed + assert json.loads(route.calls.last.request.content)["stream"] is True + await _wait_for_payload(recorder) + assert len(recorder.payloads) == 1 + _assert_spend_payload(recorder.payloads[0], recorder.message_id, tags, prompt_tokens=11, completion_tokens=7) diff --git a/tests/unit/llms/bedrock/image/test_amazon_nova_canvas_transformation.py b/tests/unit/llms/bedrock/image/test_amazon_nova_canvas_transformation.py index 2802016c04a..66001ad86be 100644 --- a/tests/unit/llms/bedrock/image/test_amazon_nova_canvas_transformation.py +++ b/tests/unit/llms/bedrock/image/test_amazon_nova_canvas_transformation.py @@ -1,4 +1,11 @@ +import json +from typing import Final + +import httpx import pytest +import respx + +import litellm from litellm.llms.bedrock.image_generation.amazon_nova_canvas_transformation import ( AmazonNovaCanvasConfig, ) @@ -79,3 +86,30 @@ def test_transform_response_dict_to_openai_response(): assert hasattr(result, "data") assert len(result.data) == 2 assert result.data[0].b64_json == "b64img1" + + +_NOVA_CANVAS_PROMPT: Final = "A serene mountain landscape at sunset with a lake reflection" +_NOVA_CANVAS_IMAGES: Final = ("b64-first-image", "b64-second-image") + + +def test_nova_canvas_image_gen_reports_positive_response_cost(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post( + url__regex=r"^https://bedrock-runtime\.us-east-1\.amazonaws\.com/model/amazon\.nova-canvas-v1(:|%3A)0/invoke$" + ).mock(return_value=httpx.Response(200, json={"images": list(_NOVA_CANVAS_IMAGES)})) + + response: Final = litellm.image_generation( + model="bedrock/amazon.nova-canvas-v1:0", + prompt=_NOVA_CANVAS_PROMPT, + aws_region_name="us-east-1", + aws_access_key_id="fake-access-key", + aws_secret_access_key="fake-secret-key", + ) + + assert route.call_count == 1 + sent: Final = json.loads(route.calls[0].request.content) + assert sent["taskType"] == "TEXT_IMAGE" + assert sent["textToImageParams"]["text"] == _NOVA_CANVAS_PROMPT + assert [image.b64_json for image in response.data] == list(_NOVA_CANVAS_IMAGES) + per_image: Final = litellm.model_cost["amazon.nova-canvas-v1:0"]["output_cost_per_image"] + assert per_image > 0 + assert response._hidden_params["response_cost"] == pytest.approx(len(_NOVA_CANVAS_IMAGES) * per_image) # pyright: ignore[reportPrivateUsage] # cost is only surfaced on _hidden_params diff --git a/tests/unit/llms/duckduckgo/search/test_duckduckgo_search_transformation.py b/tests/unit/llms/duckduckgo/search/test_duckduckgo_search_transformation.py index 474ffe0e519..68419891166 100644 --- a/tests/unit/llms/duckduckgo/search/test_duckduckgo_search_transformation.py +++ b/tests/unit/llms/duckduckgo/search/test_duckduckgo_search_transformation.py @@ -1,8 +1,12 @@ +from collections.abc import Iterator from typing import Final from unittest.mock import AsyncMock, MagicMock, patch -import litellm +import httpx import pytest +import respx + +import litellm class TestDuckDuckGoSearchMocked: @@ -223,3 +227,73 @@ class TestDuckDuckGoSearchMocked: urls = [result.url for result in response.results] assert any("India" in url for url in urls) assert any("Indus" in url for url in urls) + + +_DDG_INSTANT_ANSWER: Final = { + "AbstractText": "India is a country in South Asia.", + "AbstractURL": "https://en.wikipedia.org/wiki/India", + "Heading": "India", + "RelatedTopics": [ + {"FirstURL": f"https://example.com/{index}", "Text": f"Topic {index} - snippet text for topic {index}."} + for index in range(10) + ], + "Results": [], + "Type": "D", +} + + +@pytest.fixture +def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +async def test_duckduckgo_search_response_structure_and_max_results( + respx_mock: respx.MockRouter, httpx_transport: None +) -> None: + route: Final = respx_mock.get(url__startswith="https://api.duckduckgo.com/").mock( + return_value=httpx.Response(200, json=_DDG_INSTANT_ANSWER) + ) + + response: Final = await litellm.asearch(query="india", search_provider="duckduckgo", max_results=5) + + assert route.call_count == 1 + sent_params: Final = route.calls[0].request.url.params + assert sent_params["q"] == "india" + assert sent_params["format"] == "json" + assert sent_params["_max_results"] == "5" + assert response.object == "search" + assert [result.url for result in response.results] == [ + "https://en.wikipedia.org/wiki/India", + "https://example.com/0", + "https://example.com/1", + "https://example.com/2", + "https://example.com/3", + ] + first_result: Final = response.results[0] + assert first_result.title == "India" + assert first_result.snippet == "India is a country in South Asia." + assert response.results[1].title == "Topic 0" + assert response.results[1].snippet == "snippet text for topic 0." + assert response._hidden_params["response_cost"] == litellm.model_cost["duckduckgo/search"]["input_cost_per_query"] # pyright: ignore[reportPrivateUsage] # cost is only surfaced on _hidden_params + + +def test_duckduckgo_sync_search_returns_typed_results_without_a_limit(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.get(url__startswith="https://api.duckduckgo.com/").mock( + return_value=httpx.Response(200, json=_DDG_INSTANT_ANSWER) + ) + + response: Final = litellm.search(query="india", search_provider="duckduckgo") + + assert route.call_count == 1 + assert "_max_results" not in route.calls[0].request.url.params + assert response.object == "search" + assert len(response.results) == 11 + assert all( + isinstance(result.title, str) and isinstance(result.url, str) and isinstance(result.snippet, str) + for result in response.results + ) + assert response.results[-1].url == "https://example.com/9" diff --git a/tests/unit/llms/exa_ai/search/test_transformation.py b/tests/unit/llms/exa_ai/search/test_transformation.py index 5e5eb24f23b..6b556ff241f 100644 --- a/tests/unit/llms/exa_ai/search/test_transformation.py +++ b/tests/unit/llms/exa_ai/search/test_transformation.py @@ -1,11 +1,22 @@ +import json from typing import Final from unittest.mock import Mock import httpx import pytest +import respx +import litellm from litellm.llms.exa_ai.search.transformation import ExaAISearchConfig +EXA_SEARCH_URL: Final = "https://api.exa.ai/search" + + +@pytest.fixture +def exa_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("EXA_API_KEY", "test-exa-key") + monkeypatch.delenv("EXA_API_BASE", raising=False) + @pytest.mark.parametrize( ("content_fields", "expected_snippet"), @@ -31,3 +42,49 @@ def test_transform_search_response_snippet_falls_back_through_content_modes( response: Final = ExaAISearchConfig().transform_search_response(raw_response, logging_obj=Mock()) assert response.results[0].snippet == expected_snippet + + +@pytest.mark.usefixtures("exa_api_key") +def test_search_maps_exa_results_to_search_response(respx_mock: respx.MockRouter) -> None: + respx_mock.post(EXA_SEARCH_URL).respond( + json={ + "results": [ + { + "title": "AI news roundup", + "url": "https://example.com/ai-news", + "text": "The latest in artificial intelligence.", + "publishedDate": "2026-01-15T00:00:00.000Z", + }, + {"title": "Second", "url": "https://example.com/second", "text": "Second text."}, + ] + } + ) + + response: Final = litellm.search(query="artificial intelligence recent news", search_provider="exa_ai") + + assert response.object == "search" + assert isinstance(response.results, list) + assert len(response.results) == 2 + first: Final = response.results[0] + assert first.title == "AI news roundup" + assert first.url == "https://example.com/ai-news" + assert first.snippet == "The latest in artificial intelligence." + assert first.date == "2026-01-15T00:00:00.000Z" + assert response.results[1].url == "https://example.com/second" + + +@pytest.mark.usefixtures("exa_api_key") +def test_search_sends_max_results_as_num_results(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(EXA_SEARCH_URL).respond( + json={"results": [{"title": "ML", "url": "https://example.com/ml", "text": "Machine learning."}]} + ) + + response: Final = litellm.search(query="machine learning", search_provider="exa_ai", max_results=5) + + assert json.loads(route.calls.last.request.content) == { + "query": "machine learning", + "numResults": 5, + "contents": {"text": True}, + } + assert route.calls.last.request.headers["x-api-key"] == "test-exa-key" + assert [result.url for result in response.results] == ["https://example.com/ml"] diff --git a/tests/unit/llms/firecrawl/__init__.py b/tests/unit/llms/firecrawl/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/firecrawl/search/__init__.py b/tests/unit/llms/firecrawl/search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/firecrawl/search/test_transformation.py b/tests/unit/llms/firecrawl/search/test_transformation.py new file mode 100644 index 00000000000..7d18b40064b --- /dev/null +++ b/tests/unit/llms/firecrawl/search/test_transformation.py @@ -0,0 +1,32 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm + + +def test_firecrawl_search_request_body(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("FIRECRAWL_API_KEY", "test-api-key") + route: Final = respx_mock.post("https://api.firecrawl.dev/v2/search").mock( + return_value=httpx.Response( + 200, + json={ + "success": True, + "data": {"web": [{"title": "Test Title", "url": "https://example.com", "markdown": "Test content"}]}, + }, + ) + ) + + response: Final = litellm.search(query="test query", search_provider="firecrawl", max_results=10, country="US") + + assert route.call_count == 1 + sent: Final = route.calls[0].request + assert sent.headers["authorization"] == "Bearer test-api-key" + body: Final = json.loads(sent.content) + assert body["query"] == "test query" + assert body["limit"] == 10 + assert body["country"] == "US" + assert [(result.title, result.url) for result in response.results] == [("Test Title", "https://example.com")] diff --git a/tests/unit/llms/perplexity/search/__init__.py b/tests/unit/llms/perplexity/search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/perplexity/search/test_perplexity_search_transformation.py b/tests/unit/llms/perplexity/search/test_perplexity_search_transformation.py new file mode 100644 index 00000000000..432ed8248e6 --- /dev/null +++ b/tests/unit/llms/perplexity/search/test_perplexity_search_transformation.py @@ -0,0 +1,57 @@ +import json +from typing import Final + +import pytest +import respx + +import litellm + +PERPLEXITY_SEARCH_URL: Final = "https://api.perplexity.ai/search" + + +@pytest.fixture(autouse=True) +def perplexity_api_key(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PERPLEXITYAI_API_KEY", "test-perplexity-key") + monkeypatch.delenv("PERPLEXITY_API_BASE", raising=False) + + +def test_search_maps_perplexity_results_to_search_response(respx_mock: respx.MockRouter) -> None: + respx_mock.post(PERPLEXITY_SEARCH_URL).respond( + json={ + "results": [ + { + "title": "AI news roundup", + "url": "https://example.com/ai-news", + "snippet": "The latest in artificial intelligence.", + "date": "2026-01-15", + "last_updated": "2026-01-16", + }, + {"title": "Second", "url": "https://example.com/second", "snippet": "Second snippet."}, + ] + } + ) + + response: Final = litellm.search(query="artificial intelligence recent news", search_provider="perplexity") + + assert response.object == "search" + assert isinstance(response.results, list) + assert len(response.results) == 2 + first: Final = response.results[0] + assert first.title == "AI news roundup" + assert first.url == "https://example.com/ai-news" + assert first.snippet == "The latest in artificial intelligence." + assert first.date == "2026-01-15" + assert first.last_updated == "2026-01-16" + assert response.results[1].snippet == "Second snippet." + + +def test_search_sends_max_results_in_request_body(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(PERPLEXITY_SEARCH_URL).respond( + json={"results": [{"title": "ML", "url": "https://example.com/ml", "snippet": "Machine learning."}]} + ) + + response: Final = litellm.search(query="machine learning", search_provider="perplexity", max_results=5) + + assert json.loads(route.calls.last.request.content) == {"query": "machine learning", "max_results": 5} + assert route.calls.last.request.headers["Authorization"] == "Bearer test-perplexity-key" + assert [result.url for result in response.results] == ["https://example.com/ml"] diff --git a/tests/unit/llms/vertex_ai/test_vertex_ai_batch_transformation.py b/tests/unit/llms/vertex_ai/test_vertex_ai_batch_transformation.py index 8da95f839b9..d1cd640e590 100644 --- a/tests/unit/llms/vertex_ai/test_vertex_ai_batch_transformation.py +++ b/tests/unit/llms/vertex_ai/test_vertex_ai_batch_transformation.py @@ -1,6 +1,11 @@ +import json +from types import MappingProxyType +from typing import Final from unittest.mock import MagicMock, patch +import httpx import pytest +import respx import litellm from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction @@ -197,3 +202,134 @@ async def test_litellm_cancel_batch_vertex_ai(): assert mock_instance.cancel_batch.called assert response.id == "batch_123" assert response.status == "cancelling" + + +_MOCK_GCS_FILE_RESPONSE: Final = MappingProxyType( + { + "kind": "storage#object", + "id": "litellm-local/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/5f7b99ad-9203-4430-98bf-3b45451af4cb/1739598666670574", + "selfLink": "https://www.googleapis.com/storage/v1/b/litellm-local/o/litellm-vertex-files%2Fpublishers%2Fgoogle%2Fmodels%2Fgemini-1.5-flash-001%2F5f7b99ad-9203-4430-98bf-3b45451af4cb", + "name": "litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/5f7b99ad-9203-4430-98bf-3b45451af4cb", + "bucket": "litellm-local", + "generation": "1739598666670574", + "metageneration": "1", + "contentType": "application/json", + "storageClass": "STANDARD", + "size": "416", + "md5Hash": "hbBNj7C8KJ7oVH+JmyRM6A==", + "crc32c": "oDmiUA==", + "etag": "CO7D0IT+xIsDEAE=", + "timeCreated": "2025-02-15T05:51:06.741Z", + "updated": "2025-02-15T05:51:06.741Z", + "timeStorageClassUpdated": "2025-02-15T05:51:06.741Z", + "timeFinalized": "2025-02-15T05:51:06.741Z", + } +) + +_MOCK_VERTEX_BATCH_RESPONSE: Final = MappingProxyType( + { + "name": "projects/123456789/locations/us-central1/batchPredictionJobs/test-batch-id-456", + "displayName": "litellm_batch_job", + "model": "projects/123456789/locations/us-central1/models/gemini-1.5-flash-001", + "modelVersionId": "v1", + "inputConfig": { + "gcsSource": { + "uris": [ + "gs://litellm-local/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/5f7b99ad-9203-4430-98bf-3b45451af4cb" + ] + } + }, + "outputConfig": {"gcsDestination": {"outputUriPrefix": "gs://litellm-local/batch-outputs/"}}, + "dedicatedResources": { + "machineSpec": { + "machineType": "n1-standard-4", + "acceleratorType": "NVIDIA_TESLA_T4", + "acceleratorCount": 1, + }, + "startingReplicaCount": 1, + "maxReplicaCount": 1, + }, + "state": "JOB_STATE_RUNNING", + "createTime": "2025-02-15T05:51:06.741Z", + "startTime": "2025-02-15T05:51:07.741Z", + "updateTime": "2025-02-15T05:51:08.741Z", + "labels": {"key1": "value1", "key2": "value2"}, + "completionStats": {"successfulCount": 0, "failedCount": 0, "remainingCount": 100}, + } +) + + +@pytest.mark.asyncio +async def test_vertex_file_upload_create_and_retrieve_batch( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +): + monkeypatch.setenv("GCS_BUCKET_NAME", "litellm-local") + monkeypatch.setenv("VERTEXAI_PROJECT", "mock-project") + monkeypatch.setenv("VERTEXAI_LOCATION", "us-central1") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + mock_creds: Final = MagicMock(token="mock-token", valid=True, expiry=None) + monkeypatch.setattr("google.auth.default", lambda *args, **kwargs: (mock_creds, "mock-project")) + jobs_url: Final = ( + "https://us-central1-aiplatform.googleapis.com/v1/projects/mock-project/locations/us-central1" + "/batchPredictionJobs" + ) + upload_route: Final = respx_mock.post( + url__startswith="https://storage.googleapis.com/upload/storage/v1/b/litellm-local/o" + ).mock(return_value=httpx.Response(200, json=dict(_MOCK_GCS_FILE_RESPONSE))) + create_route: Final = respx_mock.post(jobs_url).mock( + return_value=httpx.Response(200, json=dict(_MOCK_VERTEX_BATCH_RESPONSE)) + ) + retrieve_route: Final = respx_mock.get(f"{jobs_url}/test-batch-id-456").mock( + return_value=httpx.Response(200, json=dict(_MOCK_VERTEX_BATCH_RESPONSE)) + ) + gcs_object_uri: Final = ( + "gs://litellm-local/litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/" + "5f7b99ad-9203-4430-98bf-3b45451af4cb" + ) + + file_obj: Final = await litellm.acreate_file( + file=( + "vertex_batch.jsonl", + b'{"custom_id": "request-1", "method": "POST", "url": "/v1/chat/completions", ' + b'"body": {"model": "gemini-1.5-flash-001", "messages": [{"role": "user", "content": "hi"}]}}\n', + "application/jsonl", + ), + purpose="batch", + custom_llm_provider="vertex_ai", + ) + + assert file_obj.id == gcs_object_uri + assert upload_route.call_count == 1 + upload_request: Final = upload_route.calls.last.request + assert upload_request.url.params["uploadType"] == "media" + assert upload_request.url.params["name"].startswith( + "litellm-vertex-files/publishers/google/models/gemini-1.5-flash-001/" + ) + assert upload_request.headers["Content-Type"] == "application/json" + uploaded_row: Final = json.loads(upload_request.content) + assert uploaded_row["request"]["contents"] == [{"role": "user", "parts": [{"text": "hi"}]}] + assert uploaded_row["request"]["labels"]["litellm_custom_id"] == "request-1" + + create_batch_response: Final = await litellm.acreate_batch( + completion_window="24h", + endpoint="/v1/chat/completions", + input_file_id=file_obj.id, + custom_llm_provider="vertex_ai", + metadata={"key1": "value1", "key2": "value2"}, + ) + + create_body: Final = json.loads(create_route.calls.last.request.content) + assert create_body["inputConfig"] == {"gcsSource": {"uris": [gcs_object_uri]}, "instancesFormat": "jsonl"} + assert create_body["model"] == "publishers/google/models/gemini-1.5-flash-001" + assert create_body["outputConfig"]["predictionsFormat"] == "jsonl" + assert create_body["outputConfig"]["gcsDestination"]["outputUriPrefix"].startswith("gs://litellm-local/") + assert create_batch_response.id == "test-batch-id-456" + assert create_batch_response.input_file_id == gcs_object_uri + + retrieved_batch: Final = await litellm.aretrieve_batch( + batch_id=create_batch_response.id, custom_llm_provider="vertex_ai" + ) + + assert retrieve_route.call_count == 1 + assert retrieved_batch.id == "test-batch-id-456" + assert retrieved_batch.input_file_id == gcs_object_uri diff --git a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py index 8c73b72a65a..8ca0a4a4335 100644 --- a/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py +++ b/tests/unit/llms/vertex_ai/text_to_speech/test_transformation.py @@ -1,9 +1,13 @@ import base64 from typing import Final +import json +from collections.abc import Iterator, Mapping from unittest.mock import MagicMock, Mock, patch import httpx import pytest +import respx +import responses from pydantic import ValidationError import litellm @@ -663,3 +667,97 @@ def test_transform_text_to_speech_response_rejects_malformed_payloads_without_ec ) assert "input_value" not in str(exc_info.value) + + +_SYNTHESIZE_URL: Final = "https://texttospeech.googleapis.com/v1/text:synthesize" +_AUTHORIZED_USER: Final = json.dumps( + { + "type": "authorized_user", + "client_id": "synthetic-client-id", + "client_secret": "synthetic-client-secret", + "refresh_token": "synthetic-refresh-token", + "quota_project_id": "test-project", + } +) +_ASYNC_INPUT: Final = "async hello what llm guardrail do you have" +_UK_VOICE: Final = {"languageCode": "en-UK", "name": "en-UK-Studio-O"} +_UK_AUDIO_CONFIG: Final = {"audioEncoding": "LINEAR22", "speakingRate": "10"} + + +@pytest.fixture +def google_token_endpoint(monkeypatch: pytest.MonkeyPatch) -> Iterator[responses.RequestsMock]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + with responses.RequestsMock(assert_all_requests_are_fired=False) as token_endpoint: + token_endpoint.post( + "https://oauth2.googleapis.com/token", + json={"access_token": "minted-google-token", "expires_in": 3600, "token_type": "Bearer"}, + ) + yield token_endpoint + litellm.in_memory_llm_clients_cache.flush_cache() + + +async def _aspeech_vertex( + respx_mock: respx.MockRouter, speech_input: str, voice_params: Mapping[str, object] +) -> httpx.Request: + route: Final = respx_mock.post(_SYNTHESIZE_URL).mock( + return_value=httpx.Response(200, json={"audioContent": base64.b64encode(b"vertex-audio").decode()}) + ) + response: Final = await litellm.aspeech( + model="vertex_ai/test", + input=speech_input, + vertex_credentials=_AUTHORIZED_USER, + **voice_params, + ) + assert response.content == b"vertex-audio" + assert route.call_count == 1 + sent: Final = route.calls[0].request + assert sent.headers["x-goog-user-project"] == "test-project" + assert sent.headers["authorization"] == "Bearer minted-google-token" + return sent + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_default_voice_posts_synthesize_request( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + sent: Final = await _aspeech_vertex(respx_mock, _ASYNC_INPUT, {}) + + assert json.loads(sent.content) == { + "input": {"text": _ASYNC_INPUT}, + "voice": {"languageCode": "en-US", "name": "en-US-Studio-O"}, + "audioConfig": {"audioEncoding": "LINEAR16", "speakingRate": "1"}, + } + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_forwards_caller_voice_and_audio_config( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + sent: Final = await _aspeech_vertex(respx_mock, _ASYNC_INPUT, {"voice": _UK_VOICE, "audioConfig": _UK_AUDIO_CONFIG}) + + assert json.loads(sent.content) == { + "input": {"text": _ASYNC_INPUT}, + "voice": _UK_VOICE, + "audioConfig": _UK_AUDIO_CONFIG, + } + + +@pytest.mark.asyncio +async def test_aspeech_vertex_ai_sends_ssml_input( + respx_mock: respx.MockRouter, google_token_endpoint: responses.RequestsMock +) -> None: + ssml: Final = """ + +

    Hello, world!

    +

    This is a test of the text-to-speech API.

    +     + """ + + sent: Final = await _aspeech_vertex(respx_mock, ssml, {"voice": _UK_VOICE, "audioConfig": _UK_AUDIO_CONFIG}) + + assert json.loads(sent.content) == { + "input": {"ssml": ssml}, + "voice": _UK_VOICE, + "audioConfig": _UK_AUDIO_CONFIG, + } diff --git a/tests/unit/passthrough/test_main.py b/tests/unit/passthrough/test_main.py new file mode 100644 index 00000000000..78fa83d126b --- /dev/null +++ b/tests/unit/passthrough/test_main.py @@ -0,0 +1,53 @@ +import json +from typing import Final + +import httpx +import pytest +import respx + +import litellm + +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler +from litellm.llms.vllm.passthrough.transformation import VLLMPassthroughConfig +from litellm.passthrough.main import allm_passthrough_route +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + +_API_BASE: Final = "http://vllm-upstream.test:8090" + + +@pytest.fixture(autouse=True) +def _httpx_transport(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + + +def test_hosted_vllm_resolves_vllm_passthrough_config() -> None: + cfg: Final = ProviderConfigManager.get_provider_passthrough_config( + model="hosted_vllm/my-deployment", + provider=LlmProviders.HOSTED_VLLM, + ) + assert isinstance(cfg, VLLMPassthroughConfig) + + +@pytest.mark.asyncio +async def test_allm_passthrough_route_hosted_vllm_sends_normalized_model(respx_mock: respx.MockRouter) -> None: + route: Final = respx_mock.post(f"{_API_BASE}/v1/chat/completions").mock( + return_value=httpx.Response(200, json={"ok": True}) + ) + client: Final = AsyncHTTPHandler() + response: Final = await allm_passthrough_route( + method="POST", + endpoint="v1/chat/completions", + model="hosted_vllm/my-deployment", + api_base=_API_BASE, + json={ + "model": "anything", + "messages": [{"role": "user", "content": "Hello"}], + }, + client=client, + ) + assert response.status_code == 200 + assert route.call_count == 1 + outbound: Final = json.loads(route.calls[0].request.content) + assert outbound["model"] == "my-deployment" + assert outbound["messages"] == [{"role": "user", "content": "Hello"}] diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py index 01be28470d1..f9795197f90 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_bedrock_guardrails.py @@ -4,12 +4,14 @@ Unit tests for Bedrock Guardrails import json import asyncio +from typing import Final from datetime import datetime, timezone import sys from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest +import respx from fastapi import HTTPException @@ -7483,3 +7485,142 @@ async def test_should_raise_guardrail_blocked_exception_null_fields(): guardrail._should_raise_guardrail_blocked_exception(response_null_grounding) is False ) + + +_MASKING_GUARDRAIL_URL: Final = "https://bedrock-runtime.us-east-1.amazonaws.com/guardrail/wf0hkdb5x07f/version/DRAFT/apply" + + +def _masking_guardrail() -> BedrockGuardrail: + return BedrockGuardrail( + guardrailIdentifier="wf0hkdb5x07f", + guardrailVersion="DRAFT", + aws_access_key_id="fake-access-key", + aws_secret_access_key="fake-secret-key", + aws_region_name="us-east-1", + ) + + +def _anonymized_reply(masked_texts: tuple[str, ...], entity_types: tuple[str, ...]) -> httpx.Response: + return httpx.Response( + 200, + json={ + "action": "GUARDRAIL_INTERVENED", + "outputs": [{"text": text} for text in masked_texts], + "assessments": [ + { + "sensitiveInformationPolicy": { + "piiEntities": [ + {"type": entity_type, "match": "redacted", "action": "ANONYMIZED"} + for entity_type in entity_types + ] + } + } + ], + }, + ) + + +def _sent_texts(route: respx.Route) -> tuple[str, ...]: + body: Final = json.loads(route.calls[0].request.content) + assert body["source"] == "INPUT" + return tuple(item["text"]["text"] for item in body["content"]) + + +@pytest.mark.asyncio +async def test_during_call_masking_rewrites_pii_in_messages( + respx_mock: respx.MockRouter, httpx_transport: None +) -> None: + guardrail: Final = _masking_guardrail() + route: Final = respx_mock.post(_MASKING_GUARDRAIL_URL).mock( + return_value=_anonymized_reply( + ( + "Hello, my phone number is {PHONE}", + "Hello, how can I help you today?", + "I need to cancel my order", + "ok, my credit card number is {CREDIT_DEBIT_CARD_NUMBER}", + ), + ("PHONE", "CREDIT_DEBIT_CARD_NUMBER"), + ) + ) + + response: Final = await guardrail.async_moderation_hook( + data={ + "model": "gpt-5.5", + "messages": [ + {"role": "user", "content": "Hello, my phone number is +1 412 555 1212"}, + {"role": "assistant", "content": "Hello, how can I help you today?"}, + {"role": "user", "content": "I need to cancel my order"}, + {"role": "user", "content": "ok, my credit card number is 1234-5678-9012-3456"}, + ], + }, + user_api_key_dict=UserAPIKeyAuth(), + call_type="completion", + ) + + assert route.call_count == 1 + assert _sent_texts(route) == ( + "Hello, my phone number is +1 412 555 1212", + "Hello, how can I help you today?", + "I need to cancel my order", + "ok, my credit card number is 1234-5678-9012-3456", + ) + assert response is not None + assert [message["content"] for message in response["messages"]] == [ + "Hello, my phone number is {PHONE}", + "Hello, how can I help you today?", + "I need to cancel my order", + "ok, my credit card number is {CREDIT_DEBIT_CARD_NUMBER}", + ] + + +@pytest.mark.asyncio +async def test_during_call_masking_rewrites_only_pii_block_in_content_list( + respx_mock: respx.MockRouter, httpx_transport: None +) -> None: + guardrail: Final = _masking_guardrail() + route: Final = respx_mock.post(_MASKING_GUARDRAIL_URL).mock( + return_value=_anonymized_reply( + ( + "Hello, my phone number is {PHONE}", + "what time is it?", + "Hello, how can I help you today?", + "who is the president of the united states?", + ), + ("PHONE",), + ) + ) + + response: Final = await guardrail.async_moderation_hook( + data={ + "model": "gpt-5.5", + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": "Hello, my phone number is +1 412 555 1212"}, + {"type": "text", "text": "what time is it?"}, + ], + }, + {"role": "assistant", "content": "Hello, how can I help you today?"}, + {"role": "user", "content": "who is the president of the united states?"}, + ], + }, + user_api_key_dict=UserAPIKeyAuth(), + call_type="completion", + ) + + assert route.call_count == 1 + assert _sent_texts(route) == ( + "Hello, my phone number is +1 412 555 1212", + "what time is it?", + "Hello, how can I help you today?", + "who is the president of the united states?", + ) + assert response is not None + messages: Final = response["messages"] + assert messages[0]["content"] == [ + {"type": "text", "text": "Hello, my phone number is {PHONE}"}, + {"type": "text", "text": "what time is it?"}, + ] + assert messages[1]["content"] == "Hello, how can I help you today?" + assert messages[2]["content"] == "who is the president of the united states?" diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py index 0cac6228085..bd760ed27b1 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -8,13 +8,18 @@ import copy import json import os import re +from collections.abc import Iterable, Sequence from contextlib import asynccontextmanager from typing import Final, Literal from unittest.mock import MagicMock, patch +import aiohttp from aiohttp import web +from aiohttp.client_proto import ResponseHandler from aiohttp.test_utils import TestServer import pytest +from pydantic import TypeAdapter +from typing_extensions import ReadOnly, TypedDict import litellm @@ -27,6 +32,7 @@ from litellm.proxy.guardrails.guardrail_hooks.presidio import ( ) from litellm.exceptions import GuardrailRaisedException from litellm.types.guardrails import LitellmParams, PiiAction, PiiEntityType +from litellm.types.proxy.guardrails.guardrail_hooks.presidio import PresidioAnalyzeRequest, PresidioAnalyzeResponseItem from litellm.types.utils import Choices, Delta, Message, ModelResponse, StreamingChoices from litellm.exceptions import BlockedPiiEntityError @@ -4575,3 +4581,221 @@ async def test_presidio_language_configuration_with_per_request_override(): assert analyze_request_default["language"] == "de" assert analyze_request_default["text"] == test_text + + +_CARD_NUMBER: Final = "4111-1111-1111-1111" +_EMAIL: Final = "test@example.com" +_BLOCK_CARD_MASK_EMAIL: Final = { + PiiEntityType.CREDIT_CARD: PiiAction.BLOCK, + PiiEntityType.EMAIL_ADDRESS: PiiAction.MASK, +} + + +class _PresidioAnonymizeRequest(TypedDict): + text: ReadOnly[str] + + +class _PresidioAnonymizeReply(TypedDict): + text: ReadOnly[str] + items: ReadOnly[tuple[()]] + + +_ANALYZE_REQUEST: Final = TypeAdapter(PresidioAnalyzeRequest) +_ANONYMIZE_REQUEST: Final = TypeAdapter(_PresidioAnonymizeRequest) + + +def _card_and_email_spans(text: str) -> tuple[PresidioAnalyzeResponseItem, ...]: + return tuple( + PresidioAnalyzeResponseItem( + entity_type=entity_type, + start=text.index(value), + end=text.index(value) + len(value), + score=1.0, + analysis_explanation=None, + ) + for entity_type, value in (("CREDIT_CARD", _CARD_NUMBER), ("EMAIL_ADDRESS", _EMAIL)) + if value in text + ) + + +class _InMemoryPresidioTransport(asyncio.Transport): + def __init__(self, protocol: ResponseHandler, analyzed: asyncio.Queue[PresidioAnalyzeRequest]) -> None: + super().__init__() + self._protocol: Final = protocol + self._analyzed: Final = analyzed + self._received = b"" + self._closing = False + + def write(self, data: bytes | bytearray | memoryview) -> None: + self._received += bytes(data) + head, separator, body = self._received.partition(b"\r\n\r\n") + if not separator: + return + request_line, *header_lines = head.decode().split("\r\n") + headers: Final = {name.lower(): value.strip() for name, _, value in (line.partition(":") for line in header_lines)} + if len(body) < int(headers.get("content-length", "0")): + return + self._received = b"" + reply: Final = json.dumps(self._reply(request_line.split(" ")[1], body)).encode() + response: Final = ( + b"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nConnection: close\r\n" + + f"Content-Length: {len(reply)}\r\n\r\n".encode() + + reply + ) + asyncio.get_running_loop().call_soon(self._protocol.data_received, response) + + def _reply(self, path: str, body: bytes) -> tuple[PresidioAnalyzeResponseItem, ...] | _PresidioAnonymizeReply: + if path == "/analyze": + analyze_request: Final = _ANALYZE_REQUEST.validate_json(body) + self._analyzed.put_nowait(analyze_request) + return _card_and_email_spans(analyze_request.get("text") or "") + assert path == "/anonymize", path + return _PresidioAnonymizeReply(text=_ANONYMIZE_REQUEST.validate_json(body)["text"], items=()) + + def writelines(self, list_of_data: Iterable[bytes | bytearray | memoryview]) -> None: + self.write(b"".join(bytes(chunk) for chunk in list_of_data)) + + def is_closing(self) -> bool: + return self._closing + + def close(self) -> None: + if not self._closing: + self._closing = True + asyncio.get_running_loop().call_soon(self._protocol.connection_lost, None) + + def abort(self) -> None: + self.close() + + def get_extra_info(self, name: str, default: object = None) -> object: + return default + + def can_write_eof(self) -> bool: + return False + + def get_write_buffer_size(self) -> int: + return 0 + + def pause_reading(self) -> None: + return None + + def resume_reading(self) -> None: + return None + + +class _InMemoryPresidioConnector(aiohttp.BaseConnector): + def __init__(self, analyzed: asyncio.Queue[PresidioAnalyzeRequest]) -> None: + super().__init__() + self._analyzed: Final = analyzed + + async def _create_connection( # pyright: ignore[reportImplicitOverride] # aiohttp's connector extension point + self, req: aiohttp.ClientRequest, traces: Sequence[object], timeout: aiohttp.ClientTimeout + ) -> ResponseHandler: + protocol: Final = ResponseHandler(asyncio.get_running_loop()) + protocol.connection_made(_InMemoryPresidioTransport(protocol, self._analyzed)) + return protocol + + +def _drain(analyzed: asyncio.Queue[PresidioAnalyzeRequest]) -> tuple[PresidioAnalyzeRequest, ...]: + return tuple(analyzed.get_nowait() for _ in range(analyzed.qsize())) + + +def _guardrail_with_in_memory_presidio( + analyzed: asyncio.Queue[PresidioAnalyzeRequest], +) -> OPTIONAL_PresidioPIIMasking: + guardrail: Final = OPTIONAL_PresidioPIIMasking( + pii_entities_config=_BLOCK_CARD_MASK_EMAIL, + presidio_analyzer_api_base="http://presidio-analyzer.test/", + presidio_anonymizer_api_base="http://presidio-anonymizer.test/", + ) + guardrail._http_session = aiohttp.ClientSession(connector=_InMemoryPresidioConnector(analyzed)) + return guardrail + + +@pytest.mark.asyncio +async def test_check_pii_raises_blocked_entity_for_card() -> None: + text: Final = f"My credit card number is {_CARD_NUMBER} and my email is {_EMAIL}" + analyzed: Final[asyncio.Queue[PresidioAnalyzeRequest]] = asyncio.Queue() + guardrail: Final = _guardrail_with_in_memory_presidio(analyzed) + try: + with pytest.raises(BlockedPiiEntityError) as excinfo: + await guardrail.check_pii(text=text, output_parse_pii=True, presidio_config=None, request_data={}) + finally: + await guardrail._close_http_session() + + analyze_requests: Final = _drain(analyzed) + assert len(analyze_requests) == 1 + assert analyze_requests[0].get("text") == text + assert set(analyze_requests[0].get("entities") or ()) == set(_BLOCK_CARD_MASK_EMAIL) + assert excinfo.value.entity_type == PiiEntityType.CREDIT_CARD + assert excinfo.value.guardrail_name == guardrail.guardrail_name + + +@pytest.mark.asyncio +async def test_pre_call_hook_raises_blocked_entity_for_card_message( + mock_user_api_key: UserAPIKeyAuth, mock_cache: DualCache +) -> None: + user_text: Final = f"My credit card is {_CARD_NUMBER} and my email is {_EMAIL}." + analyzed: Final[asyncio.Queue[PresidioAnalyzeRequest]] = asyncio.Queue() + guardrail: Final = _guardrail_with_in_memory_presidio(analyzed) + try: + with pytest.raises(BlockedPiiEntityError) as excinfo: + await guardrail.async_pre_call_hook( + user_api_key_dict=mock_user_api_key, + cache=mock_cache, + data={ + "messages": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": user_text}, + ], + "model": "gpt-5-mini", + }, + call_type="completion", + ) + finally: + await guardrail._close_http_session() + + analyze_requests: Final = _drain(analyzed) + assert user_text in [payload.get("text") for payload in analyze_requests] + assert all(set(payload.get("entities") or ()) == set(_BLOCK_CARD_MASK_EMAIL) for payload in analyze_requests) + assert excinfo.value.entity_type == PiiEntityType.CREDIT_CARD + assert excinfo.value.guardrail_name == guardrail.guardrail_name + + +@pytest.mark.asyncio +async def test_legacy_pii_masking_config_registers_logging_only_guardrail(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("PRESIDIO_ANALYZER_API_BASE", "http://localhost:5002") + monkeypatch.setenv("PRESIDIO_ANONYMIZER_API_BASE", "http://localhost:5001") + monkeypatch.setattr(litellm, "guardrail_name_config_map", {}) + monkeypatch.setattr(litellm, "callbacks", []) + + from litellm.proxy.guardrails.init_guardrails import initialize_guardrails + from litellm.types.guardrails import GuardrailEventHooks + + guardrails_config: Final = [ + { + "pii_masking": { + "callbacks": ["presidio"], + "default_on": True, + "logging_only": True, + } + } + ] + assert len(litellm.guardrail_name_config_map) == 0 + initialize_guardrails( + guardrails_config=guardrails_config, + premium_user=True, + config_file_path="", + litellm_settings={"guardrails": guardrails_config}, + ) + assert len(litellm.guardrail_name_config_map) == 1 + + pii_masking_obj: Final = next( + (c for c in litellm.callbacks if isinstance(c, OPTIONAL_PresidioPIIMasking)), + None, + ) + assert pii_masking_obj is not None + assert hasattr(pii_masking_obj, "logging_only") + assert pii_masking_obj.event_hook == GuardrailEventHooks.logging_only + assert pii_masking_obj.should_run_guardrail( + data={}, event_type=GuardrailEventHooks.logging_only + ) diff --git a/tests/unit/proxy/hooks/test_batch_rate_limiter.py b/tests/unit/proxy/hooks/test_batch_rate_limiter.py index a3e60a89c9f..2f71b177705 100644 --- a/tests/unit/proxy/hooks/test_batch_rate_limiter.py +++ b/tests/unit/proxy/hooks/test_batch_rate_limiter.py @@ -6,14 +6,19 @@ batch under a per-minute RPM/TPM budget. Scopes that configure `tpd_limit` are charged against a 24h token window instead of their minute counters. """ +import json import time -from collections.abc import Iterator +from collections.abc import Iterator, Sequence from datetime import datetime, timezone -from typing import Final +from typing import Final, Literal +import httpx import pytest +import respx from fastapi import HTTPException +from typing_extensions import ReadOnly, TypedDict +import litellm from litellm import DualCache from litellm.constants import BATCH_TPD_WINDOW_SECONDS from litellm.proxy._types import UserAPIKeyAuth @@ -22,6 +27,7 @@ from litellm.proxy.hooks.parallel_request_limiter_v3 import ( PROXY_MaxParallelRequestsHandler_v3, ) from litellm.proxy.utils import InternalUsageCache, hash_token +from litellm.types.llms.openai import ChatCompletionUserMessage, LiteLLMBatchCreateRequest class _Clock: @@ -296,3 +302,242 @@ async def test_batch_rate_limit_error_reports_reset_time_in_utc_on_a_non_utc_pro assert exc.value.headers["retry-after"] == str(BATCH_TPD_WINDOW_SECONDS - 3 * 3600) assert exc.value.headers["reset_at"] == "2026-09-14 08:00:00 UTC" assert str(exc.value.detail).endswith("Limit resets at: 2026-09-14 08:00:00 UTC") + + +_BATCH_MODEL: Final = "gpt-3.5-turbo" +_MANAGED_FILE_ID: Final = ( + "bGl0ZWxsbV9wcm94eTphcHBsaWNhdGlvbi9vY3RldC1zdHJlYW07dW5pZmllZF9pZCxyZWdyZXNzaW9uLXRlc3QtZmlsZQ==" +) + + +class _BatchBody(TypedDict): + model: ReadOnly[str] + messages: ReadOnly[Sequence[ChatCompletionUserMessage]] + + +class _BatchLine(TypedDict): + custom_id: ReadOnly[str] + method: ReadOnly[Literal["POST"]] + url: ReadOnly[Literal["/v1/chat/completions"]] + body: ReadOnly[_BatchBody] + + +@pytest.fixture +def openai_files(monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter) -> respx.MockRouter: + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + return respx_mock + + +def _batch_rows(messages: Sequence[str]) -> tuple[_BatchLine, ...]: + return tuple( + _BatchLine( + custom_id=f"request-{i}", + method="POST", + url="/v1/chat/completions", + body=_BatchBody(model=_BATCH_MODEL, messages=(ChatCompletionUserMessage(role="user", content=message),)), + ) + for i, message in enumerate(messages, start=1) + ) + + +def _serve_file(router: respx.MockRouter, file_id: str, rows: Sequence[_BatchLine]) -> respx.Route: + jsonl: Final = "\n".join(json.dumps(row) for row in rows) + return router.get(f"https://api.openai.com/v1/files/{file_id}/content").mock( + return_value=httpx.Response(200, content=jsonl.encode()) + ) + + +def _token_counter_total(rows: Sequence[_BatchLine]) -> int: + return sum(litellm.token_counter(model=row["body"]["model"], messages=row["body"]["messages"]) for row in rows) + + +def _create_batch_data(input_file_id: str) -> LiteLLMBatchCreateRequest: + return LiteLLMBatchCreateRequest(model=_BATCH_MODEL, input_file_id=input_file_id) + + +@pytest.mark.asyncio +async def test_count_input_file_usage_matches_token_counter(openai_files: respx.MockRouter): + _, _, batch_limiter = _make_limiters() + rows: Final = _batch_rows(("Hello", "Hi there", "Hey")) + content_route: Final = _serve_file(openai_files, "file-abc123", rows) + + usage: Final = await batch_limiter.count_input_file_usage(file_id="file-abc123", custom_llm_provider="openai") + + assert content_route.call_count == 1 + assert usage.request_count == 3 + assert usage.total_tokens == _token_counter_total(rows) + + +@pytest.mark.asyncio +async def test_batch_rate_limit_single_file_under_and_over_tpm(openai_files: respx.MockRouter): + small_rows: Final = _batch_rows(("Hello", "Hi", "Hey")) + big_rows: Final = _batch_rows( + ("This is a longer message that will consume more tokens from the rate limit. " * 100,) * 3 + ) + _serve_file(openai_files, "file-small", small_rows) + _serve_file(openai_files, "file-big", big_rows) + user_api_key_dict: Final = UserAPIKeyAuth(api_key="test-key-123", tpm_limit=200, rpm_limit=10) + _, _, small_limiter = _make_limiters() + + data_small: Final = dict(_create_batch_data("file-small")) + result: Final = await small_limiter.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=DualCache(), + data=data_small, + call_type="acreate_batch", + ) + + assert result is data_small + assert data_small["_batch_token_count"] == _token_counter_total(small_rows) + assert data_small["_batch_request_count"] == 3 + + _, _, big_limiter = _make_limiters() + with pytest.raises(HTTPException) as exc_info: + await big_limiter.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=DualCache(), + data=dict(_create_batch_data("file-big")), + call_type="acreate_batch", + ) + assert exc_info.value.status_code == 429 + assert "tokens" in str(exc_info.value.detail).lower() + + +@pytest.mark.asyncio +async def test_batch_rate_limit_cumulative_tpm_rejects_second_request(openai_files: respx.MockRouter): + _, _, batch_limiter = _make_limiters() + user_api_key_dict: Final = UserAPIKeyAuth(api_key="test-key-456", tpm_limit=200, rpm_limit=10) + first_rows: Final = _batch_rows(("This message has some content to reach about 100 tokens total. " * 4,) * 2) + second_rows: Final = _batch_rows( + ("This is another message with more content to exceed the remaining limit. " * 11,) * 2 + ) + _serve_file(openai_files, "file-1", first_rows) + _serve_file(openai_files, "file-2", second_rows) + first_tokens: Final = _token_counter_total(first_rows) + assert first_tokens <= 200 < first_tokens + _token_counter_total(second_rows) + + first_data: Final = dict(_create_batch_data("file-1")) + first_result: Final = await batch_limiter.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=DualCache(), + data=first_data, + call_type="acreate_batch", + ) + assert first_result is first_data + assert first_data["_batch_token_count"] == first_tokens + + with pytest.raises(HTTPException) as exc_info: + await batch_limiter.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=DualCache(), + data=dict(_create_batch_data("file-2")), + call_type="acreate_batch", + ) + assert exc_info.value.status_code == 429 + assert "tokens" in str(exc_info.value.detail).lower() + + +@pytest.mark.asyncio +async def test_batch_rate_limiter_reads_a_provider_file_with_user_context(openai_files: respx.MockRouter): + _, _, batch_limiter = _make_limiters() + user_api_key_dict: Final = UserAPIKeyAuth( + api_key="test-key-managed-files", user_id="test-user-abc123", tpm_limit=500, rpm_limit=10 + ) + rows: Final = _batch_rows(("This is a test message for batch rate limiting with managed files. " * 5,) * 3) + content_route: Final = _serve_file(openai_files, "file-abc123", rows) + + data: Final = dict(_create_batch_data("file-abc123")) + result: Final = await batch_limiter.async_pre_call_hook( + user_api_key_dict=user_api_key_dict, + cache=DualCache(), + data=data, + call_type="acreate_batch", + ) + + assert content_route.call_count == 1 + assert result is data + assert data["_batch_token_count"] == _token_counter_total(rows) + assert data["_batch_request_count"] == 3 + + +@pytest.mark.asyncio +async def test_batch_rate_limiter_without_user_context(openai_files: respx.MockRouter): + _, _, batch_limiter = _make_limiters() + rows: Final = _batch_rows(("Hello",)) + content_route: Final = _serve_file(openai_files, "file-abc123", rows) + + usage_without_context: Final = await batch_limiter.count_input_file_usage( + file_id="file-abc123", custom_llm_provider="openai", user_api_key_dict=None + ) + usage_with_context: Final = await batch_limiter.count_input_file_usage( + file_id="file-abc123", + custom_llm_provider="openai", + user_api_key_dict=UserAPIKeyAuth(api_key="test-key", user_id="test-user-123"), + ) + + assert content_route.call_count == 2 + assert usage_without_context.request_count == usage_with_context.request_count == 1 + assert usage_without_context.total_tokens == usage_with_context.total_tokens == _token_counter_total(rows) + + +@pytest.mark.asyncio +async def test_managed_file_is_read_through_the_managed_files_hook_with_user_context( + openai_files: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles + + from litellm import Router + from litellm.models.managed_files import LiteLLM_ManagedFileTable + from litellm.proxy import proxy_server + from litellm.proxy.openai_files_endpoints.common_utils import is_base64_encoded_unified_file_id + from litellm.proxy.utils import ProxyLogging + + assert is_base64_encoded_unified_file_id(_MANAGED_FILE_ID) + rows: Final = _batch_rows(("Test message for regression",)) + provider_route: Final = _serve_file(openai_files, "file-provider-1", rows) + standard_route: Final = _serve_file(openai_files, "file-abc123", rows) + unrouted_managed_read: Final = _serve_file(openai_files, _MANAGED_FILE_ID, rows) + file_cache: Final = InternalUsageCache(dual_cache=DualCache()) + await file_cache.async_set_cache( + key=_MANAGED_FILE_ID, + value=LiteLLM_ManagedFileTable( + unified_file_id=_MANAGED_FILE_ID, + model_mappings={"deployment-1": "file-provider-1"}, + flat_model_file_ids=["file-provider-1"], + created_by="test-user-regression", + ).model_dump(), + litellm_parent_otel_span=None, + ) + proxy_logging: Final = ProxyLogging(user_api_key_cache=DualCache()) + proxy_logging.proxy_hook_mapping["managed_files"] = _PROXY_LiteLLMManagedFiles( + internal_usage_cache=file_cache, prisma_client=None + ) + router: Final = Router( + model_list=[ + { + "model_name": _BATCH_MODEL, + "litellm_params": {"model": f"openai/{_BATCH_MODEL}", "api_key": "sk-test"}, + "model_info": {"id": "deployment-1"}, + } + ] + ) + monkeypatch.setattr(proxy_server, "proxy_logging_obj", proxy_logging) + monkeypatch.setattr(proxy_server, "llm_router", router) + _, _, batch_limiter = _make_limiters() + user_api_key_dict: Final = UserAPIKeyAuth( + api_key="test-key-regression", user_id="test-user-regression", tpm_limit=1000, rpm_limit=10 + ) + + managed_usage: Final = await batch_limiter.count_input_file_usage( + file_id=_MANAGED_FILE_ID, custom_llm_provider="openai", user_api_key_dict=user_api_key_dict + ) + standard_usage: Final = await batch_limiter.count_input_file_usage( + file_id="file-abc123", custom_llm_provider="openai", user_api_key_dict=user_api_key_dict + ) + + assert provider_route.call_count == 1 + assert standard_route.call_count == 1 + assert not unrouted_managed_read.called + assert managed_usage.request_count == standard_usage.request_count == 1 + assert managed_usage.total_tokens == standard_usage.total_tokens == _token_counter_total(rows) diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py index 50768e48d43..180ece77a6e 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_common_utils.py @@ -3,7 +3,9 @@ from typing import Final from unittest.mock import AsyncMock, MagicMock import pytest +from litellm_enterprise.proxy.hooks.managed_files import _PROXY_LiteLLMManagedFiles +from litellm.caching.caching import DualCache from litellm.proxy.openai_files_endpoints.common_utils import ( apply_unified_file_ids, get_credentials_for_model, @@ -11,7 +13,7 @@ from litellm.proxy.openai_files_endpoints.common_utils import ( map_raw_file_ids_to_unified, ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError -from litellm.proxy.utils import handle_exception_on_proxy +from litellm.proxy.utils import InternalUsageCache, handle_exception_on_proxy from litellm.types.utils import LiteLLMBatch _RAW_MODEL_WITH_PROMPT: Final = "opus-4.6 Please summarize my medical records\nPatient has diabetes" @@ -514,3 +516,139 @@ class TestCompletedBatchSafeToRetire: ) def test_is_litellm_executed_batch_reads_the_llm_batch_id_prefix(decoded_unified_batch_id: str, executed: bool): assert is_litellm_executed_batch(decoded_unified_batch_id) is executed + + +def _managed_files_hook(prisma_client: MagicMock) -> _PROXY_LiteLLMManagedFiles: + return _PROXY_LiteLLMManagedFiles(internal_usage_cache=InternalUsageCache(DualCache()), prisma_client=prisma_client) + + +def _managed_object_prisma(db_batch_object: MagicMock | None) -> MagicMock: + prisma_client: Final = MagicMock() + prisma_client.db.litellm_managedobjecttable.find_first = AsyncMock(return_value=db_batch_object) + prisma_client.db.litellm_managedobjecttable.update = AsyncMock() + return prisma_client + + +@pytest.mark.asyncio +async def test_batch_status_sync_from_provider_to_database(caplog: pytest.LogCaptureFixture): + import json + import logging + + from litellm.proxy.openai_files_endpoints.common_utils import ( + get_batch_from_database, + update_batch_in_database, + ) + + batch_id: Final = "batch_test123" + unified_batch_id: Final = "litellm_proxy:test_unified_batch" + stored_row: Final = MagicMock( + unified_object_id=batch_id, + status="validating", + file_object=json.dumps( + { + "id": batch_id, + "object": "batch", + "status": "validating", + "endpoint": "/v1/chat/completions", + "input_file_id": "file-test123", + "completion_window": "24h", + "created_at": 1234567890, + } + ), + ) + prisma_client: Final = _managed_object_prisma(stored_row) + managed_files: Final = _managed_files_hook(prisma_client) + logger: Final = logging.getLogger("test_batch_status_sync") + + db_batch_object, response_batch = await get_batch_from_database( + batch_id=batch_id, + unified_batch_id=unified_batch_id, + managed_files_obj=managed_files, + prisma_client=prisma_client, + verbose_proxy_logger=logger, + ) + + prisma_client.db.litellm_managedobjecttable.find_first.assert_awaited_once_with( + where={"unified_object_id": batch_id} + ) + assert db_batch_object is stored_row + assert isinstance(response_batch, LiteLLMBatch) + assert response_batch.id == batch_id + assert response_batch.status == "validating" + assert response_batch.input_file_id == "file-test123" + + completed: Final = LiteLLMBatch( + id=batch_id, + object="batch", + status="completed", + endpoint="/v1/chat/completions", + input_file_id="file-test123", + completion_window="24h", + created_at=1234567890, + output_file_id="file-output123", + ) + with caplog.at_level(logging.INFO, logger=logger.name): + await update_batch_in_database( + batch_id=batch_id, + unified_batch_id=unified_batch_id, + response=completed, + managed_files_obj=managed_files, + prisma_client=prisma_client, + verbose_proxy_logger=logger, + db_batch_object=db_batch_object, + operation="retrieve", + poller_owns_accounting=False, + ) + + update: Final = prisma_client.db.litellm_managedobjecttable.update + update.assert_awaited_once() + assert update.await_args.kwargs["where"] == {"unified_object_id": batch_id} + written: Final = update.await_args.kwargs["data"] + assert written["status"] == "complete" + assert written["batch_processed"] is True + assert written["updated_at"] is not None + assert json.loads(written["file_object"])["status"] == "completed" + assert json.loads(written["file_object"])["output_file_id"] == "file-output123" + assert f"Updating batch {batch_id} status from validating to completed" in caplog.messages + + +@pytest.mark.asyncio +async def test_batch_cancel_updates_database(caplog: pytest.LogCaptureFixture): + import json + import logging + + from litellm.proxy.openai_files_endpoints.common_utils import update_batch_in_database + + batch_id: Final = "batch_cancel_test" + cancelled: Final = LiteLLMBatch( + id=batch_id, + object="batch", + status="cancelled", + endpoint="/v1/chat/completions", + input_file_id="file-test123", + completion_window="24h", + created_at=1234567890, + cancelled_at=1234567999, + ) + prisma_client: Final = _managed_object_prisma(None) + logger: Final = logging.getLogger("test_batch_cancel_updates_database") + + with caplog.at_level(logging.INFO, logger=logger.name): + await update_batch_in_database( + batch_id=batch_id, + unified_batch_id="litellm_proxy:cancel_test", + response=cancelled, + managed_files_obj=_managed_files_hook(prisma_client), + prisma_client=prisma_client, + verbose_proxy_logger=logger, + operation="cancel", + ) + + update: Final = prisma_client.db.litellm_managedobjecttable.update + update.assert_awaited_once() + assert update.await_args.kwargs["where"] == {"unified_object_id": batch_id} + written: Final = update.await_args.kwargs["data"] + assert written["status"] == "cancelled" + assert "batch_processed" not in written + assert json.loads(written["file_object"])["cancelled_at"] == 1234567999 + assert f"Updating batch {batch_id} status to cancelled after cancel" in caplog.messages diff --git a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py index 4ce31cbb449..793921255ec 100644 --- a/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py +++ b/tests/unit/proxy/openai_files_endpoint/test_files_endpoint.py @@ -2145,6 +2145,54 @@ def test_get_file_content_streams_openai_direct_path( proxy_logging_obj.post_call_failure_hook.assert_not_called() +@respx.mock +def test_get_file_content_forwards_upstream_download_headers(monkeypatch, llm_router: Router): + import litellm.proxy.proxy_server as ps + from litellm.proxy._types import LitellmUserRoles + + setup_proxy_logging_object(monkeypatch, llm_router) + monkeypatch.setattr("litellm.proxy.proxy_server.master_key", None) + monkeypatch.setattr("litellm.proxy.proxy_server.prisma_client", None) + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") + + body: Final = b'{"prompt": "Hello", "completion": "Hi"}' + upstream_filename: Final = "mydata.jsonl" + upstream_request_id: Final = "req_upstream_123" + upstream_route: Final = respx.get("https://api.openai.com/v1/files/file-abc123/content").mock( + return_value=httpx.Response( + status_code=200, + content=body, + headers={ + "content-type": "application/octet-stream", + "content-length": str(len(body)), + "content-disposition": f'attachment; filename="{upstream_filename}"', + "x-request-id": upstream_request_id, + }, + ) + ) + + app.dependency_overrides[ps.user_api_key_auth] = lambda: UserAPIKeyAuth( + api_key="test-key", + user_role=LitellmUserRoles.PROXY_ADMIN, + user_id="test-user", + ) + try: + response: Final = client.get( + "/v1/files/file-abc123/content", + headers={"Authorization": "Bearer test-key"}, + ) + finally: + app.dependency_overrides.pop(ps.user_api_key_auth, None) + + assert response.status_code == 200, response.text + assert upstream_route.call_count == 1 + assert response.content == body + assert response.headers["content-type"].startswith("application/octet-stream") + assert int(response.headers["content-length"]) == len(response.content) + assert upstream_filename in response.headers["content-disposition"] + assert response.headers["x-request-id"] == upstream_request_id + + def test_get_file_content_routed_provider_skips_streaming_when_resolved_provider_is_not_supported( mocker: MockerFixture, monkeypatch, llm_router: Router ): diff --git a/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_assembly_passthrough_route.py b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_assembly_passthrough_route.py new file mode 100644 index 00000000000..a862d0fa108 --- /dev/null +++ b/tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_assembly_passthrough_route.py @@ -0,0 +1,172 @@ +import asyncio +import json +from collections.abc import Mapping +from datetime import datetime +from typing import Final, cast + +import httpx +import pytest +import respx +from fastapi import Request, Response +from starlette.types import Message + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import assemblyai_proxy_route +from litellm.types.utils import StandardLoggingPayload + +_KEY: Final = "synthetic-assemblyai-key" +_UPSTREAM: Final = "https://api.assemblyai.com/v2/transcript" +_AUDIO_URL: Final = "https://assembly.ai/wildfires.mp3" + + +def _is_transcript_log(payload: StandardLoggingPayload, transcript_id: str) -> bool: + response: Final = payload["response"] + return isinstance(response, dict) and response.get("id") == transcript_id + + +class _SuccessRecorder(CustomLogger): + def __init__(self, transcript_id: str, loop: asyncio.AbstractEventLoop) -> None: + super().__init__() + self.transcript_id: Final = transcript_id + self.loop: Final = loop + self.logged: Final = asyncio.Event() + self.payloads: tuple[StandardLoggingPayload, ...] = () + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + payload: Final = cast(StandardLoggingPayload, kwargs["standard_logging_object"]) + self.payloads = (*self.payloads, payload) + if _is_transcript_log(payload, self.transcript_id): + self.loop.call_soon_threadsafe(self.logged.set) + + +def _request(method: str, path: str, body: bytes = b"") -> Request: + scope: Final = { + "type": "http", + "http_version": "1.1", + "method": method, + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json")], + "client": ("127.0.0.1", 51234), + "server": ("proxy.local", 4000), + "state": {}, + } + + async def receive() -> Message: + return {"type": "http.request", "body": body, "more_body": False} + + return Request(scope, receive) + + +def _install_recorder(monkeypatch: pytest.MonkeyPatch, transcript_id: str) -> _SuccessRecorder: + recorder: Final = _SuccessRecorder(transcript_id, asyncio.get_running_loop()) + monkeypatch.setenv("ASSEMBLYAI_API_KEY", _KEY) + monkeypatch.setattr(litellm, "_async_success_callback", [recorder]) + monkeypatch.setattr(litellm, "success_callback", []) + return recorder + + +async def _wait_for_success_log(recorder: _SuccessRecorder) -> StandardLoggingPayload: + await asyncio.wait_for(recorder.logged.wait(), 30) + logged: Final = tuple( + payload for payload in recorder.payloads if _is_transcript_log(payload, recorder.transcript_id) + ) + assert len(logged) == 1, f"AssemblyAI success log for {recorder.transcript_id} emitted {len(logged)} times" + return logged[0] + + +@pytest.mark.asyncio +async def test_assemblyai_transcribe_create_poll_delete( + respx_mock: respx.MockRouter, httpx_transport: None, monkeypatch: pytest.MonkeyPatch +) -> None: + recorder: Final = _install_recorder(monkeypatch, "tr_1") + create_route: Final = respx_mock.post(_UPSTREAM).mock( + return_value=httpx.Response(200, json={"id": "tr_1", "status": "queued", "audio_url": _AUDIO_URL}) + ) + poll_route: Final = respx_mock.get(f"{_UPSTREAM}/tr_1").mock( + return_value=httpx.Response(200, json={"id": "tr_1", "status": "completed", "text": "fires near town"}) + ) + delete_route: Final = respx_mock.delete(f"{_UPSTREAM}/tr_1").mock( + return_value=httpx.Response(200, json={"id": "tr_1", "status": "deleted"}) + ) + admin: Final = UserAPIKeyAuth(api_key="sk-master", user_role=LitellmUserRoles.PROXY_ADMIN) + create_body: Final = {"audio_url": _AUDIO_URL, "speech_models": ["universal-2"]} + + create: Final = await assemblyai_proxy_route( + endpoint="v2/transcript", + request=_request("POST", "/assemblyai/v2/transcript", json.dumps(create_body).encode()), + fastapi_response=Response(), + user_api_key_dict=admin, + ) + await _wait_for_success_log(recorder) + poll: Final = await assemblyai_proxy_route( + endpoint="v2/transcript/tr_1", + request=_request("GET", "/assemblyai/v2/transcript/tr_1"), + fastapi_response=Response(), + user_api_key_dict=admin, + ) + delete: Final = await assemblyai_proxy_route( + endpoint="v2/transcript/tr_1", + request=_request("DELETE", "/assemblyai/v2/transcript/tr_1"), + fastapi_response=Response(), + user_api_key_dict=admin, + ) + + assert isinstance(create, Response) and isinstance(poll, Response) and isinstance(delete, Response) + assert create.status_code == 200 + assert json.loads(create.body)["id"] == "tr_1" + assert poll.status_code == 200 + assert json.loads(poll.body)["status"] == "completed" + assert delete.status_code == 200 + assert json.loads(delete.body)["status"] == "deleted" + assert create_route.call_count == 1 + assert json.loads(create_route.calls[0].request.content) == create_body + assert delete_route.call_count == 1 + client_poll: Final = poll_route.calls[-1].request + assert client_poll.headers["authorization"] == _KEY + assert create_route.calls[0].request.headers["authorization"] == _KEY + assert delete_route.calls[0].request.headers["authorization"] == _KEY + + +@pytest.mark.asyncio +async def test_assemblyai_transcribe_with_non_admin_key_logs_key_identity( + respx_mock: respx.MockRouter, httpx_transport: None, monkeypatch: pytest.MonkeyPatch +) -> None: + recorder: Final = _install_recorder(monkeypatch, "tr_9") + create_route: Final = respx_mock.post(_UPSTREAM).mock( + return_value=httpx.Response(200, json={"id": "tr_9", "status": "queued", "audio_url": _AUDIO_URL}) + ) + respx_mock.get(f"{_UPSTREAM}/tr_9").mock( + return_value=httpx.Response(200, json={"id": "tr_9", "status": "completed", "text": "fires near town"}) + ) + non_admin: Final = UserAPIKeyAuth( + api_key="hashed-non-admin-key", + token="hashed-non-admin-key", + user_id="non-admin-user", + team_id="non-admin-team", + user_role=LitellmUserRoles.INTERNAL_USER, + ) + + response: Final = await assemblyai_proxy_route( + endpoint="v2/transcript", + request=_request("POST", "/assemblyai/v2/transcript", json.dumps({"audio_url": _AUDIO_URL}).encode()), + fastapi_response=Response(), + user_api_key_dict=non_admin, + ) + logged: Final = await _wait_for_success_log(recorder) + + assert isinstance(response, Response) + assert response.status_code == 200, response.body + assert create_route.call_count == 1 + assert create_route.calls[0].request.headers["authorization"] == _KEY + assert logged["metadata"]["user_api_key_hash"] == "hashed-non-admin-key" + assert logged["metadata"]["user_api_key_user_id"] == "non-admin-user" + assert logged["metadata"]["user_api_key_team_id"] == "non-admin-team" + assert logged["custom_llm_provider"] == "assemblyai" diff --git a/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py b/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py index f6f63aca71d..9c372380749 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py +++ b/tests/unit/proxy/pass_through_endpoints/test_vertex_ai_live_passthrough.py @@ -1,17 +1,28 @@ -from collections.abc import Sequence +import asyncio +import json +import uuid +from collections.abc import Mapping, Sequence from datetime import datetime +from typing import Final, cast from unittest.mock import MagicMock, patch +import httpx import litellm import pytest +import respx +from fastapi import Request, Response +from starlette.types import Message from typing_extensions import NotRequired, ReadOnly, TypedDict +from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy.pass_through_endpoints.llm_provider_handlers.vertex_ai_live_passthrough_logging_handler import ( VertexAILivePassthroughLoggingHandler, ) +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import vertex_proxy_route from litellm.proxy.pass_through_endpoints.success_handler import PassThroughEndpointLogging -from litellm.types.utils import CostBreakdown, LlmProviders, Usage +from litellm.types.utils import CostBreakdown, LlmProviders, StandardLoggingPayload, Usage class _LiveTurn(TypedDict): @@ -887,3 +898,95 @@ class TestVertexAILivePassthroughErrorHandling: assert "result" in result assert "kwargs" in result + + +class _SuccessRecorder(CustomLogger): + def __init__(self, api_base: str, loop: asyncio.AbstractEventLoop) -> None: + super().__init__() + self.api_base: Final = api_base + self.loop: Final = loop + self.logged: Final = asyncio.Event() + self.payloads: tuple[StandardLoggingPayload, ...] = () + + async def async_log_success_event( + self, kwargs: Mapping[str, object], response_obj: object, start_time: datetime, end_time: datetime + ) -> None: + payload: Final = cast(StandardLoggingPayload, kwargs["standard_logging_object"]) + self.payloads = (*self.payloads, payload) + if payload["api_base"] == self.api_base: + self.loop.call_soon_threadsafe(self.logged.set) + + +def _generate_content_endpoint(project: str) -> str: + return f"v1/projects/{project}/locations/us-central1/publishers/google/models/gemini-2.0-flash:generateContent" + + +def _vertex_request(endpoint: str, body: bytes) -> Request: + path: Final = f"/vertex_ai/{endpoint}" + scope: Final = { + "type": "http", + "http_version": "1.1", + "method": "POST", + "scheme": "http", + "path": path, + "raw_path": path.encode(), + "root_path": "", + "query_string": b"", + "headers": [(b"content-type", b"application/json"), (b"authorization", b"Bearer client-google-token")], + "client": ("127.0.0.1", 51234), + "server": ("proxy.local", 4000), + "state": {}, + } + + async def receive() -> Message: + return {"type": "http.request", "body": body, "more_body": False} + + return Request(scope, receive) + + +@pytest.mark.asyncio +async def test_vertex_ai_generate_content_spendlog( + respx_mock: respx.MockRouter, httpx_transport: None, monkeypatch: pytest.MonkeyPatch +) -> None: + endpoint: Final = _generate_content_endpoint(f"p-{uuid.uuid4().hex}") + upstream_url: Final = f"https://us-central1-aiplatform.googleapis.com/{endpoint}" + recorder: Final = _SuccessRecorder(upstream_url, asyncio.get_running_loop()) + monkeypatch.setattr(litellm, "_async_success_callback", [recorder]) + monkeypatch.setattr(litellm, "success_callback", []) + contents: Final = [{"role": "user", "parts": [{"text": "hi"}]}] + route: Final = respx_mock.post(upstream_url).mock( + return_value=httpx.Response( + 200, + json={ + "candidates": [ + {"content": {"role": "model", "parts": [{"text": "hello vertex"}]}, "finishReason": "STOP"} + ], + "usageMetadata": {"promptTokenCount": 9, "candidatesTokenCount": 6, "totalTokenCount": 15}, + }, + ) + ) + + response: Final = await vertex_proxy_route( + endpoint=endpoint, + request=_vertex_request(endpoint, json.dumps({"contents": contents}).encode()), + fastapi_response=Response(), + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-token", user_role=LitellmUserRoles.PROXY_ADMIN), + ) + assert isinstance(response, Response) + assert response.status_code == 200 + call_id: Final = response.headers.get("x-litellm-call-id") + assert call_id + await asyncio.wait_for(recorder.logged.wait(), 30) + assert route.call_count == 1 + outbound: Final = route.calls[0].request + assert outbound.headers["authorization"] == "Bearer client-google-token" + assert json.loads(outbound.content)["contents"] == contents + matching: Final = tuple(payload for payload in recorder.payloads if payload["api_base"] == upstream_url) + assert len(matching) == 1, recorder.payloads + logged: Final = matching[0] + assert logged["id"] == call_id + assert logged["response_cost"] > 0 + assert "gemini" in logged["model"] + assert logged["custom_llm_provider"] == "vertex_ai" + assert logged["prompt_tokens"] == 9 + assert logged["completion_tokens"] == 6 diff --git a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py index 0df6eecd4b9..c5bb1ed5b61 100644 --- a/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/unit/proxy/spend_tracking/test_spend_tracking_utils.py @@ -8,18 +8,21 @@ from typing import Any, Final, cast from unittest.mock import AsyncMock, MagicMock, patch import pytest -from typing_extensions import ReadOnly, TypedDict +from pydantic import TypeAdapter +from typing_extensions import NotRequired, ReadOnly, TypedDict import litellm import litellm.constants as litellm_constants import litellm.proxy.spend_tracking.spend_tracking_utils as spend_tracking_utils from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, LITTELM_CLI_SERVICE_ACCOUNT_NAME, LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, MAX_SPEND_LOG_MODEL_NAME_LENGTH, REDACTED_BY_LITELM_STRING, SESSION_ID_OMITTED_METADATA_KEY, UNKNOWN_MODEL_SPEND_LOG_MODEL from litellm.litellm_core_utils.litellm_logging import StandardLoggingPayloadSetup +from litellm.llms.base_llm.ocr.transformation import OCRResponse, OCRUsageInfo from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy._types import SpendLogsPayload, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import LiteLLMProxyRequestSetup from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.proxy.spend_tracking.spend_tracking_utils import ( + _extract_usage_for_ocr_call, _get_messages_for_spend_logs_payload, _get_proxy_server_request_for_spend_logs_payload, _get_request_duration_ms, @@ -5805,3 +5808,236 @@ def test_untrusted_agent_label_cannot_replace_verified_billing_identity(billing_ ) assert payload["agent_id"] == "header-selected-agent" assert payload["billing_agent_id"] == billing_agent + + +class _OcrUsageInfoDict(TypedDict, total=False): + pages_processed: ReadOnly[int] + doc_size_bytes: ReadOnly[int] + + +class _OcrResponseDict(TypedDict): + id: ReadOnly[str] + object: ReadOnly[str] + model: ReadOnly[str] + usage_info: ReadOnly[NotRequired[_OcrUsageInfoDict]] + + +class _TokenUsageDict(TypedDict): + prompt_tokens: ReadOnly[int] + completion_tokens: ReadOnly[int] + total_tokens: ReadOnly[int] + + +class _CompletionResponseDict(TypedDict): + id: ReadOnly[str] + object: ReadOnly[str] + model: ReadOnly[str] + usage: ReadOnly[_TokenUsageDict] + + +class _LoggingMetadata(TypedDict, total=False): + user_api_key_user_id: ReadOnly[str] + user_api_key_team_id: ReadOnly[str] + + +class _LoggingLitellmParams(TypedDict, total=False): + metadata: ReadOnly[_LoggingMetadata] + + +class _LoggingKwargs(TypedDict): + model: ReadOnly[str] + call_type: ReadOnly[str] + litellm_params: ReadOnly[_LoggingLitellmParams] + response_cost: ReadOnly[float] + + +class _AdditionalUsageValues(TypedDict, total=False): + pages_processed: ReadOnly[int | None] + doc_size_bytes: ReadOnly[int | None] + + +class _SpendLogMetadata(TypedDict): + additional_usage_values: ReadOnly[_AdditionalUsageValues] + + +_SPEND_LOG_METADATA: Final = TypeAdapter(_SpendLogMetadata) +_OCR_LOGGED_AT: Final = datetime.datetime(2026, 1, 1, tzinfo=timezone.utc) +_OCR_RESPONSE_COST: Final = 0.05 + + +def _ocr_logging_kwargs(call_type: str = "ocr", metadata: _LoggingMetadata | None = None) -> _LoggingKwargs: + return _LoggingKwargs( + model="test-ocr-model", + call_type=call_type, + litellm_params=_LoggingLitellmParams() if metadata is None else _LoggingLitellmParams(metadata=metadata), + response_cost=_OCR_RESPONSE_COST, + ) + + +def _ocr_payload( + kwargs: _LoggingKwargs, response_obj: _OcrResponseDict | _CompletionResponseDict | OCRResponse +) -> SpendLogsPayload: + return get_logging_payload( + kwargs=dict(kwargs), + response_obj=response_obj, + start_time=_OCR_LOGGED_AT, + end_time=_OCR_LOGGED_AT, + ) + + +def _additional_usage_values(payload: SpendLogsPayload) -> _AdditionalUsageValues: + return _SPEND_LOG_METADATA.validate_json(payload["metadata"])["additional_usage_values"] + + +class TestExtractUsageForOCRCall: + def test_extract_usage_from_dict(self) -> None: + response_obj_dict: Final = {"usage_info": _OcrUsageInfoDict(pages_processed=5)} + + usage: Final = _extract_usage_for_ocr_call(response_obj_dict, response_obj_dict) + + assert usage == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "pages_processed": 5} + + def test_extract_usage_from_pydantic_model(self) -> None: + response_obj: Final = OCRResponse( + pages=[], + model="test-ocr-model", + usage_info=OCRUsageInfo(pages_processed=10, doc_size_bytes=1024), + ) + + usage: Final = _extract_usage_for_ocr_call(response_obj, response_obj.model_dump()) + + assert usage["prompt_tokens"] == 0 + assert usage["completion_tokens"] == 0 + assert usage["total_tokens"] == 0 + assert usage["pages_processed"] == 10 + assert usage["doc_size_bytes"] == 1024 + + def test_extract_usage_with_object_attributes(self) -> None: + class _SimpleUsageInfo: + def __init__(self, pages_processed: int) -> None: + self.pages_processed = pages_processed + + class _SimpleOCRResponse: + def __init__(self) -> None: + self.usage_info = _SimpleUsageInfo(pages_processed=3) + + usage: Final = _extract_usage_for_ocr_call(_SimpleOCRResponse(), {}) + + assert usage == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "pages_processed": 3} + + def test_extract_usage_missing_usage_info(self) -> None: + assert _extract_usage_for_ocr_call({}, {}) == {} + + def test_extract_usage_empty_usage_info(self) -> None: + usage: Final = _extract_usage_for_ocr_call({"usage_info": {}}, {"usage_info": {}}) + + assert usage == {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0, "pages_processed": 0} + + +class TestGetLoggingPayloadOCR: + def test_ocr_call_with_dict_response(self) -> None: + payload: Final = _ocr_payload( + _ocr_logging_kwargs(), + { + "id": "ocr-test-123", + "object": "ocr", + "model": "test-ocr-model", + "usage_info": {"pages_processed": 7, "doc_size_bytes": 2048}, + }, + ) + + assert payload["call_type"] == "ocr" + assert payload["request_id"] == "ocr-test-123" + assert payload["prompt_tokens"] == 0 + assert payload["completion_tokens"] == 0 + assert payload["total_tokens"] == 0 + assert payload["spend"] == _OCR_RESPONSE_COST + assert _additional_usage_values(payload)["pages_processed"] == 7 + assert _additional_usage_values(payload)["doc_size_bytes"] == 2048 + + def test_aocr_call_with_pydantic_response(self) -> None: + payload: Final = _ocr_payload( + _ocr_logging_kwargs(call_type="aocr"), + OCRResponse(pages=[], model="test-ocr-model", usage_info=OCRUsageInfo(pages_processed=12)), + ) + + assert payload["call_type"] == "aocr" + assert payload["prompt_tokens"] == 0 + assert payload["completion_tokens"] == 0 + assert payload["total_tokens"] == 0 + assert payload["spend"] == _OCR_RESPONSE_COST + assert _additional_usage_values(payload)["pages_processed"] == 12 + + def test_ocr_call_missing_usage_info(self) -> None: + payload: Final = _ocr_payload( + _ocr_logging_kwargs(), + {"id": "ocr-test-789", "object": "ocr", "model": "test-ocr-model"}, + ) + + assert payload["call_type"] == "ocr" + assert payload["prompt_tokens"] == 0 + assert payload["completion_tokens"] == 0 + assert payload["total_tokens"] == 0 + assert payload["spend"] == _OCR_RESPONSE_COST + assert "pages_processed" not in _additional_usage_values(payload) + + def test_ocr_call_with_zero_pages(self) -> None: + payload: Final = _ocr_payload( + _ocr_logging_kwargs(), + { + "id": "ocr-test-000", + "object": "ocr", + "model": "test-ocr-model", + "usage_info": {"pages_processed": 0}, + }, + ) + + assert payload["call_type"] == "ocr" + assert payload["prompt_tokens"] == 0 + assert payload["completion_tokens"] == 0 + assert payload["total_tokens"] == 0 + assert payload["spend"] == _OCR_RESPONSE_COST + assert _additional_usage_values(payload)["pages_processed"] == 0 + + def test_non_ocr_call_uses_token_based_usage(self) -> None: + payload: Final = _ocr_payload( + _LoggingKwargs( + model="gpt-5.5", call_type="completion", litellm_params=_LoggingLitellmParams(), response_cost=0.02 + ), + { + "id": "completion-test-123", + "object": "chat.completion", + "model": "gpt-5.5", + "usage": {"prompt_tokens": 50, "completion_tokens": 100, "total_tokens": 150}, + }, + ) + + assert payload["call_type"] == "completion" + assert payload["prompt_tokens"] == 50 + assert payload["completion_tokens"] == 100 + assert payload["total_tokens"] == 150 + assert payload["spend"] == 0.02 + assert "pages_processed" not in _additional_usage_values(payload) + + def test_ocr_with_metadata(self) -> None: + payload: Final = _ocr_payload( + _ocr_logging_kwargs( + metadata=_LoggingMetadata(user_api_key_user_id="test-user", user_api_key_team_id="test-team") + ), + { + "id": "ocr-metadata-test", + "object": "ocr", + "model": "test-ocr-model", + "usage_info": {"pages_processed": 5, "doc_size_bytes": 1024}, + }, + ) + + assert payload["call_type"] == "ocr" + assert payload["user"] == "test-user" + assert payload["team_id"] == "test-team" + assert payload["prompt_tokens"] == 0 + assert payload["completion_tokens"] == 0 + assert payload["total_tokens"] == 0 + assert payload["spend"] == _OCR_RESPONSE_COST + assert _additional_usage_values(payload)["pages_processed"] == 5 + assert _additional_usage_values(payload)["doc_size_bytes"] == 1024 diff --git a/tests/unit/test_main.py b/tests/unit/test_main.py index 2d0d8740e12..dd7073bb381 100644 --- a/tests/unit/test_main.py +++ b/tests/unit/test_main.py @@ -25,6 +25,8 @@ import litellm from litellm import acompletion, completion from litellm import main as litellm_main from litellm.constants import CONTROL_OPTIONS_KEY +from litellm.caching.base_cache import BaseCache +from litellm.caching.caching import Cache from litellm.integrations.custom_logger import CustomLogger from litellm.integrations.custom_prompt_management import CustomPromptManagement from litellm.litellm_core_utils.core_helpers import get_litellm_metadata_from_kwargs @@ -33,7 +35,7 @@ from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.prompt_templates.factory import anthropic_messages_pt from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.types.litellm_params import ControlOptions -from litellm.types.llms.openai import AllMessageValues +from litellm.types.llms.openai import AllMessageValues, HttpxBinaryResponseContent from litellm.types.prompts.init_prompts import PromptSpec from litellm.types.utils import Delta, ModelResponseStream, StandardCallbackDynamicParams, StreamingChoices, Usage @@ -5831,3 +5833,148 @@ def test_completion_openai_metadata(monkeypatch, enable_preview_features): } else: assert "metadata" not in mock_completion.call_args.kwargs + + +AZURE_TTS_BASE: Final = "https://tts.example.azure.com" +SPEECH_INPUT: Final = "the quick brown fox jumped over the lazy dogs" + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_speech_azure_returns_binary_audio( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync_mode: bool +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post( + url__regex=rf"{AZURE_TTS_BASE}/openai/deployments/tts/audio/speech\?api-version=.+" + ).mock(return_value=httpx.Response(200, content=b"ID3-fake-mp3")) + + speech_kwargs: Final = { + "model": "azure/tts", + "input": SPEECH_INPUT, + "voice": "alloy", + "api_base": AZURE_TTS_BASE, + "api_key": "fake-key", + "max_retries": 1, + "timeout": 60, + } + response: Final = litellm.speech(**speech_kwargs) if sync_mode else await litellm.aspeech(**speech_kwargs) + + assert route.call_count == 1 + assert route.calls[0].request.headers["api-key"] == "fake-key" + assert json.loads(route.calls[0].request.content) == {"model": "tts", "input": SPEECH_INPUT, "voice": "alloy"} + assert isinstance(response, HttpxBinaryResponseContent) + assert response.content == b"ID3-fake-mp3" + + +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_speech_openai_returns_binary_audio( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, sync_mode: bool +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/audio/speech").mock( + return_value=httpx.Response(200, content=b"ID3-fake-mp3") + ) + + speech_kwargs: Final = { + "model": "openai/tts-1", + "input": SPEECH_INPUT, + "voice": "alloy", + "api_key": "fake-key", + "max_retries": 1, + "timeout": 60, + } + response: Final = litellm.speech(**speech_kwargs) if sync_mode else await litellm.aspeech(**speech_kwargs) + + assert route.call_count == 1 + assert route.calls[0].request.headers["authorization"] == "Bearer fake-key" + assert json.loads(route.calls[0].request.content) == {"model": "tts-1", "input": SPEECH_INPUT, "voice": "alloy"} + assert isinstance(response, HttpxBinaryResponseContent) + assert response.content == b"ID3-fake-mp3" + + +class _SignallingCache(Cache): + def __init__(self, loop: asyncio.AbstractEventLoop) -> None: + super().__init__() + self.loop: Final = loop + self.written: Final = asyncio.Event() + + async def async_add_cache( + self, result: object, dynamic_cache_object: BaseCache | None = None, **kwargs: object + ) -> None: + await super().async_add_cache(result, dynamic_cache_object=dynamic_cache_object, **kwargs) + self.loop.call_soon_threadsafe(self.written.set) + + +GETTYSBURG_WAV: Final = ("gettysburg.wav", b"RIFF\x00\x00\x00\x00WAVE-gettysburg", "audio/wav") +EAGLE_WAV: Final = ("eagle.wav", b"RIFF\x00\x00\x00\x00WAVE-eagle", "audio/wav") + + +async def test_transcription_caching_hit_same_file_miss_different_file( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + cache: Final = _SignallingCache(asyncio.get_running_loop()) + monkeypatch.setattr(litellm, "cache", cache) + route: Final = respx_mock.post("https://api.openai.com/v1/audio/transcriptions").mock( + side_effect=[ + httpx.Response(200, json={"text": "gettysburg transcript"}), + httpx.Response(200, json={"text": "eagle transcript"}), + ] + ) + + response_1: Final = await litellm.atranscription(model="openai/whisper-1", file=GETTYSBURG_WAV, api_key="fake-key") + await asyncio.wait_for(cache.written.wait(), 30) + + response_2: Final = await litellm.atranscription(model="openai/whisper-1", file=GETTYSBURG_WAV, api_key="fake-key") + assert response_2._hidden_params["cache_hit"] is True + assert response_2.text == response_1.text == "gettysburg transcript" + + response_3: Final = await litellm.atranscription(model="openai/whisper-1", file=EAGLE_WAV, api_key="fake-key") + assert response_3._hidden_params.get("cache_hit") is not True + assert response_3.text == "eagle transcript" + assert route.call_count == 2 + + +async def test_whisper_log_pre_call_fires_once(respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + respx_mock.post("https://api.openai.com/v1/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) + + class _PreCallRecorder(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.models: tuple[str, ...] = () + + def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: + self.models = (*self.models, model) + + recorder: Final = _PreCallRecorder() + monkeypatch.setattr(litellm, "callbacks", [recorder]) + + await litellm.atranscription(model="openai/whisper-1", file=GETTYSBURG_WAV, api_key="fake-key") + + assert recorder.models == ("whisper-1",) + + +@pytest.mark.parametrize("model", ["gpt-4o-mini-transcribe", "gpt-4o-transcribe", "whisper-1"]) +async def test_transcription_model_names_pass_through( + respx_mock: respx.MockRouter, monkeypatch: pytest.MonkeyPatch, model: str +): + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + route: Final = respx_mock.post("https://api.openai.com/v1/audio/transcriptions").mock( + return_value=httpx.Response(200, json={"text": "hello"}) + ) + + response: Final = await litellm.atranscription( + model=f"openai/{model}", + file=GETTYSBURG_WAV, + api_key="fake-key", + response_format="json", + ) + + assert response._hidden_params["model"] == model + assert response._hidden_params["custom_llm_provider"] == "openai" + assert response.text == "hello" + assert route.call_count == 1 + assert f'name="model"\r\n\r\n{model}\r\n'.encode() in route.calls[0].request.content From 3a3b8cad809f1a2d530d7572cd770dc35b796d30 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:04:07 -0700 Subject: [PATCH 31/65] feat(ui): add TypeSafe and Strands Decider to the Add Model provider list (#45335) * feat(ui): add TypeSafe and Strands Decider to the Add Model provider list * test(public_endpoints): type the provider fields helper against the route's model --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../provider_create_fields.json | 56 ++++++++++++++++++ .../public_endpoints/test_public_endpoints.py | 51 +++++++++++++++- .../public/assets/logos/typesafe.png | Bin 0 -> 4311 bytes .../components/provider_info_helpers.test.tsx | 49 +++++++++++++++ .../src/components/provider_info_helpers.tsx | 10 ++++ 5 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 ui/litellm-dashboard/public/assets/logos/typesafe.png diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index 2dd9f50bec8..ae07d5d6bd9 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3189,6 +3189,34 @@ ], "default_model_placeholder": "soniox/stt-async-v5" }, + { + "provider": "StrandsDecider", + "provider_display_name": "Strands Decider", + "litellm_provider": "strands_decider", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "http://localhost:8000", + "tooltip": "URL of your self-hosted Strands Decider server", + "required": true, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": false, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "strands_decider/strands-decider-2B-hobson-v19" + }, { "provider": "Tencent", "provider_display_name": "Tencent", @@ -3319,6 +3347,34 @@ ], "default_model_placeholder": "gpt-3.5-turbo" }, + { + "provider": "TypeSafe", + "provider_display_name": "TypeSafe", + "litellm_provider": "typesafe", + "credential_fields": [ + { + "key": "api_base", + "label": "API Base", + "placeholder": "https://api.typesafe.ai", + "tooltip": null, + "required": false, + "field_type": "text", + "options": null, + "default_value": null + }, + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "typesafe/jev-latest" + }, { "provider": "V0", "provider_display_name": "V0", diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 30538c1167f..7e20b5f8c3c 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -9,6 +9,7 @@ import pytest from fastapi import FastAPI from fastapi.testclient import TestClient +from pydantic import TypeAdapter from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.public_endpoints import router @@ -16,6 +17,7 @@ from litellm.router_strategy.complexity_router.fuse_presets import get_fuse_pres from litellm.types.proxy.management_endpoints.model_management_endpoints import ( ModelGroupInfoProxy, ) +from litellm.types.proxy.public_endpoints.public_endpoints import ProviderCreateInfo from litellm.types.utils import LlmProviders @@ -402,6 +404,53 @@ def test_tencent_provider_fields(): assert fields_by_key["api_base"]["required"] is False +def _decisions_provider_entry(provider: str) -> ProviderCreateInfo: + app_instance: Final = FastAPI() + app_instance.include_router(router) + test_client: Final = TestClient(app_instance) + + response: Final = test_client.get("/public/providers/fields") + assert response.status_code == 200 + providers: Final = TypeAdapter(list[ProviderCreateInfo]).validate_python(response.json()) + entry: Final = next((p for p in providers if p.provider == provider), None) + assert entry is not None, f"{provider} provider entry not found" + return entry + + +def test_typesafe_provider_fields(): + typesafe: Final = _decisions_provider_entry("TypeSafe") + + assert typesafe.provider_display_name == "TypeSafe" + assert typesafe.litellm_provider == LlmProviders.TYPESAFE.value + assert typesafe.default_model_placeholder is not None + assert typesafe.default_model_placeholder.startswith("typesafe/") + + fields_by_key: Final = {f.key: f for f in typesafe.credential_fields} + + assert fields_by_key["api_key"].required is True + assert fields_by_key["api_key"].field_type == "password" + + assert fields_by_key["api_base"].required is False + assert fields_by_key["api_base"].field_type == "text" + + +def test_strands_decider_provider_fields(): + strands: Final = _decisions_provider_entry("StrandsDecider") + + assert strands.provider_display_name == "Strands Decider" + assert strands.litellm_provider == LlmProviders.STRANDS_DECIDER.value + assert strands.default_model_placeholder is not None + assert strands.default_model_placeholder.startswith("strands_decider/") + + fields_by_key: Final = {f.key: f for f in strands.credential_fields} + + assert fields_by_key["api_base"].required is True + assert fields_by_key["api_base"].field_type == "text" + + assert fields_by_key["api_key"].required is False + assert fields_by_key["api_key"].field_type == "password" + + ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( { "a2a", @@ -436,12 +485,10 @@ ADD_MODEL_UNLISTED_PROVIDERS: Final = frozenset( "sagemaker_nova", "scaleway", "stability", - "strands_decider", "synthetic", "tensormesh", "text-completion-inception", "transcribe", - "typesafe", "valkey", "xiaomi_mimo", "zai", diff --git a/ui/litellm-dashboard/public/assets/logos/typesafe.png b/ui/litellm-dashboard/public/assets/logos/typesafe.png new file mode 100644 index 0000000000000000000000000000000000000000..45107c7d09d7e53da800df531d92a5eaafd08f99 GIT binary patch literal 4311 zcmW+)c{J3G_nytp7$ekRe1?x@4BmJVq6R}VL$Yrnp-3f$q=}kA%uvh>X|tpV5!oWC zER!WAS>vrN*-4hsg7owK{c-Pop68rWyu_ngaibT}j>juD4IAW{@E$r%EHa{d>h z@ExV@c-!_)(R8$TwXr`HG3E4p%JKQMQ^S-~!%kLEP}l+8)Z)A5j0*;V_8OM|s_0^l zBp=yn5sq?paDjLz{>N#kFX+C$qlkr*y&@qH3EBSz6mlnf_l}5+vhj>^Id?MZ!g1OO zi0!H1v&VBQvjQOyls1KA=^8WgE$6>GUyG5p3D7{LSAW+W9zB0(R>C(H!To6o-ci~l!ht#x`b<3E07^Sc1rT+daRN0%!~Lk;#Q5{KlBZu>C& z!Z4Mgq;lq09hxsZSt@i;h!fHh0R>0GqtZxfX=&V$*B|g`-TkG$VBCOu22CmFM#`+) z?*Y!Op00V-8v@j0GphOvQmNlW*7W1B(6Fr!?y$@}t1(-q0+r5pJw3 zpIA%ppc3RaAe~YdJrHsh#wF_`vidnn=4$3Jw3#|Y%S{QO`2OwA| zB_rs=P``m^;aHa`FA%`Pu zStqaNZn-B^0U1GSj7WVW9-Z7r-V!{pgVD4o{p-Jbnr_6~5RoAJQNuMIXz9b1Oz|KU zy6gHWzU2Z}C@B3>zu;6*%b=Qc2Xdo$?o#>dK{W-k=Uzt4$7{J&Ll$AiB?#v`LwAD( zo>wx6nw*wb*R8a6mZ)|JZKg`XH(ca-EOh*foyL?mlr_>KkljSjb246A*3 zm52S{Bmo|kN%a>00pkaVTE)d5{)c04QzmQ?Z7foX1l;{lp%y=kMV^0_*8;p!JbN4{ z+F#t@C;~y$4S!4i&jGu5A<$YTbp+qZx}4`Mvl~Uw6s^EM9~PA)SE;(c!0zYWgH{^E za~7Op3CdFB`>OIaTu++OPdBs);Ae%xTO1(6L_ms6yJS%6*>mLw$J^i&LN60g(bRyc z#DnI5SRPWDz!{`BX0aJWrN7B753lphDG+A1cEE1tCLU7AfB9op||I&oab|{XVc=@YQB# z^4iT7ZQ%B?Js#JII%`D&ao~5{L@WjJ<(6?{`^7Hit~gt-b!bFmup*V2dD*se@d*Sn ze0bq3^6xHK$*(uJ+@1mu{jv;xDx*K{#M_h0NBl zgF*S>FXK28iStdT?t>_z`OJ+Hdmw72vJYfJR`pNr5e2xb=Fy*?9LqIN^YRnplv&;< zY$>*F>Jy3~yiJMg`v}HHC-=w^12Pim5Z>;Wr{@xhZzf)dQN}9ze-XPmX-QUq;sN5>b9$=0mA^Sz||?7}mIp8p2)Vk&L9{{*L^@ z7!he8-b6}AlRbXtH3g6Kspa1llsOkkcZ}y%g~qmi)IV$=CLH8}=C!VF@jM^?PWi@&-A>S2;&sF}+=;RE ztV67h|GYJ=$6j98PiHn>7xtX-`er?vl+PW&cBF!t^yQz<=I;rjkC|H;1+)r+9(!t! z_7`kN9T*W-?f#=ItOov@2hKfmn;6X)fhuMNqvn}9m5afDo}Su>kx-b;KiA6b!!tNR z$%niuBXu#;J}JQ$hgmSu=50LIo<=mi8Q_H#&xbmD9p8G~@KZ#DK{kjmH?lz#P zRSIJ=O*lr%ps0`0eKGP(b-Xs<~tV@w_)!^aUek7j>y|_8eC3Hfp#6kBXwpdLCG3MeY+b&)ZBp;5(`|)~_ z{3eKLc5SX+XT6FQ0z@|=O&J_0Wq009lSteB95E<)VzIII3LuN%OwRG}ybuwZtO(D@ z+aMZAI$2};Zh4?j`AI0?e)5<7=hG<7{r&^Q#eub!+&2K%O#Y~{BY*wlHZ?_Pk^r+ zBuWxEu9OM$zUY*L+lkVT6v6;oaq=Bd(FT9Z(Pya$Fa>1yafq~k%+TMS{j5Um);BRGR|3=zuez6gcwiEtaNajW06m6#!dF^3v8ddyCK{ z`DK{?Kg5SXy+L%7)OyTc>1EhO3hvXlvR>t7jQ9M?QA>AV`$n=qnsPI-z=eksBcRqK zSX?2VDkZ4rs}0&6c%|yTS(-GU@Ie0IowYnLo49xm+pE7|&yz14yS+m{DWYUy<={`- z!m)_NNkqUGhCTH7V{`f${i!i!P0oLtt0+>yx5NB?7wxMU{j+qZ0CywKH5cs@>?GZQ ze14f3aIwgh3XYEmTKHucV7SOt92`&e1oTT`Xg$9bUJSXX7L zcdpLpOebM}kD5IBtS6#i>A-t!DgGklb`7lL#rq7L*bx$Ojh zw25&sYhgb_0J`fYcB+kdWg<5ch^HGuXxc16_fJXD0sniwp*g;&xthzc#D-mOqti_b zy9hMV#OPJABgb88XM`LlMOpRO*Ci!X)bK>O3ab&Tj}WsAk{cF6&wLS(qwWneOa z37gU!Yl~-TukIoD#VYQTu`obzT_^j}52R$pUl(-9kesV!GXxCFzkVN(Kz4q$c$#9BRx}0wSbJ!5<0pdbA}5TT(o#| zGyd_q*2h)x>GhANiwZ)!2)kAQlXN|JRD`_;Kq zfAgQY=}IF=5k-j_{*=k}k3bxd8E*)+143*z?!kKSI0;YKKS@r1Rn_L4RG$1wa0ij= z1RfRvO|=NI>3wh8l~f=?X_`QKciGmdcs~)`6Z2fOBrK>qDi;E0nPx~E!B5L{d_Za?9zx@Np%~Lt`M91pp$PWTXKU$O^+7yjYass z`$LgjH2FJhnBkOb;ikTs+=i0I!dB~?EFY<6^V9?qR^jkJ$YGcxR@ku zi)|tGUTEUc4fRAT+jKRv*!n(6$%3fST@-gG3vY$l@b_BTl&a48=J(dw9hG1UoxBW* z2@SFyuuc+b+9VSmVx7$)tl0U&GrcC0F@U*L!-OFkx&vZ=X?ObiNH1-`ODfjC*s;gw z%>mZdXgH#$HD!$ftX>e2ye9&amQ?K$zb7J^E+X^$&_i4nd>p#6xDS@D*n=SjbxHHl z5z9?>-_&wmp;m^b#-SH|Ou6-nvj%E$*GLAv?Zq-qrOh%=pR^9hwPxjnVX_p*#dXgN z%aq%Osk|PmRoCBlqVyakbMC_naQY!_U7Jm{*to&Jb9Z7O4-9LV^NW*P^4p)*J(2b7 zd?lTIG^}C0icXx^Z2n-ay)r%cUvD<`%oP(uwYgo%?x6D59#%o5(g?=~lB@x4V@$Sz z2za}eT!r@yO||ChK7&5eBH@z_o@KBhe#2b!$_XW;Y;ag$oc_x0{7{V6_Eo)uA(9#i z!T|>&5`@#VE(fg`YAD_Rm@kaXgg!u}tW3I=?StDEYP-4$OHe%?J^6U(N3L+HhEG+K zIkEn@5d4bNJdSQlHn^G|?KY#d1CR{rt-2X_{e ABLDyZ literal 0 HcmV?d00001 diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx index 7cfdaf3275d..cecd6ce01bf 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.test.tsx @@ -214,6 +214,26 @@ describe("provider_info_helpers", () => { const { logo } = getProviderLogoAndName("tencent"); expect(logo).toContain("tencent"); }); + + it("should resolve the typesafe slug and TypeSafe enum key to the TypeSafe name and bundled logo", () => { + const fromSlug = getProviderLogoAndName("typesafe"); + expect(fromSlug.displayName).toBe(Providers.TypeSafe); + expect(fromSlug.logo).toContain("typesafe"); + + const fromEnumKey = getProviderLogoAndName("TypeSafe"); + expect(fromEnumKey.displayName).toBe(Providers.TypeSafe); + expect(fromEnumKey.logo).toBe(fromSlug.logo); + }); + + it("should resolve the strands_decider slug and StrandsDecider enum key to the Strands Decider name and bundled logo", () => { + const fromSlug = getProviderLogoAndName("strands_decider"); + expect(fromSlug.displayName).toBe("Strands Decider"); + expect(fromSlug.logo).toContain("strands"); + + const fromEnumKey = getProviderLogoAndName("StrandsDecider"); + expect(fromEnumKey.displayName).toBe(Providers.StrandsDecider); + expect(fromEnumKey.logo).toBe(fromSlug.logo); + }); }); describe("getPlaceholder", () => { @@ -318,6 +338,11 @@ describe("provider_info_helpers", () => { expect(getPlaceholder(Providers.Tencent)).toBe("tencent/deepseek-v4-pro"); }); + it("should return decision model placeholders for the TypeSafe and StrandsDecider dropdown keys", () => { + expect(getPlaceholder("TypeSafe")).toBe("typesafe/jev-latest"); + expect(getPlaceholder("StrandsDecider")).toBe("strands_decider/strands-decider-2B-hobson-v19"); + }); + it("should return default gpt-3.5-turbo placeholder for unknown provider", () => { expect(getPlaceholder("UnknownProvider" as any)).toBe("gpt-3.5-turbo"); }); @@ -421,6 +446,30 @@ describe("provider_info_helpers", () => { expect(getProviderModels("Sail" as Providers, modelMap)).toEqual(["sail/openai/gpt-oss-120b"]); }); + it("should list only typesafe decision models for the 'TypeSafe' provider key, not the OpenRouter-hosted one", () => { + const modelMap = { + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + "typesafe/jev-preview": { litellm_provider: "typesafe", mode: "evaluation" }, + "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + }; + expect(getProviderModels("TypeSafe" as Providers, modelMap)).toEqual([ + "typesafe/jev-latest", + "typesafe/jev-preview", + ]); + }); + + it("should list only strands_decider models for the 'StrandsDecider' provider key", () => { + const modelMap = { + "strands_decider/strands-decider-2B-hobson-v19": { litellm_provider: "strands_decider", mode: "evaluation" }, + "typesafe/jev-latest": { litellm_provider: "typesafe", mode: "evaluation" }, + "openrouter/typesafe/jev-1.13": { litellm_provider: "openrouter", mode: "evaluation" }, + }; + expect(getProviderModels("StrandsDecider" as Providers, modelMap)).toEqual([ + "strands_decider/strands-decider-2B-hobson-v19", + ]); + }); + it("should include bedrock converse but exclude standalone bedrock_mantle when called with 'Bedrock' provider key", () => { const modelMap = { "bedrock-base": { litellm_provider: "bedrock" }, diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 5ea693bea10..5a57fc23dde 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -55,9 +55,11 @@ import sapLogo from "../../public/assets/logos/sap.png"; import scxAiLogo from "../../public/assets/logos/scx_ai.svg"; import snowflakeLogo from "../../public/assets/logos/snowflake.svg"; import sonioxLogo from "../../public/assets/logos/soniox.svg"; +import strandsLogo from "../../public/assets/logos/strands.svg"; import tencentLogo from "../../public/assets/logos/tencent.svg"; import togetheraiLogo from "../../public/assets/logos/togetherai.svg"; import topazLogo from "../../public/assets/logos/topaz.svg"; +import typesafeLogo from "../../public/assets/logos/typesafe.png"; import v0Logo from "../../public/assets/logos/v0.svg"; import vercelLogo from "../../public/assets/logos/vercel.svg"; import vllmLogo from "../../public/assets/logos/vllm.png"; @@ -167,11 +169,13 @@ export enum Providers { SCX_AI = "SCX.ai", Snowflake = "Snowflake", Soniox = "Soniox", + StrandsDecider = "Strands Decider", TEXT_COMPLETION_CODESTRAL = "Text-Completion-Codestral", Tencent = "Tencent", TogetherAI = "TogetherAI", TOPAZ = "Topaz", Triton = "Triton", + TypeSafe = "TypeSafe", V0 = "V0", VERCEL_AI_GATEWAY = "Vercel Ai Gateway", Vertex_AI = "Vertex AI (Anthropic, Gemini, etc.)", @@ -287,11 +291,13 @@ export const provider_map: Record = { SCX_AI: "scx-ai", Snowflake: "snowflake", Soniox: "soniox", + StrandsDecider: "strands_decider", TEXT_COMPLETION_CODESTRAL: "text-completion-codestral", Tencent: "tencent", TogetherAI: "together_ai", TOPAZ: "topaz", Triton: "triton", + TypeSafe: "typesafe", V0: "v0", VERCEL_AI_GATEWAY: "vercel_ai_gateway", Vertex_AI: "vertex_ai", @@ -387,11 +393,13 @@ export const providerLogoMap: Partial> = { [Providers.SCX_AI]: scxAiLogo.src, [Providers.Snowflake]: snowflakeLogo.src, [Providers.Soniox]: sonioxLogo.src, + [Providers.StrandsDecider]: strandsLogo.src, [Providers.Tencent]: tencentLogo.src, [Providers.TEXT_COMPLETION_CODESTRAL]: mistralLogo.src, [Providers.TogetherAI]: togetheraiLogo.src, [Providers.TOPAZ]: topazLogo.src, [Providers.Triton]: nvidiaTritonLogo.src, + [Providers.TypeSafe]: typesafeLogo.src, [Providers.V0]: v0Logo.src, [Providers.VERCEL_AI_GATEWAY]: vercelLogo.src, [Providers.Vertex_AI]: googleLogo.src, @@ -457,7 +465,9 @@ const providerPlaceholderMap: Partial> = { [Providers.Sail]: "sail/openai/gpt-oss-120b", [Providers.SCX_AI]: "scx-ai/GLM-5.2", [Providers.Snowflake]: "snowflake/mistral-7b", + [Providers.StrandsDecider]: "strands_decider/strands-decider-2B-hobson-v19", [Providers.Tencent]: "tencent/deepseek-v4-pro", + [Providers.TypeSafe]: "typesafe/jev-latest", [Providers.Vertex_AI]: "gemini-pro", [Providers.VolcEngine]: "volcengine/", [Providers.Voyage]: "voyage/", From d0202ac364ea5e88fcf63d83634f47547677d685 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Thu, 8 Oct 2026 04:10:34 -0700 Subject: [PATCH 32/65] feat(mcp): support upstream OAuth client metadata identities (#45231) * feat(mcp): support upstream OAuth client metadata identities * fix(mcp): load client metadata on cold OAuth requests * fix(mcp): discover client metadata with configured OAuth endpoints * fix(mcp): preserve configured endpoints when discovery fails * fix(mcp): retain CIMD identities across refresh and catalog reloads * fix(mcp): reuse saved CIMD identity for token endpoint refresh * fix(mcp): keep dynamic client registration ahead of CIMD unless the deployment opts in A provider that advertises both a registration endpoint and client ID metadata documents now gets the registration flow the gateway used before, so a gateway on a private network keeps working against it; the metadata document identity applies when the provider offers no registration, or when general_settings.mcp_prefer_client_id_metadata_document is true. The saved CIMD refresh identity compares tokens as bytes so a non-ASCII refresh token cannot crash the token route. * chore(ui): regenerate dashboard API types for the new MCP general setting --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .github/workflows/test-unit-proxy-db.yml | 10 + .../proxy/_experimental/mcp_server/catalog.py | 1 + litellm/proxy/_experimental/mcp_server/db.py | 5 + .../mcp_server/discoverable_endpoints.py | 113 ++++- .../mcp_server/mcp_server_manager.py | 79 +++- .../_experimental/mcp_server/oauth_utils.py | 59 ++- .../authz_code_refresher.py | 28 +- .../outbound_credentials/oauth_token_store.py | 1 + .../per_user_oauth_store.py | 2 + .../outbound_credentials/v2_token_store.py | 2 + litellm/proxy/_lazy_features.py | 1 + litellm/proxy/_types.py | 4 + litellm/proxy/proxy_server.py | 1 + .../types/mcp_server/mcp_server_manager.py | 2 + .../test_authz_code_refresher.py | 82 +++- .../mcp_server/test_db_credentials.py | 63 ++- .../mcp_server/test_discoverable_endpoints.py | 417 +++++++++++++++++- .../mcp_server/test_mcp_server_manager.py | 159 +++++++ tests/unit/proxy/test__lazy_features.py | 14 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 5 + 20 files changed, 997 insertions(+), 51 deletions(-) diff --git a/.github/workflows/test-unit-proxy-db.yml b/.github/workflows/test-unit-proxy-db.yml index ba88464d652..3a989dc6651 100644 --- a/.github/workflows/test-unit-proxy-db.yml +++ b/.github/workflows/test-unit-proxy-db.yml @@ -141,6 +141,16 @@ jobs: dist: loadscope timeout: 15 + - test-group: mcp-oauth + test-path: >- + tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py + tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py + tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py + tests/unit/proxy/_experimental/mcp_server/outbound_credentials + workers: 4 + dist: loadscope + timeout: 15 + # ---- logging: split into 2 shards ---- - test-group: custom-logging test-path: >- diff --git a/litellm/proxy/_experimental/mcp_server/catalog.py b/litellm/proxy/_experimental/mcp_server/catalog.py index d2d6291233e..d0383be61a8 100644 --- a/litellm/proxy/_experimental/mcp_server/catalog.py +++ b/litellm/proxy/_experimental/mcp_server/catalog.py @@ -72,6 +72,7 @@ def _configuration_identity(server: MCPServer) -> str: "token_url", "registration_url", "authorization_response_iss_parameter_supported", + "client_id_metadata_document_supported", ) ) | (frozenset() if server.issuer_is_anchored else frozenset(("issuer",))), diff --git a/litellm/proxy/_experimental/mcp_server/db.py b/litellm/proxy/_experimental/mcp_server/db.py index e2d26054c80..823f0ed675c 100644 --- a/litellm/proxy/_experimental/mcp_server/db.py +++ b/litellm/proxy/_experimental/mcp_server/db.py @@ -223,6 +223,7 @@ class _OAuthCredentialAccessToken(TypedDict): class OAuthCredentialPayload(_OAuthCredentialAccessToken, total=False): identity_binding_proof: ReadOnly[str] + cimd_client_id: ReadOnly[str] type: str refresh_token: str expires_at: str @@ -1765,6 +1766,7 @@ async def store_user_oauth_credential( scopes: list[str] | None = None, skip_byok_guard: bool = False, identity_binding_proof: str | None = None, + cimd_client_id: str | None = None, ) -> None: """Persist an OAuth2 access token for a user+server pair. @@ -1782,6 +1784,7 @@ async def store_user_oauth_credential( "access_token": access_token, "connected_at": datetime.now(timezone.utc).isoformat(), **({"identity_binding_proof": identity_binding_proof} if identity_binding_proof else {}), + **({"cimd_client_id": cimd_client_id} if cimd_client_id else {}), } if refresh_token: payload["refresh_token"] = refresh_token @@ -2063,6 +2066,7 @@ async def refresh_user_oauth_token( auth_method=getattr(server, "token_endpoint_auth_method", None), client_id=client_id, client_secret=client_secret, + cimd_client_id=cred.get("cimd_client_id"), ) token_data: Final[dict[str, str]] = { "grant_type": "refresh_token", @@ -2132,6 +2136,7 @@ async def refresh_user_oauth_token( expires_in=expires_in, scopes=scopes, identity_binding_proof=binding_proof, + cimd_client_id=cred.get("cimd_client_id"), skip_byok_guard=True, # Row is already OAuth2; skip the extra find_unique check ) diff --git a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py index da6ce954b1d..0253719f410 100644 --- a/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py +++ b/litellm/proxy/_experimental/mcp_server/discoverable_endpoints.py @@ -79,9 +79,13 @@ from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( enforce_oauth_identity_binding, ) from litellm.proxy._experimental.mcp_server.oauth_utils import ( + CIMD_METADATA_PATH, TOKEN_NO_CACHE_HEADERS, build_upstream_oauth2_token_request, + get_cimd_client_id, + get_cimd_document_url, get_request_base_url, + needs_cimd_discovery, oauth_client_registration_matches, resolve_upstream_resource, validate_trusted_redirect_uri, @@ -638,6 +642,7 @@ async def _store_per_user_token_server_side( user_id: str, token_response: dict[str, Any], identity_binding_proof: str | None = None, + cimd_client_id: str | None = None, ) -> None: """Persist the OAuth token server-side and warm the Redis cache. @@ -680,6 +685,7 @@ async def _store_per_user_token_server_side( expires_in=expires_in, scopes=scopes, identity_binding_proof=identity_binding_proof, + **({"cimd_client_id": cimd_client_id} if cimd_client_id is not None else {}), ) verbose_logger.info( "_store_per_user_token_server_side: stored token for user=%s server=%s", @@ -778,20 +784,20 @@ async def _server_with_oauth_endpoints( mcp_server: MCPServer, needed_endpoint: Callable[[MCPServer], str | None], ) -> MCPServer: - """Join deferred OAuth discovery only when the endpoint this caller needs is still missing. + """Join deferred discovery for missing endpoints or unknown public-client metadata. - Admin-entered endpoints live on ``configured_*`` after an anchored issuer empties the - resolved fields. A caller whose needed endpoint already resolves never awaits discovery - and cannot 503 over a leftover pin. A server still missing it joins the deferred task; - no slot is a no-op and the caller 400s. + Capability discovery is optional when manual endpoints already resolve; the manager + preserves those endpoints if discovery fails. No discovery slot remains a no-op. """ - if needed_endpoint(mcp_server) is not None: + if needed_endpoint(mcp_server) is not None and not needs_cimd_discovery(mcp_server): return mcp_server from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( # noqa: PLC0415 # circular import with mcp_server_manager at module load global_mcp_server_manager, ) - return await global_mcp_server_manager.ensure_oauth_metadata_discovered(mcp_server) + if needed_endpoint(mcp_server) is None: + return await global_mcp_server_manager.ensure_oauth_metadata_discovered(mcp_server) + return await global_mcp_server_manager.ensure_oauth_metadata_discovered(mcp_server, needed_endpoint=needed_endpoint) def _raise_unless_oauth2_discovery_server( @@ -979,7 +985,8 @@ async def authorize_with_server( binding: Final = resolved_server.oauth_identity_binding enforce_binding: Final = binding is not None and binding.mode == "enforce" - if enforce_binding: + cimd_client_id: Final = get_cimd_client_id(resolved_server) + if enforce_binding or cimd_client_id: _require_s256_pkce(code_challenge, code_challenge_method) if resolved_server.is_dcr_bridge: @@ -1043,7 +1050,7 @@ async def authorize_with_server( relay_state: Final = secrets.token_urlsafe(_OAUTH_STATE_HANDLE_BYTES) params: Final = { - "client_id": resolved_server.client_id if resolved_server.client_id else client_id, + "client_id": resolved_server.client_id or cimd_client_id or client_id, "redirect_uri": f"{request_base_url}/callback", "state": relay_state, "response_type": response_type or "code", @@ -1080,7 +1087,37 @@ def _token_credential_source(mcp_server: MCPServer) -> CredentialSource: """Mirrors the resolved-client rule in :func:`exchange_token_with_server`: when the server has a stored client_id the gateway presents its own credentials upstream, so a credential rejection is the operator's fault, not the caller's.""" - return "gateway_stored" if mcp_server.client_id else "caller_supplied" + return "gateway_stored" if mcp_server.client_id or get_cimd_client_id(mcp_server) else "caller_supplied" + + +async def _saved_cimd_refresh_client_id( + server: MCPServer, user_id: str | None, refresh_token: str | None +) -> str | None: + """Reuse only the client identity bound to this caller's presented refresh grant.""" + if ( + not user_id + or not refresh_token + or not server.needs_user_oauth_token + or server.auth_type != MCPAuth.oauth2 + or server.client_id + or server.client_secret + ): + return None + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server.db import get_user_oauth_credential + + if proxy_server.prisma_client is None: + return None + try: + credential: Final = await get_user_oauth_credential(proxy_server.prisma_client, user_id, server.server_id) + except Exception: # noqa: BLE001 # optional storage must not prevent a caller-owned OAuth exchange + return None + if credential is None: + return None + stored_refresh: Final = credential.get("refresh_token") + if not stored_refresh or not secrets.compare_digest(stored_refresh.encode(), refresh_token.encode()): + return None + return credential.get("cimd_client_id") async def exchange_token_with_server( @@ -1117,6 +1154,17 @@ async def exchange_token_with_server( ), ) + request_user_id: Final = ( + await extract_user_id_from_request(request) + if resolved_server.needs_user_oauth_token or resolved_server.oauth_identity_binding is not None + else None + ) + cimd_client_id: Final = ( + await _saved_cimd_refresh_client_id(resolved_server, request_user_id, refresh_token) + if grant_type == "refresh_token" + else None + ) or get_cimd_client_id(resolved_server) + # The id, secret, and token-endpoint auth method must come from the same source. When the # server-side client_id wins, falling back to the caller's secret pairs the persisted client # with a foreign secret; the register short-circuit hands clients a placeholder secret @@ -1138,16 +1186,11 @@ async def exchange_token_with_server( auth_method=resolved_auth_method, client_id=resolved_client_id, client_secret=resolved_client_secret, + cimd_client_id=cimd_client_id, ) except TokenEndpointAuthConfigError as exc: raise HTTPException(status_code=400, detail=str(exc)) from exc - request_user_id: Final = ( - await extract_user_id_from_request(request) - if resolved_server.needs_user_oauth_token or resolved_server.oauth_identity_binding is not None - else None - ) - bridge_identity: _BridgeAuthorizationCode | None = None bridge_mint_ready: _BridgeMintReady | None = None bridge_upstream_refresh: SecretStr | None = None @@ -1265,7 +1308,7 @@ async def exchange_token_with_server( except httpx.HTTPStatusError as exc: fault: Final = classify_upstream_token_rejection( exc.response, - credential_source=_token_credential_source(resolved_server), + credential_source="gateway_stored" if cimd_client_id else _token_credential_source(resolved_server), log_context=resolved_server.server_id, ) upstream_rejected_bridge_refresh: Final = ( @@ -1336,6 +1379,7 @@ async def exchange_token_with_server( user_id=user_id, token_response=token_response, identity_binding_proof=binding_proof, + **({"cimd_client_id": cimd_client_id} if cimd_client_id is not None else {}), ) else: verbose_logger.warning( @@ -1991,8 +2035,21 @@ async def register_client_with_server( ), ) + cimd_client_id: Final = get_cimd_client_id(resolved_server) + if cimd_client_id: + return { + "client_id": cimd_client_id, + "token_endpoint_auth_method": "none", + "redirect_uris": client_facing_redirect_uris, + } registration_url: Final = resolved_server.effective_registration_url if registration_url is None: + if resolved_server.client_id_metadata_document_supported and resolved_server.is_gateway_managed_oauth2: + raise HTTPException( + status_code=400, + detail="CIMD requires a stable HTTPS PROXY_BASE_URL and public-client authentication; " + "configure these or provide a pre-registered OAuth client", + ) return dummy_return bridge_relay: Final = _dcr_bridge_relays_client_registration(resolved_server) @@ -3162,3 +3219,25 @@ async def register_client(request: Request, mcp_server_name: str | None = None): client_redirect_uris=client_redirect_uris, client_application_type=client_application_type, ) + + +@router.get(CIMD_METADATA_PATH, include_in_schema=False) +async def oauth_client_metadata() -> JSONResponse: + from mcp.shared.auth import OAuthClientInformationFull + from pydantic import AnyUrl + + document_url: Final = get_cimd_document_url() + if document_url is None: + raise HTTPException(status_code=404, detail="CIMD requires a configured HTTPS PROXY_BASE_URL") + base_url: Final = document_url.removesuffix(CIMD_METADATA_PATH) + metadata: Final = OAuthClientInformationFull( + client_id=document_url, + client_name="LiteLLM MCP Gateway", + redirect_uris=[AnyUrl(f"{base_url}/callback")], + token_endpoint_auth_method="none", + grant_types=["authorization_code", "refresh_token"], + response_types=["code"], + ) + return JSONResponse( + metadata.model_dump(mode="json", exclude_none=True), headers={"Cache-Control": "public, max-age=300"} + ) diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index 701a8f74813..c42ab60c48b 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -105,6 +105,7 @@ from litellm.proxy._experimental.mcp_server.oauth_utils import ( # noqa: F401 _redact_mcp_resource_url, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export canonicalize_url_identity, get_byok_www_authenticate, + needs_cimd_discovery, redact_mcp_resource_url, ) from litellm.proxy._experimental.mcp_server.outbound_credentials import ( @@ -452,6 +453,7 @@ class _AuthorizationServerMetadataPayload(TypedDict, total=False): authorization_endpoint: str token_endpoint: str registration_endpoint: str + client_id_metadata_document_supported: ReadOnly[object] scopes_supported: Sequence[str] grant_types_supported: Sequence[str] token_endpoint_auth_methods_supported: Sequence[str] @@ -767,14 +769,16 @@ def _flow_endpoints_missing( return authorization_url is None or token_url is None -def oauth_endpoints_unresolved(server: MCPServer) -> bool: - """``_flow_endpoints_missing`` over a built registry entry, for the reload fast-path check. +def oauth_endpoints_unresolved(server: MCPServer, *, include_client_metadata: bool = True) -> bool: + """Whether endpoint or eligible client metadata discovery is still pending. The flow comes from ``effective_oauth2_flow``, the one column-first, shape-fallback judge every flow decision uses, not from the raw column: a legacy row the startup backfill deliberately left unstamped (the ambiguous M2M shape) serves M2M at request time, and reading the bare column here would classify it as interactive-missing-endpoints and re-run discovery on every reload. """ + if include_client_metadata and needs_cimd_discovery(server): + return True if ( server.auth_type == MCPAuth.oauth2_token_exchange and server.token_exchange_profile == "entra_obo" @@ -868,6 +872,8 @@ def carry_forward_resolved_oauth_endpoints(new_server: MCPServer, previous_serve may_carry: Final = _endpoints_corroborate_authorization_url( previous_server.authorization_url, new_server.authorization_url ) + if may_carry and new_server.client_id_metadata_document_supported is None: + new_server.client_id_metadata_document_supported = previous_server.client_id_metadata_document_supported if may_carry and new_server.issuer is None: new_server.issuer = previous_server.issuer new_server.authorization_response_iss_parameter_supported = ( # rebind-ok: publish on the existing rebuild object @@ -908,7 +914,7 @@ def _restrict_discovery_to_corroborated_authorization_server( return metadata if _endpoints_corroborate_authorization_url(metadata.authorization_url, manual_authorization_url): return metadata - if not metadata.token_url and not metadata.registration_url: + if not metadata.token_url and not metadata.registration_url and not metadata.client_id_metadata_document_supported: return metadata bridge_note: Final = ( " The discovered registration_url is rejected with it, so this dcr_bridge server stays on the" @@ -926,7 +932,9 @@ def _restrict_discovery_to_corroborated_authorization_server( _normalized_authorize_endpoint(manual_authorization_url), bridge_note, ) - return metadata.model_copy(update={"token_url": None, "registration_url": None}) + return metadata.model_copy( + update={"token_url": None, "registration_url": None, "client_id_metadata_document_supported": False} + ) def _redacted_origin_list(urls: Sequence[str]) -> str: @@ -2060,6 +2068,7 @@ class MCPServerManager: update={ "scopes": server.scopes or metadata.scopes, "issuer": server.issuer or discovered_issuer, + "client_id_metadata_document_supported": metadata.client_id_metadata_document_supported, "authorization_response_iss_parameter_supported": ( metadata.authorization_response_iss_parameter_supported if discovered_issuer is not None @@ -2224,12 +2233,27 @@ class MCPServerManager: if should_defer != has_slot: self._set_oauth_discovery_deferred(server.server_id, should_defer) - async def ensure_oauth_metadata_discovered(self, server: MCPServer, *, _retry_stale: bool = True) -> MCPServer: + async def ensure_oauth_metadata_discovered( + self, + server: MCPServer, + *, + needed_endpoint: Callable[[MCPServer], str | None] | None = None, + _retry_stale: bool = True, + ) -> MCPServer: return await self.catalog.resolve_oauth_metadata( - server, lambda selected: self._ensure_oauth_metadata_discovered(selected, _retry_stale=_retry_stale) + server, + lambda selected: self._ensure_oauth_metadata_discovered( + selected, needed_endpoint=needed_endpoint, _retry_stale=_retry_stale + ), ) - async def _ensure_oauth_metadata_discovered(self, server: MCPServer, *, _retry_stale: bool = True) -> MCPServer: + async def _ensure_oauth_metadata_discovered( + self, + server: MCPServer, + *, + needed_endpoint: Callable[[MCPServer], str | None] | None = None, + _retry_stale: bool = True, + ) -> MCPServer: """Join the bounded discovery task and return the resolved server. Concurrent callers share one task per server. A failed attempt remains @@ -2237,9 +2261,12 @@ class MCPServerManager: Args: server: The MCP server whose OAuth metadata must be resolved. + needed_endpoint: A caller-specific endpoint that may remain usable when + optional capability discovery fails. Returns: - The resolved server; the registered server when no discovery is + The resolved server; a configured caller endpoint remains usable on + optional capability-discovery failure. The registered server when no discovery is pending, or when discovery failed for a client-forwarded-token server, whose session consumes no discovered endpoint. @@ -2257,18 +2284,26 @@ class MCPServerManager: outcome: Final = await asyncio.shield(task) except asyncio.CancelledError: if task.cancelled() and not self._oauth_discovery_slot_is_current(server.server_id, generation): - return await self._rejoin_oauth_metadata_discovery(server, retry_stale=_retry_stale) + return await self._rejoin_oauth_metadata_discovery( + server, needed_endpoint=needed_endpoint, retry_stale=_retry_stale + ) raise match outcome: case _OAuthDiscoveryResolved(resolved_server): self.catalog.assert_current(resolved_server) return resolved_server case _OAuthDiscoveryStale(): - return await self._rejoin_oauth_metadata_discovery(server, retry_stale=_retry_stale) + return await self._rejoin_oauth_metadata_discovery( + server, needed_endpoint=needed_endpoint, retry_stale=_retry_stale + ) case _OAuthDiscoveryFailed(timed_out=timed_out): current: Final = self._registered_server(server) self.catalog.assert_current(current) - if current.is_client_forwarded_token: + if ( + current.is_client_forwarded_token + or not oauth_endpoints_unresolved(current, include_client_metadata=False) + or (needed_endpoint is not None and needed_endpoint(current) is not None) + ): return current server_ref: Final = current.alias or current.server_name or current.name or current.server_id reason: Final = "timed out" if timed_out else "returned incomplete metadata" @@ -2279,11 +2314,19 @@ class MCPServerManager: return assert_never(outcome) - async def _rejoin_oauth_metadata_discovery(self, server: MCPServer, *, retry_stale: bool) -> MCPServer: + async def _rejoin_oauth_metadata_discovery( + self, server: MCPServer, *, needed_endpoint: Callable[[MCPServer], str | None] | None = None, retry_stale: bool + ) -> MCPServer: if retry_stale: - return await self.ensure_oauth_metadata_discovered(server, _retry_stale=False) + return await self.ensure_oauth_metadata_discovered( + server, needed_endpoint=needed_endpoint, _retry_stale=False + ) current: Final = self._registered_server(server) - if not oauth_endpoints_unresolved(current) or current.is_client_forwarded_token: + if ( + not oauth_endpoints_unresolved(current, include_client_metadata=False) + or current.is_client_forwarded_token + or (needed_endpoint is not None and needed_endpoint(current)) + ): return current raise HTTPException(status_code=503, detail="OAuth metadata discovery changed repeatedly; retry shortly") @@ -2616,6 +2659,9 @@ class MCPServerManager: scopes=resolved_scopes, configured_scopes=tuple(configured_scopes) if configured_scopes else None, issuer=effective_issuer, + client_id_metadata_document_supported=( + gated_oauth_metadata.client_id_metadata_document_supported if gated_oauth_metadata else None + ), authorization_response_iss_parameter_supported=( gated_oauth_metadata.authorization_response_iss_parameter_supported if gated_oauth_metadata @@ -3194,6 +3240,9 @@ class MCPServerManager: scopes=resolved_scopes, configured_scopes=configured_scopes, issuer=effective_issuer, + client_id_metadata_document_supported=( + gated_oauth_metadata.client_id_metadata_document_supported if gated_oauth_metadata else None + ), authorization_response_iss_parameter_supported=( gated_oauth_metadata.authorization_response_iss_parameter_supported if gated_oauth_metadata else False ), @@ -5304,6 +5353,7 @@ class MCPServerManager: authorization_url=data.get("authorization_endpoint"), token_url=data.get("token_endpoint"), registration_url=data.get("registration_endpoint"), + client_id_metadata_document_supported=data.get("client_id_metadata_document_supported") is True, discovered_issuer=claimed_issuer if isinstance(claimed_issuer, str) and claimed_issuer else None, authorization_response_iss_parameter_supported=data.get( "authorization_response_iss_parameter_supported" @@ -5336,6 +5386,7 @@ class MCPServerManager: return MCPOAuthMetadata( authorization_url=f"{base}/oauth2/v2.0/authorize", token_url=f"{base}/oauth2/v2.0/token", + client_id_metadata_document_supported=False, ) @staticmethod diff --git a/litellm/proxy/_experimental/mcp_server/oauth_utils.py b/litellm/proxy/_experimental/mcp_server/oauth_utils.py index e60c4ef1c30..276cf250b30 100644 --- a/litellm/proxy/_experimental/mcp_server/oauth_utils.py +++ b/litellm/proxy/_experimental/mcp_server/oauth_utils.py @@ -130,6 +130,57 @@ def _resolve_proxy_base_url_env() -> str | None: return None +CIMD_METADATA_PATH: Final = "/oauth/client-metadata.json" + + +def get_cimd_document_url() -> str | None: + try: + configured: Final = _resolve_proxy_base_url_env() + parsed: Final = urlparse(configured or "") + except ValueError: + return None + if parsed.scheme != "https" or not parsed.hostname or parsed.username is not None or parsed.password is not None: + return None + return f"{configured}{CIMD_METADATA_PATH}" + + +def _can_use_cimd(server: "MCPServer") -> bool: + return ( + server.is_gateway_managed_oauth2 + and server.needs_user_oauth_token + and not server.client_id + and not server.client_secret + and server.token_endpoint_auth_method != "client_secret_basic" + ) + + +def needs_cimd_discovery(server: "MCPServer") -> bool: + """Resolve unknown client metadata support even when OAuth endpoints are configured.""" + return ( + getattr(server, "client_id_metadata_document_supported", False) is None + and _can_use_cimd(server) + and get_cimd_document_url() is not None + ) + + +def _deployment_prefers_cimd() -> bool: + from litellm.proxy.proxy_server import general_settings + + return general_settings.get("mcp_prefer_client_id_metadata_document") is True + + +def _dynamic_registration_takes_precedence(server: "MCPServer") -> bool: + return server.effective_registration_url is not None and not _deployment_prefers_cimd() + + +def get_cimd_client_id(server: "MCPServer") -> str | None: + if getattr(server, "client_id_metadata_document_supported", False) is not True or not _can_use_cimd(server): + return None + if _dynamic_registration_takes_precedence(server): + return None + return get_cimd_document_url() + + BYOK_RESOURCE_METADATA_PATH: Final = "/v1/mcp/oauth/protected-resource" @@ -748,6 +799,7 @@ def build_upstream_oauth2_token_request( auth_method: object, client_id: str | None, client_secret: str | None, + cimd_client_id: str | None = None, ) -> TokenEndpointClientAuth: """Client auth plus the RFC 8707 ``resource`` for one upstream plain-OAuth2 token request. @@ -757,10 +809,11 @@ def build_upstream_oauth2_token_request( authenticate as the caller's own client rather than the server's; ``resource`` always comes from the server, so no leg can choose or forget it. """ + selected_cimd_id: Final = cimd_client_id or get_cimd_client_id(mcp_server) client_auth: Final = build_token_endpoint_client_auth( - auth_method=normalize_token_endpoint_auth_method(auth_method), - client_id=client_id, - client_secret=client_secret, + auth_method=None if selected_cimd_id else normalize_token_endpoint_auth_method(auth_method), + client_id=selected_cimd_id or client_id, + client_secret=None if selected_cimd_id else client_secret, ) resource: Final = resolve_upstream_resource(mcp_server) if not resource: diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py index af1e82eab82..b926ca8a6a4 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/authz_code_refresher.py @@ -25,7 +25,7 @@ from litellm.proxy._experimental.mcp_server.oauth_identity_binding import ( RefreshTokenPresented, enforce_oauth_identity_binding, ) -from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request +from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request, get_cimd_client_id from litellm.proxy._experimental.mcp_server.outbound_credentials.oauth_token_store import ( OAuthToken, ) @@ -47,6 +47,7 @@ class CredentialPersist(Protocol): expires_in: int | None, scopes: tuple[str, ...] | None, identity_binding_proof: str | None = None, + cimd_client_id: str | None = None, ) -> None: ... @@ -112,12 +113,14 @@ class AuthorizationCodeRefresher: if not token_url: return None + cimd_client_id: Final = token.cimd_client_id or get_cimd_client_id(server) try: token_request: Final = build_upstream_oauth2_token_request( server, auth_method=server.token_endpoint_auth_method, client_id=server.client_id, client_secret=server.client_secret, + cimd_client_id=cimd_client_id, ) except TokenEndpointAuthConfigError as exc: verbose_logger.warning("MCP OAuth refresh misconfigured for server %s: %s", server_id, exc) @@ -155,22 +158,21 @@ class AuthorizationCodeRefresher: expires_in: Final = _parse_expires_in(body.get("expires_in")) scopes: Final = _parse_scopes(body.get("scope")) or token.scopes - if binding_proof is not None: - await self._persist( - user_id, - server_id, - access_token, - new_refresh, - expires_in, - scopes or None, - identity_binding_proof=binding_proof, - ) - else: - await self._persist(user_id, server_id, access_token, new_refresh, expires_in, scopes or None) + await self._persist( + user_id, + server_id, + access_token, + new_refresh, + expires_in, + scopes or None, + **({"identity_binding_proof": binding_proof} if binding_proof is not None else {}), + **({"cimd_client_id": cimd_client_id} if cimd_client_id is not None else {}), + ) return OAuthToken( access_token=access_token, expires_at=self._clock() + expires_in if expires_in is not None else None, refresh_token=new_refresh, scopes=scopes, identity_binding_proof=binding_proof, + cimd_client_id=cimd_client_id, ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py index 0ac4296f498..b9f09c266d5 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/oauth_token_store.py @@ -42,6 +42,7 @@ class OAuthToken: refresh_token: str | None = None scopes: tuple[str, ...] = () identity_binding_proof: str | None = None + cimd_client_id: str | None = None def __repr__(self) -> str: has_refresh: Final = self.refresh_token is not None diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py index 2897c3e8e4a..d8094d5facb 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/per_user_oauth_store.py @@ -72,6 +72,7 @@ async def _persist_credential( expires_in: int | None, scopes: tuple[str, ...] | None, identity_binding_proof: str | None = None, + cimd_client_id: str | None = None, ) -> None: from litellm.proxy._experimental.mcp_server.db import ( # noqa: PLC0415 store_user_oauth_credential, @@ -90,6 +91,7 @@ async def _persist_credential( scopes=list(scopes) if scopes else None, skip_byok_guard=True, identity_binding_proof=identity_binding_proof, + **({"cimd_client_id": cimd_client_id} if cimd_client_id is not None else {}), ) diff --git a/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py b/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py index 0f18931d118..56ab8e6bbdc 100644 --- a/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py +++ b/litellm/proxy/_experimental/mcp_server/outbound_credentials/v2_token_store.py @@ -47,12 +47,14 @@ def _to_oauth_token(payload: Mapping[str, object]) -> OAuthToken | None: refresh_token: Final = payload.get("refresh_token") expires_at: Final = payload.get("expires_at") binding_proof: Final = payload.get("identity_binding_proof") + cimd_client_id: Final = payload.get("cimd_client_id") return OAuthToken( access_token=access_token, expires_at=_iso_to_epoch(expires_at) if isinstance(expires_at, str) else None, refresh_token=refresh_token if isinstance(refresh_token, str) else None, scopes=_to_scopes(payload.get("scopes")), identity_binding_proof=binding_proof if isinstance(binding_proof, str) else None, + cimd_client_id=cimd_client_id if isinstance(cimd_client_id, str) else None, ) diff --git a/litellm/proxy/_lazy_features.py b/litellm/proxy/_lazy_features.py index 517c0916e13..68188800438 100644 --- a/litellm/proxy/_lazy_features.py +++ b/litellm/proxy/_lazy_features.py @@ -166,6 +166,7 @@ LAZY_FEATURES: Final[tuple[LazyFeature, ...]] = ( name="mcp_discoverable", module_path="litellm.proxy._experimental.mcp_server.discoverable_endpoints", path_prefixes=( + "/oauth/client-metadata.json", "/.well-known/oauth-", "/.well-known/openid-configuration", "/.well-known/jwks.json", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index bc27b2352a9..f009390576a 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3185,6 +3185,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase): ge=1, description="Number of trusted reverse proxies/load balancers in front of the gateway that append to X-Forwarded-For. When set (and mcp_trusted_proxy_ranges validates the direct peer), the client IP for MCP access control is read this many entries from the right of the chain instead of the spoofable leftmost value, defeating append-style X-Forwarded-For forgery.", ) + mcp_prefer_client_id_metadata_document: bool | None = Field( + None, + description="When true, a gateway-managed OAuth2 MCP server whose authorization server advertises Client ID Metadata Document support identifies itself with the gateway's public metadata document URL even when that authorization server also offers dynamic client registration. Requires a public HTTPS PROXY_BASE_URL the authorization server can fetch. Default false: dynamic client registration is used whenever the authorization server offers it, and the metadata document only when it does not.", + ) trusted_proxy_ranges: list[str] | None = Field( None, description="CIDR ranges of trusted reverse proxies allowed to provide identity headers for header-based auth paths such as enable_oauth2_proxy_auth and custom_ui_sso_sign_in_handler, and whose X-Forwarded-For is used to attribute Admin UI sign-in attempts to a source address. Set it to an empty list when clients connect directly, so the peer address is the source. Left unset, or containing an entry that is not an address or CIDR range, the per-source sign-in limit is off.", diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index b4b23f111d1..000ffa9c305 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -18538,6 +18538,7 @@ _GENERAL_SETTINGS_CONFIG_LIST_FIELD_TYPES: Final[Mapping[str, str]] = MappingPro "mcp_client_id_header": "String", "mcp_trusted_proxy_ranges": "List", "mcp_xff_num_trusted_hops": "Integer", + "mcp_prefer_client_id_metadata_document": "Boolean", "always_include_stream_usage": "Boolean", "forward_client_headers_to_llm_api": "Boolean", "mcp_required_fields": "List", diff --git a/litellm/types/mcp_server/mcp_server_manager.py b/litellm/types/mcp_server/mcp_server_manager.py index 2f5d983cdc0..a7027102250 100644 --- a/litellm/types/mcp_server/mcp_server_manager.py +++ b/litellm/types/mcp_server/mcp_server_manager.py @@ -39,6 +39,7 @@ class MCPOAuthMetadata(LiteLLMBaseModel): authorization_url: str | None = None token_url: str | None = None registration_url: str | None = None + client_id_metadata_document_supported: bool | None = None authorization_response_iss_parameter_supported: bool = False discovered_issuer: str | None = None """The ``issuer`` the authorization-server metadata document self-attests (RFC 8414). Persisted @@ -120,6 +121,7 @@ class MCPServer(LiteLLMBaseModel): client_secret: str | None = None issuer: str | None = None issuer_is_anchored: bool = False + client_id_metadata_document_supported: bool | None = None authorization_response_iss_parameter_supported: bool = False dcr_issuer: str | None = None dcr_server_url: str | None = None diff --git a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py b/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py index df068b60338..350e7f1f910 100644 --- a/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py +++ b/tests/unit/proxy/_experimental/mcp_server/outbound_credentials/test_authz_code_refresher.py @@ -52,7 +52,7 @@ def _endpoint(body, sink=None): def _recording_persist(sink): async def persist( - user_id, server_id, access_token, refresh_token, expires_in, scopes + user_id, server_id, access_token, refresh_token, expires_in, scopes, cimd_client_id=None ): sink.append( (user_id, server_id, access_token, refresh_token, expires_in, scopes) @@ -325,3 +325,83 @@ async def test_verified_refresh_preserves_binding_proof_in_storage(): assert token.refresh_token == "rotated" assert token.identity_binding_proof == "verified-binding" assert persist.await_args.kwargs["identity_binding_proof"] == "verified-binding" + + +@pytest.mark.asyncio +async def test_cimd_refresh_on_fresh_replica_preserves_user_and_client_identity(monkeypatch): + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + server = MCPServer.model_validate( + { + "server_id": "srv", + "name": "srv", + "server_name": "srv", + "transport": "http", + "url": "https://mcp.example.com/mcp", + "auth_type": "oauth2", + "oauth2_flow": "authorization_code", + "token_url": "https://idp.example.com/token", + "client_id_metadata_document_supported": True, + } + ) + posted = [] + persisted = [] + refreshed = await _refresher( + server=server, + body={"access_token": "new-at", "expires_in": 3600}, + post_sink=posted, + persist_sink=persisted, + ).refresh("alice", "srv", OAuthToken(access_token="old-at", refresh_token="old-rt")) + assert refreshed is not None + assert refreshed.access_token == "new-at" + assert posted == [ + ( + "https://idp.example.com/token", + { + "grant_type": "refresh_token", + "refresh_token": "old-rt", + "client_id": "https://gateway.example.com/oauth/client-metadata.json", + }, + {}, + ) + ] + assert persisted == [("alice", "srv", "new-at", "old-rt", 3600, None)] + assert server.client_id is None + + +@pytest.mark.asyncio +async def test_saved_cimd_identity_refreshes_without_discovery(monkeypatch): + from litellm.proxy._experimental.mcp_server.outbound_credentials.v2_token_store import V2PerUserTokenStore + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + identity = "https://gateway.example.com/oauth/client-metadata.json" + stored = {"access_token": "old", "refresh_token": "old-rt", "cimd_client_id": identity} + + async def read_credential(user_id, server_id): + assert (user_id, server_id) == ("alice", "srv") + return stored + + async def persist(user_id, server_id, access_token, refresh_token, expires_in, scopes, **metadata): + assert (user_id, server_id, access_token) == ("alice", "srv", "new") + assert metadata["cimd_client_id"] == identity + + async def post(url, form, headers): + assert url == "https://idp.example.com/token" + assert form["client_id"] == identity + assert "client_secret" not in form + assert "Authorization" not in headers + return {"access_token": "new", "refresh_token": "rotated"} + + server = MCPServer( + server_id="srv", name="srv", transport="http", auth_type="oauth2", oauth2_flow="authorization_code", + authorization_url="https://idp.example.com/authorize", token_url="https://idp.example.com/token", + ) + token = await V2PerUserTokenStore(read_credential).fetch("alice", "srv") + assert token is not None + refreshed = await AuthorizationCodeRefresher(lambda _: server, post, persist).refresh("alice", "srv", token) + assert refreshed is not None + assert refreshed.access_token == "new" + assert refreshed.refresh_token == "rotated" + assert refreshed.cimd_client_id == identity diff --git a/tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py b/tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py index de86c1b62f4..10993280e89 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_db_credentials.py @@ -708,7 +708,8 @@ async def test_store_user_oauth_credential_does_not_persist_plaintext(): @pytest.mark.asyncio -async def test_oauth_round_trip_returns_payload(): +@pytest.mark.parametrize("cimd_client_id", [None, "https://gateway.example.com/oauth/client-metadata.json"]) +async def test_oauth_round_trip_returns_payload(cimd_client_id): access_token = "ya29.a0AfH6SMBverysecretaccesstoken" prisma = _make_prisma_with_existing(row=None) await store_user_oauth_credential( @@ -719,6 +720,7 @@ async def test_oauth_round_trip_returns_payload(): refresh_token="rfr-xyz", scopes=["a", "b"], identity_binding_proof="verified-proof", + cimd_client_id=cimd_client_id, ) stored = _stored_value(prisma) @@ -734,6 +736,7 @@ async def test_oauth_round_trip_returns_payload(): assert result["refresh_token"] == "rfr-xyz" assert result["scopes"] == ["a", "b"] assert result["identity_binding_proof"] == "verified-proof" + assert result.get("cimd_client_id") == cimd_client_id @pytest.mark.asyncio @@ -1795,3 +1798,61 @@ async def test_unverified_legacy_cache_cannot_bypass_enforcement(monkeypatch): await mcp_per_user_token_cache.set("alice", "srv", "bob", 60) assert await module.resolve_user_oauth_access_token("alice", server) is None assert await mcp_per_user_token_cache.get("alice", "srv") is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize("owner", ["native", "legacy"]) +@pytest.mark.parametrize("capability", [None, False]) +async def test_saved_cimd_grant_refreshes_from_encrypted_storage_on_a_fresh_replica(monkeypatch, respx_mock, owner, capability): + from urllib.parse import parse_qs + + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import db as db_module + from litellm.proxy._experimental.mcp_server.discoverable_endpoints import _store_per_user_token_server_side + from litellm.proxy._experimental.mcp_server.outbound_credentials.per_user_oauth_store import ( + LazyPerUserOAuthTokenStore, + ) + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + identity = "https://gateway.example.com/oauth/client-metadata.json" + prisma = _make_prisma_with_existing(row=None) + monkeypatch.setattr(proxy_server, "prisma_client", prisma) + server = MCPServer( + server_id="saved-cimd", name="saved_cimd", transport="http", auth_type="oauth2", + client_id_metadata_document_supported=capability, + oauth2_flow="authorization_code", authorization_url="https://idp.example.com/authorize", + token_url="https://idp.example.com/token", + ) + await _store_per_user_token_server_side( + server, "alice", {"access_token": "old", "refresh_token": "old-refresh", "expires_in": -10}, + cimd_client_id=identity, + ) + + async def read_row(**kwargs): + return SimpleNamespace(credential_b64=_stored_value(prisma), user_id="alice", server_id=server.server_id) + + prisma.db.litellm_mcpusercredentials.find_unique.side_effect = read_row + saved = await get_user_oauth_credential(prisma, "alice", server.server_id) + assert saved is not None and saved["cimd_client_id"] == identity + assert identity not in _stored_value(prisma) + monkeypatch.setenv("PROXY_BASE_URL", "https://renamed-gateway.example.com") + upstream = respx_mock.post("https://idp.example.com/token").respond( + 200, json={"access_token": "fresh", "refresh_token": "rotated", "expires_in": 3600} + ) + if owner == "native": + refreshed = await LazyPerUserOAuthTokenStore(lambda _: server).fetch("alice", server.server_id) + assert refreshed is not None and refreshed.access_token == "fresh" + assert refreshed.cimd_client_id == identity + else: + legacy = await db_module.refresh_user_oauth_token(prisma, "alice", server, saved) + assert legacy is not None and legacy["access_token"] == "fresh" + persisted = await get_user_oauth_credential(prisma, "alice", server.server_id) + assert persisted is not None + assert persisted["cimd_client_id"] == identity + assert persisted["refresh_token"] == "rotated" + assert upstream.call_count == 1 + assert parse_qs(upstream.calls[0].request.content.decode())["client_id"] == [identity] + assert "client_secret" not in parse_qs(upstream.calls[0].request.content.decode()) + assert server.client_id is None and server.client_id_metadata_document_supported is capability diff --git a/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py b/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py index 23bdfa3eadf..4f22200145e 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_discoverable_endpoints.py @@ -20,9 +20,9 @@ if TYPE_CHECKING: import httpx from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from fastapi import APIRouter + from respx import MockRouter from litellm.proxy.auth.handle_jwt import JWTHandler - from litellm.types.mcp_server.mcp_server_manager import MCPServer @@ -12568,10 +12568,12 @@ async def test_oauth_write_denial_does_not_erase_identity_binding( @pytest.mark.asyncio @pytest.mark.parametrize("admin_only", [False, True]) +@pytest.mark.parametrize("cimd", [False, True]) async def test_signed_oauth_callback_honors_credential_write_policy( jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], monkeypatch: pytest.MonkeyPatch, admin_only: bool, + cimd: bool, ) -> None: import httpx import litellm @@ -12586,7 +12588,8 @@ async def test_signed_oauth_callback_honors_credential_write_policy( server: Final = MCPServer( server_id="signed-server", name="signed-server", transport=MCPTransport.http, - auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", client_id="client", + auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", client_id=None if cimd else "client", + client_id_metadata_document_supported=cimd, token_url="https://upstream.example.test/token", ) monkeypatch.setattr(proxy_server, "general_settings", { @@ -12594,6 +12597,7 @@ async def test_signed_oauth_callback_honors_credential_write_policy( "admin_only_routes": [f"/v1/mcp/server/{server.server_id}/oauth-user-credential"] if admin_only else [], }) monkeypatch.setenv("LITELLM_SALT_KEY", "signed-oauth-test-salt") + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") manager: Final = MagicMock() manager.get_allowed_mcp_servers = AsyncMock(return_value=[server.server_id]) manager.invalidate_user_oauth_token_cache = AsyncMock() @@ -12630,6 +12634,16 @@ async def test_signed_oauth_callback_honors_credential_write_policy( "user_id": "jwt-owner", "server_id": server.server_id, } + from litellm.proxy.common_utils.encrypt_decrypt_utils import decrypt_value_helper + + payload: Final = json.loads(decrypt_value_helper( + table.upsert.call_args.kwargs["data"]["create"]["credential_b64"], "credential_b64" + )) + if cimd: + assert payload["cimd_client_id"] == "https://gateway.example.com/oauth/client-metadata.json" + else: + assert "cimd_client_id" not in payload + @pytest.mark.asyncio @pytest.mark.parametrize("allowed", [False, True]) @@ -13362,3 +13376,402 @@ async def test_register_application_type_keeps_no_registration_endpoint_fallback "redirect_uris": ["https://gateway.example/callback"], } assert len(upstream.calls) == 0 + + +def _cimd_oauth_server(): + from litellm.types.mcp_server.mcp_server_manager import MCPServer + + return MCPServer.model_validate( + { + "server_id": "cimd-server", + "name": "cimd-server", + "server_name": "cimd-server", + "url": "https://mcp.example.com/mcp", + "transport": "http", + "auth_type": "oauth2", + "oauth2_flow": "authorization_code", + "authorization_url": "https://idp.example.com/authorize", + "token_url": "https://idp.example.com/token", + "client_id_metadata_document_supported": True, + } + ) + + +def _cimd_request(): + from starlette.requests import Request + + return Request( + { + "type": "http", + "method": "GET", + "scheme": "https", + "path": "/", + "root_path": "", + "query_string": b"", + "headers": [], + "server": ("gateway.example.com", 443), + "client": ("127.0.0.1", 10000), + } + ) + + +@pytest.mark.asyncio +async def test_cimd_registration_returns_https_identity_without_dcr(monkeypatch, respx_mock): + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + response = await endpoints.register_client_with_server( + _cimd_request(), _cimd_oauth_server(), "Gateway", None, None, None + ) + body = json.loads(response.body) if hasattr(response, "body") else response + assert body["client_id"] == "https://gateway.example.com/oauth/client-metadata.json" + assert "client_secret" not in body + assert len(respx_mock.calls) == 0 + + +@pytest.mark.asyncio +async def test_cimd_authorization_uses_metadata_identity_and_s256(monkeypatch): + from urllib.parse import parse_qs, urlparse + + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("LITELLM_SALT_KEY", "cimd-test-state-signing-key") + response = await endpoints.authorize_with_server( + _cimd_request(), + _cimd_oauth_server(), + "placeholder", + "https://gateway.example.com/ui/", + code_challenge="a" * 43, + code_challenge_method="S256", + ) + params = parse_qs(urlparse(response.headers["location"]).query) + assert params["client_id"] == ["https://gateway.example.com/oauth/client-metadata.json"] + assert params["redirect_uri"] == ["https://gateway.example.com/callback"] + assert params["code_challenge_method"] == ["S256"] + + +@pytest.mark.asyncio +async def test_cimd_authorization_rejects_missing_pkce(monkeypatch): + from fastapi import HTTPException + + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("LITELLM_SALT_KEY", "cimd-test-state-signing-key") + with pytest.raises(HTTPException) as exc: + await endpoints.authorize_with_server( + _cimd_request(), _cimd_oauth_server(), "placeholder", "https://gateway.example.com/ui/" + ) + assert exc.value.status_code == 400 + + +def test_cimd_refresh_request_uses_same_identity_without_caller_secret(monkeypatch): + from litellm.proxy._experimental.mcp_server.oauth_utils import build_upstream_oauth2_token_request + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + request = build_upstream_oauth2_token_request( + _cimd_oauth_server(), auth_method=None, client_id="placeholder", client_secret="dummy" + ) + assert request.body["client_id"] == "https://gateway.example.com/oauth/client-metadata.json" + assert "client_secret" not in request.body + assert "Authorization" not in request.headers + + +@pytest.mark.parametrize( + "base", [None, "http://gateway.example.com", "invalid", "https://user:secret@gateway.example.com"] +) +@pytest.mark.asyncio +async def test_cimd_without_stable_https_origin_reports_actionable_error(monkeypatch, base, respx_mock): + from fastapi import HTTPException + + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + if base is not None: + monkeypatch.setenv("PROXY_BASE_URL", base) + with pytest.raises(HTTPException) as exc: + await endpoints.register_client_with_server(_cimd_request(), _cimd_oauth_server(), "Gateway", None, None, None) + assert exc.value.status_code == 400 + assert "HTTPS PROXY_BASE_URL" in str(exc.value.detail) + assert len(respx_mock.calls) == 0 + + +@pytest.mark.parametrize("base", [None, "http://gateway.example.com"]) +@pytest.mark.asyncio +async def test_cimd_without_https_origin_falls_back_to_available_dcr(monkeypatch, base, respx_mock): + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + if base is not None: + monkeypatch.setenv("PROXY_BASE_URL", base) + server = _cimd_oauth_server().model_copy(update={"registration_url": "https://idp.example.com/register"}) + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + post = respx_mock.post("https://idp.example.com/register").respond(201, json={"client_id": "registered-client"}) + response = await endpoints.register_client_with_server(_cimd_request(), server, "Gateway", None, None, None) + assert json.loads(response.body)["client_id"] == "registered-client" + assert post.call_count == 1 + + +@pytest.mark.asyncio +async def test_cimd_yields_to_dynamic_registration_when_the_authorization_server_offers_both(monkeypatch, respx_mock): + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy._experimental.mcp_server.oauth_utils import get_cimd_client_id + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + server = _cimd_oauth_server().model_copy(update={"registration_url": "https://idp.example.com/register"}) + post = respx_mock.post("https://idp.example.com/register").respond(201, json={"client_id": "registered-client"}) + response = await endpoints.register_client_with_server(_cimd_request(), server, "Gateway", None, None, None) + assert json.loads(response.body)["client_id"] == "registered-client" + assert post.call_count == 1 + assert get_cimd_client_id(server) is None + + +@pytest.mark.asyncio +async def test_cimd_is_preferred_over_dynamic_registration_when_the_deployment_opts_in(monkeypatch, respx_mock): + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setattr(proxy_server, "general_settings", {"mcp_prefer_client_id_metadata_document": True}) + server = _cimd_oauth_server().model_copy(update={"registration_url": "https://idp.example.com/register"}) + post = respx_mock.post("https://idp.example.com/register").respond(201, json={"client_id": "registered-client"}) + response = await endpoints.register_client_with_server(_cimd_request(), server, "Gateway", None, None, None) + body = json.loads(response.body) if hasattr(response, "body") else response + assert body["client_id"] == "https://gateway.example.com/oauth/client-metadata.json" + assert "client_secret" not in body + assert post.call_count == 0 + + +@pytest.mark.parametrize( + "updates", + [ + {"client_id": "static-client", "client_secret": "static-secret"}, + {"client_id": "persisted-dcr-client", "dcr_issuer": "https://idp.example.com"}, + {"client_id_metadata_document_supported": False}, + {"auth_type": "oauth_delegate", "dcr_bridge": True}, + {"auth_type": "true_passthrough", "dcr_bridge": True}, + {"delegate_auth_to_upstream": True}, + {"oauth2_flow": "client_credentials"}, + {"client_secret": "configured-secret"}, + {"token_endpoint_auth_method": "client_secret_basic"}, + ], +) +def test_cimd_preserves_existing_identity_and_other_auth_modes(monkeypatch, updates): + from litellm.proxy._experimental.mcp_server.oauth_utils import get_cimd_client_id + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + assert get_cimd_client_id(_cimd_oauth_server().model_copy(update=updates)) is None + + +@pytest.mark.asyncio +async def test_cimd_document_is_public_and_binds_configured_origin(monkeypatch): + import httpx + from fastapi import FastAPI + + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com/proxy") + app = FastAPI() + app.include_router(endpoints.router) + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url="https://attacker.example") as client: + response = await client.get("/oauth/client-metadata.json", headers={"X-Forwarded-Host": "attacker.example"}) + assert response.status_code == 200 + assert response.json()["client_id"] == "https://gateway.example.com/proxy/oauth/client-metadata.json" + assert response.json()["redirect_uris"] == ["https://gateway.example.com/proxy/callback"] + assert response.json()["token_endpoint_auth_method"] == "none" + assert "client_secret" not in response.json() + assert response.headers["cache-control"] == "public, max-age=300" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("base", [None, "https://[invalid"]) +async def test_cimd_document_is_unavailable_without_configured_https_origin(monkeypatch, base): + from fastapi import HTTPException + + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.delenv("PROXY_BASE_URL", raising=False) + if base is not None: + monkeypatch.setenv("PROXY_BASE_URL", base) + with pytest.raises(HTTPException) as exc: + await endpoints.oauth_client_metadata() + assert exc.value.status_code == 404 + + +@pytest.mark.asyncio +async def test_cimd_upstream_client_rejection_is_gateway_fault(monkeypatch, respx_mock): + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + upstream = respx_mock.post("https://idp.example.com/token").respond( + 401, json={"error": "invalid_client", "error_description": "provider-private-detail"} + ) + response = await endpoints.exchange_token_with_server( + request=_cimd_request(), + mcp_server=_cimd_oauth_server(), + grant_type="authorization_code", + code="code", + redirect_uri="https://gateway.example.com/callback", + client_id="caller-placeholder", + client_secret="dummy", + code_verifier="verifier", + ) + assert upstream.call_count == 1 + assert response.status_code == 502 + body = json.loads(response.body) + assert body["error"] == "server_error" + assert "provider-private-detail" not in body["error_description"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flow", ["refresh", "authorize"]) +async def test_optional_cimd_discovery_preserves_the_callers_configured_endpoint( + monkeypatch: pytest.MonkeyPatch, respx_mock: "MockRouter", flow: str +) -> None: + from urllib.parse import parse_qs + + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("LITELLM_MCP_OAUTH_DISCOVERY_ON_STARTUP", "0") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + monkeypatch.setenv("LITELLM_SALT_KEY", "cimd-test-state-signing-key") + token: Final = respx_mock.post("https://idp.example.com/token").respond( + 200, json={"access_token": "upstream-access", "token_type": "Bearer", "expires_in": 3600} + ) + discovery: Final = respx_mock.route().respond(503) + manager: Final = manager_module.MCPServerManager() + monkeypatch.setattr(manager_module, "global_mcp_server_manager", manager) + await manager.load_servers_from_config({"manual": { + "url": "https://mcp.example.com/mcp", "transport": "http", "auth_type": "oauth2", + "oauth2_flow": "authorization_code", + **({"token_url": "https://idp.example.com/token"} if flow == "refresh" + else {"authorization_url": "https://idp.example.com/authorize"}), + }}) + server: Final = next(iter(manager.config_mcp_servers.values())) + async with manager.catalog.operation(): + if flow == "refresh": + response: Final = await endpoints.exchange_token_with_server( + request=_cimd_request(), mcp_server=server, grant_type="refresh_token", + refresh_token="existing-refresh", client_id="existing-client", + code=None, redirect_uri=None, client_secret=None, code_verifier=None, + ) + assert response.status_code == 200 + assert json.loads(response.body)["access_token"] == "upstream-access" + assert parse_qs(token.calls[0].request.content.decode())["refresh_token"] == ["existing-refresh"] + else: + redirect: Final = await endpoints.authorize_with_server( + _cimd_request(), server, "existing-client", "https://gateway.example.com/ui/", + code_challenge="a" * 43, code_challenge_method="S256", + ) + assert redirect.status_code == 307 + assert redirect.headers["location"].startswith("https://idp.example.com/authorize?") + assert discovery.call_count > 0 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("origin", [None, "https://renamed-gateway.example.com"]) +async def test_token_route_preserves_saved_cimd_grant_after_origin_change( + jwt_oauth_identity: tuple["JWTHandler", "RSAPrivateKey"], + monkeypatch: pytest.MonkeyPatch, + respx_mock: "MockRouter", + origin: str | None, +) -> None: + from types import SimpleNamespace + from urllib.parse import parse_qs + + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import db, mcp_server_manager + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("LITELLM_SALT_KEY", "saved-cimd-route-test-salt") + monkeypatch.setenv("DISABLE_AIOHTTP_TRANSPORT", "True") + server: Final = _cimd_oauth_server() + mcp_server_manager.global_mcp_server_manager.registry[server.server_id] = server + identity: Final = "https://gateway.example.com/oauth/client-metadata.json" + table: Final = proxy_server.prisma_client.db.litellm_mcpusercredentials + table.find_unique = AsyncMock(return_value=None) + table.upsert = AsyncMock() + await endpoints._store_per_user_token_server_side( + server, "jwt-owner", {"access_token": "expired", "refresh_token": "saved-refresh", "expires_in": -1}, + cimd_client_id=identity, + ) + + async def saved_row(**kwargs): + return SimpleNamespace(credential_b64=table.upsert.call_args.kwargs["data"]["create"]["credential_b64"]) + + table.find_unique.side_effect = saved_row + if origin is None: + monkeypatch.delenv("PROXY_BASE_URL") + else: + monkeypatch.setenv("PROXY_BASE_URL", origin) + _, key = jwt_oauth_identity + upstream: Final = respx_mock.post(server.token_url).respond( + 200, json={"access_token": "fresh", "refresh_token": "rotated", "expires_in": 3600} + ) + response: Final = await endpoints.exchange_token_with_server( + _token_request({"Authorization": f"Bearer {_oauth_identity_jwt(key, scope='litellm_proxy_admin')}"}), + server, "refresh_token", None, None, identity, None, None, refresh_token="saved-refresh", + ) + assert response.status_code == 200 + assert parse_qs(upstream.calls[0].request.content.decode())["client_id"] == [identity] + persisted: Final = await db.get_user_oauth_credential(proxy_server.prisma_client, "jwt-owner", server.server_id) + assert persisted is not None and persisted["cimd_client_id"] == identity + assert persisted["refresh_token"] == "rotated" + assert table.upsert.await_count == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("state", [ + "matching", "unicode_grant", "foreign_refresh", "missing_refresh", "missing_grant", "database_missing", + "database_outage", "static_client", "anonymous", +]) +async def test_saved_cimd_refresh_identity_is_bound_to_the_callers_stored_grant( + monkeypatch: pytest.MonkeyPatch, state: str, +) -> None: + from types import SimpleNamespace + + from litellm.proxy import proxy_server + from litellm.proxy._experimental.mcp_server import db + from litellm.proxy._experimental.mcp_server import discoverable_endpoints as endpoints + + monkeypatch.setenv("LITELLM_SALT_KEY", "saved-cimd-owner-test-salt") + server: Final = _cimd_oauth_server() + identity: Final = "https://original-gateway.example.com/oauth/client-metadata.json" + grant: Final = "alice-refresh-\u00e9" if state == "unicode_grant" else "alice-refresh" + database: Final = MagicMock() + table: Final = database.db.litellm_mcpusercredentials + table.find_unique = AsyncMock(return_value=None) + table.upsert = AsyncMock() + monkeypatch.setattr(proxy_server, "prisma_client", database) + await db.store_user_oauth_credential( + database, "alice", server.server_id, "expired", + refresh_token=None if state == "missing_refresh" else grant, cimd_client_id=identity, + ) + table.find_unique.reset_mock() + table.find_unique.return_value = ( + None if state == "missing_grant" else SimpleNamespace( + credential_b64=table.upsert.call_args.kwargs["data"]["create"]["credential_b64"] + ) + ) + if state == "database_missing": + monkeypatch.setattr(proxy_server, "prisma_client", None) + if state == "database_outage": + table.find_unique.side_effect = RuntimeError("database unavailable") + if state == "static_client": + server.client_id = "configured-client" + resolved: Final = await endpoints._saved_cimd_refresh_client_id( + server, None if state == "anonymous" else "alice", + "foreign-refresh" if state == "foreign_refresh" else grant, + ) + assert resolved == (identity if state in ("matching", "unicode_grant") else None) + if state in ("anonymous", "static_client", "database_missing"): + table.find_unique.assert_not_awaited() + else: + table.find_unique.assert_awaited_once_with(where={"user_id_server_id": {"user_id": "alice", "server_id": server.server_id}}) diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 5bd4ff5d894..5731738f137 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -1461,6 +1461,7 @@ class TestMCPServerManager: manager = MCPServerManager() metadata = MCPOAuthMetadata( + client_id_metadata_document_supported=True, authorization_url="https://attacker.example.com/authorize", token_url="https://attacker.example.com/token", scopes=["read", "admin"], @@ -1478,6 +1479,8 @@ class TestMCPServerManager: assert server.token_url is None assert server.scopes == ["read", "admin"] + assert server.client_id_metadata_document_supported is False + @pytest.mark.asyncio async def test_load_servers_from_config_fills_token_url_when_metadata_corroborates_manual_authorization_url(self): """Corroborated metadata keeps the self-heal on the config path: when the discovered document @@ -1487,6 +1490,7 @@ class TestMCPServerManager: manager = MCPServerManager() metadata = MCPOAuthMetadata( + client_id_metadata_document_supported=True, authorization_url="https://idp.example.com/authorize", token_url="https://idp.example.com/token", scopes=["read", "admin"], @@ -1503,6 +1507,8 @@ class TestMCPServerManager: assert server.token_url == "https://idp.example.com/token" assert server.scopes == ["read", "admin"] + assert server.client_id_metadata_document_supported is True + @pytest.mark.asyncio @pytest.mark.parametrize("blank_authorization_url", ["", " "]) async def test_load_servers_from_config_blank_authorization_url_is_not_a_pin(self, blank_authorization_url): @@ -2535,6 +2541,7 @@ class TestMCPServerManager: ) metadata = MCPOAuthMetadata( + client_id_metadata_document_supported=True, authorization_url="https://idp.example.com/authorize", token_url="https://idp.example.com/token", registration_url="https://idp.example.com/register", @@ -2548,6 +2555,8 @@ class TestMCPServerManager: assert built.registration_url == "https://idp.example.com/register" assert built.scopes == ["read"] + assert built.client_id_metadata_document_supported is True + @pytest.mark.asyncio async def test_build_from_table_uses_issuer_anchored_endpoints_when_issuer_configured(self): """When an admin configures an issuer, the build takes its endpoints from the issuer-anchored @@ -4507,6 +4516,7 @@ class TestMCPServerManager: "authorization_endpoint": "https://idp.example.com/authorize", "token_endpoint": "https://idp.example.com/token", "scopes_supported": ["read", "write"], + "client_id_metadata_document_supported": True, }, ) mock_client = MagicMock() @@ -4522,6 +4532,8 @@ class TestMCPServerManager: assert result.token_url == "https://idp.example.com/token" assert result.scopes == ["read", "write"] + assert result.client_id_metadata_document_supported is True + @pytest.mark.asyncio async def test_fetch_single_authorization_server_metadata_rejects_issuer_mismatch(self): """RFC 8414 §3.3 fail-closed: a document self-attesting a DIFFERENT issuer than the one it was @@ -19108,3 +19120,150 @@ def test_discovery_keys_bind_static_auth_to_caller_and_configuration() -> None: manager._discovery_key(updated, first, None, None, None, None), ) assert len(set(keys)) == 3 + + +@pytest.mark.parametrize("advertised", [False, True]) +def test_untrusted_metadata_cannot_enable_cimd_without_matching_authorization_endpoint(advertised): + from litellm.proxy._experimental.mcp_server.mcp_server_manager import ( + _restrict_discovery_to_corroborated_authorization_server, + ) + + metadata = MCPOAuthMetadata(scopes=["read"], client_id_metadata_document_supported=advertised) + result = _restrict_discovery_to_corroborated_authorization_server( + metadata, "https://trusted.example.com/authorize", "server", False + ) + assert result is not None + assert result.client_id_metadata_document_supported is False + assert result.scopes == ["read"] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("source", ["config", "database"]) +@pytest.mark.parametrize("advertised", [True, False]) +@pytest.mark.parametrize("startup", [True, False]) +async def test_manual_oauth_endpoints_discover_client_metadata_once( + source: str, advertised: bool, startup: bool, respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + from starlette.requests import Request + + from litellm.proxy._experimental.mcp_server import mcp_server_manager as manager_module + from litellm.proxy._experimental.mcp_server.oauth_utils import get_cimd_client_id + + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("LITELLM_MCP_OAUTH_DISCOVERY_ON_STARTUP", "1" if startup else "0") + await _mock_oauth_discovery(respx_mock, monkeypatch, server_url="https://up.example.com/mcp", scopes=["read"]) + metadata: Final = respx_mock.get("https://up.example.com/.well-known/oauth-authorization-server").respond( + json={ + "issuer": "https://up.example.com", + "authorization_endpoint": "https://up.example.com/authorize", + "token_endpoint": "https://up.example.com/token", + "client_id_metadata_document_supported": advertised, + } + ) + manager: Final = MCPServerManager() + monkeypatch.setattr(manager_module, "global_mcp_server_manager", manager) + configured: Final[MCPServer] + if source == "config": + await manager.load_servers_from_config({"manual": { + "url": "https://up.example.com/mcp", "transport": "http", "auth_type": "oauth2", + "oauth2_flow": "authorization_code", "authorization_url": "https://up.example.com/authorize", + "token_url": "https://up.example.com/token", "scopes": ["read"], + }}) + configured = next(iter(manager.config_mcp_servers.values())) + else: + row: Final = LiteLLM_MCPServerTable( + server_id="manual", alias="manual", url="https://up.example.com/mcp", transport=MCPTransport.http, + auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", + authorization_url="https://up.example.com/authorize", token_url="https://up.example.com/token", + credentials={"scopes": ["read"]}, created_at=datetime.now(), updated_at=datetime.now(), + ) + built: Final = await manager.build_mcp_server_from_table(row, credentials_are_encrypted=False) + manager.registry[built.server_id] = built + configured = built + async with manager.catalog.operation(): + response: Final = await discoverable_endpoints.register_client_with_server( + Request({"type": "http", "scheme": "https", "server": ("gateway.example.com", 443), + "path": "/register", "root_path": "", "headers": [], "query_string": b""}), + configured, "Gateway", None, None, None, + ) + body: Final = json.loads(response.body) if hasattr(response, "body") else response + assert body["client_id"] == ( + "https://gateway.example.com/oauth/client-metadata.json" if advertised else configured.server_name + ) + resolved: Final = await manager.ensure_oauth_metadata_discovered(configured) + again: Final = await manager.ensure_oauth_metadata_discovered(resolved) + assert metadata.call_count == 1 + assert resolved.client_id_metadata_document_supported is advertised + assert again == resolved + assert get_cimd_client_id(resolved) == ( + "https://gateway.example.com/oauth/client-metadata.json" if advertised else None + ) + assert resolved.effective_authorization_url == "https://up.example.com/authorize" + assert resolved.effective_token_url == "https://up.example.com/token" + + +@pytest.mark.asyncio +async def test_optional_client_metadata_discovery_failure_preserves_manual_endpoints( + respx_mock: MockRouter, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + monkeypatch.setenv("LITELLM_MCP_OAUTH_DISCOVERY_ON_STARTUP", "0") + await _mock_oauth_discovery(respx_mock, monkeypatch, server_url="https://up.example.com/mcp", scopes=["read"]) + respx_mock.get("https://up.example.com/.well-known/oauth-authorization-server").respond(503) + respx_mock.route().respond(404) + manager: Final = MCPServerManager() + await manager.load_servers_from_config({"manual": { + "url": "https://up.example.com/mcp", "transport": "http", "auth_type": "oauth2", + "oauth2_flow": "authorization_code", "authorization_url": "https://up.example.com/authorize", + "token_url": "https://up.example.com/token", "scopes": ["read"], + }}) + configured: Final = next(iter(manager.config_mcp_servers.values())) + resolved: Final = await manager.ensure_oauth_metadata_discovered(configured) + attempts: Final = len(respx_mock.calls) + retry: Final = await manager.ensure_oauth_metadata_discovered(resolved) + assert attempts > 0 + assert len(respx_mock.calls) == attempts + assert retry == resolved + assert resolved.client_id_metadata_document_supported is None + assert resolved.effective_authorization_url == "https://up.example.com/authorize" + assert resolved.effective_token_url == "https://up.example.com/token" + assert manager.oauth_discovery_slot(resolved.server_id) is not None + + +@pytest.mark.parametrize("capability", [True, False]) +@pytest.mark.parametrize("rebuild", ["same", "repointed", "fresh_discovery", "anchored"]) +def test_oauth_rebuild_retains_only_corroborated_cimd_capability(capability: bool, rebuild: str) -> None: + from litellm.proxy._experimental.mcp_server.mcp_server_manager import carry_forward_resolved_oauth_endpoints + + previous: Final = MCPServer( + server_id="cimd-rebuild", name="cimd_rebuild", url="https://mcp.example.com/mcp", + transport=MCPTransport.http, auth_type=MCPAuth.oauth2, + authorization_url="https://idp.example.com/authorize", token_url="https://idp.example.com/token", + client_id_metadata_document_supported=capability, + ) + rebuilt: Final = previous.model_copy(update={ + "client_id_metadata_document_supported": not capability if rebuild == "fresh_discovery" else None, + "authorization_url": "https://changed.example.com/authorize" if rebuild == "repointed" else previous.authorization_url, + "token_url": None, + "issuer": "https://idp.example.com" if rebuild == "anchored" else None, + "issuer_is_anchored": rebuild == "anchored", + }) + carry_forward_resolved_oauth_endpoints(rebuilt, previous) + expected: Final = not capability if rebuild == "fresh_discovery" else None if rebuild in ("repointed", "anchored") else capability + assert rebuilt.client_id_metadata_document_supported is expected + + +@pytest.mark.asyncio +@pytest.mark.parametrize("endpoint", ["authorization_url", "token_url"]) +async def test_repeated_stale_discovery_uses_current_callers_endpoint(endpoint: str) -> None: + manager: Final = MCPServerManager() + original: Final = MCPServer( + server_id="partial-replacement", name="replacement", url="https://old.example.com/mcp", + transport=MCPTransport.http, auth_type=MCPAuth.oauth2, oauth2_flow="authorization_code", + ) + replacement: Final = original.model_copy(update={endpoint: "https://new.example.com/oauth"}) + manager.registry[original.server_id] = replacement + resolved: Final = await manager._rejoin_oauth_metadata_discovery( + original, needed_endpoint=lambda server: getattr(server, endpoint), retry_stale=False, + ) + assert resolved is replacement diff --git a/tests/unit/proxy/test__lazy_features.py b/tests/unit/proxy/test__lazy_features.py index c3c5068b890..c8e34ad750c 100644 --- a/tests/unit/proxy/test__lazy_features.py +++ b/tests/unit/proxy/test__lazy_features.py @@ -21,6 +21,20 @@ FLAG: Final = "LITELLM_DISABLE_LAZY_ROUTES" WARMUP_PATH: Final = "/lazy/warm/{name}" +def test_cimd_metadata_is_available_before_any_oauth_request(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(FLAG, "false") + monkeypatch.setenv("PROXY_BASE_URL", "https://gateway.example.com") + app: Final = FastAPI() + attach_lazy_features(app) + + with TestClient(app) as client: + response: Final = client.get("/oauth/client-metadata.json") + + assert response.status_code == 200 + assert response.json()["client_id"] == "https://gateway.example.com/oauth/client-metadata.json" + assert response.json()["redirect_uris"] == ["https://gateway.example.com/callback"] + + class _Operation(BaseModel): tags: tuple[str, ...] diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 1108d4a8743..bf863323e77 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -30314,6 +30314,11 @@ export interface components { * @description Custom CIDR ranges that define internal/private networks for MCP access control. When set, only these ranges are treated as internal. Defaults to RFC 1918 private ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, 127.0.0.0/8). */ mcp_internal_ip_ranges?: string[] | null; + /** + * Mcp Prefer Client Id Metadata Document + * @description When true, a gateway-managed OAuth2 MCP server whose authorization server advertises Client ID Metadata Document support identifies itself with the gateway's public metadata document URL even when that authorization server also offers dynamic client registration. Requires a public HTTPS PROXY_BASE_URL the authorization server can fetch. Default false: dynamic client registration is used whenever the authorization server offers it, and the metadata document only when it does not. + */ + mcp_prefer_client_id_metadata_document?: boolean | null; /** * Mcp Required Fields * @description List of MCP server fields that must be filled in for a submission to pass standards checks (e.g. ['description', 'source_url', 'alias']). From 9a2c9e60992ec78cec8cc5f082da09a96b0cc015 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:18:25 -0700 Subject: [PATCH 33/65] fix(proxy-extras): name the database error when the Lens rename check cannot run (#45341) * fix(proxy-extras): name the database error when the Lens rename check cannot run The Lens rename check runs before prisma db push and raised a generic "Cannot verify Lens data safety" RuntimeError on any psycopg error, with the cause only in the chained exception. proxy_cli prints the message and exits, so a Postgres at its connection limit read as a Lens problem. The raised message now ends with the redacted database error text, and the connection opener is an injected parameter so the unit test drives the check without a database. * test(proxy-extras): drop the Lens check test class docstring --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../litellm_proxy_extras/utils.py | 22 ++++++---- .../test_litellm_proxy_extras_utils.py | 43 +++++++++++++++++++ 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/litellm-proxy-extras/litellm_proxy_extras/utils.py b/litellm-proxy-extras/litellm_proxy_extras/utils.py index 1fd292b8137..72462fa9ae4 100644 --- a/litellm-proxy-extras/litellm_proxy_extras/utils.py +++ b/litellm-proxy-extras/litellm_proxy_extras/utils.py @@ -11,7 +11,7 @@ import time from collections.abc import Callable from dataclasses import dataclass, replace from pathlib import Path -from typing import TYPE_CHECKING, Final, Optional, Union +from typing import TYPE_CHECKING, Final, Optional, Protocol, Union from urllib.parse import unquote, urlsplit from litellm_proxy_extras import prisma_toolchain @@ -35,6 +35,12 @@ if TYPE_CHECKING: import psycopg.sql +class LensCheckConnect(Protocol): + def __call__( + self, conninfo: str, *, connect_timeout: int, autocommit: bool + ) -> "psycopg.Connection[tuple[object, ...]]": ... + + def str_to_bool(value: Optional[str]) -> bool: if value is None: return False @@ -243,11 +249,9 @@ def _redact_credentials(text: str) -> str: passwords: Final = sorted(_configured_database_passwords(), key=len, reverse=True) alternation: Final = "|".join(re.escape(password) for password in passwords) password_pattern: Final = ( - re.compile(rf"(?P:|password=)(?:{alternation})(?=@|&|$|[\s'\"\]),])", re.IGNORECASE) - if passwords - else None + re.compile(rf"(?{_REDACTED}", text) if password_pattern is not None else text + result: Final = password_pattern.sub(_REDACTED, text) if password_pattern is not None else text return _secret_shape_redactor()(result) @@ -690,7 +694,7 @@ class ProxyExtrasDBManager: ) @staticmethod - def raise_if_lens_rename_pending() -> None: + def raise_if_lens_rename_pending(connect: LensCheckConnect | None = None) -> None: database_url: Final = os.environ.get("DATABASE_URL") if not database_url: return @@ -698,8 +702,9 @@ class ProxyExtrasDBManager: import psycopg except ImportError as exc: raise RuntimeError("Install psycopg to verify Lens data safety before prisma db push.") from exc + open_connection: Final = connect if connect is not None else psycopg.connect try: - with psycopg.connect( + with open_connection( ProxyExtrasDBManager._strip_prisma_query_params(database_url), connect_timeout=10, autocommit=True ) as connection: legacy: Final = connection.execute( @@ -710,7 +715,8 @@ class ProxyExtrasDBManager: ).fetchone() except psycopg.Error as exc: raise RuntimeError( - "Cannot verify Lens data safety; refusing prisma db push. Check database connectivity and psycopg installation." + "Cannot verify Lens data safety; refusing prisma db push. " + f"The database check failed: {_redact_credentials(str(exc)).strip()}" ) from exc if legacy is not None: raise RuntimeError( diff --git a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py index 70c6ec44790..7edb531a63b 100644 --- a/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py +++ b/tests/unit/litellm_proxy_extras/test_litellm_proxy_extras_utils.py @@ -655,6 +655,49 @@ class _FakePsycopgConn: return _FakeCursor() +class TestLensRenamePendingCheck: + _DATABASE_URL: Final = "postgresql://litellm:hunter2@localhost:5432/litellm" + + def test_database_failure_text_reaches_the_raised_message(self, monkeypatch: pytest.MonkeyPatch) -> None: + import psycopg + + monkeypatch.setenv("DATABASE_URL", self._DATABASE_URL) + + def refuse(conninfo: str, *, connect_timeout: int, autocommit: bool) -> NoReturn: + raise psycopg.OperationalError("FATAL: sorry, too many clients already") + + with pytest.raises(RuntimeError) as err: + ProxyExtrasDBManager.raise_if_lens_rename_pending(connect=refuse) + assert "FATAL: sorry, too many clients already" in str(err.value) + + def test_password_libpq_echoes_is_redacted_from_the_message(self, monkeypatch: pytest.MonkeyPatch) -> None: + import psycopg + + password: Final = "p%zzword" + database_url: Final = f"postgresql://litellm:{password}@localhost:5432/litellm" + monkeypatch.setenv("DATABASE_URL", database_url) + monkeypatch.delenv("DIRECT_URL", raising=False) + with pytest.raises(psycopg.Error) as libpq: + psycopg.connect(database_url, connect_timeout=10, autocommit=True) + + with pytest.raises(RuntimeError) as err: + ProxyExtrasDBManager.raise_if_lens_rename_pending() + assert password not in str(err.value) + assert str(err.value).endswith(str(libpq.value).strip().replace(password, "REDACTED")) + + def test_legacy_tables_keep_their_own_message(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("DATABASE_URL", self._DATABASE_URL) + executed: Final[list[tuple[str, tuple[str]]]] = [] + + def legacy_tables_present(conninfo: str, *, connect_timeout: int, autocommit: bool) -> _FakePsycopgConn: + return _FakePsycopgConn(executed) + + with pytest.raises(RuntimeError) as err: + ProxyExtrasDBManager.raise_if_lens_rename_pending(connect=legacy_tables_present) + assert str(err.value).startswith("Legacy Lens tables exist.") + assert executed[0][1] == ("public",) + + class TestSpendLogsPartitionDetectionSchemaScope: """A same-named LiteLLM_SpendLogs in another schema must not trip the detector: the catalog lookup has to be scoped to Prisma's target schema.""" From 01166906fb7f36d2b45a8aa43b10affbeb6e57c9 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:24:13 -0700 Subject: [PATCH 34/65] test(integration): run the response cache tool-call tests on a proxy without the message cap (#45350) * test(integration): run the response cache tool-call tests on a proxy without the message cap Since #43878 the response cache skips any request past 4 messages by default, so the three providers-lane tests that post a seven-message tool conversation twice never saw a cache hit. They now boot one owned proxy from the lane config with max_messages set to null and keep their assertions; the shared proxy and the caching-lane cap test are unchanged * test(integration): give the uncapped-proxy cache tests the owned-proxy timeout budget The providers lane runs pytest with a 90 second per-test timeout that includes fixture setup, and the first of the three tests to run pays the owned proxy boot before the spend-log test can poll for up to 70 seconds. The sibling owned-proxy tests already carry pytest.mark.timeout(240), so these three take the same budget --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- ...test_cache_control_tool_call_marks_wire.py | 61 +++++++++++++++---- 1 file changed, 48 insertions(+), 13 deletions(-) diff --git a/tests/integration/providers/test_cache_control_tool_call_marks_wire.py b/tests/integration/providers/test_cache_control_tool_call_marks_wire.py index de5d4170fc8..21fee0554ea 100644 --- a/tests/integration/providers/test_cache_control_tool_call_marks_wire.py +++ b/tests/integration/providers/test_cache_control_tool_call_marks_wire.py @@ -1,16 +1,26 @@ import asyncio import json -from collections.abc import Iterable +from collections.abc import Iterable, Iterator from datetime import datetime, timedelta, timezone +from pathlib import Path from typing import Final, cast import anthropic import httpx import openai import pytest +import yaml from anthropic.types import MessageParam, TextBlockParam, ToolParam -from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.client import ( + Gateway, + Scenario, + eventually, + gateway_from_environment, + object_value, + string_value, +) from integration._support.database import read_rows +from integration._support.process import owned_proxy from integration._support.wire import Reply, Request, Wire, wire_server from integration.providers._cache_control_marks_support import ( ANTHROPIC_MODEL, @@ -52,6 +62,7 @@ from pydantic import JsonValue, TypeAdapter from litellm.utils import get_prompt_cache_min_tokens _JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_LANE_CONFIG: Final = Path(__file__).resolve().parents[1] / "proxy_config.yaml" _CLIENT_MARKED: Final = client_marked() _GEMINI_REPLY: Final = json.dumps( { @@ -71,6 +82,25 @@ _OPENAI_REPLY: Final = json.dumps( ).encode() +def _uncapped_cache_config(directory: Path) -> Path: + lane: Final = object_value(yaml.safe_load(_LANE_CONFIG.read_text())) + settings: Final = object_value(lane["litellm_settings"]) + cache_params: Final = object_value(settings["cache_params"]) + config: Final = {**lane, "litellm_settings": {**settings, "cache_params": {**cache_params, "max_messages": None}}} + path: Final = directory / "proxy_config_uncapped_cache.yaml" + path.write_text(yaml.safe_dump(config, sort_keys=False)) + return path + + +@pytest.fixture(scope="module") +def uncapped_gateway(tmp_path_factory: pytest.TempPathFactory) -> Iterator[Gateway]: + """A proxy whose response cache has no `max_messages` cap, so a seven-message tool conversation is cached""" + with gateway_from_environment() as lane: + directory: Final = tmp_path_factory.mktemp("uncapped_cache") + with owned_proxy(lane, directory, {}, config=_uncapped_cache_config(directory)) as owned: + yield owned + + def _anthropic_deployment(scenario: Scenario, wire: Wire, **fields: JsonValue) -> str: return scenario.model(model=f"anthropic/{ANTHROPIC_MODEL}", api_base=wire.url, api_key=PROVIDER_KEY, **fields) @@ -485,13 +515,14 @@ def test_responses_bridge_keeps_system_and_user_marks_within_the_cap(gateway: Ga assert anthropic_labels(_only_request(wire)) == [SYSTEM_LABEL, ASK_LABEL] -def test_response_cache_serves_the_capped_request_once(gateway: Gateway) -> None: +@pytest.mark.timeout(240) +def test_response_cache_serves_the_capped_request_once(uncapped_gateway: Gateway) -> None: marker: Final = new_marker() - with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario: + with wire_server(anthropic_peer) as wire, uncapped_gateway.scenario() as scenario: model: Final = _anthropic_deployment(scenario, wire, cache_control_injection_points=POINTS) body: Final = chat_body(model, conversation(marker, marked_calls())) - first: Final = post_chat(gateway, body) - second: Final = post_chat(gateway, body) + first: Final = post_chat(uncapped_gateway, body) + second: Final = post_chat(uncapped_gateway, body) received: Final = wire.drain() assert (first[0], second[0]) == (200, 200), (first[2], second[2]) assert first[1].startswith("chatcmpl-"), first[2] @@ -828,14 +859,15 @@ def _cache_hit_rows(response_id: str) -> list[dict[str, JsonValue]]: ) -def test_response_cache_hit_records_the_injection_on_the_first_row_only(gateway: Gateway) -> None: +@pytest.mark.timeout(240) +def test_response_cache_hit_records_the_injection_on_the_first_row_only(uncapped_gateway: Gateway) -> None: marker: Final = new_marker() unmarked: Final = [tool_call(city) for city in CITIES] - with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario: + with wire_server(anthropic_peer) as wire, uncapped_gateway.scenario() as scenario: model: Final = _anthropic_deployment(scenario, wire, cache_control_injection_points=POINTS) body: Final = chat_body(model, conversation(marker, unmarked, ask_marked=False)) - first: Final = post_chat(gateway, body) - second: Final = post_chat(gateway, body) + first: Final = post_chat(uncapped_gateway, body) + second: Final = post_chat(uncapped_gateway, body) received: Final = wire.drain() assert (first[0], second[0]) == (200, 200), (first[2], second[2]) assert first[1] == second[1], (first[2], second[2]) @@ -845,15 +877,18 @@ def test_response_cache_hit_records_the_injection_on_the_first_row_only(gateway: assert hit_rows[0]["injected"] is None, hit_rows +@pytest.mark.timeout(240) @pytest.mark.parametrize("path", ("/v1/messages", "/v1/responses")) -def test_response_cache_twins_on_messages_and_responses_stay_within_the_cap(gateway: Gateway, path: str) -> None: +def test_response_cache_twins_on_messages_and_responses_stay_within_the_cap( + uncapped_gateway: Gateway, path: str +) -> None: marker: Final = new_marker() - with wire_server(anthropic_peer) as wire, gateway.scenario() as scenario: + with wire_server(anthropic_peer) as wire, uncapped_gateway.scenario() as scenario: model: Final = _anthropic_deployment(scenario, wire, cache_control_injection_points=POINTS) body: Final = ( messages_body(model, marker, stream=False) if path == "/v1/messages" else responses_body(model, marker) ) - responses: Final = tuple(gateway.request("POST", path, body) for _ in range(2)) + responses: Final = tuple(uncapped_gateway.request("POST", path, body) for _ in range(2)) received: Final = wire.drain() assert [response.status_code for response in responses] == [200, 200], [response.text for response in responses] expected: Final = _CLIENT_MARKED if path == "/v1/messages" else [SYSTEM_LABEL, ASK_LABEL] From df13d5930634e75b40a6eb3d61b6ebc1548532cf Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:45:21 -0700 Subject: [PATCH 35/65] fix(logging): skip sync success callbacks for internal sub-calls, deflake RAG and Langfuse tests (#45345) * fix(logging): skip sync success callbacks for internal sub-calls Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(langfuse): assert no retry sleep instead of a wall-clock budget Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: add types to deflake regression coverage Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(tests): avoid unrelated formatting changes Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/utils.py | 23 ++-- .../langfuse/test_langfuse_sdk.py | 23 ++-- tests/unit/rag/test_main.py | 102 +++++++++++++++++- 3 files changed, 127 insertions(+), 21 deletions(-) diff --git a/litellm/utils.py b/litellm/utils.py index 9ded4ff1131..7a334f8974b 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -1895,17 +1895,18 @@ def client(original_function): # LOG SUCCESS - handle streaming success logging in the _next_ object, remove `handle_success` once it's deprecated verbose_logger.info("Wrapper: Completed Call, calling success_handler") - # Copy the current context to propagate it to the background thread - # This is essential for OpenTelemetry span context propagation - ctx: Final = contextvars.copy_context() - executor: Final[BoundedLoggingThreadPoolExecutor] = getattr(sys.modules[__name__], "executor") - executor.submit( - ctx.run, - logging_obj.success_handler, - result, - start_time, - end_time, - ) + if not is_internal_call.get(): + # Copy the current context to propagate it to the background thread + # This is essential for OpenTelemetry span context propagation + ctx: Final = contextvars.copy_context() + executor: Final[BoundedLoggingThreadPoolExecutor] = getattr(sys.modules[__name__], "executor") + executor.submit( + ctx.run, + logging_obj.success_handler, + result, + start_time, + end_time, + ) # RETURN RESULT return result except Exception as e: diff --git a/tests/unit/integrations/langfuse/test_langfuse_sdk.py b/tests/unit/integrations/langfuse/test_langfuse_sdk.py index 5f01b35a338..cf13667608b 100644 --- a/tests/unit/integrations/langfuse/test_langfuse_sdk.py +++ b/tests/unit/integrations/langfuse/test_langfuse_sdk.py @@ -8,17 +8,19 @@ otherwise record its own duration instead of the call's. import json import logging import threading +import time import uuid from base64 import b64encode from datetime import datetime, timedelta, timezone from time import monotonic, sleep -from types import MappingProxyType +from types import MappingProxyType, SimpleNamespace from typing import Final import httpx import opentelemetry.trace as otel_trace import pytest from langfuse import LangfuseOtelSpanAttributes as A +from langfuse.api.core import http_client as langfuse_http_client from langfuse.api.core.api_error import ApiError from langfuse.api.core.request_options import RequestOptions from opentelemetry.proto.collector.trace.v1.trace_service_pb2 import ExportTraceServiceRequest @@ -1042,28 +1044,35 @@ def test_auth_check_fails_when_the_keys_reach_no_project(): @pytest.mark.parametrize("status", [500, 503, 429], ids=["http-500", "http-503", "http-429"]) -def test_auth_check_and_project_id_make_one_round_trip_when_langfuse_is_down(status): +def test_auth_check_and_project_id_make_one_round_trip_when_langfuse_is_down( + status: int, monkeypatch: pytest.MonkeyPatch +) -> None: """Both run on the event loop; the generated client's default retries sleep for seconds, or for Retry-After.""" - requests: list[httpx.Request] = [] + requests: Final[list[httpx.Request]] = [] + sleeps: Final[list[float]] = [] + monkeypatch.setattr( + langfuse_http_client, + "time", + SimpleNamespace(sleep=sleeps.append, time=time.time), + ) def fail(request: httpx.Request) -> httpx.Response: requests.append(request) return httpx.Response(status, request=request, headers={"retry-after": "20"}, json={"message": "down"}) - client = build_langfuse_client( + client: Final = build_langfuse_client( public_key="pk", secret_key="sk", base_url="http://127.0.0.1:1", httpx_client=httpx.Client(transport=httpx.MockTransport(fail)), ) - started = monotonic() - failure = client.auth_check() + failure: Final = client.auth_check() with pytest.raises(ApiError): client.project_id() assert failure is not None and f"status_code: {status}" in failure.reason assert len(requests) == 2 - assert monotonic() - started < 0.5 + assert sleeps == [], "failed auth checks should not sleep for REST retries" @pytest.mark.parametrize( diff --git a/tests/unit/rag/test_main.py b/tests/unit/rag/test_main.py index 81318a1e113..87599d32c22 100644 --- a/tests/unit/rag/test_main.py +++ b/tests/unit/rag/test_main.py @@ -11,9 +11,12 @@ aquery carries the completion response with real usage and cost. """ import asyncio +import datetime import json +from collections.abc import Callable, Mapping +from concurrent.futures import Future from types import MappingProxyType -from typing import Final +from typing import Final, cast from unittest.mock import patch import httpx @@ -22,8 +25,10 @@ import respx from pydantic import ValidationError import litellm +import litellm.utils as litellm_utils from litellm._internal_context import is_internal_call from litellm.integrations.custom_logger import CustomLogger +from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.logging_worker import GLOBAL_LOGGING_WORKER from litellm.types.utils import CallTypes, ModelResponse @@ -45,11 +50,27 @@ async def _drain_logging_worker() -> None: class RecordingLogger(CustomLogger): def __init__(self): super().__init__() - self.success_events = [] + self.success_events: Final[list[dict[str, object]]] = [] + self.sync_success_events: Final[list[dict[str, object]]] = [] - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): + async def async_log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: self.success_events.append({"kwargs": kwargs, "response_obj": response_obj}) + def log_success_event( + self, + kwargs: Mapping[str, object], + response_obj: object, + start_time: datetime.datetime, + end_time: datetime.datetime, + ) -> None: + self.sync_success_events.append({"kwargs": kwargs, "response_obj": response_obj}) + @pytest.mark.asyncio @pytest.mark.parametrize("use_router", [False, True]) @@ -107,6 +128,81 @@ async def test_aquery_single_billing_event_carries_completion_usage_and_cost(use assert standard_logging_object["response_cost"] > 0 +@pytest.mark.asyncio +@pytest.mark.parametrize("use_router", [False, True]) +async def test_aquery_vector_store_search_sub_call_logs_no_sync_success_event_on_the_parent( + use_router: bool, monkeypatch: pytest.MonkeyPatch +) -> None: + await _drain_logging_worker() + recording_logger: Final = RecordingLogger() + + class InlineExecutor: + def submit( + self, + fn: Callable[..., object], + *args: object, + **kwargs: object, + ) -> Future[object]: + future: Final = Future[object]() + future.set_result(fn(*args, **kwargs)) + return future + + monkeypatch.setattr(litellm_utils, "executor", InlineExecutor()) + monkeypatch.setattr(litellm_logging, "executor", InlineExecutor()) + monkeypatch.setattr(litellm, "callbacks", [recording_logger]) + + router_kwargs: Final = ( + { + "router": litellm.Router( + model_list=[ + { + "model_name": "gpt-4o-mini", + "litellm_params": {"model": "openai/gpt-4o-mini", "api_key": "test-key"}, + } + ] + ) + } + if use_router + else {} + ) + + response: Final = await litellm.aquery( + model="gpt-4o-mini", + messages=[{"role": "user", "content": "What is the secret project codename?"}], + retrieval_config={"vector_store_id": "vs_test_123", "custom_llm_provider": "openai"}, + mock_response="The secret project codename is AZURE-FALCON-42.", + **router_kwargs, + ) + assert isinstance(response, ModelResponse), "aquery should return its completion response" + assert is_internal_call.get() is False, "aquery should restore the internal-call context" + await _drain_logging_worker() + + assert len(recording_logger.success_events) == 1, "aquery should run one async success callback" + event: Final = recording_logger.success_events[0] + response_obj: Final = event["response_obj"] + assert isinstance(response_obj, ModelResponse), "the async callback should receive the completion response" + assert response_obj.usage.total_tokens > 0, "the async callback response should include completion usage" + + event_kwargs: Final = cast(Mapping[str, object], event["kwargs"]) + standard_logging_object: Final = cast(Mapping[str, object], event_kwargs["standard_logging_object"]) + assert cast(str, standard_logging_object["call_type"]) == "aquery", "the async event should be for aquery" + assert cast(int, standard_logging_object["total_tokens"]) > 0, ( + "the async event should include total completion tokens" + ) + assert cast(int, standard_logging_object["prompt_tokens"]) > 0, "the async event should include prompt tokens" + assert cast(int, standard_logging_object["completion_tokens"]) > 0, ( + "the async event should include completion tokens" + ) + assert cast(float, standard_logging_object["response_cost"]) > 0, "the async event should include completion cost" + sync_response_types: Final = [ + type(event["response_obj"]).__name__ for event in recording_logger.sync_success_events + ] + assert recording_logger.sync_success_events == [], ( + "an internal sub-call must not log on the parent's logging object; " + f"sync success event response types: {sync_response_types}" + ) + + @pytest.mark.asyncio async def test_aquery_response_hidden_params_carry_completion_cost(): """ From d8c0e2c7153d82234ec46f0231bf9b9714e7d52a Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 04:53:13 -0700 Subject: [PATCH 36/65] fix(azure): move sora-2 retirement date to the later Models API date (#45355) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 2 +- model_prices_and_context_window.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 32fd15de9e7..97760a4f0b9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -53201,7 +53201,7 @@ ] }, "azure/sora-2": { - "deprecation_date": "2026-10-15", + "deprecation_date": "2026-11-02", "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 32fd15de9e7..97760a4f0b9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -53201,7 +53201,7 @@ ] }, "azure/sora-2": { - "deprecation_date": "2026-10-15", + "deprecation_date": "2026-11-02", "litellm_provider": "azure", "mode": "video_generation", "output_cost_per_video_per_second": 0.1, From f824d11a22383abc6cb316cb51cb4e739afeecbd Mon Sep 17 00:00:00 2001 From: ishaan-berri <155045088+ishaan-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 07:54:11 -0700 Subject: [PATCH 37/65] fix(lens): block teamless feedback writes and fix retention test (#45266) * fix(lens): reject feedback writes from callers with no team or key * test(lens): teamless callers cannot overwrite feedback * test(lens): expect lens_feedback retention during schema setup * refactor(lens): flatten feedback summaries without a stacked comprehension * chore(lens-ui): drop routine comment on the feedback panel * chore(lens-ui): drop routine comment on the feedback view * chore(lens-ui): drop routine comment on the low feedback check * chore(lens-ui): drop routine comment on the post stub --- litellm/proxy/lens/feedback_endpoints.py | 2 + litellm/proxy/lens/feedback_repository.py | 3 +- tests/test_litellm_rust/test_traces.py | 3 +- .../proxy/lens/test_feedback_endpoints.py | 49 ++++++++++++------- .../traces/detail/feedback/FeedbackPanel.tsx | 1 - .../lens/traces/detail/feedback/feedback.ts | 1 - .../AgentTracesSection.integration.test.tsx | 1 - .../lens/traces/list/useTraceFeedback.ts | 1 - 8 files changed, 36 insertions(+), 25 deletions(-) diff --git a/litellm/proxy/lens/feedback_endpoints.py b/litellm/proxy/lens/feedback_endpoints.py index f9479873a84..59dac3f35ec 100644 --- a/litellm/proxy/lens/feedback_endpoints.py +++ b/litellm/proxy/lens/feedback_endpoints.py @@ -56,6 +56,8 @@ def write_scope(auth: UserAPIKeyAuth) -> Scope: return Scope(all_teams=True) if auth.user_role == LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY: raise HTTPException(403, "Admin viewers cannot write feedback") + if not auth.team_id and not auth.token: + raise HTTPException(403, "Feedback requires a team or API key") return Scope(team_id=auth.team_id or "", api_key_hash="" if auth.team_id else auth.token or "") diff --git a/litellm/proxy/lens/feedback_repository.py b/litellm/proxy/lens/feedback_repository.py index 9ee1af164e2..3228cd2792a 100644 --- a/litellm/proxy/lens/feedback_repository.py +++ b/litellm/proxy/lens/feedback_repository.py @@ -1,6 +1,7 @@ import hashlib from collections.abc import Mapping from datetime import datetime, timezone +from itertools import chain from typing import Final, Protocol from litellm.proxy.lens.feedback_models import Feedback, FeedbackInput, TraceFeedback, TraceFeedbackSummary @@ -165,7 +166,7 @@ class ClickHouseFeedbackStore: **access_parameters(scope).model_dump(), trace_ids=sorted({t.trace_id for t in traces}) ), ) - return tuple(summary for trace in traces for summary in _summaries(trace, rows)) + return tuple(chain.from_iterable(_summaries(trace, rows) for trace in traces)) def _summaries(trace: TraceIdentity, rows: tuple[FeedbackSummaryRow, ...]) -> tuple[TraceFeedbackSummary, ...]: diff --git a/tests/test_litellm_rust/test_traces.py b/tests/test_litellm_rust/test_traces.py index f07bc776bce..3596d1d343c 100644 --- a/tests/test_litellm_rust/test_traces.py +++ b/tests/test_litellm_rust/test_traces.py @@ -174,10 +174,11 @@ async def test_schema_setup_uses_configured_retention(recording_server: Recordin request.raw_body for request in recording_server.requests if b"MODIFY TTL" in request.raw_body ) assert all(b"INTERVAL 7 DAY" in statement for statement in ttl_statements) - assert tuple(request.raw_body.strip() for request in recording_server.requests[-3:]) == ( + assert tuple(request.raw_body.strip() for request in recording_server.requests[-4:]) == ( b"ALTER TABLE `trace_test`.otel_traces MODIFY TTL toDateTime(Timestamp) + INTERVAL 7 DAY", b"ALTER TABLE `trace_test`.agent_traces_by_key MODIFY TTL toDateTime(StartTs) + INTERVAL 7 DAY", b"ALTER TABLE `trace_test`.spend_logs MODIFY TTL toDateTime(start_time) + INTERVAL 7 DAY", + b"ALTER TABLE `trace_test`.lens_feedback MODIFY TTL toDateTime(CreatedAt) + INTERVAL 7 DAY", ) diff --git a/tests/unit/proxy/lens/test_feedback_endpoints.py b/tests/unit/proxy/lens/test_feedback_endpoints.py index 3ee7c79f169..71d60e22eea 100644 --- a/tests/unit/proxy/lens/test_feedback_endpoints.py +++ b/tests/unit/proxy/lens/test_feedback_endpoints.py @@ -94,34 +94,31 @@ class FakeClickHouse(ClickHouseStorage): and self._visible(str(r["TeamId"]), str(r["ApiKeyHash"]), parameters) ) case LensFeedbackSummaryParams(): - live = tuple( + live: Final = tuple( r for r in self._latest() if r["TraceId"] in parameters.trace_ids and self._visible(str(r["TeamId"]), str(r["ApiKeyHash"]), parameters) ) - keys = sorted({(str(r["TeamId"]), str(r["ApiKeyHash"]), str(r["TraceId"])) for r in live}) - return tuple( - FeedbackSummaryRow( - trace_id=trace, - trace_ref=ref(team, key, trace), - count=len(scores), - average=sum(scores) / len(scores), - lowest=min(scores), - ) - for team, key, trace in keys - for scores in [ - [ - int(str(r["Score"])) - for r in live - if (r["TeamId"], r["ApiKeyHash"], r["TraceId"]) == (team, key, trace) - ] - ] - ) + keys: Final = sorted({(str(r["TeamId"]), str(r["ApiKeyHash"]), str(r["TraceId"])) for r in live}) + return tuple(_summary_row(live, team, key, trace) for team, key, trace in keys) case _: raise AssertionError(f"unexpected query {query.name}") +def _summary_row(live: tuple[Mapping[str, object], ...], team: str, key: str, trace: str) -> FeedbackSummaryRow: + scores: Final = tuple( + int(str(r["Score"])) for r in live if (r["TeamId"], r["ApiKeyHash"], r["TraceId"]) == (team, key, trace) + ) + return FeedbackSummaryRow( + trace_id=trace, + trace_ref=ref(team, key, trace), + count=len(scores), + average=sum(scores) / len(scores), + lowest=min(scores), + ) + + def store(**traces: tuple[tuple[str, str], ...]) -> ClickHouseFeedbackStore: return ClickHouseFeedbackStore(FakeClickHouse(traces or {"t1": (("team-a", "key-a"),)})) @@ -207,6 +204,20 @@ async def test_viewers_can_read_but_not_write_and_non_admins_cannot_read_in_lens assert (write.value.status_code, read.value.status_code) == (403, 403) +@pytest.mark.asyncio +async def test_a_caller_without_a_team_or_key_cannot_write_on_a_teamless_trace() -> None: + feedback: Final = store(t1=(("", "key-a"),)) + await submit_feedback(submission(9, "mine", user="customer-1"), ADMIN, feedback, T0) + + with pytest.raises(HTTPException) as write: + await submit_feedback(submission(1, "overwrite", user="customer-1"), INTERNAL, feedback, T0) + with pytest.raises(HTTPException) as delete: + await delete_feedback(FeedbackDeletion(trace_id="t1", user="customer-1"), INTERNAL, feedback, T0) + + assert (write.value.status_code, delete.value.status_code) == (403, 403) + assert [f.score for f in (await read_feedback(FeedbackTarget(trace_id="t1"), ADMIN, feedback)).feedback] == [9] + + @pytest.mark.asyncio async def test_tenant_comes_from_the_trace_and_author_defaults_to_the_caller() -> None: feedback: Final = store() diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.tsx b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.tsx index f06591acce8..74ea709a731 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/FeedbackPanel.tsx @@ -46,7 +46,6 @@ function Entry({ entry }: { entry: Feedback }) { ); } -/** End-user feedback on this run, shown first so a developer reads what the user said before the steps. */ export function FeedbackPanel({ summary, accessToken }: FeedbackPanelProps) { const api = useTracesApi(accessToken); const traceRef = summary.trace_ref ?? ""; diff --git a/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.ts b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.ts index 6a572997657..01a26e788a7 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/detail/feedback/feedback.ts @@ -6,7 +6,6 @@ export interface FeedbackView { readonly lowest: number; } -/** End-user feedback on one run, newest first, or null when nobody has rated it. */ export function feedbackView(feedback: readonly Feedback[]): FeedbackView | null { if (feedback.length === 0) return null; const scores = feedback.map((entry) => entry.score); diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx index f0ba76367b3..5a32aa4e610 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx +++ b/ui/litellm-dashboard/src/components/lens/traces/list/AgentTracesSection.integration.test.tsx @@ -81,7 +81,6 @@ const unrated = (traces: TraceKey[]): TraceFeedbackSummary[] => lowest: null, })); -/** Routes the shared POST mock: findings get `findings`, the feedback summary gets `feedback`. */ const stubPost = ( findings: (traces: TraceKey[]) => Promise, feedback: (traces: TraceKey[]) => Promise = async (traces) => unrated(traces), diff --git a/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.ts b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.ts index b35542a0943..fa091c784f0 100644 --- a/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.ts +++ b/ui/litellm-dashboard/src/components/lens/traces/list/useTraceFeedback.ts @@ -13,7 +13,6 @@ export type TraceFeedbackState = export const traceFeedbackKey = (accessToken: string) => ["traceFeedback", accessToken] as const; -/** A run some end user scored at or below the low-score threshold. */ export const isLowFeedback = (state: TraceFeedbackState | undefined): boolean => state?.status === "ready" && state.summary.count > 0 && (state.summary.lowest ?? Infinity) <= LOW_SCORE; From 98465fe2ba4fc26ab9e6ab366c7f90f8e4d9c685 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 15:34:20 +0000 Subject: [PATCH 38/65] fix(ci): trim rust-test debuginfo so the job fits on the runner disk (#45318) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .github/workflows/test-rust.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-rust.yml b/.github/workflows/test-rust.yml index c07487d5b49..a9f75b5f87e 100644 --- a/.github/workflows/test-rust.yml +++ b/.github/workflows/test-rust.yml @@ -98,6 +98,9 @@ jobs: rust-test: runs-on: ubuntu-latest timeout-minutes: 30 + # Trim debuginfo to keep the job's target dir within the runner's disk + env: + CARGO_PROFILE_DEV_DEBUG: line-tables-only defaults: run: working-directory: litellm-rust From 82a5d9842aef933fea5e640167d2fec274af5ce1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:18:29 -0500 Subject: [PATCH 39/65] fix(ui): list only catalog models in the Add Model picker (#44877) The Add Model selector read the merged runtime cost map, so deployment ids and provider-prefixed backend keys registered for proxy deployments showed up as duplicate model choices. The public cost map endpoint now accepts catalog_only=true to return the catalog as loaded, and the Add Model panel requests that view. The default response is unchanged. Resolves LIT-9263 Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../public_endpoints/public_endpoints.py | 6 ++-- .../public_endpoints/test_public_endpoints.py | 26 ++++++++++++++ .../hooks/models/useModelCostMap.test.ts | 19 +++++++++++ .../hooks/models/useModelCostMap.ts | 8 ++--- .../PriceDataManagementTab.test.tsx | 34 +++++++++++++++---- .../components/PriceDataManagementTab.tsx | 7 ++-- .../panels/AddModelPanel.integration.test.tsx | 27 ++++++++++++++- .../panels/AddModelPanel.tsx | 2 +- .../src/components/networking.test.ts | 17 ++++++++++ .../src/components/networking.tsx | 5 +-- ui/litellm-dashboard/src/lib/http/schema.d.ts | 14 +++++++- 11 files changed, 145 insertions(+), 20 deletions(-) diff --git a/litellm/proxy/public_endpoints/public_endpoints.py b/litellm/proxy/public_endpoints/public_endpoints.py index 6b6ee7a9b67..1cccc6b2cfb 100644 --- a/litellm/proxy/public_endpoints/public_endpoints.py +++ b/litellm/proxy/public_endpoints/public_endpoints.py @@ -18,6 +18,7 @@ from litellm.litellm_core_utils.get_blog_posts import ( GetBlogPosts, get_blog_posts, ) +from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap from litellm.proxy._types import ( CommonProxyErrors, ) @@ -453,15 +454,16 @@ async def get_public_fuse_presets() -> FusePresetCatalog: "/public/litellm_model_cost_map", tags=["public", "model management"], ) -async def get_litellm_model_cost_map(): +async def get_litellm_model_cost_map(catalog_only: bool = False): """ Public endpoint to get the LiteLLM model cost map. Returns pricing information for all supported models. + With catalog_only=true, returns the catalog as loaded, without entries registered at runtime for proxy deployments. """ import litellm try: - _model_cost_map: Final = litellm.model_cost + _model_cost_map: Final = GetModelCostMap.loaded_model_cost_map() if catalog_only else litellm.model_cost return _model_cost_map except Exception as e: raise HTTPException( diff --git a/tests/unit/proxy/public_endpoints/test_public_endpoints.py b/tests/unit/proxy/public_endpoints/test_public_endpoints.py index 7e20b5f8c3c..b720ba24c3e 100644 --- a/tests/unit/proxy/public_endpoints/test_public_endpoints.py +++ b/tests/unit/proxy/public_endpoints/test_public_endpoints.py @@ -76,6 +76,32 @@ def test_get_provider_create_fields(): ), "Expected at least one provider to have detailed credential fields" +def test_get_litellm_model_cost_map_catalog_only_excludes_runtime_registered_entries( + monkeypatch: pytest.MonkeyPatch, +) -> None: + import litellm + from litellm.litellm_core_utils.get_model_cost_map import GetModelCostMap + + monkeypatch.setattr(litellm, "model_cost", dict(litellm.model_cost)) + runtime_key: Final = "lit-9263-deployment-alias" + litellm.register_model({runtime_key: {"litellm_provider": "openai", "mode": "chat"}}, persist_across_reloads=False) + app: Final = FastAPI() + app.include_router(router) + client: Final = TestClient(app) + + live_response: Final = client.get("/public/litellm_model_cost_map") + catalog_response: Final = client.get("/public/litellm_model_cost_map", params={"catalog_only": "true"}) + + assert live_response.status_code == 200 + assert live_response.json()[runtime_key]["litellm_provider"] == "openai" + assert catalog_response.status_code == 200 + catalog_payload: Final = catalog_response.json() + assert runtime_key not in catalog_payload + assert catalog_payload == json.loads( + json.dumps({key: dict(entry) for key, entry in GetModelCostMap.loaded_model_cost_map().items()}) + ) + + def test_get_litellm_model_cost_map_returns_cost_map(): app = FastAPI() app.include_router(router) diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.test.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.test.ts index f79ca33bc5d..68983f5146e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.test.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.test.ts @@ -141,4 +141,23 @@ describe("useModelCostMap", () => { expect(result.current).toHaveProperty("isSuccess"); expect(result.current).toHaveProperty("error"); }); + + it("fetches the catalog-only map under its own cache entry when catalogOnly is set", async () => { + (modelCostMap as any).mockImplementation(async (catalogOnly: boolean) => + catalogOnly ? { catalog: { litellm_provider: "bedrock" } } : mockModelCostData, + ); + + const { result: live } = renderHook(() => useModelCostMap(), { wrapper }); + const { result: catalog } = renderHook(() => useModelCostMap(true, true), { wrapper }); + + await waitFor(() => { + expect(live.current.isSuccess).toBe(true); + expect(catalog.current.isSuccess).toBe(true); + }); + + expect(live.current.data).toEqual(mockModelCostData); + expect(catalog.current.data).toEqual({ catalog: { litellm_provider: "bedrock" } }); + expect(modelCostMap).toHaveBeenCalledWith(false); + expect(modelCostMap).toHaveBeenCalledWith(true); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts index d9824b4753e..cf97a504e1e 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts +++ b/ui/litellm-dashboard/src/app/(dashboard)/hooks/models/useModelCostMap.ts @@ -2,13 +2,13 @@ import { modelCostMap } from "@/components/networking"; import { useQuery } from "@tanstack/react-query"; import { createQueryKeys } from "../common/queryKeysFactory"; -const modelCostMapKeys = createQueryKeys("modelCostMap"); +export const modelCostMapKeys = createQueryKeys("modelCostMap"); -export const useModelCostMap = (enabled = true) => { +export const useModelCostMap = (enabled = true, catalogOnly = false) => { return useQuery>({ enabled, - queryKey: modelCostMapKeys.list({}), - queryFn: async () => await modelCostMap(), + queryKey: modelCostMapKeys.list(catalogOnly ? { filters: { catalog_only: "true" } } : {}), + queryFn: async () => await modelCostMap(catalogOnly), staleTime: 60 * 1000, // 1 minute gcTime: 60 * 1000, // 1 minute }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx index 7cd418bc176..b2beb5beeed 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.test.tsx @@ -1,17 +1,39 @@ /* @vitest-environment jsdom */ -import { render, screen } from "@testing-library/react"; +import { QueryClient, QueryClientProvider } from "@tanstack/react-query"; +import { fireEvent, render, screen } from "@testing-library/react"; import { describe, expect, it, vi } from "vitest"; +import { modelCostMapKeys } from "../../hooks/models/useModelCostMap"; import PriceDataManagementTab from "./PriceDataManagementTab"; -vi.mock("@/components/price_data_reload", () => ({ default: () =>
    reload
    })); -vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => ({ accessToken: "sk-test" }) })); -vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ - useModelCostMap: () => ({ refetch: vi.fn() }), +vi.mock("@/components/price_data_reload", () => ({ + default: ({ onReloadSuccess }: { onReloadSuccess: () => void }) => , })); +vi.mock("@/app/(dashboard)/hooks/useAuthorized", () => ({ default: () => ({ accessToken: "sk-test" }) })); + +const renderTab = (queryClient: QueryClient) => + render( + + + , + ); describe("PriceDataManagementTab", () => { it("renders its content standalone, without a tab-panel ancestor", () => { - render(); + renderTab(new QueryClient()); expect(screen.getByText("Price Data Management")).toBeInTheDocument(); }); + + it("invalidates both the live and the catalog-only cost map after a reload", () => { + const queryClient = new QueryClient(); + const liveKey = modelCostMapKeys.list({}); + const catalogKey = modelCostMapKeys.list({ filters: { catalog_only: "true" } }); + queryClient.setQueryData(liveKey, {}); + queryClient.setQueryData(catalogKey, {}); + renderTab(queryClient); + + fireEvent.click(screen.getByRole("button", { name: "reload" })); + + expect(queryClient.getQueryState(liveKey)?.isInvalidated).toBe(true); + expect(queryClient.getQueryState(catalogKey)?.isInvalidated).toBe(true); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.tsx index 126f6970c2e..ce6f8e77d38 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/components/PriceDataManagementTab.tsx @@ -1,11 +1,12 @@ import PriceDataReload from "@/components/price_data_reload"; import React from "react"; import useAuthorized from "@/app/(dashboard)/hooks/useAuthorized"; -import { useModelCostMap } from "../../hooks/models/useModelCostMap"; +import { useQueryClient } from "@tanstack/react-query"; +import { modelCostMapKeys } from "../../hooks/models/useModelCostMap"; const PriceDataManagementTab = () => { const { accessToken } = useAuthorized(); - const { refetch: refetchModelCostMap } = useModelCostMap(); + const queryClient = useQueryClient(); return (
    @@ -19,7 +20,7 @@ const PriceDataManagementTab = () => { { - refetchModelCostMap(); + queryClient.invalidateQueries({ queryKey: modelCostMapKeys.all }); }} buttonText="Reload Price Data" size="middle" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx index efba26734ff..410758491fe 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.integration.test.tsx @@ -22,7 +22,11 @@ vi.mock("@/app/(dashboard)/hooks/uiSettings/usePtuCostAttributionEnabled", () => usePtuCostAttributionEnabled: () => mockPtuEnabled(), })); -vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ useModelCostMap: () => ({ data: {} }) })); +const mockUseModelCostMap = vi.fn(); + +vi.mock("@/app/(dashboard)/hooks/models/useModelCostMap", () => ({ + useModelCostMap: (...args: unknown[]) => mockUseModelCostMap(...args), +})); vi.mock("@/app/(dashboard)/hooks/credentials/useCredentials", () => ({ useCredentials: () => ({ data: { credentials: [] } }), @@ -138,10 +142,29 @@ const setup = async () => { describe("AddModelPanel submit payload contract", () => { beforeEach(() => { vi.clearAllMocks(); + mockUseModelCostMap.mockReturnValue({ data: {} }); mockPtuEnabled.mockReturnValue(false); mockAuthorized.mockReturnValue(PROXY_ADMIN); }); + it("lists catalog models in the model picker, not entries registered at runtime for deployments", async () => { + mockUseModelCostMap.mockImplementation((_enabled: boolean, catalogOnly: boolean) => ({ + data: catalogOnly + ? { "gpt-4o-2024-08-06": { litellm_provider: "openai" } } + : { + "gpt-4o-2024-08-06": { litellm_provider: "openai" }, + "openai-gpt-4o-deployment-id": { litellm_provider: "openai" }, + }, + })); + const { user } = await setup(); + await user.click(screen.getByRole("combobox", { name: /provider/i })); + await user.click(await screen.findByText("OpenAI")); + await user.click(await screen.findByPlaceholderText("Select models")); + + expect(await screen.findByText("gpt-4o-2024-08-06")).toBeInTheDocument(); + expect(screen.queryByText("openai-gpt-4o-deployment-id")).not.toBeInTheDocument(); + }); + it("sends only the always-mounted fields while Advanced Settings stays closed", async () => { const { fillRequired, submit } = await setup(); await fillRequired(); @@ -295,6 +318,7 @@ describe("AddModelPanel submit payload contract", () => { describe("AddModelPanel empty-string skip", () => { beforeEach(() => { vi.clearAllMocks(); + mockUseModelCostMap.mockReturnValue({ data: {} }); mockPtuEnabled.mockReturnValue(false); mockAuthorized.mockReturnValue(PROXY_ADMIN); }); @@ -376,6 +400,7 @@ describe("AddModelPanel validation gates", () => { describe("AddModelPanel behaviours the removed Advanced Settings form instance never drove", () => { beforeEach(() => { vi.clearAllMocks(); + mockUseModelCostMap.mockReturnValue({ data: {} }); mockPtuEnabled.mockReturnValue(false); mockAuthorized.mockReturnValue(PROXY_ADMIN); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx index 1cdce04d07c..9dacabe506b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/models-and-endpoints/panels/AddModelPanel.tsx @@ -23,7 +23,7 @@ export default function AddModelPanel() { const form = useForm({ mode: "onChange", defaultValues: INITIAL_VALUES }); const registry = useMountRegistry(); const queryClient = useQueryClient(); - const { data: modelCostMapData } = useModelCostMap(); + const { data: modelCostMapData } = useModelCostMap(true, true); const { data: credentialsResponse } = useCredentials(); const { data: teams } = useTeams(); const [selectedProvider, setSelectedProvider] = useState(Providers.Anthropic); diff --git a/ui/litellm-dashboard/src/components/networking.test.ts b/ui/litellm-dashboard/src/components/networking.test.ts index 62670467827..0fdf7a4799d 100644 --- a/ui/litellm-dashboard/src/components/networking.test.ts +++ b/ui/litellm-dashboard/src/components/networking.test.ts @@ -871,3 +871,20 @@ describe("schema-bound dashboard responses", () => { expect(result.users[0]).toEqual(user); }); }); + +describe("modelCostMap", () => { + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("requests the catalog-only map when catalogOnly is set and the full map otherwise", async () => { + const mockFetch = vi.fn().mockImplementation(async () => new Response(JSON.stringify({}))); + vi.stubGlobal("fetch", mockFetch); + + await Networking.modelCostMap(true); + await Networking.modelCostMap(); + + expect(mockFetch.mock.calls[0][0]).toMatch(/\/public\/litellm_model_cost_map\?catalog_only=true$/); + expect(mockFetch.mock.calls[1][0]).toMatch(/\/public\/litellm_model_cost_map$/); + }); +}); diff --git a/ui/litellm-dashboard/src/components/networking.tsx b/ui/litellm-dashboard/src/components/networking.tsx index bf35e22f228..a874f58f868 100644 --- a/ui/litellm-dashboard/src/components/networking.tsx +++ b/ui/litellm-dashboard/src/components/networking.tsx @@ -553,9 +553,10 @@ export const getOpenAPISchema = async () => { return jsonData; }; -export const modelCostMap = async () => { +export const modelCostMap = async (catalogOnly = false) => { try { - const url = proxyBaseUrl ? `${proxyBaseUrl}/public/litellm_model_cost_map` : `/public/litellm_model_cost_map`; + const path = catalogOnly ? "/public/litellm_model_cost_map?catalog_only=true" : "/public/litellm_model_cost_map"; + const url = proxyBaseUrl ? `${proxyBaseUrl}${path}` : path; const response = await fetch(url, { method: "GET", headers: { diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index bf863323e77..769798b5134 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -13959,6 +13959,7 @@ export interface paths { * Get Litellm Model Cost Map * @description Public endpoint to get the LiteLLM model cost map. * Returns pricing information for all supported models. + * With catalog_only=true, returns the catalog as loaded, without entries registered at runtime for proxy deployments. */ get: operations["get_litellm_model_cost_map_public_litellm_model_cost_map_get"]; put?: never; @@ -70662,7 +70663,9 @@ export interface operations { }; get_litellm_model_cost_map_public_litellm_model_cost_map_get: { parameters: { - query?: never; + query?: { + catalog_only?: boolean; + }; header?: never; path?: never; cookie?: never; @@ -70678,6 +70681,15 @@ export interface operations { "application/json": unknown; }; }; + /** @description Validation Error */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["HTTPValidationError"]; + }; + }; }; }; get_mcp_servers_public_mcp_hub_get: { From 35b0992533ff05b8cab3fe342a1024b4fa6a0ee6 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 8 Oct 2026 09:27:12 -0700 Subject: [PATCH 40/65] feat(rust): add standalone typed LLM wire contracts (#45188) * refactor(rust): add typed LLM wire contracts to llms-types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(rust): drop blank lines left in new wire types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(rust): complete additive wire type contracts * feat(rust): complete additive nested wire contracts * feat(rust): give Messages content blocks concrete per-block contracts Replace the shared all-optional ContentBlockPayload with one struct per documented block and nested tagged unions for server tool results, so required fields from the official Messages reference are enforced. Make CustomTool a plain struct with required name and input_schema, add MessagesToolParam plus the toolset and undated tool-search tags, model MiniMax media sources as a tagged union, require documented fields on chat audio, image URLs, and logprobs, and rename the new block enum to MessagesContentPart so it no longer shadows the streaming struct. Co-Authored-By: Claude Opus 5.5 * fix(rust): tighten new wire contracts against the official references Give each Messages builtin tool a concrete contract with its documented required fields, fixed names, and typed allowed callers in place of the shared all-optional ToolDefinition bag. Make MCP tool result content and fallback triggers optional as the request params document, and narrow tool search references, web fetch content, and bash results to their documented block types. Keep unmodeled Responses output items and new usage iteration and stop detail types as Recognized values instead of rejecting the whole payload, accept find_in_page and optional open_page URLs, preserve JSON Schema property order, and move prompt_cache_breakpoint to Chat Completions content parts where OpenAI documents it. Drop OCR table and key/value types that only matched Azure Document Intelligence and would reject other providers' normalized passthrough, along with unsourced bounding box and applied edit fields. Co-Authored-By: Claude Opus 5.5 * feat(rust): type every documented Responses output item and close wire gaps Add the 18 remaining documented Responses output item variants with concrete nested actions, outputs, and safety checks, and add the documented namespace, async, caller, and image generation fields to the existing items. Restrict MCP tool result content to a string or text blocks, model the Messages diagnostics cache-miss union and its request param with null distinct from absent, and accept draft-07 tuple items and prefixItems in JSON Schema. Co-Authored-By: Claude Opus 5.5 * test(rust): keep Messages contract assertions in test bodies Compare built-in tool decodes against fully built expected values and split the required-field rejections per type, so no case carries its assertions in a callback. Co-Authored-By: Claude Opus 5.5 * docs(rust): cite the Mistral OCR reference beside the OCR wire types Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 --- litellm-rust/Cargo.lock | 2 + litellm-rust/crates/llms-types/AGENTS.md | 6 + litellm-rust/crates/llms-types/Cargo.toml | 3 +- .../crates/llms-types/src/formats/AGENTS.md | 36 + .../src/formats/chat_completions.rs | 6 + .../src/formats/chat_completions/content.rs | 155 ++ .../llms-types/src/formats/messages/AGENTS.md | 14 + .../src/formats/messages/content.rs | 712 ++++++++ .../src/formats/messages/metadata.rs | 331 ++++ .../llms-types/src/formats/messages/mod.rs | 45 +- .../src/formats/messages/request.rs | 15 + .../llms-types/src/formats/messages/tools.rs | 564 ++++++ .../llms-types/src/formats/messages/usage.rs | 75 + .../crates/llms-types/src/formats/mod.rs | 3 + .../crates/llms-types/src/formats/ocr.rs | 12 + .../llms-types/src/formats/responses/mod.rs | 2 + .../src/formats/responses/output.rs | 831 +++++++++ .../formats/responses/streaming_websocket.rs | 14 + .../crates/llms-types/src/json_schema.rs | 51 + litellm-rust/crates/llms-types/src/lib.rs | 1 + .../crates/llms-types/src/providers/AGENTS.md | 17 + .../llms-types/src/providers/minimax.rs | 59 + .../crates/llms-types/src/providers/mod.rs | 1 + litellm-rust/crates/llms-types/tests/chat.rs | 163 ++ .../crates/llms-types/tests/json_schema.rs | 148 ++ .../llms-types/tests/messages_request.rs | 9 + .../crates/llms-types/tests/messages_types.rs | 1512 +++++++++++++++++ .../crates/llms-types/tests/minimax.rs | 82 + litellm-rust/crates/llms-types/tests/ocr.rs | 38 +- .../crates/llms-types/tests/responses.rs | 671 +++++++- 30 files changed, 5574 insertions(+), 4 deletions(-) create mode 100644 litellm-rust/crates/llms-types/src/formats/AGENTS.md create mode 100644 litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs create mode 100644 litellm-rust/crates/llms-types/src/formats/messages/content.rs create mode 100644 litellm-rust/crates/llms-types/src/formats/messages/metadata.rs create mode 100644 litellm-rust/crates/llms-types/src/formats/messages/tools.rs create mode 100644 litellm-rust/crates/llms-types/src/formats/messages/usage.rs create mode 100644 litellm-rust/crates/llms-types/src/formats/responses/output.rs create mode 100644 litellm-rust/crates/llms-types/src/json_schema.rs create mode 100644 litellm-rust/crates/llms-types/src/providers/AGENTS.md create mode 100644 litellm-rust/crates/llms-types/src/providers/minimax.rs create mode 100644 litellm-rust/crates/llms-types/tests/chat.rs create mode 100644 litellm-rust/crates/llms-types/tests/json_schema.rs create mode 100644 litellm-rust/crates/llms-types/tests/messages_types.rs create mode 100644 litellm-rust/crates/llms-types/tests/minimax.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 987491d514b..057faa279e3 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4248,6 +4248,7 @@ dependencies = [ name = "litellm-llms-types" version = "0.1.0" dependencies = [ + "indexmap 2.14.0", "macro_rules_attribute", "rstest", "schemars 1.2.2", @@ -6512,6 +6513,7 @@ checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" dependencies = [ "chrono", "dyn-clone", + "indexmap 2.14.0", "ref-cast", "schemars_derive 1.2.2", "serde", diff --git a/litellm-rust/crates/llms-types/AGENTS.md b/litellm-rust/crates/llms-types/AGENTS.md index 0d418822d06..e5bd83b1d6e 100644 --- a/litellm-rust/crates/llms-types/AGENTS.md +++ b/litellm-rust/crates/llms-types/AGENTS.md @@ -67,3 +67,9 @@ The same ownership rule applies to Messages, Responses, Chat Completions, OCR, a - Follow the workspace test-placement and `rstest` rules - Test provider transformations, header policy, and stream execution in their owning crates - Do not test import locations or Rust source structure as substitutes for behavior + +# references + +## json_schema.rs + +- https://json-schema.org/draft/2020-12/json-schema-core diff --git a/litellm-rust/crates/llms-types/Cargo.toml b/litellm-rust/crates/llms-types/Cargo.toml index 2d880b87faf..4f190f34026 100644 --- a/litellm-rust/crates/llms-types/Cargo.toml +++ b/litellm-rust/crates/llms-types/Cargo.toml @@ -6,9 +6,10 @@ license.workspace = true repository.workspace = true [features] -schema = ["dep:schemars"] +schema = ["dep:schemars", "schemars/indexmap2"] [dependencies] +indexmap = { version = "2.14.0", features = ["serde"] } macro_rules_attribute.workspace = true schemars = { workspace = true, optional = true } serde.workspace = true diff --git a/litellm-rust/crates/llms-types/src/formats/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/AGENTS.md new file mode 100644 index 00000000000..e49c286f058 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/AGENTS.md @@ -0,0 +1,36 @@ +# references + +These links describe shared format fields and discriminators. The official API reference is authoritative, and SDK sources only clarify shapes the reference leaves implicit. These contracts are partial typed projections, not exhaustive upstream schemas + +## chat_completions.rs and chat_completions/content.rs + +- https://developers.openai.com/api/reference/resources/chat + +SDK wire definitions: + +- https://github.com/openai/openai-python/blob/main/src/openai/types/chat/chat_completion_content_part_param.py + +## responses/output.rs and responses/streaming_websocket.rs + +- https://developers.openai.com/api/reference/resources/responses +- https://developers.openai.com/api/reference/resources/responses/streaming-events + +SDK wire definitions: + +- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response.py +- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/response_output_item.py +- https://github.com/openai/openai-python/blob/main/src/openai/types/responses/mcp_tool_call_error.py + +## batches.rs + +- https://developers.openai.com/api/reference/resources/batches + +## audio_transcription.rs + +- https://developers.openai.com/api/reference/resources/audio/subresources/transcriptions + +## ocr.rs + +- https://docs.mistral.ai/api/endpoint/ocr + +Messages references live in `messages/AGENTS.md`. `LiteLLMOcrResponse` follows the Mistral OCR response shape, and `OcrBoundingBox` describes the corner coordinates providers copy into the normalized `bbox`. Normalized `tables` and `keyValuePairs` stay open because providers pass through different native shapes diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions.rs index 31b5046469a..b6bcd614ab4 100644 --- a/litellm-rust/crates/llms-types/src/formats/chat_completions.rs +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions.rs @@ -1,6 +1,12 @@ use serde_json::{Map, Value}; use strum::IntoStaticStr; +mod content; +pub use content::{ + ChatContentPart, ChatFile, ChatInputAudio, ChatLogprobs, ChatMediaUrl, ChatMediaUrlParameters, + ChatTokenLogprob, ChatTopLogprob, ChatVideoMetadata, PromptCacheBreakpoint, PromptCacheMode, +}; + /// Reasoning effort level accepted or applied by the model. #[macro_rules_attribute::apply(wire_type)] #[derive(Copy, Eq, IntoStaticStr)] diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs new file mode 100644 index 00000000000..bc0d63fab97 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs @@ -0,0 +1,155 @@ +use serde_json::{Map, Value}; + +use crate::formats::messages::{CacheControl, CitationsConfig, ContentSource}; + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ChatContentPart { + Text { + text: String, + #[serde(skip_serializing_if = "Option::is_none")] + cache_control: Option, + #[serde(skip_serializing_if = "Option::is_none")] + prompt_cache_breakpoint: Option, + #[serde(flatten)] + extra: Map, + }, + ImageUrl { + image_url: ChatMediaUrl, + #[serde(skip_serializing_if = "Option::is_none")] + prompt_cache_breakpoint: Option, + #[serde(flatten)] + extra: Map, + }, + VideoUrl { + video_url: ChatMediaUrl, + #[serde(flatten)] + extra: Map, + }, + InputAudio { + input_audio: ChatInputAudio, + #[serde(skip_serializing_if = "Option::is_none")] + prompt_cache_breakpoint: Option, + #[serde(flatten)] + extra: Map, + }, + File { + file: Box, + #[serde(skip_serializing_if = "Option::is_none")] + prompt_cache_breakpoint: Option, + #[serde(flatten)] + extra: Map, + }, + Document { + source: Box, + #[serde(skip_serializing_if = "Option::is_none")] + title: Option, + #[serde(skip_serializing_if = "Option::is_none")] + context: Option, + #[serde(skip_serializing_if = "Option::is_none")] + citations: Option, + #[serde(flatten)] + extra: Map, + }, + Refusal { + refusal: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct PromptCacheBreakpoint { + pub mode: PromptCacheMode, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum PromptCacheMode { + Explicit, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum ChatMediaUrl { + Url(String), + Parameters(Box), +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ChatMediaUrlParameters { + pub url: String, + pub detail: Option, + pub format: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ChatInputAudio { + pub data: String, + pub format: String, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ChatFile { + pub file_data: Option, + pub file_id: Option, + pub filename: Option, + pub format: Option, + pub detail: Option, + pub video_metadata: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ChatVideoMetadata { + pub fps: Option, + pub start_offset: Option, + pub end_offset: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ChatLogprobs { + pub content: Option>, + pub refusal: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ChatTokenLogprob { + pub token: String, + pub logprob: serde_json::Number, + pub bytes: Option>, + pub top_logprobs: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ChatTopLogprob { + pub token: String, + pub logprob: serde_json::Number, + pub bytes: Option>, + #[serde(flatten)] + pub extra: Map, +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md b/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md index 72558688269..20a5632f267 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md +++ b/litellm-rust/crates/llms-types/src/formats/messages/AGENTS.md @@ -3,3 +3,17 @@ This directory owns shared Messages API data contracts and serialization: reques Adapter contracts and execution inputs such as `MessagesTransformContext` belong in `llms/src/base_llm/messages`. Provider rewriting and interpretation belong in `llms/src//messages`. Call envelopes, live streams, and call orchestration belong in `inference-messages` Represent web-search results, encrypted-content fields, and thinking configuration as data here. Decisions to flatten results, remove encrypted content, select thinking budgets, or require beta headers belong to provider transformations. Data-shape validation belongs here, while model capability checks and request adaptation do not + +# references + +These upstream contracts document fields and discriminators. They are documentation links, not saved golden snapshots. Beta-only blocks and tools (MCP, compaction, tool changes, advisor, fallback, browser state, and toolsets) follow the beta reference + +## Messages bodies, content, tools, and streaming + +- https://platform.claude.com/docs/en/api/http/messages/create +- https://platform.claude.com/docs/en/api/messages/create.md +- https://platform.claude.com/docs/en/api/beta/messages/create.md +- https://platform.claude.com/docs/en/build-with-claude/streaming.md +- https://platform.claude.com/docs/en/build-with-claude/context-editing + +Anthropic-compatible hosts document their deviations in the `llms/src//messages` guides. Copy a host reference into `../../providers/AGENTS.md` only when that host gets a typed extension in `providers` diff --git a/litellm-rust/crates/llms-types/src/formats/messages/content.rs b/litellm-rust/crates/llms-types/src/formats/messages/content.rs new file mode 100644 index 00000000000..53d0ebdcafb --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/messages/content.rs @@ -0,0 +1,712 @@ +use serde_json::{Map, Value}; + +use super::{CacheControl, MessagesToolParam}; +use crate::json_schema::JsonSchema; + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ContentSource { + Base64 { + media_type: String, + data: String, + #[serde(flatten)] + extra: Map, + }, + Url { + url: String, + #[serde(flatten)] + extra: Map, + }, + File { + file_id: String, + #[serde(flatten)] + extra: Map, + }, + Text { + media_type: String, + data: String, + #[serde(flatten)] + extra: Map, + }, + Content { + content: BlockContent, + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum BlockContent { + Text(String), + Blocks(Vec), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolCaller { + Direct { + #[serde(flatten)] + extra: Map, + }, + #[serde(rename = "code_execution_20250825")] + CodeExecution { + tool_id: String, + #[serde(flatten)] + extra: Map, + }, + #[serde(rename = "code_execution_20260120")] + CodeExecution20260120 { + tool_id: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct CitationsConfig { + pub enabled: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct PageCitation { + pub cited_text: String, + pub document_index: u64, + pub document_title: Option, + pub start_page_number: u64, + pub end_page_number: u64, + pub file_id: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct CharCitation { + pub cited_text: String, + pub document_index: u64, + pub document_title: Option, + pub start_char_index: u64, + pub end_char_index: u64, + pub file_id: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ContentBlockCitation { + pub cited_text: String, + pub document_index: u64, + pub document_title: Option, + pub start_block_index: u64, + pub end_block_index: u64, + pub file_id: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebSearchCitation { + pub cited_text: String, + pub url: String, + pub encrypted_index: String, + pub title: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct SearchResultCitation { + pub cited_text: String, + pub search_result_index: u64, + pub source: String, + pub title: Option, + pub start_block_index: u64, + pub end_block_index: u64, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Citation { + PageLocation(PageCitation), + CharLocation(CharCitation), + WebSearchResultLocation(WebSearchCitation), + ContentBlockLocation(ContentBlockCitation), + SearchResultLocation(SearchResultCitation), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MessagesContentPart { + Text(TextBlock), + Image(ImageBlock), + Document(Box), + SearchResult(SearchResultBlock), + Thinking(ThinkingBlock), + RedactedThinking(RedactedThinkingBlock), + ToolUse(ToolUseBlock), + ToolResult(ToolResultBlock), + ToolReference(ToolReferenceBlock), + BrowserState(BrowserStateBlock), + ServerToolUse(ServerToolUseBlock), + WebSearchToolResult(ServerToolResultBlock), + WebFetchToolResult(ServerToolResultBlock), + CodeExecutionToolResult(ServerToolResultBlock), + BashCodeExecutionToolResult(ServerToolResultBlock), + TextEditorCodeExecutionToolResult( + ServerToolResultBlock, + ), + ToolSearchToolResult(ServerToolResultBlock), + AdvisorToolResult(ServerToolResultBlock), + ContainerUpload(ContainerUploadBlock), + McpToolUse(McpToolUseBlock), + McpToolResult(McpToolResultBlock), + McpToolListing(McpToolListingBlock), + Compaction(CompactionBlock), + ToolAddition(ToolChangeBlock), + ToolRemoval(ToolChangeBlock), + Fallback(FallbackBlock), +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct TextBlock { + pub text: String, + pub citations: Option>, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ImageBlock { + pub source: ContentSource, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct DocumentBlock { + pub source: ContentSource, + pub title: Option, + pub context: Option, + pub citations: Option, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct SearchResultBlock { + pub source: String, + pub title: String, + pub content: Vec, + pub citations: Option, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct ThinkingBlock { + pub thinking: String, + pub signature: String, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct RedactedThinkingBlock { + pub data: String, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ToolUseBlock { + pub id: String, + pub name: String, + pub input: Map, + pub caller: Option, + pub toolset_name: Option, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ToolResultBlock { + pub tool_use_id: String, + pub content: Option, + pub is_error: Option, + pub toolset_name: Option, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ToolReferenceBlock { + pub tool_name: String, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct BrowserStateBlock { + pub tabs: Vec, + pub state_changes: Option>, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct BrowserTab { + pub tab_id: String, + pub title: String, + pub url: String, + pub active: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BrowserStateChange { + TabOpened { + tab_id: String, + #[serde(flatten)] + extra: Map, + }, + DownloadStarted { + download_id: String, + url: String, + #[serde(flatten)] + extra: Map, + }, + DownloadCompleted { + download_id: String, + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + path: Option, + #[serde(skip_serializing_if = "Option::is_none")] + size_bytes: Option, + #[serde(flatten)] + extra: Map, + }, + DownloadFailed { + download_id: String, + url: String, + #[serde(skip_serializing_if = "Option::is_none")] + error: Option, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ServerToolUseBlock { + pub id: String, + pub name: String, + pub input: Map, + pub caller: Option, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ServerToolResultBlock { + pub tool_use_id: String, + pub content: C, + pub caller: Option, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ServerToolError { + pub error_code: String, + pub error_message: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum WebSearchToolResultContent { + Error(WebSearchResultError), + Results(Vec), +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct WebSearchResultError { + #[serde(rename = "type")] + pub error_type: WebSearchResultErrorType, + pub error_code: String, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum WebSearchResultErrorType { + WebSearchToolResultError, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebSearchResult { + #[serde(rename = "type")] + pub result_type: WebSearchResultType, + pub url: String, + pub title: String, + pub encrypted_content: String, + pub page_age: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum WebSearchResultType { + WebSearchResult, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum WebFetchToolResultContent { + WebFetchToolResultError(ServerToolError), + WebFetchResult(WebFetchResult), +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebFetchResult { + pub url: String, + pub content: WebFetchDocument, + pub retrieved_at: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum WebFetchDocument { + Document(Box), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum CodeExecutionToolResultContent { + CodeExecutionToolResultError(ServerToolError), + CodeExecutionResult(CodeExecutionResult), + EncryptedCodeExecutionResult(EncryptedCodeExecutionResult), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BashCodeExecutionToolResultContent { + BashCodeExecutionToolResultError(ServerToolError), + BashCodeExecutionResult(CodeExecutionResult), +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct CodeExecutionResult { + pub stdout: String, + pub stderr: String, + pub return_code: i64, + pub content: Vec, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct EncryptedCodeExecutionResult { + pub encrypted_stdout: String, + pub stderr: String, + pub return_code: i64, + pub content: Vec, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum CodeExecutionOutput { + CodeExecutionOutput { + file_id: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum BashCodeExecutionOutput { + BashCodeExecutionOutput { + file_id: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum TextEditorCodeExecutionToolResultContent { + TextEditorCodeExecutionToolResultError(ServerToolError), + TextEditorCodeExecutionViewResult(TextEditorViewResult), + TextEditorCodeExecutionCreateResult(TextEditorCreateResult), + TextEditorCodeExecutionStrReplaceResult(TextEditorStrReplaceResult), +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct TextEditorViewResult { + pub content: String, + pub file_type: TextEditorFileType, + pub num_lines: Option, + pub start_line: Option, + pub total_lines: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum TextEditorFileType { + Text, + Image, + Pdf, +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct TextEditorCreateResult { + pub is_file_update: bool, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct TextEditorStrReplaceResult { + pub lines: Option>, + pub new_lines: Option, + pub new_start: Option, + pub old_lines: Option, + pub old_start: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolSearchToolResultContent { + ToolSearchToolResultError(ServerToolError), + ToolSearchToolSearchResult(ToolSearchResult), +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct ToolSearchResult { + pub tool_references: Vec, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolSearchReference { + ToolReference(ToolReferenceBlock), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum AdvisorToolResultContent { + AdvisorToolResultError(ServerToolError), + AdvisorResult { + text: String, + #[serde(skip_serializing_if = "Option::is_none")] + stop_reason: Option, + #[serde(flatten)] + extra: Map, + }, + AdvisorRedactedResult { + encrypted_content: String, + #[serde(skip_serializing_if = "Option::is_none")] + stop_reason: Option, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ContainerUploadBlock { + pub file_id: String, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct McpToolUseBlock { + pub id: String, + pub name: String, + pub server_name: String, + pub input: Map, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct McpToolResultBlock { + pub tool_use_id: String, + pub content: Option, + pub is_error: Option, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum McpToolResultContent { + Text(String), + Blocks(Vec), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum McpToolResultText { + Text(TextBlock), +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct McpToolListingBlock { + pub mcp_server_name: String, + pub tools: Vec, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct McpListedTool { + pub name: String, + pub description: Option, + pub input_schema: JsonSchema, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct CompactionBlock { + pub content: Option, + pub encrypted_content: Option, + pub signature: Option, + pub tool_changes: Option>, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolChange { + ToolAddition(ToolChangeBlock), + ToolRemoval(ToolChangeBlock), +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ToolChangeBlock { + pub tool: ToolChangeTarget, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolChangeTarget { + ToolReference { + name: String, + #[serde(flatten)] + extra: Map, + }, + McpToolReference { + server_name: String, + name: String, + #[serde(flatten)] + extra: Map, + }, + McpToolsetReference { + server_name: String, + #[serde(flatten)] + extra: Map, + }, + ToolDefinition { + definition: Box, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct FallbackBlock { + pub from: FallbackModel, + pub to: FallbackModel, + pub trigger: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct FallbackModel { + pub model: String, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum FallbackTrigger { + Refusal { + #[serde(skip_serializing_if = "Option::is_none")] + category: Option, + #[serde(flatten)] + extra: Map, + }, +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs b/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs new file mode 100644 index 00000000000..28a5aaa149a --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs @@ -0,0 +1,331 @@ +use serde_json::{Map, Value}; + +use crate::json_schema::JsonSchema; +use crate::recognized::Recognized; + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct MessagesMetadata { + pub user_id: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct OutputFormat { + #[serde(rename = "type")] + pub format_type: OutputFormatType, + pub schema: JsonSchema, + pub strict: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum OutputFormatType { + JsonSchema, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct MessagesCompaction { + #[serde(rename = "type")] + pub compaction_type: CompactionType, + pub instructions: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum CompactionType { + Summarize, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct MessagesContainer { + pub id: Option, + pub expires_at: Option, + pub skills: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ContainerSkill { + #[serde(rename = "type")] + pub skill_type: SkillType, + pub skill_id: String, + pub version: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum SkillType { + Anthropic, + Custom, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct McpServer { + #[serde(rename = "type")] + pub server_type: McpServerType, + pub url: String, + pub name: String, + pub authorization_token: Option, + pub tool_configuration: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum McpServerType { + Url, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct McpToolConfiguration { + pub allowed_tools: Option>, + pub enabled: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct StopDetails { + #[serde(rename = "type")] + pub detail_type: Recognized, + pub category: Option, + pub explanation: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum StopDetailsType { + Refusal, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ContextManagementResponse { + pub applied_edits: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct AppliedEdit { + #[serde(rename = "type")] + pub edit_type: Option, + pub cleared_input_tokens: Option, + pub cleared_tool_uses: Option, + pub cleared_thinking_turns: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct Safeguard { + #[serde(rename = "type")] + pub safeguard_type: String, + pub classifier_context: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[derive( + Clone, + Debug, + PartialEq, + Eq, + strum::Display, + strum::EnumString, + strum::AsRefStr, + serde_with::DeserializeFromStr, + serde_with::SerializeDisplay, +)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))] +#[strum(serialize_all = "snake_case")] +pub enum MessageRole { + User, + Assistant, + System, + #[strum(default, transparent)] + Other(String), +} + +impl MessageRole { + pub fn as_str(&self) -> &str { + self.as_ref() + } +} + +impl From for MessageRole { + fn from(value: String) -> Self { + value.parse().unwrap_or_else(|never| match never {}) + } +} + +impl From for String { + fn from(value: MessageRole) -> Self { + value.to_string() + } +} + +#[derive( + Clone, + Debug, + PartialEq, + Eq, + strum::Display, + strum::EnumString, + strum::AsRefStr, + serde_with::DeserializeFromStr, + serde_with::SerializeDisplay, +)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))] +#[strum(serialize_all = "snake_case")] +pub enum MessageType { + Message, + #[strum(default, transparent)] + Other(String), +} + +impl MessageType { + pub fn as_str(&self) -> &str { + self.as_ref() + } +} + +impl From for MessageType { + fn from(value: String) -> Self { + value.parse().unwrap_or_else(|never| match never {}) + } +} + +impl From for String { + fn from(value: MessageType) -> Self { + value.to_string() + } +} + +#[derive( + Clone, + Debug, + PartialEq, + Eq, + strum::Display, + strum::EnumString, + strum::AsRefStr, + serde_with::DeserializeFromStr, + serde_with::SerializeDisplay, +)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[cfg_attr(feature = "schema", schemars(from = "String", into = "String"))] +#[strum(serialize_all = "snake_case")] +pub enum StopReason { + EndTurn, + MaxTokens, + StopSequence, + ToolUse, + Refusal, + Compaction, + PauseTurn, + ModelContextWindowExceeded, + #[strum(default, transparent)] + Other(String), +} + +impl StopReason { + pub fn as_str(&self) -> &str { + self.as_ref() + } +} + +impl From for StopReason { + fn from(value: String) -> Self { + value.parse().unwrap_or_else(|never| match never {}) + } +} + +impl From for String { + fn from(value: StopReason) -> Self { + value.to_string() + } +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum ContainerReference { + Id(String), + Parameters(Box), +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct MessagesDiagnosticsParam { + #[serde( + default, + skip_serializing_if = "Option::is_none", + with = "::serde_with::rust::double_option" + )] + #[cfg_attr(feature = "schema", schemars(with = "Option"))] + pub previous_message_id: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct MessagesDiagnostics { + pub cache_miss_reason: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum CacheMissReason { + ModelChanged(CacheMissedTokens), + SystemChanged(CacheMissedTokens), + ToolsChanged(CacheMissedTokens), + MessagesChanged(CacheMissedTokens), + PreviousMessageNotFound { + #[serde(flatten)] + extra: Map, + }, + Unavailable { + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct CacheMissedTokens { + pub cache_missed_input_tokens: u64, + #[serde(flatten)] + pub extra: Map, +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/mod.rs b/litellm-rust/crates/llms-types/src/formats/messages/mod.rs index 219e0ae63a0..08695ca7ecc 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/mod.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/mod.rs @@ -1,11 +1,54 @@ +mod content; +mod metadata; mod request; mod response; pub mod streaming; +mod tools; +mod usage; +pub use content::{ + AdvisorToolResultContent, BashCodeExecutionOutput, BashCodeExecutionToolResultContent, + BlockContent, BrowserStateBlock, BrowserStateChange, BrowserTab, CharCitation, Citation, + CitationsConfig, CodeExecutionOutput, CodeExecutionResult, CodeExecutionToolResultContent, + CompactionBlock, ContainerUploadBlock, ContentBlockCitation, ContentSource, DocumentBlock, + EncryptedCodeExecutionResult, FallbackBlock, FallbackModel, FallbackTrigger, ImageBlock, + McpListedTool, McpToolListingBlock, McpToolResultBlock, McpToolResultContent, + McpToolResultText, McpToolUseBlock, MessagesContentPart, PageCitation, RedactedThinkingBlock, + SearchResultBlock, SearchResultCitation, ServerToolError, ServerToolResultBlock, + ServerToolUseBlock, TextBlock, TextEditorCodeExecutionToolResultContent, + TextEditorCreateResult, TextEditorFileType, TextEditorStrReplaceResult, TextEditorViewResult, + ThinkingBlock, ToolCaller, ToolChange, ToolChangeBlock, ToolChangeTarget, ToolReferenceBlock, + ToolResultBlock, ToolSearchReference, ToolSearchResult, ToolSearchToolResultContent, + ToolUseBlock, WebFetchDocument, WebFetchResult, WebFetchToolResultContent, WebSearchCitation, + WebSearchResult, WebSearchResultError, WebSearchResultErrorType, WebSearchResultType, + WebSearchToolResultContent, +}; +pub use metadata::{ + AppliedEdit, CacheMissReason, CacheMissedTokens, CompactionType, ContainerReference, + ContainerSkill, ContextManagementResponse, McpServer, McpServerType, McpToolConfiguration, + MessageRole, MessageType, MessagesCompaction, MessagesContainer, MessagesDiagnostics, + MessagesDiagnosticsParam, MessagesMetadata, OutputFormat, OutputFormatType, Safeguard, + SkillType, StopDetails, StopDetailsType, StopReason, +}; pub use request::{ AdaptiveThinking, CacheControl, ContentBlock, ContentBlockType, ContextEdit, ContextManagement, - DisabledThinking, EffortLevel, EnabledThinking, Message, MessageContent, + ContextTrigger, DisabledThinking, EffortLevel, EnabledThinking, Message, MessageContent, MessagesOptionalParams, MessagesRequest, MessagesTool, OutputConfig, Speed, SystemPrompt, ThinkingConfig, ThinkingDisplay, }; pub use response::MessagesResponse; +pub use tools::{ + AdvisorTool, AdvisorToolName, AllowedCaller, BashToolName, BrowserToolsetConfigs, + BuiltinMessagesTool, ClientTool, CodeExecutionToolName, ComputerTool, ComputerTool20251124, + ComputerToolName, ComputerToolsetConfigs, CustomTool, CustomToolType, McpToolset, + MemoryToolName, MessagesToolParam, ResponseInclusion, ServerTool, StrReplaceBasedEditToolName, + StrReplaceEditorName, TextEditorTool20250728, ToolChoice, ToolChoiceType, ToolResultUrlSource, + ToolSearchBm25ToolName, ToolSearchRegexToolName, Toolset, ToolsetToolConfig, + UrlSourceToolReference, UserInputUrlSource, UserLocationType, WebFetchTool, + WebFetchTool20260309, WebFetchTool20260318, WebFetchToolName, WebFetchUrlSources, + WebSearchTool, WebSearchTool20260318, WebSearchToolName, WebSearchUserLocation, +}; +pub use usage::{ + CacheCreationUsage, MessagesOutputTokensDetails, MessagesUsage, ServerToolUsage, + UsageIteration, UsageIterationType, +}; diff --git a/litellm-rust/crates/llms-types/src/formats/messages/request.rs b/litellm-rust/crates/llms-types/src/formats/messages/request.rs index 16cc55e4f5a..1474dcf2428 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/request.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/request.rs @@ -190,6 +190,21 @@ pub enum MessagesTool { }, } +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ContextTrigger { + InputTokens { + value: u64, + #[serde(flatten)] + extra: Map, + }, + ToolUses { + value: u64, + #[serde(flatten)] + extra: Map, + }, +} + #[macro_rules_attribute::apply(wire_type)] #[serde(tag = "type")] pub enum ContextEdit { diff --git a/litellm-rust/crates/llms-types/src/formats/messages/tools.rs b/litellm-rust/crates/llms-types/src/formats/messages/tools.rs new file mode 100644 index 00000000000..7d32c7123da --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/messages/tools.rs @@ -0,0 +1,564 @@ +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +use super::{CacheControl, CitationsConfig, McpListedTool}; +use crate::json_schema::JsonSchema; + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +pub enum AllowedCaller { + #[serde(rename = "direct")] + Direct, + #[serde(rename = "code_execution_20250825")] + CodeExecution20250825, + #[serde(rename = "code_execution_20260120")] + CodeExecution20260120, + #[serde(rename = "code_execution_20260521")] + CodeExecution20260521, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebSearchUserLocation { + #[serde(rename = "type")] + pub location_type: UserLocationType, + pub city: Option, + pub country: Option, + pub region: Option, + pub timezone: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum UserLocationType { + Approximate, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ToolChoice { + #[serde(rename = "type")] + pub choice_type: ToolChoiceType, + pub name: Option, + pub disable_parallel_tool_use: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum ToolChoiceType { + Auto, + Any, + Tool, + None, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct CustomTool { + #[serde(rename = "type")] + pub tool_type: Option, + pub name: String, + pub input_schema: JsonSchema, + pub description: Option, + pub strict: Option, + pub cache_control: Option, + pub defer_loading: Option, + pub allowed_callers: Option>, + pub input_examples: Option>>, + pub eager_input_streaming: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum CustomToolType { + Custom, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BashToolName { + Bash, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum StrReplaceEditorName { + StrReplaceEditor, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum StrReplaceBasedEditToolName { + StrReplaceBasedEditTool, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum MemoryToolName { + Memory, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ComputerToolName { + Computer, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum CodeExecutionToolName { + CodeExecution, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ToolSearchRegexToolName { + ToolSearchToolRegex, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ToolSearchBm25ToolName { + ToolSearchToolBm25, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum WebSearchToolName { + WebSearch, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum WebFetchToolName { + WebFetch, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum AdvisorToolName { + Advisor, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ServerTool { + pub name: N, + pub allowed_callers: Option>, + pub cache_control: Option, + pub defer_loading: Option, + pub strict: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ClientTool { + pub name: N, + pub allowed_callers: Option>, + pub cache_control: Option, + pub defer_loading: Option, + pub input_examples: Option>>, + pub strict: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct TextEditorTool20250728 { + pub name: StrReplaceBasedEditToolName, + pub allowed_callers: Option>, + pub cache_control: Option, + pub defer_loading: Option, + pub input_examples: Option>>, + pub max_characters: Option, + pub strict: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ComputerTool { + pub name: ComputerToolName, + pub display_width_px: u64, + pub display_height_px: u64, + pub display_number: Option, + pub allowed_callers: Option>, + pub cache_control: Option, + pub defer_loading: Option, + pub input_examples: Option>>, + pub strict: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ComputerTool20251124 { + pub name: ComputerToolName, + pub display_width_px: u64, + pub display_height_px: u64, + pub display_number: Option, + pub enable_zoom: Option, + pub allowed_callers: Option>, + pub cache_control: Option, + pub defer_loading: Option, + pub input_examples: Option>>, + pub strict: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[derive(Copy, Eq)] +#[serde(rename_all = "snake_case")] +pub enum ResponseInclusion { + Full, + Excluded, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebSearchTool { + pub name: WebSearchToolName, + pub allowed_callers: Option>, + pub allowed_domains: Option>, + pub blocked_domains: Option>, + pub cache_control: Option, + pub defer_loading: Option, + pub max_uses: Option, + pub strict: Option, + pub user_location: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebSearchTool20260318 { + pub name: WebSearchToolName, + pub allowed_callers: Option>, + pub allowed_domains: Option>, + pub blocked_domains: Option>, + pub cache_control: Option, + pub defer_loading: Option, + pub max_uses: Option, + pub response_inclusion: Option, + pub strict: Option, + pub user_location: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum UrlSourceToolReference { + ToolReference { + name: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ToolResultUrlSource { + All { + #[serde(flatten)] + extra: Map, + }, + None { + #[serde(flatten)] + extra: Map, + }, + Only { + tools: Vec, + #[serde(flatten)] + extra: Map, + }, + Except { + tools: Vec, + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum UserInputUrlSource { + All { + #[serde(flatten)] + extra: Map, + }, + None { + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebFetchUrlSources { + pub client_tool_results: Option, + pub server_tool_results: Option, + pub user_input: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebFetchTool { + pub name: WebFetchToolName, + pub allowed_callers: Option>, + pub allowed_domains: Option>, + pub blocked_domains: Option>, + pub cache_control: Option, + pub citations: Option, + pub defer_loading: Option, + pub max_content_tokens: Option, + pub max_uses: Option, + pub strict: Option, + pub url_sources: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebFetchTool20260309 { + pub name: WebFetchToolName, + pub allowed_callers: Option>, + pub allowed_domains: Option>, + pub blocked_domains: Option>, + pub cache_control: Option, + pub citations: Option, + pub defer_loading: Option, + pub max_content_tokens: Option, + pub max_uses: Option, + pub strict: Option, + pub url_sources: Option, + pub use_cache: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct WebFetchTool20260318 { + pub name: WebFetchToolName, + pub allowed_callers: Option>, + pub allowed_domains: Option>, + pub blocked_domains: Option>, + pub cache_control: Option, + pub citations: Option, + pub defer_loading: Option, + pub max_content_tokens: Option, + pub max_uses: Option, + pub response_inclusion: Option, + pub strict: Option, + pub url_sources: Option, + pub use_cache: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct AdvisorTool { + pub name: AdvisorToolName, + pub model: String, + pub allowed_callers: Option>, + pub cache_control: Option, + pub caching: Option, + pub defer_loading: Option, + pub max_tokens: Option, + pub max_uses: Option, + pub strict: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ToolsetToolConfig { + pub defer_loading: Option, + pub enabled: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct McpToolset { + pub mcp_server_name: String, + pub cache_control: Option, + pub configs: Option>, + pub default_config: Option, + pub tools: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct BrowserToolsetConfigs { + #[serde(rename = "type")] + pub type_text: Option, + pub close_tab: Option, + pub double_click: Option, + pub file_upload: Option, + pub find: Option, + pub form_input: Option, + pub get_page_text: Option, + pub hold_key: Option, + pub hover: Option, + pub javascript_exec: Option, + pub key: Option, + pub left_click: Option, + pub left_click_drag: Option, + pub left_mouse_down: Option, + pub left_mouse_up: Option, + pub list_tabs: Option, + pub middle_click: Option, + pub mouse_move: Option, + pub navigate: Option, + pub new_tab: Option, + pub read_console: Option, + pub read_network: Option, + pub read_page: Option, + pub right_click: Option, + pub screenshot: Option, + pub scroll: Option, + pub scroll_to: Option, + pub switch_tab: Option, + pub triple_click: Option, + pub wait: Option, + pub zoom: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct ComputerToolsetConfigs { + #[serde(rename = "type")] + pub type_text: Option, + pub cursor_position: Option, + pub double_click: Option, + pub hold_key: Option, + pub key: Option, + pub left_click: Option, + pub left_click_drag: Option, + pub left_mouse_down: Option, + pub left_mouse_up: Option, + pub middle_click: Option, + pub mouse_move: Option, + pub right_click: Option, + pub screenshot: Option, + pub scroll: Option, + pub triple_click: Option, + pub wait: Option, + pub zoom: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct Toolset { + pub cache_control: Option, + pub configs: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type")] +pub enum BuiltinMessagesTool { + #[serde(rename = "bash_20241022")] + Bash20241022(ClientTool), + #[serde(rename = "bash_20250124")] + Bash20250124(ClientTool), + #[serde(rename = "text_editor_20241022")] + TextEditor20241022(ClientTool), + #[serde(rename = "text_editor_20250124")] + TextEditor20250124(ClientTool), + #[serde(rename = "text_editor_20250429")] + TextEditor20250429(ClientTool), + #[serde(rename = "text_editor_20250728")] + TextEditor20250728(TextEditorTool20250728), + #[serde(rename = "memory_20250818")] + Memory20250818(ClientTool), + #[serde(rename = "computer_20241022")] + Computer20241022(ComputerTool), + #[serde(rename = "computer_20250124")] + Computer20250124(ComputerTool), + #[serde(rename = "computer_20251124")] + Computer20251124(ComputerTool20251124), + #[serde(rename = "code_execution_20250522")] + CodeExecution20250522(ServerTool), + #[serde(rename = "code_execution_20250825")] + CodeExecution20250825(ServerTool), + #[serde(rename = "code_execution_20260120")] + CodeExecution20260120(ServerTool), + #[serde(rename = "code_execution_20260521")] + CodeExecution20260521(ServerTool), + #[serde(rename = "tool_search_tool_regex_20251119")] + ToolSearchRegex20251119(ServerTool), + #[serde(rename = "tool_search_tool_regex")] + ToolSearchRegex(ServerTool), + #[serde(rename = "tool_search_tool_bm25_20251119")] + ToolSearchBm2520251119(ServerTool), + #[serde(rename = "tool_search_tool_bm25")] + ToolSearchBm25(ServerTool), + #[serde(rename = "web_search_20250305")] + WebSearch20250305(WebSearchTool), + #[serde(rename = "web_search_20260209")] + WebSearch20260209(WebSearchTool), + #[serde(rename = "web_search_20260318")] + WebSearch20260318(WebSearchTool20260318), + #[serde(rename = "web_fetch_20250910")] + WebFetch20250910(WebFetchTool), + #[serde(rename = "web_fetch_20260209")] + WebFetch20260209(WebFetchTool), + #[serde(rename = "web_fetch_20260309")] + WebFetch20260309(WebFetchTool20260309), + #[serde(rename = "web_fetch_20260318")] + WebFetch20260318(WebFetchTool20260318), + #[serde(rename = "advisor_20260301")] + Advisor20260301(AdvisorTool), + #[serde(rename = "browser_toolset_20260801")] + BrowserToolset20260801(Toolset), + #[serde(rename = "computer_toolset_20260801")] + ComputerToolset20260801(Toolset), + #[serde(rename = "mcp_toolset")] + McpToolset(McpToolset), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum MessagesToolParam { + Builtin(Box), + Custom(Box), +} diff --git a/litellm-rust/crates/llms-types/src/formats/messages/usage.rs b/litellm-rust/crates/llms-types/src/formats/messages/usage.rs new file mode 100644 index 00000000000..aa235c437e8 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/messages/usage.rs @@ -0,0 +1,75 @@ +use serde_json::{Map, Value}; + +use crate::recognized::Recognized; + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ServerToolUsage { + pub web_search_requests: Option, + pub web_fetch_requests: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct UsageIteration { + #[serde(rename = "type")] + pub iteration_type: Recognized, + pub input_tokens: Option, + pub output_tokens: Option, + pub cache_creation_input_tokens: Option, + pub cache_read_input_tokens: Option, + pub cache_creation: Option, + pub model: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum UsageIterationType { + Compaction, + Message, + AdvisorMessage, + FallbackMessage, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct MessagesUsage { + pub input_tokens: Option, + pub output_tokens: Option, + pub cache_creation_input_tokens: Option, + pub cache_read_input_tokens: Option, + pub server_tool_use: Option, + pub cache_creation: Option, + pub output_tokens_details: Option, + pub service_tier: Option, + pub inference_geo: Option, + pub speed: Option, + pub iterations: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct CacheCreationUsage { + pub ephemeral_1h_input_tokens: Option, + pub ephemeral_5m_input_tokens: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct MessagesOutputTokensDetails { + pub thinking_tokens: Option, + #[serde(flatten)] + pub extra: Map, +} diff --git a/litellm-rust/crates/llms-types/src/formats/mod.rs b/litellm-rust/crates/llms-types/src/formats/mod.rs index 53f2577090b..0353077c3a9 100644 --- a/litellm-rust/crates/llms-types/src/formats/mod.rs +++ b/litellm-rust/crates/llms-types/src/formats/mod.rs @@ -1,3 +1,6 @@ +//! Wire types for each provider API format (OpenAI Chat Completions, +//! Anthropic Messages, Responses, ...), one submodule per format. + pub mod audio_transcription; pub mod batches; pub mod chat_completions; diff --git a/litellm-rust/crates/llms-types/src/formats/ocr.rs b/litellm-rust/crates/llms-types/src/formats/ocr.rs index b491f5d82f1..eaecf4feda7 100644 --- a/litellm-rust/crates/llms-types/src/formats/ocr.rs +++ b/litellm-rust/crates/llms-types/src/formats/ocr.rs @@ -150,3 +150,15 @@ impl LiteLLMOcrResponse { fn ocr_object() -> String { "ocr".into() } + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct OcrBoundingBox { + pub top_left_x: Option, + pub top_left_y: Option, + pub bottom_right_x: Option, + pub bottom_right_y: Option, + #[serde(flatten)] + pub extra: Map, +} diff --git a/litellm-rust/crates/llms-types/src/formats/responses/mod.rs b/litellm-rust/crates/llms-types/src/formats/responses/mod.rs index 0aefd8a8698..a45a971bae6 100644 --- a/litellm-rust/crates/llms-types/src/formats/responses/mod.rs +++ b/litellm-rust/crates/llms-types/src/formats/responses/mod.rs @@ -1,4 +1,6 @@ +mod output; mod response; pub mod streaming_websocket; +pub use output::*; pub use response::ResponsesApiResponse; diff --git a/litellm-rust/crates/llms-types/src/formats/responses/output.rs b/litellm-rust/crates/llms-types/src/formats/responses/output.rs new file mode 100644 index 00000000000..a5c79ff1fdc --- /dev/null +++ b/litellm-rust/crates/llms-types/src/formats/responses/output.rs @@ -0,0 +1,831 @@ +use crate::formats::chat_completions::PromptCacheBreakpoint; +use serde_json::{Map, Value}; +use std::collections::BTreeMap; + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesOutputItem { + Message(ResponsesMessage), + FunctionCall(ResponsesFunctionCall), + CustomToolCall(ResponsesCustomToolCall), + Reasoning(ResponsesReasoning), + WebSearchCall(ResponsesWebSearchCall), + FileSearchCall(ResponsesFileSearchCall), + ImageGenerationCall(ResponsesImageGenerationCall), + CodeInterpreterCall(ResponsesCodeInterpreterCall), + McpCall(ResponsesMcpCall), + McpListTools(ResponsesMcpListTools), + FunctionCallOutput(ResponsesFunctionCallOutput), + ComputerCall(ResponsesComputerCall), + ComputerCallOutput(ResponsesComputerCallOutput), + Program(ResponsesProgram), + ProgramOutput(ResponsesProgramOutput), + ToolSearchCall(ResponsesToolSearchCall), + ToolSearchOutput(ResponsesToolSearchOutput), + AdditionalTools(ResponsesAdditionalTools), + Compaction(ResponsesCompaction), + LocalShellCall(ResponsesLocalShellCall), + LocalShellCallOutput(ResponsesLocalShellCallOutput), + ShellCall(ResponsesShellCall), + ShellCallOutput(ResponsesShellCallOutput), + ApplyPatchCall(ResponsesApplyPatchCall), + ApplyPatchCallOutput(ResponsesApplyPatchCallOutput), + McpApprovalRequest(ResponsesMcpApprovalRequest), + McpApprovalResponse(ResponsesMcpApprovalResponse), + CustomToolCallOutput(ResponsesCustomToolCallOutput), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesContentPart { + OutputText { + text: String, + #[serde(skip_serializing_if = "Option::is_none")] + annotations: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + logprobs: Option>, + #[serde(flatten)] + extra: Map, + }, + Refusal { + refusal: String, + #[serde(flatten)] + extra: Map, + }, + SummaryText { + text: String, + #[serde(flatten)] + extra: Map, + }, + ReasoningText { + text: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesAnnotation { + UrlCitation(ResponsesUrlCitation), + FileCitation(ResponsesFileCitation), + FilePath(ResponsesFileCitation), + ContainerFileCitation(ResponsesFileCitation), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesCodeOutput { + Logs { + logs: String, + #[serde(flatten)] + extra: Map, + }, + Image { + url: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesMessage { + pub id: Option, + pub status: Option, + pub role: Option, + pub phase: Option, + pub content: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesFunctionCall { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub name: Option, + pub arguments: Option, + pub namespace: Option, + pub r#async: Option, + pub caller: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesCustomToolCall { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub name: Option, + pub input: Option, + pub namespace: Option, + pub r#async: Option, + pub caller: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesReasoning { + pub id: Option, + pub status: Option, + pub summary: Option>, + pub content: Option>, + pub encrypted_content: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesWebSearchCall { + pub id: Option, + pub status: Option, + pub action: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesFileSearchCall { + pub id: Option, + pub status: Option, + pub queries: Option>, + pub results: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesFileSearchResult { + pub file_id: Option, + pub filename: Option, + pub score: Option, + pub text: Option, + pub attributes: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesImageGenerationCall { + pub id: Option, + pub status: Option, + pub result: Option, + pub action: Option, + pub background: Option, + pub output_format: Option, + pub quality: Option, + pub revised_prompt: Option, + pub size: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesCodeInterpreterCall { + pub id: Option, + pub status: Option, + pub code: Option, + pub container_id: Option, + pub outputs: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesMcpCall { + pub id: Option, + pub status: Option, + pub name: Option, + pub server_label: Option, + pub arguments: Option, + pub output: Option, + pub error: Option, + pub approval_request_id: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesUrlCitation { + pub url: Option, + pub title: Option, + pub start_index: Option, + pub end_index: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesFileCitation { + pub file_id: Option, + pub filename: Option, + pub index: Option, + pub start_index: Option, + pub end_index: Option, + pub container_id: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesWebSearchAction { + Search { + #[serde(skip_serializing_if = "Option::is_none")] + query: Option, + #[serde(skip_serializing_if = "Option::is_none")] + queries: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + sources: Option>, + #[serde(flatten)] + extra: Map, + }, + OpenPage { + #[serde(skip_serializing_if = "Option::is_none")] + url: Option, + #[serde(flatten)] + extra: Map, + }, + #[serde(alias = "find")] + FindInPage { + url: String, + pattern: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesWebSearchSource { + #[serde(rename = "type")] + pub source_type: Option, + pub url: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesMcpListTools { + pub id: Option, + pub server_label: Option, + pub tools: Option>, + pub error: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesMcpTool { + pub name: Option, + pub description: Option, + pub input_schema: Option, + pub annotations: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum ResponsesMcpError { + Message(String), + Detail(ResponsesMcpErrorDetail), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesMcpErrorDetail { + McpProtocolError { + code: i64, + message: String, + #[serde(flatten)] + extra: Map, + }, + HttpError { + code: i64, + message: String, + #[serde(flatten)] + extra: Map, + }, + McpToolExecutionError { + content: Value, + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesToolCaller { + Direct { + #[serde(flatten)] + extra: Map, + }, + Program { + caller_id: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum ResponsesToolOutput { + Text(String), + Content(Vec), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesInputContent { + InputText { + text: String, + #[serde(skip_serializing_if = "Option::is_none")] + prompt_cache_breakpoint: Option, + #[serde(flatten)] + extra: Map, + }, + InputImage { + detail: String, + #[serde(skip_serializing_if = "Option::is_none")] + file_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + image_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + prompt_cache_breakpoint: Option, + #[serde(flatten)] + extra: Map, + }, + InputFile { + #[serde(skip_serializing_if = "Option::is_none")] + detail: Option, + #[serde(skip_serializing_if = "Option::is_none")] + file_data: Option, + #[serde(skip_serializing_if = "Option::is_none")] + file_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + file_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] + filename: Option, + #[serde(skip_serializing_if = "Option::is_none")] + prompt_cache_breakpoint: Option, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesFunctionCallOutput { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub output: Option, + pub caller: Option, + pub created_by: Option, + pub name: Option, + pub namespace: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesCustomToolCallOutput { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub output: Option, + pub caller: Option, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesSafetyCheck { + pub id: Option, + pub code: Option, + pub message: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesComputerCall { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub pending_safety_checks: Option>, + pub action: Option, + pub actions: Option>, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesComputerAction { + Click { + button: String, + x: i64, + y: i64, + #[serde(skip_serializing_if = "Option::is_none")] + keys: Option>, + #[serde(flatten)] + extra: Map, + }, + DoubleClick { + x: i64, + y: i64, + #[serde(skip_serializing_if = "Option::is_none")] + keys: Option>, + #[serde(flatten)] + extra: Map, + }, + Drag { + path: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + keys: Option>, + #[serde(flatten)] + extra: Map, + }, + Keypress { + keys: Vec, + #[serde(flatten)] + extra: Map, + }, + Move { + x: i64, + y: i64, + #[serde(skip_serializing_if = "Option::is_none")] + keys: Option>, + #[serde(flatten)] + extra: Map, + }, + Screenshot { + #[serde(flatten)] + extra: Map, + }, + Scroll { + scroll_x: i64, + scroll_y: i64, + x: i64, + y: i64, + #[serde(skip_serializing_if = "Option::is_none")] + keys: Option>, + #[serde(flatten)] + extra: Map, + }, + Type { + text: String, + #[serde(flatten)] + extra: Map, + }, + Wait { + #[serde(flatten)] + extra: Map, + }, +} + +#[macro_rules_attribute::apply(wire_type)] +pub struct ResponsesCoordinate { + pub x: i64, + pub y: i64, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesComputerCallOutput { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub output: Option, + pub acknowledged_safety_checks: Option>, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesComputerOutput { + ComputerScreenshot { + #[serde(skip_serializing_if = "Option::is_none")] + file_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + image_url: Option, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesProgram { + pub id: Option, + pub call_id: Option, + pub code: Option, + pub fingerprint: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesProgramOutput { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub result: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesToolSearchCall { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub arguments: Option, + pub execution: Option, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesToolSearchOutput { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub execution: Option, + pub tools: Option>>, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesAdditionalTools { + pub id: Option, + pub role: Option, + pub tools: Option>>, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesCompaction { + pub id: Option, + pub encrypted_content: Option, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesLocalShellCall { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub action: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesLocalShellAction { + Exec { + command: Vec, + env: BTreeMap, + #[serde(skip_serializing_if = "Option::is_none")] + timeout_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + user: Option, + #[serde(skip_serializing_if = "Option::is_none")] + working_directory: Option, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesLocalShellCallOutput { + pub id: Option, + pub status: Option, + pub output: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesShellCall { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub action: Option, + pub environment: Option, + pub caller: Option, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesShellAction { + pub commands: Option>, + pub max_output_length: Option, + pub timeout_ms: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesShellEnvironment { + Local { + #[serde(flatten)] + extra: Map, + }, + ContainerReference { + container_id: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesShellCallOutput { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub max_output_length: Option, + pub output: Option>, + pub caller: Option, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesShellOutputChunk { + pub outcome: Option, + pub stdout: Option, + pub stderr: Option, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesShellOutcome { + Timeout { + #[serde(flatten)] + extra: Map, + }, + Exit { + exit_code: i64, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesApplyPatchCall { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub operation: Option, + pub caller: Option, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponsesApplyPatchOperation { + CreateFile { + path: String, + diff: String, + #[serde(flatten)] + extra: Map, + }, + DeleteFile { + path: String, + #[serde(flatten)] + extra: Map, + }, + UpdateFile { + path: String, + diff: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesApplyPatchCallOutput { + pub id: Option, + pub status: Option, + pub call_id: Option, + pub output: Option, + pub caller: Option, + pub created_by: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesMcpApprovalRequest { + pub id: Option, + pub server_label: Option, + pub name: Option, + pub arguments: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesMcpApprovalResponse { + pub id: Option, + pub approval_request_id: Option, + pub approve: Option, + pub reason: Option, + #[serde(flatten)] + pub extra: Map, +} diff --git a/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs b/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs index 6b648a21bd1..b3c204d7383 100644 --- a/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs +++ b/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs @@ -1,3 +1,5 @@ +use super::ResponsesOutputItem; +use crate::recognized::Recognized; use serde_json::{Map, Value}; #[derive( @@ -90,6 +92,18 @@ pub struct ResponsesErrorBody { pub message: String, } +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct ResponsesEventResponse { + pub id: Option, + pub model: Option, + pub status: Option, + pub output: Option>>, + #[serde(flatten)] + pub extra: Map, +} + #[cfg(test)] mod tests { use rstest::rstest; diff --git a/litellm-rust/crates/llms-types/src/json_schema.rs b/litellm-rust/crates/llms-types/src/json_schema.rs new file mode 100644 index 00000000000..702e1bb8a80 --- /dev/null +++ b/litellm-rust/crates/llms-types/src/json_schema.rs @@ -0,0 +1,51 @@ +use indexmap::IndexMap; +use serde_json::{Map, Value}; + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum JsonSchema { + Boolean(bool), + Object(Box), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum JsonSchemaType { + Name(String), + Names(Vec), +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(untagged)] +pub enum JsonSchemaItems { + Schema(Box), + Tuple(Vec), +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct JsonSchemaObject { + #[serde(rename = "type")] + pub schema_type: Option, + pub properties: Option>, + pub required: Option>, + #[serde(rename = "additionalProperties")] + pub additional_properties: Option>, + pub items: Option, + #[serde(rename = "prefixItems")] + pub prefix_items: Option>, + #[serde(rename = "$defs")] + pub defs: Option>, + #[serde(rename = "$ref")] + pub reference: Option, + #[serde(rename = "anyOf")] + pub any_of: Option>, + #[serde(rename = "allOf")] + pub all_of: Option>, + #[serde(rename = "oneOf")] + pub one_of: Option>, + pub strict: Option, + #[serde(flatten)] + pub extra: Map, +} diff --git a/litellm-rust/crates/llms-types/src/lib.rs b/litellm-rust/crates/llms-types/src/lib.rs index 116c11c0f88..814122f2ab9 100644 --- a/litellm-rust/crates/llms-types/src/lib.rs +++ b/litellm-rust/crates/llms-types/src/lib.rs @@ -6,6 +6,7 @@ macro_rules_attribute::attribute_alias! { pub mod formats; pub mod headers; +pub mod json_schema; pub mod providers; pub mod recognized; pub mod serde_compat; diff --git a/litellm-rust/crates/llms-types/src/providers/AGENTS.md b/litellm-rust/crates/llms-types/src/providers/AGENTS.md new file mode 100644 index 00000000000..69712edbcbb --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/AGENTS.md @@ -0,0 +1,17 @@ +# references + +These links describe the provider-specific wire types in this directory. Shared Messages payload references live in `../formats/messages/AGENTS.md` + +## anthropic.rs + +`AnthropicBeta` and `BetaSet` represent the `anthropic-beta` header values and their wire spelling + +- https://platform.claude.com/docs/en/api/beta-headers.md + +## minimax.rs + +MiniMax's Messages reference documents its image, video, and mid-conversation system content extensions. Its cache reference documents `cache_control` on those blocks + +- https://platform.minimax.io/docs/api-reference/text-chat-anthropic +- https://platform.minimax.io/docs/api-reference/text-chat-anthropic.md +- https://platform.minimax.io/docs/api-reference/anthropic-api-compatible-cache.md diff --git a/litellm-rust/crates/llms-types/src/providers/minimax.rs b/litellm-rust/crates/llms-types/src/providers/minimax.rs new file mode 100644 index 00000000000..efc4133e1af --- /dev/null +++ b/litellm-rust/crates/llms-types/src/providers/minimax.rs @@ -0,0 +1,59 @@ +use serde_json::{Map, Value}; + +use crate::formats::messages::CacheControl; + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MinimaxMessagesContentBlock { + Image(MinimaxMediaBlock), + Video(MinimaxMediaBlock), + MidConvSystem { + text: String, + #[serde(flatten)] + extra: Map, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +pub struct MinimaxMediaBlock { + pub source: MinimaxMediaSource, + pub cache_control: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum MinimaxMediaSource { + Base64 { + media_type: String, + data: String, + #[serde(flatten)] + options: MinimaxMediaOptions, + }, + Url { + url: String, + #[serde(flatten)] + options: MinimaxMediaOptions, + }, +} + +#[serde_with::skip_serializing_none] +#[macro_rules_attribute::apply(wire_type)] +#[derive(Default)] +pub struct MinimaxMediaOptions { + pub detail: Option, + pub fps: Option, + pub max_long_side_pixel: Option, + #[serde(flatten)] + pub extra: Map, +} + +#[macro_rules_attribute::apply(wire_type)] +#[serde(rename_all = "snake_case")] +pub enum MinimaxMediaDetail { + Low, + Default, + High, +} diff --git a/litellm-rust/crates/llms-types/src/providers/mod.rs b/litellm-rust/crates/llms-types/src/providers/mod.rs index e529997219e..c9eafe964ef 100644 --- a/litellm-rust/crates/llms-types/src/providers/mod.rs +++ b/litellm-rust/crates/llms-types/src/providers/mod.rs @@ -1 +1,2 @@ pub mod anthropic; +pub mod minimax; diff --git a/litellm-rust/crates/llms-types/tests/chat.rs b/litellm-rust/crates/llms-types/tests/chat.rs new file mode 100644 index 00000000000..76aa791b510 --- /dev/null +++ b/litellm-rust/crates/llms-types/tests/chat.rs @@ -0,0 +1,163 @@ +use litellm_llms_types::formats::chat_completions::{ + ChatContentPart, ChatLogprobs, ChatMediaUrl, PromptCacheMode, +}; +use litellm_llms_types::formats::messages::ContentSource; +use rstest::rstest; +use serde_json::{Value, json}; + +#[rstest] +#[case::parts(json!([ + {"type":"text","text":"hello","cache_control":{"type":"ephemeral"},"prompt_cache_breakpoint":{"mode":"explicit"}}, + {"type":"image_url","image_url":{"url":"https://example.test/image","detail":"high","format":"image/png"}}, + {"type":"video_url","video_url":"https://example.test/video"}, + {"type":"input_audio","input_audio":{"data":"AA==","format":"wav"}}, + {"type":"file","file":{"file_id":"file_1","file_data":"JVBE","filename":"a.mp4","format":"video/mp4","detail":"low","video_metadata":{"fps":1,"start_offset":"1s","end_offset":"2s"}}}, + {"type":"document","source":{"type":"text","media_type":"text/plain","data":"doc"},"title":"T","context":"C","citations":{"enabled":true}}, + {"type":"refusal","refusal":"refused"} +]))] +fn content_parts_round_trip(#[case] wire: Value) { + let parts: Vec = serde_json::from_value(wire.clone()).unwrap(); + let [ + ChatContentPart::Text { + text, + cache_control, + prompt_cache_breakpoint: Some(breakpoint), + extra: text_extra, + }, + ChatContentPart::ImageUrl { + image_url: ChatMediaUrl::Parameters(image), + prompt_cache_breakpoint: None, + .. + }, + ChatContentPart::VideoUrl { + video_url: ChatMediaUrl::Url(video), + .. + }, + ChatContentPart::InputAudio { input_audio, .. }, + ChatContentPart::File { file, .. }, + ChatContentPart::Document { + source, + title: Some(title), + context: Some(context), + citations: Some(citations), + .. + }, + ChatContentPart::Refusal { refusal, .. }, + ] = parts.as_slice() + else { + panic!("expected typed content parts"); + }; + assert_eq!(text, "hello"); + assert_eq!( + cache_control.as_ref().unwrap().cache_type.as_deref(), + Some("ephemeral") + ); + assert_eq!(breakpoint.mode, PromptCacheMode::Explicit); + assert!(breakpoint.extra.is_empty()); + assert!(text_extra.is_empty()); + assert_eq!(image.url, "https://example.test/image"); + assert_eq!(image.detail.as_deref(), Some("high")); + assert_eq!(image.format.as_deref(), Some("image/png")); + assert!(image.extra.is_empty()); + assert_eq!(video, "https://example.test/video"); + assert_eq!(input_audio.data, "AA=="); + assert_eq!(input_audio.format, "wav"); + assert_eq!(file.file_id.as_deref(), Some("file_1")); + assert_eq!(file.file_data.as_deref(), Some("JVBE")); + assert_eq!(file.filename.as_deref(), Some("a.mp4")); + assert_eq!(file.format.as_deref(), Some("video/mp4")); + assert_eq!(file.detail.as_deref(), Some("low")); + assert!(file.extra.is_empty()); + let metadata = file.video_metadata.as_ref().unwrap(); + assert_eq!(metadata.fps, Some(1.into())); + assert_eq!(metadata.start_offset.as_deref(), Some("1s")); + assert_eq!(metadata.end_offset.as_deref(), Some("2s")); + assert!(metadata.extra.is_empty()); + let ContentSource::Text { + data, media_type, .. + } = source.as_ref() + else { + panic!("expected document text source"); + }; + assert_eq!(data, "doc"); + assert_eq!(media_type, "text/plain"); + assert_eq!(title, "T"); + assert_eq!(context, "C"); + assert_eq!(citations.enabled, Some(true)); + assert_eq!(refusal, "refused"); + assert_eq!(serde_json::to_value(parts).unwrap(), wire); +} + +#[rstest] +fn logprobs_round_trip_with_tokens_and_alternatives() { + let wire = json!({ + "content":[{"token":"hi","logprob":-1,"bytes":[104,105],"top_logprobs":[{"token":"hey","logprob":-2.5,"bytes":[104]}]}], + "refusal":[{"token":"refused","logprob":-3}] + }); + let logprobs: ChatLogprobs = serde_json::from_value(wire.clone()).unwrap(); + let token = &logprobs.content.as_ref().unwrap()[0]; + assert_eq!(token.token, "hi"); + assert_eq!(token.logprob, (-1).into()); + assert_eq!(token.bytes.as_deref(), Some([104, 105].as_slice())); + let alternative = &token.top_logprobs.as_ref().unwrap()[0]; + assert_eq!(alternative.token, "hey"); + assert_eq!(alternative.bytes.as_deref(), Some([104].as_slice())); + assert_eq!(logprobs.refusal.as_ref().unwrap()[0].token, "refused"); + assert!(logprobs.refusal.as_ref().unwrap()[0].top_logprobs.is_none()); + assert_eq!(serde_json::to_value(logprobs).unwrap(), wire); +} + +#[rstest] +#[case::text(json!({"type":"text","text":false}))] +#[case::missing_image(json!({"type":"image_url"}))] +#[case::audio_shape(json!({"type":"input_audio","input_audio":{"data":7,"format":"wav"}}))] +#[case::audio_missing_format(json!({"type":"input_audio","input_audio":{"data":"AA=="}}))] +#[case::image_missing_url(json!({"type":"image_url","image_url":{"detail":"high"}}))] +#[case::file_metadata(json!({"type":"file","file":{"video_metadata":{"fps":"fast"}}}))] +#[case::document_source(json!({"type":"document","source":{"type":"url","url":false}}))] +#[case::refusal_missing_text(json!({"type":"refusal"}))] +#[case::file_missing_file(json!({"type":"file"}))] +#[case::document_missing_source(json!({"type":"document"}))] +#[case::video_missing_url(json!({"type":"video_url"}))] +#[case::cache_control_shape(json!({"type":"text","text":"t","cache_control":"ephemeral"}))] +#[case::breakpoint_missing_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{}}))] +#[case::breakpoint_unknown_mode(json!({"type":"text","text":"t","prompt_cache_breakpoint":{"mode":"auto"}}))] +#[case::unknown_tag(json!({"type":"future"}))] +fn content_parts_reject_malformed_typed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +fn partial_file_preserves_extensions_and_omits_null_optionals() { + let wire = json!({"type":"file","file":{"file_id":null,"filename":"a.pdf","extension":[1,null]},"future":true}); + let part: ChatContentPart = serde_json::from_value(wire).unwrap(); + let ChatContentPart::File { + file, + prompt_cache_breakpoint: None, + extra, + } = &part + else { + panic!("expected file") + }; + assert!(file.file_id.is_none()); + assert!(file.file_data.is_none()); + assert_eq!(file.filename.as_deref(), Some("a.pdf")); + assert_eq!(extra["future"], json!(true)); + assert_eq!( + serde_json::to_value(part).unwrap(), + json!({"type":"file","file":{"filename":"a.pdf","extension":[1,null]},"future":true}) + ); +} + +#[rstest] +#[case::missing_token(json!({"logprob":-1}))] +#[case::missing_logprob(json!({"token":"hi"}))] +#[case::wrong_bytes(json!({"token":"hi","logprob":-1,"bytes":[256]}))] +fn token_logprobs_reject_malformed_fields(#[case] wire: Value) { + assert!( + serde_json::from_value::( + wire + ) + .is_err() + ); +} diff --git a/litellm-rust/crates/llms-types/tests/json_schema.rs b/litellm-rust/crates/llms-types/tests/json_schema.rs new file mode 100644 index 00000000000..a626304910c --- /dev/null +++ b/litellm-rust/crates/llms-types/tests/json_schema.rs @@ -0,0 +1,148 @@ +use litellm_llms_types::json_schema::{ + JsonSchema, JsonSchemaItems, JsonSchemaObject, JsonSchemaType, +}; +use rstest::rstest; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::{Value, json}; + +fn round_trip(wire: Value) -> T +where + T: DeserializeOwned + Serialize, +{ + let parsed: T = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(&parsed).unwrap(), wire); + parsed +} + +#[rstest] +#[case::boolean(json!(false))] +#[case::nested(json!({ + "type":["object","null"], + "properties":{"nested":{"type":"array","items":{"$ref":"#/$defs/item"}},"free":true}, + "additionalProperties":{"type":"string"}, + "$defs":{"item":{"anyOf":[{"type":"string"},false]}}, + "required":["nested"], + "enum":[{"custom":[1,null]}], + "future":null +}))] +fn recursive_schema_round_trips(#[case] wire: Value) { + let schema = round_trip::(wire); + match schema { + JsonSchema::Boolean(allowed) => assert!(!allowed), + JsonSchema::Object(schema) => { + assert_eq!( + schema.schema_type, + Some(JsonSchemaType::Names(vec!["object".into(), "null".into()])) + ); + let properties = schema.properties.as_ref().unwrap(); + let JsonSchema::Object(nested) = &properties["nested"] else { + panic!("expected nested schema") + }; + let Some(JsonSchemaItems::Schema(items)) = &nested.items else { + panic!("expected single items schema") + }; + let JsonSchema::Object(items) = items.as_ref() else { + panic!("expected items schema") + }; + assert_eq!(items.reference.as_deref(), Some("#/$defs/item")); + assert_eq!(properties["free"], JsonSchema::Boolean(true)); + let JsonSchema::Object(definition) = &schema.defs.as_ref().unwrap()["item"] else { + panic!("expected schema definition") + }; + assert_eq!( + definition.any_of.as_ref().unwrap()[1], + JsonSchema::Boolean(false) + ); + assert_eq!(schema.extra["enum"], json!([{"custom":[1,null]}])); + } + } +} + +#[rstest] +fn schema_maps_keep_wire_key_order() { + let wire = r#"{"type":"object","properties":{"zeta":{"type":"string"},"alpha":{"type":"integer"}},"$defs":{"y":true,"b":false}}"#; + let schema: JsonSchemaObject = serde_json::from_str(wire).unwrap(); + let names: Vec<&str> = schema + .properties + .as_ref() + .unwrap() + .keys() + .map(String::as_str) + .collect(); + assert_eq!(names, ["zeta", "alpha"]); + assert_eq!(serde_json::to_string(&schema).unwrap(), wire); +} + +#[rstest] +fn schema_object_round_trips_supported_keywords() { + let schema = round_trip::(json!({ + "type":"object", + "properties":{"name":{"type":"string"}}, + "required":["name"], + "additionalProperties":false, + "$ref":"#/$defs/value", + "strict":true, + "extension":{"nested":[1,null]} + })); + assert_eq!( + schema.schema_type, + Some(JsonSchemaType::Name("object".into())) + ); + assert_eq!(schema.required.as_deref(), Some(["name".into()].as_slice())); + assert_eq!( + schema.additional_properties.as_deref(), + Some(&JsonSchema::Boolean(false)) + ); + assert_eq!(schema.reference.as_deref(), Some("#/$defs/value")); + assert_eq!(schema.strict, Some(true)); +} + +#[rstest] +#[case::scalar(json!(7))] +#[case::properties_shape(json!({"properties":[]}))] +#[case::nested_schema(json!({"properties":{"name":7}}))] +#[case::schema_type(json!({"type":["string",7]}))] +#[case::items_shape(json!({"items":7}))] +#[case::tuple_member(json!({"items":[{"type":"string"},7]}))] +#[case::prefix_items_shape(json!({"prefixItems":{"type":"string"}}))] +#[case::composite_shape(json!({"anyOf":["string"]}))] +fn schemas_reject_malformed_known_keywords(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +#[case::draft_07_tuple(json!({"type":"array","items":[{"type":"string"},true]}))] +#[case::draft_2020_12_tuple(json!({"type":"array","prefixItems":[{"type":"string"},true],"items":false}))] +fn tuple_schemas_expose_positional_members(#[case] wire: Value) { + let schema = round_trip::(wire); + let members = match (&schema.items, &schema.prefix_items) { + (Some(JsonSchemaItems::Tuple(members)), None) => members, + (Some(JsonSchemaItems::Schema(rest)), Some(members)) => { + assert_eq!(rest.as_ref(), &JsonSchema::Boolean(false)); + members + } + _ => panic!("expected positional members"), + }; + let [JsonSchema::Object(first), JsonSchema::Boolean(true)] = members.as_slice() else { + panic!("expected string schema then true"); + }; + assert_eq!( + first.schema_type, + Some(JsonSchemaType::Name("string".into())) + ); + assert!(schema.extra.is_empty()); +} + +#[rstest] +fn empty_schema_omits_null_optionals_and_preserves_arbitrary_keywords() { + let schema: JsonSchemaObject = serde_json::from_value( + json!({"type":null,"properties":null,"const":{"arbitrary":[1,null]},"future":null}), + ) + .unwrap(); + assert!(schema.schema_type.is_none()); + assert!(schema.properties.is_none()); + assert_eq!( + serde_json::to_value(schema).unwrap(), + json!({"const":{"arbitrary":[1,null]},"future":null}) + ); +} diff --git a/litellm-rust/crates/llms-types/tests/messages_request.rs b/litellm-rust/crates/llms-types/tests/messages_request.rs index a0515c21459..4825932bc02 100644 --- a/litellm-rust/crates/llms-types/tests/messages_request.rs +++ b/litellm-rust/crates/llms-types/tests/messages_request.rs @@ -33,6 +33,15 @@ fn content_block_type_schema_remains_a_string() { #[case::compaction("compaction", ContentBlockType::Compaction)] #[case::advisor_result("advisor_tool_result", ContentBlockType::AdvisorToolResult)] #[case::web_search_result("web_search_tool_result", ContentBlockType::WebSearchToolResult)] +#[case::image("image", ContentBlockType::Other("image".into()))] +#[case::document("document", ContentBlockType::Other("document".into()))] +#[case::tool_addition("tool_addition", ContentBlockType::Other("tool_addition".into()))] +#[case::tool_removal("tool_removal", ContentBlockType::Other("tool_removal".into()))] +#[case::advisor("advisor_result", ContentBlockType::Other("advisor_result".into()))] +#[case::web_search_error( + "web_search_tool_result_error", + ContentBlockType::Other("web_search_tool_result_error".into()) +)] #[case::future_block("future_block", ContentBlockType::Other("future_block".into()))] #[case::case_sensitive("Tool_Use", ContentBlockType::Other("Tool_Use".into()))] #[case::empty("", ContentBlockType::Other(String::new()))] diff --git a/litellm-rust/crates/llms-types/tests/messages_types.rs b/litellm-rust/crates/llms-types/tests/messages_types.rs new file mode 100644 index 00000000000..51785a54c98 --- /dev/null +++ b/litellm-rust/crates/llms-types/tests/messages_types.rs @@ -0,0 +1,1512 @@ +use indexmap::IndexMap; +use litellm_llms_types::formats::messages::{ + AdvisorTool, AdvisorToolName, AdvisorToolResultContent, AllowedCaller, BashCodeExecutionOutput, + BashCodeExecutionToolResultContent, BashToolName, BlockContent, BrowserStateChange, + BrowserToolsetConfigs, BuiltinMessagesTool, CacheControl, CacheMissReason, Citation, + CitationsConfig, ClientTool, CodeExecutionOutput, CodeExecutionToolName, + CodeExecutionToolResultContent, ComputerTool, ComputerTool20251124, ComputerToolName, + ComputerToolsetConfigs, ContainerReference, ContentSource, ContextManagementResponse, + ContextTrigger, CustomTool, CustomToolType, FallbackTrigger, McpListedTool, McpServer, + McpToolResultContent, McpToolResultText, McpToolset, MemoryToolName, MessageRole, MessageType, + MessagesCompaction, MessagesContainer, MessagesContentPart, MessagesDiagnostics, + MessagesDiagnosticsParam, MessagesMetadata, MessagesToolParam, MessagesUsage, OutputFormat, + ResponseInclusion, Safeguard, ServerTool, SkillType, StopDetails, StopDetailsType, StopReason, + StrReplaceBasedEditToolName, StrReplaceEditorName, TextEditorCodeExecutionToolResultContent, + TextEditorFileType, TextEditorTool20250728, ToolCaller, ToolChange, ToolChangeTarget, + ToolChoice, ToolChoiceType, ToolResultUrlSource, ToolSearchBm25ToolName, ToolSearchReference, + ToolSearchRegexToolName, ToolSearchToolResultContent, Toolset, ToolsetToolConfig, + UrlSourceToolReference, UsageIterationType, UserInputUrlSource, UserLocationType, + WebFetchDocument, WebFetchTool, WebFetchTool20260309, WebFetchTool20260318, WebFetchToolName, + WebFetchToolResultContent, WebFetchUrlSources, WebSearchTool, WebSearchTool20260318, + WebSearchToolName, WebSearchToolResultContent, WebSearchUserLocation, +}; +use litellm_llms_types::json_schema::{JsonSchema, JsonSchemaObject, JsonSchemaType}; +use litellm_llms_types::recognized::Recognized; +use rstest::rstest; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::{Map, Value, json}; + +fn round_trip(wire: Value) -> T +where + T: DeserializeOwned + Serialize, +{ + let parsed: T = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(&parsed).unwrap(), wire); + parsed +} + +#[rstest] +fn metadata_contracts_round_trip() { + let metadata = round_trip::(json!({"user_id":"user_1","future":true})); + assert_eq!(metadata.user_id.as_deref(), Some("user_1")); + assert_eq!(metadata.extra.get("future"), Some(&json!(true))); + let output_format = round_trip::(json!({ + "type":"json_schema", + "schema":{"type":"object","properties":{"name":{"type":"string"}}}, + "strict":true + })); + assert_eq!(output_format.strict, Some(true)); + assert!(output_format.extra.is_empty()); + let JsonSchema::Object(schema) = &output_format.schema else { + panic!("expected output object schema"); + }; + assert!(schema.properties.as_ref().unwrap().contains_key("name")); + let compaction = + round_trip::(json!({"type":"summarize","instructions":"briefly"})); + assert_eq!(compaction.instructions.as_deref(), Some("briefly")); + assert!(compaction.extra.is_empty()); + let container = round_trip::(json!({ + "id":"container_1", + "expires_at":"2026-01-01T00:00:00Z", + "skills":[{"type":"custom","skill_id":"skill_1","version":"1"}] + })); + assert_eq!(container.id.as_deref(), Some("container_1")); + assert!(container.extra.is_empty()); + let [skill] = container.skills.as_ref().unwrap().as_slice() else { + panic!("expected container skill"); + }; + assert_eq!(skill.skill_type, SkillType::Custom); + assert_eq!(skill.skill_id, "skill_1"); + assert_eq!(skill.version.as_deref(), Some("1")); + assert!(skill.extra.is_empty()); + let reference = round_trip::(json!({"id":"container_1"})); + let ContainerReference::Parameters(parameters) = reference else { + panic!("expected container parameters"); + }; + assert_eq!(parameters.id.as_deref(), container.id.as_deref()); + assert!(parameters.skills.is_none()); + round_trip::( + json!({"id":"container_1","skills":[{"type":"anthropic","skill_id":"pptx"}]}), + ); + let server = round_trip::(json!({ + "type":"url", + "url":"https://example.test/mcp", + "name":"search", + "authorization_token":"token", + "tool_configuration":{"allowed_tools":["search"],"enabled":true} + })); + assert_eq!(server.url, "https://example.test/mcp"); + assert_eq!(server.name, "search"); + assert_eq!(server.authorization_token.as_deref(), Some("token")); + assert!(server.extra.is_empty()); + let configuration = server.tool_configuration.as_ref().unwrap(); + assert_eq!(configuration.enabled, Some(true)); + assert_eq!( + configuration.allowed_tools.as_deref(), + Some([String::from("search")].as_slice()) + ); + assert!(configuration.extra.is_empty()); + let context = round_trip::(json!({ + "applied_edits":[ + {"type":"clear_tool_uses_20250919","cleared_input_tokens":7,"cleared_tool_uses":2}, + {"type":"clear_thinking_20251015","cleared_input_tokens":3,"cleared_thinking_turns":1} + ] + })); + let [tool_uses, thinking] = context.applied_edits.as_ref().unwrap().as_slice() else { + panic!("expected two applied edits"); + }; + assert_eq!( + tool_uses.edit_type.as_deref(), + Some("clear_tool_uses_20250919") + ); + assert_eq!( + (tool_uses.cleared_input_tokens, tool_uses.cleared_tool_uses), + (Some(7), Some(2)) + ); + assert!(tool_uses.cleared_thinking_turns.is_none()); + assert_eq!( + ( + thinking.cleared_input_tokens, + thinking.cleared_thinking_turns + ), + (Some(3), Some(1)) + ); + assert!(tool_uses.extra.is_empty() && thinking.extra.is_empty()); + let safeguard = round_trip::( + json!({"type":"classifier","classifier_context":{"source":"test"}}), + ); + assert_eq!(safeguard.safeguard_type, "classifier"); + assert_eq!( + safeguard.classifier_context.as_ref().unwrap().get("source"), + Some(&json!("test")) + ); + assert!(safeguard.extra.is_empty()); +} + +#[rstest] +fn stop_details_expose_refusal_and_keep_unknown_types() { + let refusal = round_trip::( + json!({"type":"refusal","category":"cyber","explanation":"blocked"}), + ); + assert_eq!( + refusal.detail_type, + Recognized::Known(StopDetailsType::Refusal) + ); + assert_eq!(refusal.category.as_deref(), Some("cyber")); + assert_eq!(refusal.explanation.as_deref(), Some("blocked")); + assert!(refusal.extra.is_empty()); + let safeguard = + round_trip::(json!({"type":"safeguard","safeguard_types":["classifier"]})); + assert_eq!( + safeguard.detail_type, + Recognized::Unrecognized(json!("safeguard")) + ); + assert_eq!(safeguard.extra["safeguard_types"], json!(["classifier"])); +} + +#[rstest] +fn container_rejects_skill_without_id() { + assert!( + serde_json::from_value::( + json!({"skills":[{"type":"custom","version":"1"}]}) + ) + .is_err() + ); +} + +#[rstest] +#[case::without_url(json!({"type":"url","name":"search"}))] +#[case::without_name(json!({"type":"url","url":"https://example.test/mcp"}))] +fn mcp_server_rejects_missing_required_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +fn output_format_rejects_missing_schema() { + assert!(serde_json::from_value::(json!({"type":"json_schema"})).is_err()); +} + +#[rstest] +#[case::user(json!("user"))] +#[case::assistant(json!("assistant"))] +#[case::system(json!("system"))] +#[case::future(json!("future_role"))] +fn message_roles_round_trip(#[case] wire: Value) { + round_trip::(wire); +} + +#[rstest] +#[case::message(json!("message"))] +#[case::future(json!("future_type"))] +fn message_types_round_trip(#[case] wire: Value) { + round_trip::(wire); +} + +#[rstest] +#[case::end_turn(json!("end_turn"))] +#[case::refusal(json!("refusal"))] +#[case::compaction(json!("compaction"))] +#[case::future(json!("future_reason"))] +fn stop_reasons_round_trip(#[case] wire: Value) { + round_trip::(wire); +} + +#[rstest] +fn tool_choice_round_trips() { + let choice = round_trip::( + json!({"type":"tool","name":"search","disable_parallel_tool_use":true}), + ); + assert_eq!(choice.choice_type, ToolChoiceType::Tool); + assert_eq!(choice.name.as_deref(), Some("search")); + assert_eq!(choice.disable_parallel_tool_use, Some(true)); + assert!(choice.extra.is_empty()); +} + +#[rstest] +fn usage_contracts_round_trip() { + let usage = round_trip::(json!({ + "input_tokens":10, + "output_tokens":4, + "server_tool_use":{"web_search_requests":2,"web_fetch_requests":1}, + "cache_creation":{"ephemeral_1h_input_tokens":3,"ephemeral_5m_input_tokens":1}, + "output_tokens_details":{"thinking_tokens":2}, + "iterations":[ + {"type":"compaction","input_tokens":7,"output_tokens":1}, + {"type":"message","input_tokens":3,"output_tokens":3} + ], + "service_tier":"priority", + "inference_geo":"global", + "speed":"fast" + })); + assert_eq!(usage.inference_geo.as_deref(), Some("global")); + assert_eq!(usage.input_tokens, Some(10)); + assert_eq!(usage.output_tokens, Some(4)); + assert!(usage.extra.is_empty()); + let server = usage.server_tool_use.as_ref().unwrap(); + assert_eq!(server.web_search_requests, Some(2)); + assert_eq!(server.web_fetch_requests, Some(1)); + let cache = usage.cache_creation.as_ref().unwrap(); + assert_eq!(cache.ephemeral_1h_input_tokens, Some(3)); + assert_eq!(cache.ephemeral_5m_input_tokens, Some(1)); + assert_eq!( + usage + .output_tokens_details + .as_ref() + .unwrap() + .thinking_tokens, + Some(2) + ); + let [compaction, message] = usage.iterations.as_ref().unwrap().as_slice() else { + panic!("expected usage iterations"); + }; + assert_eq!( + compaction.iteration_type, + Recognized::Known(UsageIterationType::Compaction) + ); + assert_eq!( + message.iteration_type, + Recognized::Known(UsageIterationType::Message) + ); + assert_eq!( + compaction.input_tokens.unwrap() + message.input_tokens.unwrap(), + usage.input_tokens.unwrap() + ); + assert_eq!( + compaction.output_tokens.unwrap() + message.output_tokens.unwrap(), + usage.output_tokens.unwrap() + ); +} + +#[rstest] +fn usage_iterations_expose_advisor_and_fallback_models() { + let usage = round_trip::(json!({ + "iterations":[ + {"type":"advisor_message","model":"claude-opus-5-5","input_tokens":5,"output_tokens":2, + "cache_creation_input_tokens":4,"cache_read_input_tokens":1, + "cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":4}}, + {"type":"fallback_message","model":"claude-sonnet-5-5","input_tokens":6,"output_tokens":3, + "cache_creation_input_tokens":0,"cache_read_input_tokens":2}, + {"type":"future_iteration","input_tokens":1} + ] + })); + let [advisor, fallback, future] = usage.iterations.as_deref().unwrap() else { + panic!("expected three iterations"); + }; + assert_eq!( + advisor.iteration_type, + Recognized::Known(UsageIterationType::AdvisorMessage) + ); + assert_eq!(advisor.model.as_deref(), Some("claude-opus-5-5")); + assert_eq!( + ( + advisor.cache_creation_input_tokens, + advisor.cache_read_input_tokens + ), + (Some(4), Some(1)) + ); + assert_eq!( + advisor + .cache_creation + .as_ref() + .unwrap() + .ephemeral_5m_input_tokens, + advisor.cache_creation_input_tokens + ); + assert!(advisor.extra.is_empty()); + assert_eq!( + fallback.iteration_type, + Recognized::Known(UsageIterationType::FallbackMessage) + ); + assert_eq!(fallback.model.as_deref(), Some("claude-sonnet-5-5")); + assert_eq!(fallback.cache_read_input_tokens, Some(2)); + assert!(fallback.extra.is_empty()); + assert_eq!( + future.iteration_type, + Recognized::Unrecognized(json!("future_iteration")) + ); + assert_eq!(future.input_tokens, Some(1)); +} + +#[rstest] +#[case::base64(json!({"type":"base64","media_type":"image/png","data":"AA=="}))] +#[case::url(json!({"type":"url","url":"https://example.test/image"}))] +#[case::file(json!({"type":"file","file_id":"file_1"}))] +#[case::text(json!({"type":"text","media_type":"text/plain","data":"document"}))] +#[case::content(json!({"type":"content","content":"nested"}))] +fn content_sources_round_trip(#[case] wire: Value) { + round_trip::(wire); +} + +fn client_tool(name: N) -> ClientTool { + ClientTool { + name, + allowed_callers: None, + cache_control: None, + defer_loading: None, + input_examples: None, + strict: None, + extra: Map::new(), + } +} + +fn server_tool(name: N) -> ServerTool { + ServerTool { + name, + allowed_callers: None, + cache_control: None, + defer_loading: None, + strict: None, + extra: Map::new(), + } +} + +fn computer_tool(display_width_px: u64, display_height_px: u64) -> ComputerTool { + ComputerTool { + name: ComputerToolName::Computer, + display_width_px, + display_height_px, + display_number: None, + allowed_callers: None, + cache_control: None, + defer_loading: None, + input_examples: None, + strict: None, + extra: Map::new(), + } +} + +fn web_search_tool() -> WebSearchTool { + WebSearchTool { + name: WebSearchToolName::WebSearch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + defer_loading: None, + max_uses: None, + strict: None, + user_location: None, + extra: Map::new(), + } +} + +fn web_fetch_tool() -> WebFetchTool { + WebFetchTool { + name: WebFetchToolName::WebFetch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + citations: None, + defer_loading: None, + max_content_tokens: None, + max_uses: None, + strict: None, + url_sources: None, + extra: Map::new(), + } +} + +fn web_fetch_url_sources() -> WebFetchUrlSources { + WebFetchUrlSources { + client_tool_results: None, + server_tool_results: None, + user_input: None, + extra: Map::new(), + } +} + +fn tool_reference(name: &str) -> UrlSourceToolReference { + UrlSourceToolReference::ToolReference { + name: String::from(name), + extra: Map::new(), + } +} + +fn toolset_config(enabled: Option, defer_loading: Option) -> ToolsetToolConfig { + ToolsetToolConfig { + defer_loading, + enabled, + extra: Map::new(), + } +} + +fn cache_control(cache_type: &str, ttl: &str) -> CacheControl { + CacheControl { + cache_type: Some(String::from(cache_type)), + ttl: Some(String::from(ttl)), + ..CacheControl::default() + } +} + +fn browser_toolset_configs() -> BrowserToolsetConfigs { + BrowserToolsetConfigs { + type_text: None, + close_tab: None, + double_click: None, + file_upload: None, + find: None, + form_input: None, + get_page_text: None, + hold_key: None, + hover: None, + javascript_exec: None, + key: None, + left_click: None, + left_click_drag: None, + left_mouse_down: None, + left_mouse_up: None, + list_tabs: None, + middle_click: None, + mouse_move: None, + navigate: None, + new_tab: None, + read_console: None, + read_network: None, + read_page: None, + right_click: None, + screenshot: None, + scroll: None, + scroll_to: None, + switch_tab: None, + triple_click: None, + wait: None, + zoom: None, + extra: Map::new(), + } +} + +fn computer_toolset_configs() -> ComputerToolsetConfigs { + ComputerToolsetConfigs { + type_text: None, + cursor_position: None, + double_click: None, + hold_key: None, + key: None, + left_click: None, + left_click_drag: None, + left_mouse_down: None, + left_mouse_up: None, + middle_click: None, + mouse_move: None, + right_click: None, + screenshot: None, + scroll: None, + triple_click: None, + wait: None, + zoom: None, + extra: Map::new(), + } +} + +#[rstest] +#[case::bash_20241022( + json!({"type":"bash_20241022","name":"bash","input_examples":[{"command":"ls"}]}), + BuiltinMessagesTool::Bash20241022(ClientTool { + input_examples: Some(vec![Map::from_iter([(String::from("command"), json!("ls"))])]), + ..client_tool(BashToolName::Bash) + }) +)] +#[case::bash_20250124( + json!({"type":"bash_20250124","name":"bash","allowed_callers":["direct","code_execution_20260521"]}), + BuiltinMessagesTool::Bash20250124(ClientTool { + allowed_callers: Some(vec![AllowedCaller::Direct, AllowedCaller::CodeExecution20260521]), + ..client_tool(BashToolName::Bash) + }) +)] +#[case::text_editor_20241022( + json!({"type":"text_editor_20241022","name":"str_replace_editor","defer_loading":true}), + BuiltinMessagesTool::TextEditor20241022(ClientTool { + defer_loading: Some(true), + ..client_tool(StrReplaceEditorName::StrReplaceEditor) + }) +)] +#[case::text_editor_20250124( + json!({"type":"text_editor_20250124","name":"str_replace_editor","strict":true}), + BuiltinMessagesTool::TextEditor20250124(ClientTool { + strict: Some(true), + ..client_tool(StrReplaceEditorName::StrReplaceEditor) + }) +)] +#[case::text_editor_20250429( + json!({"type":"text_editor_20250429","name":"str_replace_based_edit_tool","cache_control":{"type":"ephemeral","ttl":"1h"}}), + BuiltinMessagesTool::TextEditor20250429(ClientTool { + cache_control: Some(cache_control("ephemeral", "1h")), + ..client_tool(StrReplaceBasedEditToolName::StrReplaceBasedEditTool) + }) +)] +#[case::text_editor_20250728( + json!({"type":"text_editor_20250728","name":"str_replace_based_edit_tool","max_characters":10000}), + BuiltinMessagesTool::TextEditor20250728(TextEditorTool20250728 { + name: StrReplaceBasedEditToolName::StrReplaceBasedEditTool, + allowed_callers: None, + cache_control: None, + defer_loading: None, + input_examples: None, + max_characters: Some(10000), + strict: None, + extra: Map::new(), + }) +)] +#[case::memory_20250818( + json!({"type":"memory_20250818","name":"memory","allowed_callers":["code_execution_20250825"]}), + BuiltinMessagesTool::Memory20250818(ClientTool { + allowed_callers: Some(vec![AllowedCaller::CodeExecution20250825]), + ..client_tool(MemoryToolName::Memory) + }) +)] +#[case::computer_20241022( + json!({"type":"computer_20241022","name":"computer","display_width_px":1024,"display_height_px":768,"display_number":1}), + BuiltinMessagesTool::Computer20241022(ComputerTool { + display_number: Some(1), + ..computer_tool(1024, 768) + }) +)] +#[case::computer_20250124( + json!({"type":"computer_20250124","name":"computer","display_width_px":1280,"display_height_px":800}), + BuiltinMessagesTool::Computer20250124(computer_tool(1280, 800)) +)] +#[case::computer_20251124( + json!({"type":"computer_20251124","name":"computer","display_width_px":1920,"display_height_px":1080,"enable_zoom":true}), + BuiltinMessagesTool::Computer20251124(ComputerTool20251124 { + name: ComputerToolName::Computer, + display_width_px: 1920, + display_height_px: 1080, + display_number: None, + enable_zoom: Some(true), + allowed_callers: None, + cache_control: None, + defer_loading: None, + input_examples: None, + strict: None, + extra: Map::new(), + }) +)] +#[case::code_execution_20250522( + json!({"type":"code_execution_20250522","name":"code_execution","strict":false}), + BuiltinMessagesTool::CodeExecution20250522(ServerTool { + strict: Some(false), + ..server_tool(CodeExecutionToolName::CodeExecution) + }) +)] +#[case::code_execution_20250825( + json!({"type":"code_execution_20250825","name":"code_execution","defer_loading":false}), + BuiltinMessagesTool::CodeExecution20250825(ServerTool { + defer_loading: Some(false), + ..server_tool(CodeExecutionToolName::CodeExecution) + }) +)] +#[case::code_execution_20260120( + json!({"type":"code_execution_20260120","name":"code_execution","allowed_callers":["direct"]}), + BuiltinMessagesTool::CodeExecution20260120(ServerTool { + allowed_callers: Some(vec![AllowedCaller::Direct]), + ..server_tool(CodeExecutionToolName::CodeExecution) + }) +)] +#[case::code_execution_20260521( + json!({"type":"code_execution_20260521","name":"code_execution","allowed_callers":["code_execution_20260120"]}), + BuiltinMessagesTool::CodeExecution20260521(ServerTool { + allowed_callers: Some(vec![AllowedCaller::CodeExecution20260120]), + ..server_tool(CodeExecutionToolName::CodeExecution) + }) +)] +#[case::tool_search_regex_20251119( + json!({"type":"tool_search_tool_regex_20251119","name":"tool_search_tool_regex","defer_loading":true}), + BuiltinMessagesTool::ToolSearchRegex20251119(ServerTool { + defer_loading: Some(true), + ..server_tool(ToolSearchRegexToolName::ToolSearchToolRegex) + }) +)] +#[case::tool_search_regex( + json!({"type":"tool_search_tool_regex","name":"tool_search_tool_regex","strict":true}), + BuiltinMessagesTool::ToolSearchRegex(ServerTool { + strict: Some(true), + ..server_tool(ToolSearchRegexToolName::ToolSearchToolRegex) + }) +)] +#[case::tool_search_bm25_20251119( + json!({"type":"tool_search_tool_bm25_20251119","name":"tool_search_tool_bm25","defer_loading":false}), + BuiltinMessagesTool::ToolSearchBm2520251119(ServerTool { + defer_loading: Some(false), + ..server_tool(ToolSearchBm25ToolName::ToolSearchToolBm25) + }) +)] +#[case::tool_search_bm25( + json!({"type":"tool_search_tool_bm25","name":"tool_search_tool_bm25","strict":false}), + BuiltinMessagesTool::ToolSearchBm25(ServerTool { + strict: Some(false), + ..server_tool(ToolSearchBm25ToolName::ToolSearchToolBm25) + }) +)] +#[case::web_search_20250305( + json!({"type":"web_search_20250305","name":"web_search","max_uses":3,"allowed_domains":["example.test"], + "user_location":{"type":"approximate","city":"San Francisco","country":"US","timezone":"America/Los_Angeles"}}), + BuiltinMessagesTool::WebSearch20250305(WebSearchTool { + max_uses: Some(3), + allowed_domains: Some(vec![String::from("example.test")]), + user_location: Some(WebSearchUserLocation { + location_type: UserLocationType::Approximate, + city: Some(String::from("San Francisco")), + country: Some(String::from("US")), + region: None, + timezone: Some(String::from("America/Los_Angeles")), + extra: Map::new(), + }), + ..web_search_tool() + }) +)] +#[case::web_search_20260209( + json!({"type":"web_search_20260209","name":"web_search","blocked_domains":["blocked.test"]}), + BuiltinMessagesTool::WebSearch20260209(WebSearchTool { + blocked_domains: Some(vec![String::from("blocked.test")]), + ..web_search_tool() + }) +)] +#[case::web_search_20260318( + json!({"type":"web_search_20260318","name":"web_search","response_inclusion":"excluded"}), + BuiltinMessagesTool::WebSearch20260318(WebSearchTool20260318 { + name: WebSearchToolName::WebSearch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + defer_loading: None, + max_uses: None, + response_inclusion: Some(ResponseInclusion::Excluded), + strict: None, + user_location: None, + extra: Map::new(), + }) +)] +#[case::web_fetch_20250910( + json!({"type":"web_fetch_20250910","name":"web_fetch","max_content_tokens":5000,"citations":{"enabled":true}, + "url_sources":{"client_tool_results":{"type":"only","tools":[{"type":"tool_reference","name":"lookup"}]}, + "server_tool_results":{"type":"all"},"user_input":{"type":"none"}}}), + BuiltinMessagesTool::WebFetch20250910(WebFetchTool { + max_content_tokens: Some(5000), + citations: Some(CitationsConfig { + enabled: Some(true), + extra: Map::new(), + }), + url_sources: Some(WebFetchUrlSources { + client_tool_results: Some(ToolResultUrlSource::Only { + tools: vec![tool_reference("lookup")], + extra: Map::new(), + }), + server_tool_results: Some(ToolResultUrlSource::All { extra: Map::new() }), + user_input: Some(UserInputUrlSource::None { extra: Map::new() }), + extra: Map::new(), + }), + ..web_fetch_tool() + }) +)] +#[case::web_fetch_20260209( + json!({"type":"web_fetch_20260209","name":"web_fetch","url_sources":{"server_tool_results":{"type":"except","tools":[{"type":"tool_reference","name":"web_search"}]}}}), + BuiltinMessagesTool::WebFetch20260209(WebFetchTool { + url_sources: Some(WebFetchUrlSources { + server_tool_results: Some(ToolResultUrlSource::Except { + tools: vec![tool_reference("web_search")], + extra: Map::new(), + }), + ..web_fetch_url_sources() + }), + ..web_fetch_tool() + }) +)] +#[case::web_fetch_20260309( + json!({"type":"web_fetch_20260309","name":"web_fetch","use_cache":false}), + BuiltinMessagesTool::WebFetch20260309(WebFetchTool20260309 { + name: WebFetchToolName::WebFetch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + citations: None, + defer_loading: None, + max_content_tokens: None, + max_uses: None, + strict: None, + url_sources: None, + use_cache: Some(false), + extra: Map::new(), + }) +)] +#[case::web_fetch_20260318( + json!({"type":"web_fetch_20260318","name":"web_fetch","use_cache":true,"response_inclusion":"full"}), + BuiltinMessagesTool::WebFetch20260318(WebFetchTool20260318 { + name: WebFetchToolName::WebFetch, + allowed_callers: None, + allowed_domains: None, + blocked_domains: None, + cache_control: None, + citations: None, + defer_loading: None, + max_content_tokens: None, + max_uses: None, + response_inclusion: Some(ResponseInclusion::Full), + strict: None, + url_sources: None, + use_cache: Some(true), + extra: Map::new(), + }) +)] +#[case::advisor_20260301( + json!({"type":"advisor_20260301","name":"advisor","model":"claude-opus-5-5","max_tokens":2048,"caching":{"type":"ephemeral","ttl":"5m"}}), + BuiltinMessagesTool::Advisor20260301(AdvisorTool { + name: AdvisorToolName::Advisor, + model: String::from("claude-opus-5-5"), + allowed_callers: None, + cache_control: None, + caching: Some(cache_control("ephemeral", "5m")), + defer_loading: None, + max_tokens: Some(2048), + max_uses: None, + strict: None, + extra: Map::new(), + }) +)] +#[case::browser_toolset_20260801( + json!({"type":"browser_toolset_20260801","configs":{"type":{"enabled":false},"javascript_exec":{"defer_loading":true}}}), + BuiltinMessagesTool::BrowserToolset20260801(Toolset { + cache_control: None, + configs: Some(Box::new(BrowserToolsetConfigs { + type_text: Some(toolset_config(Some(false), None)), + javascript_exec: Some(toolset_config(None, Some(true))), + ..browser_toolset_configs() + })), + extra: Map::new(), + }) +)] +#[case::computer_toolset_20260801( + json!({"type":"computer_toolset_20260801","configs":{"zoom":{"enabled":false},"cursor_position":{"enabled":true}}}), + BuiltinMessagesTool::ComputerToolset20260801(Toolset { + cache_control: None, + configs: Some(Box::new(ComputerToolsetConfigs { + zoom: Some(toolset_config(Some(false), None)), + cursor_position: Some(toolset_config(Some(true), None)), + ..computer_toolset_configs() + })), + extra: Map::new(), + }) +)] +#[case::mcp_toolset( + json!({"type":"mcp_toolset","mcp_server_name":"kb","default_config":{"enabled":false}, + "configs":{"search":{"enabled":true,"defer_loading":true}}, + "tools":[{"name":"search","input_schema":{"type":"object"},"description":"Search"}]}), + BuiltinMessagesTool::McpToolset(McpToolset { + mcp_server_name: String::from("kb"), + cache_control: None, + configs: Some(IndexMap::from([(String::from("search"), toolset_config(Some(true), Some(true)))])), + default_config: Some(toolset_config(Some(false), None)), + tools: Some(vec![McpListedTool { + name: String::from("search"), + description: Some(String::from("Search")), + input_schema: JsonSchema::Object(Box::new(JsonSchemaObject { + schema_type: Some(JsonSchemaType::Name(String::from("object"))), + ..JsonSchemaObject::default() + })), + extra: Map::new(), + }]), + extra: Map::new(), + }) +)] +fn builtin_tools_decode_typed_definitions( + #[case] wire: Value, + #[case] expected: BuiltinMessagesTool, +) { + assert_eq!(round_trip::(wire), expected); +} + +#[rstest] +fn builtin_tools_preserve_unknown_fields() { + let tool = round_trip::( + json!({"type":"bash_20250124","name":"bash","extension":[1,null]}), + ); + let BuiltinMessagesTool::Bash20250124(bash) = tool else { + panic!("expected bash tool"); + }; + assert_eq!(bash.extra.get("extension"), Some(&json!([1, null]))); +} + +#[rstest] +#[case::input_tokens("input_tokens", false)] +#[case::tool_uses("tool_uses", true)] +fn context_trigger_exposes_typed_threshold(#[case] tag: &str, #[case] tool_uses: bool) { + let wire = json!({"type":tag,"value":1024,"extension":true}); + let trigger: ContextTrigger = serde_json::from_value(wire.clone()).unwrap(); + match &trigger { + ContextTrigger::InputTokens { value, extra } => { + assert!(!tool_uses); + assert_eq!(*value, 1024); + assert_eq!(extra.get("extension"), Some(&json!(true))); + } + ContextTrigger::ToolUses { value, extra } => { + assert!(tool_uses); + assert_eq!(*value, 1024); + assert_eq!(extra.get("extension"), Some(&json!(true))); + } + } + assert_eq!(serde_json::to_value(trigger).unwrap(), wire); +} + +#[rstest] +#[case::negative(json!({"type":"input_tokens","value":-1}))] +#[case::wrong_shape(json!({"type":"input_tokens","value":"1024"}))] +#[case::missing_value(json!({"type":"input_tokens"}))] +#[case::null_value(json!({"type":"input_tokens","value":null}))] +#[case::tool_uses_negative(json!({"type":"tool_uses","value":-1}))] +#[case::tool_uses_fractional(json!({"type":"tool_uses","value":1.5}))] +#[case::tool_uses_missing_value(json!({"type":"tool_uses"}))] +#[case::missing_discriminator(json!({"value":1}))] +#[case::unknown_discriminator(json!({"type":"other","value":1}))] +fn token_threshold_requires_unsigned_integer(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +#[case::missing_discriminator(json!({"name":"bash"}))] +#[case::unknown_discriminator(json!({"type":"future_tool","name":"bash"}))] +#[case::null_discriminator(json!({"type":null,"name":"bash"}))] +#[case::bash_without_name(json!({"type":"bash_20250124"}))] +#[case::bash_wrong_name(json!({"type":"bash_20241022","name":"shell"}))] +#[case::legacy_editor_with_new_name(json!({"type":"text_editor_20250124","name":"str_replace_based_edit_tool"}))] +#[case::new_editor_with_legacy_name(json!({"type":"text_editor_20250728","name":"str_replace_editor"}))] +#[case::editor_20250429_with_legacy_name(json!({"type":"text_editor_20250429","name":"str_replace_editor"}))] +#[case::memory_without_name(json!({"type":"memory_20250818"}))] +#[case::computer_without_width(json!({"type":"computer_20250124","name":"computer","display_height_px":768}))] +#[case::computer_without_height(json!({"type":"computer_20241022","name":"computer","display_width_px":1024}))] +#[case::zoom_computer_without_display(json!({"type":"computer_20251124","name":"computer"}))] +#[case::code_execution_without_name(json!({"type":"code_execution_20250825"}))] +#[case::regex_search_with_bm25_name(json!({"type":"tool_search_tool_regex","name":"tool_search_tool_bm25"}))] +#[case::bm25_search_with_regex_name(json!({"type":"tool_search_tool_bm25_20251119","name":"tool_search_tool_regex"}))] +#[case::web_search_with_fetch_name(json!({"type":"web_search_20250305","name":"web_fetch"}))] +#[case::web_fetch_without_name(json!({"type":"web_fetch_20260318"}))] +#[case::advisor_without_model(json!({"type":"advisor_20260301","name":"advisor"}))] +#[case::advisor_without_name(json!({"type":"advisor_20260301","model":"claude-opus-5-5"}))] +#[case::mcp_toolset_without_server(json!({"type":"mcp_toolset"}))] +#[case::unknown_allowed_caller(json!({"type":"code_execution_20250825","name":"code_execution","allowed_callers":["code_execution_20250522"]}))] +#[case::unknown_response_inclusion(json!({"type":"web_search_20260318","name":"web_search","response_inclusion":"partial"}))] +#[case::user_input_only_filter(json!({"type":"web_fetch_20250910","name":"web_fetch","url_sources":{"user_input":{"type":"only","tools":[]}}}))] +#[case::negative_limit(json!({"type":"web_search_20250305","name":"web_search","max_uses":-1}))] +#[case::wrong_mcp_config(json!({"type":"mcp_toolset","mcp_server_name":"kb","configs":{"search":{"enabled":"yes"}}}))] +#[case::wrong_browser_config(json!({"type":"browser_toolset_20260801","configs":{"navigate":{"enabled":1}}}))] +fn builtin_tools_reject_malformed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire.clone()).is_err()); + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +fn existing_content_blocks_preserve_opaque_nested_fields() { + let wire = + json!({"type":"tool_result","content":[{"type":"text","text":7}],"source":{"url":7}}); + let block: litellm_llms_types::formats::messages::ContentBlock = + serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(block.content.as_ref(), wire.get("content")); + assert_eq!(block.extra.get("source"), wire.get("source")); + assert_eq!(serde_json::to_value(block).unwrap(), wire); +} + +fn part(wire: Value) -> MessagesContentPart { + round_trip::(wire) +} + +#[rstest] +fn text_blocks_expose_every_citation_location() { + let block = part(json!({ + "type":"text", + "text":"cited", + "citations":[ + {"type":"char_location","cited_text":"a","document_index":0,"start_char_index":1,"end_char_index":6,"file_id":"file_1"}, + {"type":"page_location","cited_text":"b","document_index":1,"start_page_number":1,"end_page_number":2}, + {"type":"content_block_location","cited_text":"c","document_index":2,"start_block_index":0,"end_block_index":1}, + {"type":"web_search_result_location","cited_text":"d","url":"https://example.test","encrypted_index":"opaque","title":"result"}, + {"type":"search_result_location","cited_text":"e","search_result_index":3,"source":"kb","start_block_index":0,"end_block_index":2} + ], + "cache_control":{"type":"ephemeral"} + })); + let MessagesContentPart::Text(text) = &block else { + panic!("expected text block"); + }; + assert_eq!(text.text, "cited"); + let [ + Citation::CharLocation(chars), + Citation::PageLocation(page), + Citation::ContentBlockLocation(blocks), + Citation::WebSearchResultLocation(search), + Citation::SearchResultLocation(result), + ] = text.citations.as_deref().unwrap() + else { + panic!("expected one citation of each location type"); + }; + assert_eq!((chars.start_char_index, chars.end_char_index), (1, 6)); + assert_eq!(chars.file_id.as_deref(), Some("file_1")); + assert_eq!((page.start_page_number, page.end_page_number), (1, 2)); + assert_eq!(blocks.document_index, 2); + assert_eq!(search.encrypted_index, "opaque"); + assert_eq!(result.search_result_index, 3); + assert_eq!(result.source, "kb"); +} + +#[rstest] +fn text_block_null_optionals_are_omitted() { + let block: MessagesContentPart = serde_json::from_value( + json!({"type":"text","text":"hi","citations":null,"cache_control":null,"future":null}), + ) + .unwrap(); + let MessagesContentPart::Text(text) = &block else { + panic!("expected text block"); + }; + assert!(text.citations.is_none()); + assert!(text.cache_control.is_none()); + assert_eq!(text.extra.get("future"), Some(&Value::Null)); + assert_eq!( + serde_json::to_value(block).unwrap(), + json!({"type":"text","text":"hi","future":null}) + ); +} + +#[rstest] +fn request_media_blocks_expose_sources() { + let MessagesContentPart::Image(image) = part(json!({ + "type":"image", + "source":{"type":"base64","media_type":"image/png","data":"AA=="}, + "transformations":{"oversized_image":"error"} + })) else { + panic!("expected image block"); + }; + assert!( + matches!(&image.source, ContentSource::Base64 { media_type, .. } if media_type == "image/png") + ); + assert_eq!(image.extra["transformations"]["oversized_image"], "error"); + let MessagesContentPart::Document(document) = part(json!({ + "type":"document", + "source":{"type":"content","content":[ + {"type":"text","text":"Section 1"}, + {"type":"image","source":{"type":"url","url":"https://example.test/chart.png"}} + ]}, + "title":"Q3 report", + "context":"quarterly", + "citations":{"enabled":true} + })) else { + panic!("expected document block"); + }; + assert_eq!(document.title.as_deref(), Some("Q3 report")); + assert_eq!(document.citations.as_ref().unwrap().enabled, Some(true)); + let ContentSource::Content { + content: BlockContent::Blocks(blocks), + .. + } = &document.source + else { + panic!("expected content-block source"); + }; + assert!(matches!( + blocks.as_slice(), + [MessagesContentPart::Text(_), MessagesContentPart::Image(_)] + )); + let MessagesContentPart::SearchResult(search) = part(json!({ + "type":"search_result", + "source":"https://example.test/result", + "title":"result", + "content":[{"type":"text","text":"found"}], + "citations":{"enabled":false} + })) else { + panic!("expected search result block"); + }; + assert_eq!(search.source, "https://example.test/result"); + assert!( + matches!(search.content.as_slice(), [MessagesContentPart::Text(text)] if text.text == "found") + ); +} + +#[rstest] +fn reasoning_and_tool_blocks_expose_required_fields() { + let MessagesContentPart::Thinking(thinking) = + part(json!({"type":"thinking","thinking":"plan","signature":"sig"})) + else { + panic!("expected thinking block"); + }; + assert_eq!( + (thinking.thinking.as_str(), thinking.signature.as_str()), + ("plan", "sig") + ); + let MessagesContentPart::RedactedThinking(redacted) = + part(json!({"type":"redacted_thinking","data":"opaque"})) + else { + panic!("expected redacted thinking block"); + }; + assert_eq!(redacted.data, "opaque"); + let MessagesContentPart::ToolUse(tool_use) = part(json!({ + "type":"tool_use", + "id":"toolu_1", + "name":"lookup", + "input":{"query":[1,null]}, + "caller":{"type":"code_execution_20260120","tool_id":"srvtoolu_1"}, + "toolset_name":"browser" + })) else { + panic!("expected tool use block"); + }; + assert_eq!(tool_use.input["query"], json!([1, null])); + assert!( + matches!(&tool_use.caller, Some(ToolCaller::CodeExecution20260120 { tool_id, .. }) if tool_id == "srvtoolu_1") + ); + assert_eq!(tool_use.toolset_name.as_deref(), Some("browser")); + let MessagesContentPart::ToolResult(result) = part(json!({ + "type":"tool_result", + "tool_use_id":"toolu_1", + "is_error":false, + "content":[ + {"type":"tool_reference","tool_name":"lookup"}, + {"type":"browser_state","tabs":[{"tab_id":"1","title":"","url":"","active":true}], + "state_changes":[{"type":"download_completed","download_id":"d1","url":"https://example.test/f","size_bytes":3}]} + ] + })) else { + panic!("expected tool result block"); + }; + let Some(BlockContent::Blocks(blocks)) = &result.content else { + panic!("expected nested result blocks"); + }; + let [ + MessagesContentPart::ToolReference(reference), + MessagesContentPart::BrowserState(browser), + ] = blocks.as_slice() + else { + panic!("expected tool reference and browser state"); + }; + assert_eq!(reference.tool_name, "lookup"); + assert_eq!(browser.tabs[0].active, Some(true)); + assert!(matches!( + browser.state_changes.as_deref(), + Some([BrowserStateChange::DownloadCompleted { + size_bytes: Some(3), + path: None, + .. + }]) + )); + let MessagesContentPart::ToolResult(text_result) = + part(json!({"type":"tool_result","tool_use_id":"toolu_2","content":"done"})) + else { + panic!("expected tool result block"); + }; + assert_eq!(text_result.content, Some(BlockContent::Text("done".into()))); + assert!(text_result.is_error.is_none()); +} + +#[rstest] +fn server_tool_results_expose_nested_result_unions() { + let MessagesContentPart::ServerToolUse(server) = part( + json!({"type":"server_tool_use","id":"srvtoolu_1","name":"web_search","input":{"query":"rust"}}), + ) else { + panic!("expected server tool use"); + }; + assert_eq!(server.name, "web_search"); + let MessagesContentPart::WebSearchToolResult(search) = part(json!({ + "type":"web_search_tool_result", + "tool_use_id":"srvtoolu_1", + "content":[{"type":"web_search_result","url":"https://example.test","title":"t","encrypted_content":"e","page_age":"1d"}] + })) else { + panic!("expected web search result"); + }; + let WebSearchToolResultContent::Results(results) = &search.content else { + panic!("expected search results"); + }; + assert_eq!(results[0].page_age.as_deref(), Some("1d")); + let MessagesContentPart::WebSearchToolResult(search_error) = part(json!({ + "type":"web_search_tool_result", + "tool_use_id":"srvtoolu_1", + "content":{"type":"web_search_tool_result_error","error_code":"max_uses_exceeded"} + })) else { + panic!("expected web search error"); + }; + assert!( + matches!(&search_error.content, WebSearchToolResultContent::Error(error) if error.error_code == "max_uses_exceeded") + ); + let MessagesContentPart::WebFetchToolResult(fetch) = part(json!({ + "type":"web_fetch_tool_result", + "tool_use_id":"srvtoolu_2", + "content":{"type":"web_fetch_result","url":"https://example.test","retrieved_at":"2026-01-01T00:00:00Z", + "content":{"type":"document","source":{"type":"text","media_type":"text/plain","data":"page"}}} + })) else { + panic!("expected web fetch result"); + }; + let WebFetchToolResultContent::WebFetchResult(fetched) = &fetch.content else { + panic!("expected fetched page"); + }; + let WebFetchDocument::Document(document) = &fetched.content; + assert!(matches!(&document.source, ContentSource::Text { data, .. } if data == "page")); + assert!(fetched.extra.is_empty()); + let MessagesContentPart::CodeExecutionToolResult(code) = part(json!({ + "type":"code_execution_tool_result", + "tool_use_id":"srvtoolu_3", + "content":{"type":"encrypted_code_execution_result","encrypted_stdout":"opaque","stderr":"","return_code":-1, + "content":[{"type":"code_execution_output","file_id":"file_1"}]} + })) else { + panic!("expected code execution result"); + }; + let CodeExecutionToolResultContent::EncryptedCodeExecutionResult(encrypted) = &code.content + else { + panic!("expected encrypted execution result"); + }; + assert_eq!(encrypted.return_code, -1); + assert!( + matches!(encrypted.content.as_slice(), [CodeExecutionOutput::CodeExecutionOutput { file_id, .. }] if file_id == "file_1") + ); + let MessagesContentPart::BashCodeExecutionToolResult(bash) = part(json!({ + "type":"bash_code_execution_tool_result", + "tool_use_id":"srvtoolu_7", + "content":{"type":"bash_code_execution_result","stdout":"ok","stderr":"","return_code":0, + "content":[{"type":"bash_code_execution_output","file_id":"file_2"}]} + })) else { + panic!("expected bash code execution result"); + }; + let BashCodeExecutionToolResultContent::BashCodeExecutionResult(ran) = &bash.content else { + panic!("expected bash execution result"); + }; + assert_eq!((ran.stdout.as_str(), ran.return_code), ("ok", 0)); + assert!( + matches!(ran.content.as_slice(), [BashCodeExecutionOutput::BashCodeExecutionOutput { file_id, .. }] if file_id == "file_2") + ); + assert!(ran.extra.is_empty()); + let MessagesContentPart::TextEditorCodeExecutionToolResult(editor) = part(json!({ + "type":"text_editor_code_execution_tool_result", + "tool_use_id":"srvtoolu_4", + "content":{"type":"text_editor_code_execution_view_result","content":"fn main() {}","file_type":"text","num_lines":1} + })) else { + panic!("expected text editor result"); + }; + let TextEditorCodeExecutionToolResultContent::TextEditorCodeExecutionViewResult(view) = + &editor.content + else { + panic!("expected view result"); + }; + assert_eq!(view.file_type, TextEditorFileType::Text); + assert_eq!(view.num_lines, Some(1)); + assert!(view.start_line.is_none()); + let MessagesContentPart::ToolSearchToolResult(tool_search) = part(json!({ + "type":"tool_search_tool_result", + "tool_use_id":"srvtoolu_5", + "content":{"type":"tool_search_tool_result_error","error_code":"unavailable","error_message":"down"} + })) else { + panic!("expected tool search result"); + }; + assert!( + matches!(&tool_search.content, ToolSearchToolResultContent::ToolSearchToolResultError(error) if error.error_message.as_deref() == Some("down")) + ); + let MessagesContentPart::ToolSearchToolResult(found) = part(json!({ + "type":"tool_search_tool_result", + "tool_use_id":"srvtoolu_8", + "content":{"type":"tool_search_tool_search_result","tool_references":[{"type":"tool_reference","tool_name":"lookup"}]} + })) else { + panic!("expected tool search result"); + }; + let ToolSearchToolResultContent::ToolSearchToolSearchResult(result) = &found.content else { + panic!("expected tool search hits"); + }; + let [ToolSearchReference::ToolReference(reference)] = result.tool_references.as_slice() else { + panic!("expected one tool reference"); + }; + assert_eq!(reference.tool_name, "lookup"); + assert!(reference.extra.is_empty() && result.extra.is_empty()); + let MessagesContentPart::AdvisorToolResult(advisor) = part(json!({ + "type":"advisor_tool_result", + "tool_use_id":"srvtoolu_6", + "content":{"type":"advisor_result","text":"advice"} + })) else { + panic!("expected advisor result"); + }; + assert!( + matches!(&advisor.content, AdvisorToolResultContent::AdvisorResult { text, stop_reason: None, .. } if text == "advice") + ); +} + +#[rstest] +fn beta_blocks_expose_mcp_compaction_and_fallback_fields() { + let MessagesContentPart::McpToolUse(mcp) = part( + json!({"type":"mcp_tool_use","id":"mcptoolu_1","name":"search","server_name":"kb","input":{}}), + ) else { + panic!("expected MCP tool use"); + }; + assert_eq!(mcp.server_name, "kb"); + let MessagesContentPart::McpToolResult(mcp_result) = part( + json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_1","is_error":true,"content":"failed"}), + ) else { + panic!("expected MCP tool result"); + }; + assert_eq!(mcp_result.is_error, Some(true)); + assert_eq!( + mcp_result.content, + Some(McpToolResultContent::Text("failed".into())) + ); + let MessagesContentPart::McpToolResult(text_result) = part(json!({ + "type":"mcp_tool_result", + "tool_use_id":"mcptoolu_3", + "content":[{"type":"text","text":"found"}] + })) else { + panic!("expected MCP tool result"); + }; + let Some(McpToolResultContent::Blocks(blocks)) = &text_result.content else { + panic!("expected MCP text blocks"); + }; + let [McpToolResultText::Text(text)] = blocks.as_slice() else { + panic!("expected one text block"); + }; + assert_eq!(text.text, "found"); + let MessagesContentPart::McpToolResult(empty_result) = + part(json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_2"})) + else { + panic!("expected MCP tool result"); + }; + assert!(empty_result.content.is_none()); + assert!(empty_result.extra.is_empty()); + let MessagesContentPart::McpToolListing(listing) = part(json!({ + "type":"mcp_tool_listing", + "mcp_server_name":"kb", + "tools":[{"name":"search","input_schema":{"type":"object"}}] + })) else { + panic!("expected MCP tool listing"); + }; + assert!(listing.tools[0].description.is_none()); + let MessagesContentPart::Compaction(compaction) = part(json!({ + "type":"compaction", + "content":"summary", + "encrypted_content":"opaque", + "tool_changes":[ + {"type":"tool_addition","tool":{"type":"tool_definition","definition":{"name":"lookup","input_schema":{"type":"object"}}}}, + {"type":"tool_removal","tool":{"type":"mcp_tool_reference","server_name":"kb","name":"search"}} + ] + })) else { + panic!("expected compaction block"); + }; + let [ + ToolChange::ToolAddition(addition), + ToolChange::ToolRemoval(removal), + ] = compaction.tool_changes.as_deref().unwrap() + else { + panic!("expected tool addition and removal"); + }; + let ToolChangeTarget::ToolDefinition { definition, .. } = &addition.tool else { + panic!("expected inline tool definition"); + }; + assert!( + matches!(definition.as_ref(), MessagesToolParam::Custom(tool) if tool.name == "lookup") + ); + assert!( + matches!(&removal.tool, ToolChangeTarget::McpToolReference { server_name, .. } if server_name == "kb") + ); + let failed: MessagesContentPart = + serde_json::from_value(json!({"type":"compaction","content":null})).unwrap(); + assert_eq!(failed, MessagesContentPart::Compaction(Default::default())); + let MessagesContentPart::Fallback(fallback) = part(json!({ + "type":"fallback", + "from":{"model":"claude-opus-5-5"}, + "to":{"model":"claude-sonnet-5-5"}, + "trigger":{"type":"refusal","category":"cyber"} + })) else { + panic!("expected fallback block"); + }; + assert_eq!(fallback.from.model, "claude-opus-5-5"); + assert_eq!(fallback.to.model, "claude-sonnet-5-5"); + let Some(FallbackTrigger::Refusal { category, extra }) = &fallback.trigger else { + panic!("expected refusal trigger"); + }; + assert_eq!(category.as_deref(), Some("cyber")); + assert!(extra.is_empty() && fallback.extra.is_empty()); + let MessagesContentPart::Fallback(untriggered) = part(json!({ + "type":"fallback", + "from":{"model":"claude-opus-5-5"}, + "to":{"model":"claude-sonnet-5-5"} + })) else { + panic!("expected fallback block"); + }; + assert!(untriggered.trigger.is_none()); +} + +#[rstest] +#[case::missing_tag(json!({"text":"hello"}))] +#[case::unknown_tag(json!({"type":"future_block","text":"hello"}))] +#[case::text_without_text(json!({"type":"text"}))] +#[case::wrong_text(json!({"type":"text","text":7}))] +#[case::citation_missing_location(json!({"type":"text","text":"a","citations":[{"type":"char_location","cited_text":"a","document_index":0}]}))] +#[case::unknown_citation(json!({"type":"text","text":"a","citations":[{"type":"future_location"}]}))] +#[case::thinking_without_signature(json!({"type":"thinking","thinking":"plan"}))] +#[case::tool_use_without_id(json!({"type":"tool_use","name":"lookup","input":{}}))] +#[case::tool_use_array_input(json!({"type":"tool_use","id":"t","name":"lookup","input":[]}))] +#[case::tool_result_without_id(json!({"type":"tool_result","content":"done"}))] +#[case::malformed_nested_block(json!({"type":"tool_result","tool_use_id":"t","content":[{"type":"text","text":7}]}))] +#[case::malformed_source(json!({"type":"document","source":{"type":"url","url":7}}))] +#[case::image_without_source(json!({"type":"image"}))] +#[case::search_result_without_title(json!({"type":"search_result","source":"s","content":[]}))] +#[case::web_search_result_without_url(json!({"type":"web_search_tool_result","tool_use_id":"t","content":[{"type":"web_search_result","title":"t","encrypted_content":"e"}]}))] +#[case::unknown_fetch_result(json!({"type":"web_fetch_tool_result","tool_use_id":"t","content":{"type":"future"}}))] +#[case::fractional_return_code(json!({"type":"bash_code_execution_tool_result","tool_use_id":"t","content":{"type":"bash_code_execution_result","stdout":"","stderr":"","return_code":0.5,"content":[]}}))] +#[case::unknown_file_type(json!({"type":"text_editor_code_execution_tool_result","tool_use_id":"t","content":{"type":"text_editor_code_execution_view_result","content":"","file_type":"video"}}))] +#[case::negative_line_count(json!({"type":"text_editor_code_execution_tool_result","tool_use_id":"t","content":{"type":"text_editor_code_execution_str_replace_result","new_lines":-1}}))] +#[case::unknown_tool_change(json!({"type":"compaction","tool_changes":[{"type":"tool_addition","tool":{"type":"future"}}]}))] +#[case::tool_search_non_reference(json!({"type":"tool_search_tool_result","tool_use_id":"t","content":{"type":"tool_search_tool_search_result","tool_references":[{"type":"text","text":"lookup"}]}}))] +#[case::web_fetch_text_content(json!({"type":"web_fetch_tool_result","tool_use_id":"t","content":{"type":"web_fetch_result","url":"https://example.test","content":{"type":"text","text":"page"}}}))] +#[case::bash_result_with_code_output(json!({"type":"bash_code_execution_tool_result","tool_use_id":"t","content":{"type":"bash_code_execution_result","stdout":"","stderr":"","return_code":0,"content":[{"type":"code_execution_output","file_id":"f"}]}}))] +#[case::code_result_with_bash_output(json!({"type":"code_execution_tool_result","tool_use_id":"t","content":{"type":"code_execution_result","stdout":"","stderr":"","return_code":0,"content":[{"type":"bash_code_execution_output","file_id":"f"}]}}))] +#[case::fallback_unknown_trigger(json!({"type":"fallback","from":{"model":"a"},"to":{"model":"b"},"trigger":{"type":"overload"}}))] +#[case::unknown_state_change(json!({"type":"browser_state","tabs":[],"state_changes":[{"type":"tab_closed","tab_id":"1"}]}))] +fn content_parts_reject_malformed_known_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +fn custom_tool_exposes_schema_and_preserves_extensions() { + let tool = round_trip::(json!({ + "type":"custom", + "name":"lookup", + "input_schema":{"type":"object","properties":{"query":{"type":"string"}}}, + "strict":false, + "defer_loading":true, + "allowed_callers":["direct","code_execution_20260120"], + "extension":{"nested":[1,null]} + })); + assert_eq!( + tool.allowed_callers.as_deref(), + Some([AllowedCaller::Direct, AllowedCaller::CodeExecution20260120].as_slice()) + ); + assert_eq!(tool.tool_type, Some(CustomToolType::Custom)); + assert_eq!(tool.name, "lookup"); + assert_eq!((tool.strict, tool.defer_loading), (Some(false), Some(true))); + let JsonSchema::Object(schema) = &tool.input_schema else { + panic!("expected an object schema"); + }; + assert!(schema.properties.as_ref().unwrap().contains_key("query")); + assert_eq!(tool.extra["extension"], json!({"nested":[1,null]})); +} + +#[rstest] +fn custom_tool_omits_null_discriminator() { + let tool: CustomTool = + serde_json::from_value(json!({"type":null,"name":"lookup","input_schema":true})).unwrap(); + assert!(tool.tool_type.is_none()); + assert!(tool.description.is_none()); + assert_eq!( + serde_json::to_value(tool).unwrap(), + json!({"name":"lookup","input_schema":true}) + ); +} + +#[rstest] +#[case::missing_name(json!({"input_schema":{}}))] +#[case::missing_schema(json!({"name":"lookup"}))] +#[case::wrong_name_shape(json!({"name":7,"input_schema":{}}))] +#[case::builtin_tool(json!({"name":"lookup","input_schema":{},"type":"bash_20250124"}))] +#[case::unknown_discriminator(json!({"name":"lookup","input_schema":{},"type":"future_tool"}))] +#[case::unknown_allowed_caller(json!({"name":"lookup","input_schema":{},"allowed_callers":["anyone"]}))] +fn custom_tool_rejects_invalid_shapes(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +#[case::builtin(json!({"type":"web_search_20250305","name":"web_search","max_uses":2}), true)] +#[case::builtin_toolset(json!({"type":"mcp_toolset","mcp_server_name":"kb"}), true)] +#[case::custom(json!({"name":"lookup","input_schema":{"type":"object"}}), false)] +fn tool_params_dispatch_on_discriminator(#[case] wire: Value, #[case] builtin: bool) { + let tool = round_trip::(wire); + assert_eq!(matches!(tool, MessagesToolParam::Builtin(_)), builtin); +} + +#[rstest] +fn tool_params_reject_unknown_tool_types() { + assert!( + serde_json::from_value::( + json!({"type":"future_tool","name":"lookup","input_schema":{}}) + ) + .is_err() + ); +} + +#[rstest] +#[case::image(json!([{"type":"image","source":{"type":"url","url":"https://example.test/a.png"}}]))] +#[case::tool_reference(json!([{"type":"tool_reference","tool_name":"lookup"}]))] +#[case::untagged(json!([{"text":"found"}]))] +fn mcp_tool_result_content_rejects_non_text_blocks(#[case] content: Value) { + assert!( + serde_json::from_value::( + json!({"type":"mcp_tool_result","tool_use_id":"mcptoolu_1","content":content}) + ) + .is_err() + ); +} + +#[rstest] +#[case::model(json!({"type":"model_changed","cache_missed_input_tokens":12}), Some(12))] +#[case::system(json!({"type":"system_changed","cache_missed_input_tokens":3}), Some(3))] +#[case::tools(json!({"type":"tools_changed","cache_missed_input_tokens":0}), Some(0))] +#[case::messages(json!({"type":"messages_changed","cache_missed_input_tokens":9}), Some(9))] +#[case::not_found(json!({"type":"previous_message_not_found"}), None)] +#[case::unavailable(json!({"type":"unavailable"}), None)] +fn diagnostics_expose_cache_miss_reason(#[case] reason: Value, #[case] missed: Option) { + let diagnostics = round_trip::(json!({"cache_miss_reason":reason})); + let Some(Recognized::Known(reason)) = &diagnostics.cache_miss_reason else { + panic!("expected known cache miss reason"); + }; + let actual = match reason { + CacheMissReason::ModelChanged(tokens) + | CacheMissReason::SystemChanged(tokens) + | CacheMissReason::ToolsChanged(tokens) + | CacheMissReason::MessagesChanged(tokens) => { + assert!(tokens.extra.is_empty()); + Some(tokens.cache_missed_input_tokens) + } + CacheMissReason::PreviousMessageNotFound { extra } + | CacheMissReason::Unavailable { extra } => { + assert!(extra.is_empty()); + None + } + }; + assert_eq!(actual, missed); + assert!(diagnostics.extra.is_empty()); +} + +#[rstest] +#[case::model(json!({"type":"model_changed","cache_missed_input_tokens":12}), "model_changed")] +#[case::system(json!({"type":"system_changed","cache_missed_input_tokens":12}), "system_changed")] +#[case::tools(json!({"type":"tools_changed","cache_missed_input_tokens":12}), "tools_changed")] +#[case::messages(json!({"type":"messages_changed","cache_missed_input_tokens":12}), "messages_changed")] +fn cache_miss_reason_variants_keep_their_tags(#[case] reason: Value, #[case] tag: &str) { + let parsed: CacheMissReason = serde_json::from_value(reason).unwrap(); + assert_eq!(serde_json::to_value(parsed).unwrap()["type"], json!(tag)); +} + +#[rstest] +#[case::pending(json!({"cache_miss_reason":null}))] +#[case::future_reason(json!({"cache_miss_reason":{"type":"future_changed","cache_missed_input_tokens":1}}))] +#[case::missing_tokens(json!({"cache_miss_reason":{"type":"model_changed"}}))] +fn diagnostics_keep_pending_and_unmodeled_reasons(#[case] wire: Value) { + let diagnostics: MessagesDiagnostics = serde_json::from_value(wire.clone()).unwrap(); + match &diagnostics.cache_miss_reason { + None => assert_eq!(serde_json::to_value(&diagnostics).unwrap(), json!({})), + Some(Recognized::Unrecognized(kept)) => { + assert_eq!(kept, &wire["cache_miss_reason"]); + assert_eq!(serde_json::to_value(&diagnostics).unwrap(), wire); + } + Some(Recognized::Known(reason)) => panic!("unexpected known reason {reason:?}"), + } +} + +#[rstest] +#[case::previous(json!({"previous_message_id":"msg_1"}), Some(Some("msg_1")))] +#[case::first_turn(json!({"previous_message_id":null}), Some(None))] +#[case::absent(json!({}), None)] +fn diagnostics_param_distinguishes_null_from_absent_previous_message( + #[case] wire: Value, + #[case] expected: Option>, +) { + let param = round_trip::(wire); + assert_eq!( + param.previous_message_id.as_ref().map(Option::as_deref), + expected + ); + assert!(param.extra.is_empty()); +} + +#[rstest] +fn diagnostics_param_rejects_non_string_previous_message() { + assert!( + serde_json::from_value::(json!({"previous_message_id":7})) + .is_err() + ); +} + +#[rstest] +fn cache_miss_reason_rejects_non_numeric_tokens() { + assert!( + serde_json::from_value::( + json!({"type":"model_changed","cache_missed_input_tokens":"many"}) + ) + .is_err() + ); +} diff --git a/litellm-rust/crates/llms-types/tests/minimax.rs b/litellm-rust/crates/llms-types/tests/minimax.rs new file mode 100644 index 00000000000..a7bd46f0934 --- /dev/null +++ b/litellm-rust/crates/llms-types/tests/minimax.rs @@ -0,0 +1,82 @@ +use litellm_llms_types::providers::minimax::{ + MinimaxMediaDetail, MinimaxMediaSource, MinimaxMessagesContentBlock, +}; +use rstest::rstest; +use serde_json::{Value, json}; + +#[rstest] +#[case::image(json!({"type":"image","source":{"type":"base64","media_type":"image/png","data":"AA==","detail":"low"}}))] +#[case::video(json!({"type":"video","source":{"type":"url","url":"https://example.test/video","detail":"high","fps":1,"max_long_side_pixel":1024,"future":null},"cache_control":{"type":"ephemeral"}}))] +#[case::mid_conversation_system(json!({"type":"mid_conv_system","text":"instruction","future":null}))] +fn provider_content_blocks_round_trip(#[case] wire: Value) { + let block: MinimaxMessagesContentBlock = serde_json::from_value(wire.clone()).unwrap(); + match &block { + MinimaxMessagesContentBlock::Image(image) => { + let MinimaxMediaSource::Base64 { + media_type, + data, + options, + } = &image.source + else { + panic!("expected base64 image source"); + }; + assert_eq!((media_type.as_str(), data.as_str()), ("image/png", "AA==")); + assert_eq!(options.detail, Some(MinimaxMediaDetail::Low)); + assert!(options.extra.is_empty()); + } + MinimaxMessagesContentBlock::Video(video) => { + let MinimaxMediaSource::Url { url, options } = &video.source else { + panic!("expected URL video source"); + }; + assert_eq!(url, "https://example.test/video"); + assert_eq!(options.detail, Some(MinimaxMediaDetail::High)); + assert_eq!(options.fps, Some(1.into())); + assert_eq!(options.max_long_side_pixel, Some(1024)); + assert_eq!(options.extra["future"], Value::Null); + assert_eq!( + video.cache_control.as_ref().unwrap().cache_type.as_deref(), + Some("ephemeral") + ); + } + MinimaxMessagesContentBlock::MidConvSystem { text, extra } => { + assert_eq!(text, "instruction"); + assert_eq!(extra["future"], Value::Null); + } + } + assert_eq!(serde_json::to_value(block).unwrap(), wire); +} + +#[rstest] +#[case::missing_source(json!({"type":"video"}))] +#[case::bad_source_tag(json!({"type":"image","source":{"type":"future"}}))] +#[case::url_without_url(json!({"type":"video","source":{"type":"url"}}))] +#[case::base64_without_data(json!({"type":"image","source":{"type":"base64","media_type":"image/png"}}))] +#[case::bad_detail(json!({"type":"image","source":{"type":"url","url":"u","detail":7}}))] +#[case::bad_fps(json!({"type":"video","source":{"type":"url","url":"u","fps":"fast"}}))] +#[case::missing_text(json!({"type":"mid_conv_system"}))] +fn provider_content_rejects_malformed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +fn partial_media_options_omit_null_optionals() { + let block: MinimaxMessagesContentBlock = serde_json::from_value(json!({ + "type":"video", + "source":{"type":"url","url":"u","detail":null,"fps":null,"future":null}, + "cache_control":null + })) + .unwrap(); + let MinimaxMessagesContentBlock::Video(video) = &block else { + panic!("expected video") + }; + let MinimaxMediaSource::Url { options, .. } = &video.source else { + panic!("expected URL source"); + }; + assert!(options.detail.is_none()); + assert!(options.fps.is_none()); + assert!(video.cache_control.is_none()); + assert_eq!( + serde_json::to_value(block).unwrap(), + json!({"type":"video","source":{"type":"url","url":"u","future":null}}) + ); +} diff --git a/litellm-rust/crates/llms-types/tests/ocr.rs b/litellm-rust/crates/llms-types/tests/ocr.rs index 48c816819f2..03e8a586a09 100644 --- a/litellm-rust/crates/llms-types/tests/ocr.rs +++ b/litellm-rust/crates/llms-types/tests/ocr.rs @@ -1,4 +1,4 @@ -use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrDocument, OcrPage}; +use litellm_llms_types::formats::ocr::{LiteLLMOcrResponse, OcrBoundingBox, OcrDocument, OcrPage}; use rstest::rstest; use serde_json::{Map, Value, json}; @@ -102,3 +102,39 @@ fn response_serialization_preserves_extensions_and_native_presence( assert_eq!(decoded.provider_native_response, native); assert_eq!(decoded.into_json(), serialized); } + +#[rstest] +fn bounding_box_exposes_corner_coordinates_and_keeps_extensions() { + let wire = json!({"top_left_x":1,"top_left_y":2.5,"bottom_right_x":30,"bottom_right_y":40,"future":true}); + let bounds: OcrBoundingBox = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(bounds.top_left_x, Some(1.into())); + assert_eq!( + bounds + .top_left_y + .as_ref() + .and_then(serde_json::Number::as_f64), + Some(2.5) + ); + assert_eq!(bounds.bottom_right_x, Some(30.into())); + assert_eq!(bounds.bottom_right_y, Some(40.into())); + assert_eq!(Value::Object(bounds.extra.clone()), json!({"future":true})); + assert_eq!(serde_json::to_value(bounds).unwrap(), wire); +} + +#[rstest] +fn partial_bounding_box_omits_null_corners() { + let bounds: OcrBoundingBox = + serde_json::from_value(json!({"top_left_x":null,"bottom_right_y":4})).unwrap(); + assert!(bounds.top_left_x.is_none()); + assert_eq!( + serde_json::to_value(bounds).unwrap(), + json!({"bottom_right_y":4}) + ); +} + +#[rstest] +#[case::string_corner(json!({"top_left_x":"1"}))] +#[case::array_corner(json!({"bottom_right_y":[4]}))] +fn bounding_box_rejects_non_numeric_corners(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} diff --git a/litellm-rust/crates/llms-types/tests/responses.rs b/litellm-rust/crates/llms-types/tests/responses.rs index 92271757b3f..73cf5e15f6f 100644 --- a/litellm-rust/crates/llms-types/tests/responses.rs +++ b/litellm-rust/crates/llms-types/tests/responses.rs @@ -1,5 +1,31 @@ -use litellm_llms_types::formats::responses::streaming_websocket::ResponsesWsEventType; +use litellm_llms_types::formats::chat_completions::{PromptCacheBreakpoint, PromptCacheMode}; +use litellm_llms_types::formats::responses::{ + ResponsesAdditionalTools, ResponsesApplyPatchCall, ResponsesApplyPatchCallOutput, + ResponsesApplyPatchOperation, ResponsesCodeOutput, ResponsesCompaction, + ResponsesComputerAction, ResponsesComputerCall, ResponsesComputerCallOutput, + ResponsesComputerOutput, ResponsesContentPart, ResponsesCoordinate, ResponsesCustomToolCall, + ResponsesCustomToolCallOutput, ResponsesFunctionCall, ResponsesFunctionCallOutput, + ResponsesImageGenerationCall, ResponsesInputContent, ResponsesLocalShellAction, + ResponsesLocalShellCall, ResponsesLocalShellCallOutput, ResponsesMcpApprovalRequest, + ResponsesMcpApprovalResponse, ResponsesMcpError, ResponsesMcpErrorDetail, ResponsesOutputItem, + ResponsesProgram, ResponsesProgramOutput, ResponsesSafetyCheck, ResponsesShellAction, + ResponsesShellCall, ResponsesShellCallOutput, ResponsesShellEnvironment, ResponsesShellOutcome, + ResponsesShellOutputChunk, ResponsesToolCaller, ResponsesToolOutput, ResponsesToolSearchCall, + ResponsesToolSearchOutput, ResponsesWebSearchAction, + streaming_websocket::{ResponsesEventResponse, ResponsesWsEventType}, +}; +use litellm_llms_types::recognized::Recognized; use rstest::rstest; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::{Value, json}; + +fn round_trip(wire: Value) +where + T: DeserializeOwned + Serialize, +{ + let parsed: T = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(serde_json::to_value(parsed).unwrap(), wire); +} #[rstest] #[case::create("response.create", ResponsesWsEventType::ResponseCreate)] @@ -46,3 +72,646 @@ fn websocket_event_type_schema_is_open_string() { Some(&serde_json::json!("string")) ); } + +#[rstest] +#[case::message(json!({"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer","annotations":[{"type":"url_citation","url":"https://example.test","start_index":0,"end_index":6}]}]}))] +#[case::function_call(json!({"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{\"query\":\"q\"}"}))] +#[case::custom_tool(json!({"type":"custom_tool_call","name":"lookup","input":"q"}))] +#[case::reasoning(json!({"type":"reasoning","summary":[{"type":"summary_text","text":"summary"}],"encrypted_content":"opaque"}))] +#[case::web_search(json!({"type":"web_search_call","action":{"type":"search","queries":["q"],"sources":[{"type":"url","url":"https://example.test"}]}}))] +#[case::file_search(json!({"type":"file_search_call","queries":["q"],"results":[{"file_id":"file_1","score":1,"attributes":{"custom":[1,null]}}]}))] +#[case::code(json!({"type":"code_interpreter_call","outputs":[{"type":"logs","logs":"done"},{"type":"image","url":"https://example.test"}]}))] +#[case::image(json!({"type":"image_generation_call","result":"generated"}))] +#[case::mcp(json!({"type":"mcp_call","server_label":"server","name":"lookup","arguments":"{}","output":"done"}))] +fn output_items_round_trip(#[case] wire: Value) { + let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap(); + match &item { + ResponsesOutputItem::Message(message) => { + assert_eq!(message.role.as_deref(), Some("assistant")); + let Some(content) = &message.content else { + panic!("expected content") + }; + let [ + ResponsesContentPart::OutputText { + text, + annotations: Some(annotations), + .. + }, + ] = content.as_slice() + else { + panic!("expected output text and annotations") + }; + assert_eq!(text, "answer"); + let [ + litellm_llms_types::formats::responses::ResponsesAnnotation::UrlCitation(citation), + ] = annotations.as_slice() + else { + panic!("expected URL citation") + }; + assert_eq!(citation.url.as_deref(), Some("https://example.test")); + assert_eq!(citation.start_index, Some(0)); + } + ResponsesOutputItem::FunctionCall(call) => { + assert_eq!(call.call_id.as_deref(), Some("call_1")); + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(call.arguments.as_deref(), Some("{\"query\":\"q\"}")); + } + ResponsesOutputItem::CustomToolCall(call) => { + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(call.input.as_deref(), Some("q")); + } + ResponsesOutputItem::Reasoning(reasoning) => { + assert_eq!(reasoning.encrypted_content.as_deref(), Some("opaque")); + let Some(summary) = &reasoning.summary else { + panic!("expected summary") + }; + let [ResponsesContentPart::SummaryText { text, .. }] = summary.as_slice() else { + panic!("expected summary text") + }; + assert_eq!(text, "summary"); + } + ResponsesOutputItem::WebSearchCall(call) => { + let Some(ResponsesWebSearchAction::Search { + queries: Some(queries), + sources: Some(sources), + .. + }) = &call.action + else { + panic!("expected search action") + }; + assert_eq!(queries, &["q"]); + assert_eq!(sources[0].url.as_deref(), Some("https://example.test")); + } + ResponsesOutputItem::FileSearchCall(call) => { + assert_eq!(call.queries.as_deref(), Some(["q".to_owned()].as_slice())); + let Some(results) = &call.results else { + panic!("expected search results") + }; + assert_eq!(results[0].file_id.as_deref(), Some("file_1")); + assert_eq!(results[0].score, Some(1.into())); + assert_eq!( + results[0].attributes.as_ref().unwrap()["custom"], + json!([1, null]) + ); + } + ResponsesOutputItem::CodeInterpreterCall(call) => { + let Some(outputs) = &call.outputs else { + panic!("expected code outputs") + }; + let [ + ResponsesCodeOutput::Logs { logs, .. }, + ResponsesCodeOutput::Image { url, .. }, + ] = outputs.as_slice() + else { + panic!("expected logs and image") + }; + assert_eq!(logs, "done"); + assert_eq!(url, "https://example.test"); + } + ResponsesOutputItem::ImageGenerationCall(call) => { + assert_eq!(call.result.as_deref(), Some("generated")) + } + ResponsesOutputItem::McpCall(call) => { + assert_eq!(call.server_label.as_deref(), Some("server")); + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(call.arguments.as_deref(), Some("{}")); + assert_eq!(call.output.as_deref(), Some("done")); + } + other => panic!("unexpected variant {other:?}"), + } + assert_eq!(serde_json::to_value(item).unwrap(), wire); +} + +#[rstest] +fn nested_event_response_exposes_typed_output_and_preserves_extensions() { + let wire = json!({ + "id":"response_1", + "model":"example-model", + "status":"completed", + "output":[{"type":"function_call","call_id":"call_1","name":"lookup","arguments":"{}","extension":true}], + "extension":{"nested":[1,null]} + }); + let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(response.id.as_deref(), Some("response_1")); + assert_eq!(response.model.as_deref(), Some("example-model")); + let Some(output) = &response.output else { + panic!("expected typed output"); + }; + let [Recognized::Known(ResponsesOutputItem::FunctionCall(call))] = output.as_slice() else { + panic!("expected function call"); + }; + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(call.arguments.as_deref(), Some("{}")); + assert_eq!(serde_json::to_value(response).unwrap(), wire); +} + +#[rstest] +#[case::empty(json!({}))] +#[case::partial(json!({"id":"response_1","output":[]}))] +fn nested_event_response_accepts_partial_metadata(#[case] wire: Value) { + round_trip::(wire); +} + +#[rstest] +#[case::wrong_model(json!({"model":7}))] +#[case::wrong_status(json!({"status":false}))] +#[case::wrong_output(json!({"output":{}}))] +fn event_response_rejects_malformed_typed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +#[case::unknown_type(json!({"type":"future_item","id":"item_1","payload":[1,null]}))] +#[case::missing_tag(json!({"id":"item_1"}))] +#[case::malformed_known(json!({"type":"message","content":[{"type":"output_text","text":7}]}))] +fn event_response_keeps_unmodeled_output_items_beside_typed_ones(#[case] item: Value) { + let wire = json!({"output":[{"type":"function_call","name":"lookup"}, item.clone()]}); + let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap(); + let Some( + [ + Recognized::Known(ResponsesOutputItem::FunctionCall(call)), + Recognized::Unrecognized(kept), + ], + ) = response.output.as_deref() + else { + panic!("expected one typed item and one preserved item"); + }; + assert_eq!(call.name.as_deref(), Some("lookup")); + assert_eq!(kept, &item); + assert_eq!(serde_json::to_value(response).unwrap(), wire); +} + +#[rstest] +#[case::find_in_page( + json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}), + json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}) +)] +#[case::legacy_find( + json!({"type":"find","url":"https://example.test","pattern":"needle"}), + json!({"type":"find_in_page","url":"https://example.test","pattern":"needle"}) +)] +fn web_search_find_action_exposes_url_and_pattern(#[case] wire: Value, #[case] serialized: Value) { + let action: ResponsesWebSearchAction = serde_json::from_value(wire).unwrap(); + let ResponsesWebSearchAction::FindInPage { + url, + pattern, + extra, + } = &action + else { + panic!("expected find_in_page action"); + }; + assert_eq!(url, "https://example.test"); + assert_eq!(pattern, "needle"); + assert!(extra.is_empty()); + assert_eq!(serde_json::to_value(action).unwrap(), serialized); +} + +#[rstest] +#[case::with_url(json!({"type":"open_page","url":"https://example.test"}), Some("https://example.test"))] +#[case::without_url(json!({"type":"open_page"}), None)] +fn web_search_open_page_url_is_optional(#[case] wire: Value, #[case] expected: Option<&str>) { + let action: ResponsesWebSearchAction = serde_json::from_value(wire.clone()).unwrap(); + let ResponsesWebSearchAction::OpenPage { url, .. } = &action else { + panic!("expected open_page action"); + }; + assert_eq!(url.as_deref(), expected); + assert_eq!(serde_json::to_value(action).unwrap(), wire); +} + +#[rstest] +#[case::find_pattern(json!({"type":"find_in_page","url":"u","pattern":false}))] +#[case::find_missing_url(json!({"type":"find_in_page","pattern":"p"}))] +#[case::open_page_url(json!({"type":"open_page","url":7}))] +fn web_search_action_rejects_malformed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +#[rstest] +fn event_response_optional_fields_omit_missing_and_null() { + let response: ResponsesEventResponse = serde_json::from_value(json!({ + "id":null,"model":null,"status":null,"output":null,"future":null + })) + .unwrap(); + assert!(response.id.is_none()); + assert!(response.model.is_none()); + assert!(response.status.is_none()); + assert!(response.output.is_none()); + assert_eq!( + serde_json::to_value(response).unwrap(), + json!({"future":null}) + ); +} + +#[rstest] +#[case::refusal(json!({"type":"refusal","refusal":"refused","future":null}), ResponsesContentPart::Refusal { refusal:"refused".into(), extra:serde_json::Map::from_iter([("future".into(), Value::Null)]) })] +#[case::reasoning(json!({"type":"reasoning_text","text":"reason"}), ResponsesContentPart::ReasoningText { text:"reason".into(), extra:Default::default() })] +fn content_parts_expose_typed_variants( + #[case] wire: Value, + #[case] expected: ResponsesContentPart, +) { + let content: ResponsesContentPart = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(content, expected); + assert_eq!(serde_json::to_value(content).unwrap(), wire); +} + +#[rstest] +fn mcp_discovery_exposes_tools_and_nested_schemas() { + let wire = json!({ + "type":"mcp_list_tools","id":"item_1","server_label":"tools", + "tools":[{"name":"lookup","description":"Look up a value","input_schema":{"type":"object","properties":{"query":{"type":"string"}}},"annotations":{"read_only":false},"future":null}], + "extension":[1,null] + }); + let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap(); + let ResponsesOutputItem::McpListTools(discovery) = &item else { + panic!("expected discovery") + }; + assert_eq!(discovery.server_label.as_deref(), Some("tools")); + let tool = &discovery.tools.as_ref().unwrap()[0]; + assert_eq!(tool.name.as_deref(), Some("lookup")); + let Some(litellm_llms_types::json_schema::JsonSchema::Object(schema)) = &tool.input_schema + else { + panic!("expected tool schema") + }; + assert!(schema.properties.as_ref().unwrap().contains_key("query")); + assert_eq!(tool.annotations, Some(json!({"read_only":false}))); + assert_eq!(tool.extra["future"], Value::Null); + assert_eq!(serde_json::to_value(item).unwrap(), wire); +} + +#[rstest] +#[case::partial(json!({"server_label":"tools","tools":[]}))] +#[case::failure(json!({"server_label":"tools","error":"unavailable"}))] +fn mcp_discovery_accepts_partial_payloads(#[case] wire: Value) { + let discovery: litellm_llms_types::formats::responses::ResponsesMcpListTools = + serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(discovery.server_label.as_deref(), Some("tools")); + assert!(discovery.id.is_none()); + assert_eq!(serde_json::to_value(discovery).unwrap(), wire); +} + +#[rstest] +#[case::message(json!("unavailable"), ResponsesMcpError::Message("unavailable".into()))] +#[case::protocol(json!({"type":"mcp_protocol_error","code":-32600,"message":"invalid","extension":null}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpProtocolError {code:-32600,message:"invalid".into(),extra:serde_json::Map::from_iter([("extension".into(), Value::Null)])}))] +#[case::http(json!({"type":"http_error","code":503,"message":"unavailable"}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::HttpError {code:503,message:"unavailable".into(),extra:Default::default()}))] +#[case::tool(json!({"type":"mcp_tool_execution_error","content":{"arbitrary":[1,null]}}), ResponsesMcpError::Detail(ResponsesMcpErrorDetail::McpToolExecutionError {content:json!({"arbitrary":[1,null]}),extra:Default::default()}))] +fn mcp_call_errors_expose_typed_variants(#[case] wire: Value, #[case] expected: ResponsesMcpError) { + let error: ResponsesMcpError = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(error, expected); + assert_eq!(serde_json::to_value(&error).unwrap(), wire); + let call: ResponsesOutputItem = + serde_json::from_value(json!({"type":"mcp_call","error":wire})).unwrap(); + let ResponsesOutputItem::McpCall(call) = call else { + panic!("expected call") + }; + assert_eq!(call.error, Some(error)); +} + +#[rstest] +#[case::tools_not_array(json!({"type":"mcp_list_tools","tools":{}}))] +#[case::invalid_name(json!({"type":"mcp_list_tools","tools":[{"name":7}]}))] +#[case::invalid_schema(json!({"type":"mcp_list_tools","tools":[{"input_schema":{"properties":{"query":7}}}]}))] +#[case::invalid_error_code(json!({"type":"mcp_call","error":{"type":"http_error","code":"503","message":"unavailable"}}))] +#[case::missing_error_content(json!({"type":"mcp_call","error":{"type":"mcp_tool_execution_error"}}))] +#[case::unknown_error_tag(json!({"type":"mcp_call","error":{"type":"future"}}))] +fn mcp_output_rejects_malformed_known_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} + +fn text(value: &str) -> Option { + Some(value.to_owned()) +} + +fn program_caller() -> Option { + Some(ResponsesToolCaller::Program { + caller_id: "prog_1".into(), + extra: Default::default(), + }) +} + +fn direct_caller() -> Option { + Some(ResponsesToolCaller::Direct { + extra: Default::default(), + }) +} + +fn safety_check() -> ResponsesSafetyCheck { + ResponsesSafetyCheck { + id: text("sc_1"), + code: text("malicious_instructions"), + message: text("check"), + ..Default::default() + } +} + +#[rstest] +#[case::function_call( + json!({"type":"function_call","id":"fc_1","status":"completed","call_id":"call_1","name":"lookup","arguments":"{}","namespace":"ns","async":true,"caller":{"type":"program","caller_id":"prog_1"}}), + ResponsesOutputItem::FunctionCall(ResponsesFunctionCall { + id: text("fc_1"), status: text("completed"), call_id: text("call_1"), name: text("lookup"), + arguments: text("{}"), namespace: text("ns"), r#async: Some(true), caller: program_caller(), ..Default::default() + }) +)] +#[case::custom_tool_call( + json!({"type":"custom_tool_call","call_id":"call_1","name":"lookup","input":"q","namespace":"ns","async":false,"caller":{"type":"direct"}}), + ResponsesOutputItem::CustomToolCall(ResponsesCustomToolCall { + call_id: text("call_1"), name: text("lookup"), input: text("q"), namespace: text("ns"), + r#async: Some(false), caller: direct_caller(), ..Default::default() + }) +)] +#[case::image_generation_call( + json!({"type":"image_generation_call","id":"ig_1","status":"completed","result":"b64","action":"edit","background":"opaque","output_format":"webp","quality":"high","revised_prompt":"a cat","size":"1536x864"}), + ResponsesOutputItem::ImageGenerationCall(ResponsesImageGenerationCall { + id: text("ig_1"), status: text("completed"), result: text("b64"), action: text("edit"), background: text("opaque"), + output_format: text("webp"), quality: text("high"), revised_prompt: text("a cat"), size: text("1536x864"), ..Default::default() + }) +)] +#[case::function_call_output_text( + json!({"type":"function_call_output","id":"fco_1","status":"completed","call_id":"call_1","output":"done","caller":{"type":"direct"},"created_by":"user_1","name":"lookup","namespace":"ns"}), + ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput { + id: text("fco_1"), status: text("completed"), call_id: text("call_1"), output: Some(ResponsesToolOutput::Text("done".into())), + caller: direct_caller(), created_by: text("user_1"), name: text("lookup"), namespace: text("ns"), ..Default::default() + }) +)] +#[case::function_call_output_content( + json!({"type":"function_call_output","call_id":"call_1","output":[ + {"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"explicit"}}, + {"type":"input_image","detail":"low","file_id":"file_1","image_url":"https://example.test/i.png"}, + {"type":"input_file","detail":"high","file_data":"data","file_id":"file_2","file_url":"https://example.test/f","filename":"f.pdf"} + ]}), + ResponsesOutputItem::FunctionCallOutput(ResponsesFunctionCallOutput { + call_id: text("call_1"), + output: Some(ResponsesToolOutput::Content(vec![ + ResponsesInputContent::InputText { + text: "t".into(), + prompt_cache_breakpoint: Some(PromptCacheBreakpoint { mode: PromptCacheMode::Explicit, extra: Default::default() }), + extra: Default::default(), + }, + ResponsesInputContent::InputImage { + detail: "low".into(), file_id: text("file_1"), image_url: text("https://example.test/i.png"), + prompt_cache_breakpoint: None, extra: Default::default(), + }, + ResponsesInputContent::InputFile { + detail: text("high"), file_data: text("data"), file_id: text("file_2"), file_url: text("https://example.test/f"), + filename: text("f.pdf"), prompt_cache_breakpoint: None, extra: Default::default(), + }, + ])), + ..Default::default() + }) +)] +#[case::custom_tool_call_output( + json!({"type":"custom_tool_call_output","id":"cto_1","status":"completed","call_id":"call_1","output":[{"type":"input_text","text":"t"}],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}), + ResponsesOutputItem::CustomToolCallOutput(ResponsesCustomToolCallOutput { + id: text("cto_1"), status: text("completed"), call_id: text("call_1"), + output: Some(ResponsesToolOutput::Content(vec![ResponsesInputContent::InputText { + text: "t".into(), prompt_cache_breakpoint: None, extra: Default::default(), + }])), + caller: program_caller(), created_by: text("user_1"), ..Default::default() + }) +)] +#[case::computer_call( + json!({"type":"computer_call","id":"cu_1","status":"completed","call_id":"call_1", + "pending_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}], + "action":{"type":"click","button":"left","x":1,"y":2,"keys":["shift"]}, + "actions":[ + {"type":"double_click","x":3,"y":4}, + {"type":"drag","path":[{"x":5,"y":6},{"x":7,"y":8}],"keys":["ctrl"]}, + {"type":"keypress","keys":["enter"]}, + {"type":"move","x":-1,"y":9}, + {"type":"screenshot"}, + {"type":"scroll","scroll_x":0,"scroll_y":-10,"x":11,"y":12}, + {"type":"type","text":"hello"}, + {"type":"wait"} + ]}), + ResponsesOutputItem::ComputerCall(ResponsesComputerCall { + id: text("cu_1"), status: text("completed"), call_id: text("call_1"), pending_safety_checks: Some(vec![safety_check()]), + action: Some(ResponsesComputerAction::Click { button: "left".into(), x: 1, y: 2, keys: Some(vec!["shift".into()]), extra: Default::default() }), + actions: Some(vec![ + ResponsesComputerAction::DoubleClick { x: 3, y: 4, keys: None, extra: Default::default() }, + ResponsesComputerAction::Drag { + path: vec![ + ResponsesCoordinate { x: 5, y: 6, extra: Default::default() }, + ResponsesCoordinate { x: 7, y: 8, extra: Default::default() }, + ], + keys: Some(vec!["ctrl".into()]), + extra: Default::default(), + }, + ResponsesComputerAction::Keypress { keys: vec!["enter".into()], extra: Default::default() }, + ResponsesComputerAction::Move { x: -1, y: 9, keys: None, extra: Default::default() }, + ResponsesComputerAction::Screenshot { extra: Default::default() }, + ResponsesComputerAction::Scroll { scroll_x: 0, scroll_y: -10, x: 11, y: 12, keys: None, extra: Default::default() }, + ResponsesComputerAction::Type { text: "hello".into(), extra: Default::default() }, + ResponsesComputerAction::Wait { extra: Default::default() }, + ]), + ..Default::default() + }) +)] +#[case::computer_call_output( + json!({"type":"computer_call_output","id":"cuo_1","status":"completed","call_id":"call_1","output":{"type":"computer_screenshot","file_id":"file_1","image_url":"https://example.test/s.png"},"acknowledged_safety_checks":[{"id":"sc_1","code":"malicious_instructions","message":"check"}],"created_by":"user_1"}), + ResponsesOutputItem::ComputerCallOutput(ResponsesComputerCallOutput { + id: text("cuo_1"), status: text("completed"), call_id: text("call_1"), + output: Some(ResponsesComputerOutput::ComputerScreenshot { file_id: text("file_1"), image_url: text("https://example.test/s.png"), extra: Default::default() }), + acknowledged_safety_checks: Some(vec![safety_check()]), created_by: text("user_1"), ..Default::default() + }) +)] +#[case::program( + json!({"type":"program","id":"prog_item","call_id":"prog_1","code":"run()","fingerprint":"fp"}), + ResponsesOutputItem::Program(ResponsesProgram { + id: text("prog_item"), call_id: text("prog_1"), code: text("run()"), fingerprint: text("fp"), ..Default::default() + }) +)] +#[case::program_output( + json!({"type":"program_output","id":"po_1","status":"completed","call_id":"prog_1","result":"42"}), + ResponsesOutputItem::ProgramOutput(ResponsesProgramOutput { + id: text("po_1"), status: text("completed"), call_id: text("prog_1"), result: text("42"), ..Default::default() + }) +)] +#[case::tool_search_call( + json!({"type":"tool_search_call","id":"ts_1","status":"completed","call_id":"call_1","arguments":{"query":["weather",null]},"execution":"server","created_by":"user_1"}), + ResponsesOutputItem::ToolSearchCall(ResponsesToolSearchCall { + id: text("ts_1"), status: text("completed"), call_id: text("call_1"), arguments: Some(json!({"query":["weather",null]})), + execution: text("server"), created_by: text("user_1"), ..Default::default() + }) +)] +#[case::tool_search_output( + json!({"type":"tool_search_output","id":"tso_1","status":"completed","call_id":"call_1","execution":"client","tools":[{"type":"function","name":"lookup","parameters":null,"strict":true}],"created_by":"user_1"}), + ResponsesOutputItem::ToolSearchOutput(ResponsesToolSearchOutput { + id: text("tso_1"), status: text("completed"), call_id: text("call_1"), execution: text("client"), + tools: Some(vec![serde_json::Map::from_iter([ + ("type".into(), json!("function")), ("name".into(), json!("lookup")), + ("parameters".into(), Value::Null), ("strict".into(), json!(true)), + ])]), + created_by: text("user_1"), ..Default::default() + }) +)] +#[case::additional_tools( + json!({"type":"additional_tools","id":"at_1","role":"developer","tools":[{"type":"local_shell"}]}), + ResponsesOutputItem::AdditionalTools(ResponsesAdditionalTools { + id: text("at_1"), role: text("developer"), + tools: Some(vec![serde_json::Map::from_iter([("type".into(), json!("local_shell"))])]), ..Default::default() + }) +)] +#[case::compaction( + json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque","created_by":"user_1"}), + ResponsesOutputItem::Compaction(ResponsesCompaction { + id: text("cmp_1"), encrypted_content: text("opaque"), created_by: text("user_1"), ..Default::default() + }) +)] +#[case::local_shell_call( + json!({"type":"local_shell_call","id":"ls_1","status":"completed","call_id":"call_1","action":{"type":"exec","command":["ls","-a"],"env":{"HOME":"/home/u"},"timeout_ms":1000,"user":"u","working_directory":"/tmp"}}), + ResponsesOutputItem::LocalShellCall(ResponsesLocalShellCall { + id: text("ls_1"), status: text("completed"), call_id: text("call_1"), + action: Some(ResponsesLocalShellAction::Exec { + command: vec!["ls".into(), "-a".into()], env: [("HOME".into(), "/home/u".into())].into(), + timeout_ms: Some(1000), user: text("u"), working_directory: text("/tmp"), extra: Default::default(), + }), + ..Default::default() + }) +)] +#[case::local_shell_call_output( + json!({"type":"local_shell_call_output","id":"lso_1","status":"completed","output":"{\"stdout\":\"x\"}"}), + ResponsesOutputItem::LocalShellCallOutput(ResponsesLocalShellCallOutput { + id: text("lso_1"), status: text("completed"), output: text("{\"stdout\":\"x\"}"), ..Default::default() + }) +)] +#[case::shell_call_local( + json!({"type":"shell_call","id":"sh_1","status":"in_progress","call_id":"call_1","action":{"commands":["ls"],"max_output_length":100,"timeout_ms":500},"environment":{"type":"local"},"caller":{"type":"direct"},"created_by":"user_1"}), + ResponsesOutputItem::ShellCall(ResponsesShellCall { + id: text("sh_1"), status: text("in_progress"), call_id: text("call_1"), + action: Some(ResponsesShellAction { commands: Some(vec!["ls".into()]), max_output_length: Some(100), timeout_ms: Some(500), ..Default::default() }), + environment: Some(ResponsesShellEnvironment::Local { extra: Default::default() }), + caller: direct_caller(), created_by: text("user_1"), ..Default::default() + }) +)] +#[case::shell_call_container( + json!({"type":"shell_call","call_id":"call_1","environment":{"type":"container_reference","container_id":"cntr_1"}}), + ResponsesOutputItem::ShellCall(ResponsesShellCall { + call_id: text("call_1"), + environment: Some(ResponsesShellEnvironment::ContainerReference { container_id: "cntr_1".into(), extra: Default::default() }), + ..Default::default() + }) +)] +#[case::shell_call_output( + json!({"type":"shell_call_output","id":"sho_1","status":"completed","call_id":"call_1","max_output_length":100,"output":[ + {"outcome":{"type":"exit","exit_code":2},"stdout":"out","stderr":"err","created_by":"user_1"}, + {"outcome":{"type":"timeout"},"stdout":"","stderr":""} + ],"caller":{"type":"program","caller_id":"prog_1"},"created_by":"user_1"}), + ResponsesOutputItem::ShellCallOutput(ResponsesShellCallOutput { + id: text("sho_1"), status: text("completed"), call_id: text("call_1"), max_output_length: Some(100), + output: Some(vec![ + ResponsesShellOutputChunk { + outcome: Some(ResponsesShellOutcome::Exit { exit_code: 2, extra: Default::default() }), + stdout: text("out"), stderr: text("err"), created_by: text("user_1"), ..Default::default() + }, + ResponsesShellOutputChunk { + outcome: Some(ResponsesShellOutcome::Timeout { extra: Default::default() }), + stdout: text(""), stderr: text(""), ..Default::default() + }, + ]), + caller: program_caller(), created_by: text("user_1"), ..Default::default() + }) +)] +#[case::apply_patch_create( + json!({"type":"apply_patch_call","id":"ap_1","status":"completed","call_id":"call_1","operation":{"type":"create_file","path":"a.txt","diff":"+a"},"caller":{"type":"direct"},"created_by":"user_1"}), + ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall { + id: text("ap_1"), status: text("completed"), call_id: text("call_1"), + operation: Some(ResponsesApplyPatchOperation::CreateFile { path: "a.txt".into(), diff: "+a".into(), extra: Default::default() }), + caller: direct_caller(), created_by: text("user_1"), ..Default::default() + }) +)] +#[case::apply_patch_delete( + json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"delete_file","path":"a.txt"}}), + ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall { + call_id: text("call_1"), + operation: Some(ResponsesApplyPatchOperation::DeleteFile { path: "a.txt".into(), extra: Default::default() }), + ..Default::default() + }) +)] +#[case::apply_patch_update( + json!({"type":"apply_patch_call","call_id":"call_1","operation":{"type":"update_file","path":"a.txt","diff":"-a\n+b"}}), + ResponsesOutputItem::ApplyPatchCall(ResponsesApplyPatchCall { + call_id: text("call_1"), + operation: Some(ResponsesApplyPatchOperation::UpdateFile { path: "a.txt".into(), diff: "-a\n+b".into(), extra: Default::default() }), + ..Default::default() + }) +)] +#[case::apply_patch_call_output( + json!({"type":"apply_patch_call_output","id":"apo_1","status":"failed","call_id":"call_1","output":"conflict","caller":{"type":"direct"},"created_by":"user_1"}), + ResponsesOutputItem::ApplyPatchCallOutput(ResponsesApplyPatchCallOutput { + id: text("apo_1"), status: text("failed"), call_id: text("call_1"), output: text("conflict"), + caller: direct_caller(), created_by: text("user_1"), ..Default::default() + }) +)] +#[case::mcp_approval_request( + json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}), + ResponsesOutputItem::McpApprovalRequest(ResponsesMcpApprovalRequest { + id: text("apr_1"), server_label: text("s"), name: text("n"), arguments: text("{}"), ..Default::default() + }) +)] +#[case::mcp_approval_response( + json!({"type":"mcp_approval_response","id":"aprr_1","approval_request_id":"apr_1","approve":false,"reason":"denied"}), + ResponsesOutputItem::McpApprovalResponse(ResponsesMcpApprovalResponse { + id: text("aprr_1"), approval_request_id: text("apr_1"), approve: Some(false), reason: text("denied"), ..Default::default() + }) +)] +fn documented_output_items_parse_into_typed_variants( + #[case] wire: Value, + #[case] expected: ResponsesOutputItem, +) { + let item: ResponsesOutputItem = serde_json::from_value(wire.clone()).unwrap(); + assert_eq!(item, expected); + assert_eq!(serde_json::to_value(item).unwrap(), wire); +} + +#[rstest] +#[case::compaction(json!({"type":"compaction","id":"cmp_1","encrypted_content":"opaque"}))] +#[case::approval(json!({"type":"mcp_approval_request","id":"apr_1","server_label":"s","name":"n","arguments":"{}"}))] +#[case::shell(json!({"type":"shell_call","call_id":"call_1","action":{"commands":["ls"]}}))] +fn event_response_types_documented_output_items(#[case] item: Value) { + let wire = json!({"output":[item.clone()]}); + let response: ResponsesEventResponse = serde_json::from_value(wire.clone()).unwrap(); + let Some([Recognized::Known(known)]) = response.output.as_deref() else { + panic!("expected one typed item"); + }; + assert_eq!( + known, + &serde_json::from_value::(item).unwrap() + ); + assert_eq!(serde_json::to_value(response).unwrap(), wire); +} + +#[rstest] +#[case::unknown_caller(json!({"type":"function_call","caller":{"type":"future"}}))] +#[case::program_caller_missing_id(json!({"type":"shell_call","caller":{"type":"program"}}))] +#[case::untagged_caller(json!({"type":"custom_tool_call","caller":{}}))] +#[case::async_not_bool(json!({"type":"function_call","async":"yes"}))] +#[case::output_not_text_or_list(json!({"type":"function_call_output","output":{"text":"t"}}))] +#[case::unknown_input_content(json!({"type":"function_call_output","output":[{"type":"input_audio"}]}))] +#[case::input_text_missing_text(json!({"type":"custom_tool_call_output","output":[{"type":"input_text"}]}))] +#[case::input_image_missing_detail(json!({"type":"custom_tool_call_output","output":[{"type":"input_image","file_id":"f"}]}))] +#[case::bad_cache_mode(json!({"type":"custom_tool_call_output","output":[{"type":"input_text","text":"t","prompt_cache_breakpoint":{"mode":"implicit"}}]}))] +#[case::unknown_computer_action(json!({"type":"computer_call","action":{"type":"teleport"}}))] +#[case::click_missing_button(json!({"type":"computer_call","action":{"type":"click","x":1,"y":2}}))] +#[case::click_fractional_x(json!({"type":"computer_call","action":{"type":"click","button":"left","x":1.5,"y":2}}))] +#[case::drag_point_missing_y(json!({"type":"computer_call","actions":[{"type":"drag","path":[{"x":1}]}]}))] +#[case::keypress_keys_not_list(json!({"type":"computer_call","actions":[{"type":"keypress","keys":"enter"}]}))] +#[case::safety_check_not_object(json!({"type":"computer_call","pending_safety_checks":["sc_1"]}))] +#[case::unknown_screenshot_tag(json!({"type":"computer_call_output","output":{"type":"screenshot"}}))] +#[case::untagged_screenshot(json!({"type":"computer_call_output","output":{"file_id":"f"}}))] +#[case::tool_not_object(json!({"type":"tool_search_output","tools":["lookup"]}))] +#[case::unknown_local_shell_action(json!({"type":"local_shell_call","action":{"type":"spawn","command":[],"env":{}}}))] +#[case::local_shell_missing_env(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"]}}))] +#[case::local_shell_env_value(json!({"type":"local_shell_call","action":{"type":"exec","command":["ls"],"env":{"A":1}}}))] +#[case::shell_commands_not_list(json!({"type":"shell_call","action":{"commands":"ls"}}))] +#[case::unknown_shell_environment(json!({"type":"shell_call","environment":{"type":"container_auto"}}))] +#[case::container_missing_id(json!({"type":"shell_call","environment":{"type":"container_reference"}}))] +#[case::unknown_shell_outcome(json!({"type":"shell_call_output","output":[{"outcome":{"type":"killed"}}]}))] +#[case::exit_missing_code(json!({"type":"shell_call_output","output":[{"outcome":{"type":"exit"}}]}))] +#[case::negative_max_output(json!({"type":"shell_call_output","max_output_length":-1}))] +#[case::unknown_patch_operation(json!({"type":"apply_patch_call","operation":{"type":"rename_file","path":"a"}}))] +#[case::update_missing_diff(json!({"type":"apply_patch_call","operation":{"type":"update_file","path":"a"}}))] +#[case::approve_not_bool(json!({"type":"mcp_approval_response","approve":"true"}))] +#[case::compaction_content_not_string(json!({"type":"compaction","encrypted_content":7}))] +#[case::program_code_not_string(json!({"type":"program","code":["run()"]}))] +fn output_items_reject_malformed_typed_fields(#[case] wire: Value) { + assert!(serde_json::from_value::(wire).is_err()); +} From 92654b69a9a27804fd470cb9f001a571c74cdd35 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:43:56 -0700 Subject: [PATCH 41/65] test: delete 34 legacy tests already covered by e2e (#45340) Co-authored-by: yuneng --- .circleci/config.yml | 45 -- .../test_bedrock_token_counter.py | 107 ---- .../base_responses_api.py | 550 ------------------ tests/llm_responses_api_testing/conftest.py | 112 ---- .../test_anthropic_responses_api.py | 90 --- .../test_azure_responses_api.py | 34 -- .../test_google_ai_studio_responses_api.py | 12 - .../test_openai_responses_api.py | 64 -- tests/llm_translation/Readme.md | 1 - tests/llm_translation/base_llm_unit_tests.py | 19 - .../realtime/test_openai_realtime.py | 138 ----- .../test_anthropic_completion.py | 28 - .../test_bedrock_completion.py | 79 --- tests/local_testing/test_caching.py | 53 -- tests/local_testing/test_completion.py | 41 -- tests/local_testing/test_custom_logger.py | 48 -- tests/local_testing/test_lunary.py | 57 -- tests/local_testing/test_text_completion.py | 13 - .../test_gemini_with_spend.test.js | 51 -- .../test_anthropic_messages_passthrough.py | 78 --- tests/test_end_users.py | 10 - tests/test_keys.py | 68 --- tests/test_new_vector_store_endpoints.py | 174 ------ tests/test_team_members.py | 12 - 24 files changed, 1884 deletions(-) delete mode 100644 tests/litellm_utils_tests/test_bedrock_token_counter.py delete mode 100644 tests/llm_responses_api_testing/base_responses_api.py delete mode 100644 tests/llm_responses_api_testing/conftest.py delete mode 100644 tests/llm_responses_api_testing/test_anthropic_responses_api.py delete mode 100644 tests/llm_responses_api_testing/test_azure_responses_api.py delete mode 100644 tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py delete mode 100644 tests/llm_responses_api_testing/test_openai_responses_api.py delete mode 100644 tests/llm_translation/realtime/test_openai_realtime.py delete mode 100644 tests/local_testing/test_lunary.py diff --git a/.circleci/config.yml b/.circleci/config.yml index 58860e3ed95..e9c295aca08 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1095,50 +1095,6 @@ jobs: - google_generate_content_endpoint_coverage.xml - google_generate_content_endpoint_coverage - llm_responses_api_testing: - docker: - - *python312_image - working_directory: ~/project - resource_class: large - environment: - REQUEST_TIMEOUT: "180" - - steps: - - checkout - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - restore_cache: - keys: - - v1-uv-cache-{{ checksum "uv.lock" }} - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - save_cache: - paths: - - ~/.cache/uv - key: v1-uv-cache-{{ checksum "uv.lock" }} - # Run pytest and generate JUnit XML report - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/llm_responses_api_testing/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -v \ - --junitxml=test-results/junit.xml \ - --durations=5 \ - -n 8 \ - --reruns 1 --only-rerun Timeout" - no_output_timeout: 15m - - # Store test results - - store_test_results: - path: test-results ocr_testing: docker: - *python312_image @@ -3516,7 +3472,6 @@ workflows: - realtime_translation_testing - guardrails_testing - google_generate_content_endpoint_testing - - llm_responses_api_testing - ocr_testing - search_testing - batches_testing diff --git a/tests/litellm_utils_tests/test_bedrock_token_counter.py b/tests/litellm_utils_tests/test_bedrock_token_counter.py deleted file mode 100644 index b4c05cb0cd7..00000000000 --- a/tests/litellm_utils_tests/test_bedrock_token_counter.py +++ /dev/null @@ -1,107 +0,0 @@ -""" -Bedrock Token Counter Tests. - -Tests for the Bedrock token counter implementation using the base test suite. - -Note: Not all Bedrock models support token counting. The CountTokens API -is only available for specific models. If the model doesn't support token -counting, the test will be skipped. -""" - -import os -from typing import Any, Dict, List - -import pytest - - -from litellm.llms.base_llm.base_utils import BaseTokenCounter -from litellm.llms.bedrock.count_tokens.bedrock_token_counter import BedrockTokenCounter -from tests.litellm_utils_tests.base_token_counter_test import BaseTokenCounterTest - - -class TestBedrockTokenCounter(BaseTokenCounterTest): - """Test suite for Bedrock token counter. - - Note: Bedrock CountTokens API support varies by model. Some models - (like older Claude versions) may not support token counting. - Use amazon.nova-* models for reliable token counting support. - """ - - def get_token_counter(self) -> BaseTokenCounter: - return BedrockTokenCounter() - - def get_test_model(self) -> str: - # Use Amazon Nova model which supports token counting - # Alternatively, use environment variable to override - return os.getenv("BEDROCK_TEST_MODEL", "amazon.nova-lite-v1:0") - - def get_test_messages(self) -> List[Dict[str, Any]]: - return [{"role": "user", "content": "Hello, how are you today?"}] - - def get_deployment_config(self) -> Dict[str, Any]: - # Bedrock uses AWS credentials from environment - # Check for AWS credentials - aws_access_key = os.getenv("AWS_ACCESS_KEY_ID") - aws_secret_key = os.getenv("AWS_SECRET_ACCESS_KEY") - aws_region = os.getenv("AWS_REGION_NAME", "us-east-1") - - if not aws_access_key or not aws_secret_key: - pytest.skip( - "AWS credentials not set (AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)" - ) - - return { - "litellm_params": { - "aws_access_key_id": aws_access_key, - "aws_secret_access_key": aws_secret_key, - "aws_region_name": aws_region, - } - } - - def get_custom_llm_provider(self) -> str: - return "bedrock" - - @pytest.mark.asyncio - async def test_count_tokens_basic(self): - """ - Test basic token counting functionality. - - Override to handle models that don't support token counting. - """ - from litellm.types.utils import TokenCountResponse - - token_counter = self.get_token_counter() - model = self.get_test_model() - messages = self.get_test_messages() - deployment = self.get_deployment_config() - - result = await token_counter.count_tokens( - model_to_use=model, - messages=messages, - contents=None, - deployment=deployment, - request_model=model, - ) - - print(f"Token count result: {result}") - - assert result is not None, "Token counter should return a result" - assert isinstance( - result, TokenCountResponse - ), "Result should be TokenCountResponse" - - # Check if the model doesn't support token counting - if result.error and "doesn't support counting tokens" in str( - result.error_message - ): - pytest.skip( - f"Model {model} doesn't support token counting: {result.error_message}" - ) - - assert ( - result.total_tokens > 0 - ), f"Token count should be > 0, got {result.total_tokens}" - assert result.tokenizer_type is not None, "tokenizer_type should be set" - assert ( - result.error is not True - ), f"Token counting should not error: {result.error_message}" diff --git a/tests/llm_responses_api_testing/base_responses_api.py b/tests/llm_responses_api_testing/base_responses_api.py deleted file mode 100644 index 984ac425e74..00000000000 --- a/tests/llm_responses_api_testing/base_responses_api.py +++ /dev/null @@ -1,550 +0,0 @@ -import httpx -import json -import pytest -from typing import Any, Dict, List, Optional -from unittest.mock import MagicMock, Mock, patch -from litellm._uuid import uuid -import time -import base64 - -import litellm -from abc import ABC, abstractmethod - -from litellm.integrations.custom_logger import CustomLogger -from litellm.types.utils import StandardLoggingPayload -from litellm.types.llms.openai import ( - ResponseCompletedEvent, - ResponsesAPIResponse, - ResponseAPIUsage, - IncompleteDetails, -) -from openai.types.responses.response_create_params import ( - ResponseInputParam, -) -from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler -import openai - - -def validate_responses_api_response(response, final_chunk: bool = False): - """ - Validate that a response from litellm.responses() or litellm.aresponses() - conforms to the expected ResponsesAPIResponse structure. - - Args: - response: The response object to validate - - Raises: - AssertionError: If the response doesn't match the expected structure - """ - # Validate response structure - print("response=", json.dumps(response, indent=4, default=str)) - assert isinstance( - response, ResponsesAPIResponse - ), "Response should be an instance of ResponsesAPIResponse" - - # Required fields - assert "id" in response and isinstance( - response["id"], str - ), "Response should have a string 'id' field" - assert "created_at" in response and isinstance( - response["created_at"], int - ), "Response should have an integer 'created_at' field" - if response.get("status") == "completed": - assert "output" in response and isinstance( - response["output"], list - ), "Response should have a list 'output' field" - - # Optional fields with their expected types - optional_fields = { - "error": (dict, type(None)), # error can be dict or None - "incomplete_details": (IncompleteDetails, type(None)), - "instructions": (str, type(None)), - "metadata": dict, - "model": str, - "object": str, - "parallel_tool_calls": (bool, type(None)), - "temperature": (int, float, type(None)), - "tool_choice": (dict, str, type(None)), - "tools": (list, type(None)), - "top_p": (int, float, type(None)), - "max_output_tokens": (int, type(None)), - "previous_response_id": (str, type(None)), - "reasoning": (dict, type(None)), - "status": str, - "text": dict, - "truncation": (str, type(None)), - "usage": ResponseAPIUsage, - "user": (str, type(None)), - "store": (bool, type(None)), - } - if final_chunk is False: - optional_fields["usage"] = type(None) - - for field, expected_type in optional_fields.items(): - if field in response: - assert isinstance( - response[field], expected_type - ), f"Field '{field}' should be of type {expected_type}, but got {type(response[field])}" - - # Check if output has at least one item - if final_chunk is True and response.get("status") == "completed": - assert ( - len(response["output"]) > 0 - ), "Response 'output' field should have at least one item" - - return True # Return True if validation passes - - -class BaseResponsesAPITest(ABC): - """ - Abstract base test class that enforces a common test across all test classes. - """ - - @abstractmethod - def get_base_completion_call_args(self) -> dict: - """Must return the base completion call args""" - pass - - def get_advanced_model_for_shell_tool(self) -> Optional[str]: - """If specified, overrides the model used by test_responses_api_shell_tool_streaming_sees_shell_output (e.g. openai/gpt-5.2 for shell support).""" - return None - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - async def test_basic_openai_responses_api(self, sync_mode): - litellm.turn_on_debug() - litellm.set_verbose = True - base_completion_call_args = self.get_base_completion_call_args() - try: - if sync_mode: - response = litellm.responses( - input="Basic ping", - max_output_tokens=20, - **base_completion_call_args, - ) - else: - response = await litellm.aresponses( - input="Basic ping", - max_output_tokens=20, - **base_completion_call_args, - ) - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") - print("litellm response=", json.dumps(response, indent=4, default=str)) - - # Use the helper function to validate the response - validate_responses_api_response(response, final_chunk=True) - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.asyncio - @pytest.mark.flaky(retries=3, delay=2) - async def test_basic_openai_responses_api_streaming(self, sync_mode): - litellm.turn_on_debug() - # Enable cost calculation for streaming usage - litellm.include_cost_in_streaming_usage = True - base_completion_call_args = self.get_base_completion_call_args() - collected_content_string = "" - response_completed_event = None - if sync_mode: - response = litellm.responses( - input="Basic ping", stream=True, **base_completion_call_args - ) - for event in response: - print("litellm response=", json.dumps(event, indent=4, default=str)) - if event.type == "response.output_text.delta": - collected_content_string += event.delta - elif event.type == "response.completed": - response_completed_event = event - else: - response = await litellm.aresponses( - input="Basic ping", stream=True, **base_completion_call_args - ) - async for event in response: - print("litellm response=", json.dumps(event, indent=4, default=str)) - if event.type == "response.output_text.delta": - collected_content_string += event.delta - elif event.type == "response.completed": - response_completed_event = event - - # assert the response completed event is not None - assert response_completed_event is not None - - # assert the response completed event has a response - assert response_completed_event.response is not None - - # For async agent APIs (like Manus), the response may be in 'running' state - # without content yet - this is valid behavior - response_status = response_completed_event.response.status - if response_status in ["running", "pending"]: - # Running/pending state is acceptable - task started successfully - print( - f"Response is in '{response_status}' state - async agent API behavior" - ) - assert response_completed_event.response.id is not None - else: - # For completed responses, validate content and usage - # assert the delta chunks content had len(collected_content_string) > 0 - # this content is typically rendered on chat ui's - assert len(collected_content_string) > 0 - - # assert the response completed event includes the usage - assert response_completed_event.response.usage is not None - - # basic test assert the usage seems reasonable - print( - "response_completed_event.response.usage=", - response_completed_event.response.usage, - ) - assert ( - response_completed_event.response.usage.input_tokens > 0 - and response_completed_event.response.usage.input_tokens < 100 - ) - assert ( - response_completed_event.response.usage.output_tokens > 0 - and response_completed_event.response.usage.output_tokens < 2000 - ) - assert ( - response_completed_event.response.usage.total_tokens > 0 - and response_completed_event.response.usage.total_tokens < 2000 - ) - - # total tokens should be the sum of input and output tokens - assert ( - response_completed_event.response.usage.total_tokens - == response_completed_event.response.usage.input_tokens - + response_completed_event.response.usage.output_tokens - ) - - # assert the response completed event includes cost when include_cost_in_streaming_usage is True - assert hasattr( - response_completed_event.response.usage, "cost" - ), "Cost should be included in streaming responses API usage object" - assert ( - response_completed_event.response.usage.cost > 0 - ), "Cost should be greater than 0" - print( - f"Cost found in streaming response: {response_completed_event.response.usage.cost}" - ) - - # Reset the setting - litellm.include_cost_in_streaming_usage = False - - @pytest.mark.parametrize("sync_mode", [False, True]) - @pytest.mark.asyncio - async def test_basic_openai_responses_delete_endpoint(self, sync_mode): - litellm.turn_on_debug() - litellm.set_verbose = True - base_completion_call_args = self.get_base_completion_call_args() - if sync_mode: - response = litellm.responses( - input="Basic ping", max_output_tokens=20, **base_completion_call_args - ) - - # delete the response - if isinstance(response, ResponsesAPIResponse): - litellm.delete_responses( - response_id=response.id, **base_completion_call_args - ) - else: - raise ValueError("response is not a ResponsesAPIResponse") - else: - response = await litellm.aresponses( - input="Basic ping", max_output_tokens=20, **base_completion_call_args - ) - - # async delete the response - if isinstance(response, ResponsesAPIResponse): - await litellm.adelete_responses( - response_id=response.id, **base_completion_call_args - ) - else: - raise ValueError("response is not a ResponsesAPIResponse") - - @pytest.mark.parametrize("sync_mode", [True, False]) - @pytest.mark.flaky(retries=3, delay=2) - @pytest.mark.asyncio - async def test_basic_openai_responses_streaming_delete_endpoint(self, sync_mode): - # litellm.turn_on_debug() - # litellm.set_verbose = True - base_completion_call_args = self.get_base_completion_call_args() - response_id = None - if sync_mode: - response_id = None - response = litellm.responses( - input="Basic ping", - max_output_tokens=20, - stream=True, - **base_completion_call_args, - ) - for event in response: - print("litellm response=", json.dumps(event, indent=4, default=str)) - if "response" in event: - response_obj = event.get("response") - if response_obj is not None: - response_id = response_obj.get("id") - print("got response_id=", response_id) - - # delete the response - assert response_id is not None - litellm.delete_responses( - response_id=response_id, **base_completion_call_args - ) - else: - response = await litellm.aresponses( - input="Basic ping", - max_output_tokens=20, - stream=True, - **base_completion_call_args, - ) - async for event in response: - print("litellm response=", json.dumps(event, indent=4, default=str)) - if "response" in event: - response_obj = event.get("response") - if response_obj is not None: - response_id = response_obj.get("id") - print("got response_id=", response_id) - - # delete the response - assert response_id is not None - await litellm.adelete_responses( - response_id=response_id, **base_completion_call_args - ) - - @pytest.mark.parametrize("sync_mode", [False, True]) - @pytest.mark.flaky(retries=3, delay=2) - @pytest.mark.asyncio - async def test_basic_openai_responses_get_endpoint(self, sync_mode): - litellm.turn_on_debug() - litellm.set_verbose = True - base_completion_call_args = self.get_base_completion_call_args() - if sync_mode: - response = litellm.responses( - input="Basic ping", max_output_tokens=20, **base_completion_call_args - ) - - # get the response - if isinstance(response, ResponsesAPIResponse): - result = litellm.get_responses( - response_id=response.id, **base_completion_call_args - ) - assert result is not None - assert result.id == response.id - assert result.output_text == response.output_text - else: - raise ValueError("response is not a ResponsesAPIResponse") - else: - response = await litellm.aresponses( - input="Basic ping", max_output_tokens=20, **base_completion_call_args - ) - # async get the response - if isinstance(response, ResponsesAPIResponse): - result = await litellm.aget_responses( - response_id=response.id, **base_completion_call_args - ) - assert result is not None - assert result.id == response.id - assert result.output_text == response.output_text - else: - raise ValueError("response is not a ResponsesAPIResponse") - - @pytest.mark.asyncio - async def test_responses_api_with_tool_calls(self): - """Test that calls the Responses API with tool calls including function call and output""" - litellm.turn_on_debug() - litellm.set_verbose = True - base_completion_call_args = self.get_base_completion_call_args() - - # Define the input with message, function call, and function call output - input_data: ResponseInputParam = [ - { - "type": "message", - "role": "user", - "content": "How is the weather in São Paulo today ?", - }, - { - "type": "function_call", - "arguments": '{"location": "São Paulo, Brazil"}', - "call_id": "fc_1fe70e2a-a596-45ef-b72c-9b8567c460e5", - "name": "get_weather", - "id": "fc_1fe70e2a-a596-45ef-b72c-9b8567c460e5", - "status": "completed", - }, - { - "type": "function_call_output", - "call_id": "fc_1fe70e2a-a596-45ef-b72c-9b8567c460e5", - "output": "Rainy", - }, - ] - - # Define the tools - tools = [ - { - "type": "function", - "name": "get_weather", - "description": "Get current temperature for a given location.", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "City and country e.g. Bogotá, Colombia", - } - }, - "required": ["location"], - "additionalProperties": False, - }, - } - ] - - try: - # Make the responses API call - response = await litellm.aresponses( - input=input_data, store=False, tools=tools, **base_completion_call_args - ) - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") - - print("litellm response=", json.dumps(response, indent=4, default=str)) - - # Validate the response structure - validate_responses_api_response(response, final_chunk=True) - - # Additional assertions specific to tool calls - assert response is not None - assert "output" in response - # For async agent APIs (like Manus), the response may be in 'running' state - # without output yet - this is valid behavior - if response.get("status") in ["running", "pending"]: - print( - f"Response is in '{response.get('status')}' state - async agent API behavior" - ) - assert response.get("id") is not None - else: - assert len(response["output"]) > 0 - - @pytest.mark.parametrize("sync_mode", [False, True]) - @pytest.mark.flaky(retries=3, delay=2) - @pytest.mark.asyncio - async def test_basic_openai_responses_cancel_endpoint(self, sync_mode): - try: - litellm.turn_on_debug() - litellm.set_verbose = True - base_completion_call_args = self.get_base_completion_call_args() - if sync_mode: - response = litellm.responses( - input="Basic ping", - max_output_tokens=20, - background=True, - **base_completion_call_args, - ) - - # cancel the response - if isinstance(response, ResponsesAPIResponse): - cancel_result = litellm.cancel_responses( - response_id=response.id, **base_completion_call_args - ) - assert cancel_result is not None - assert hasattr(cancel_result, "id") - # The actual response structure depends on the provider implementation - assert isinstance(cancel_result, ResponsesAPIResponse) - else: - raise ValueError("response is not a ResponsesAPIResponse") - else: - response = await litellm.aresponses( - input="Basic ping", - max_output_tokens=20, - background=True, - **base_completion_call_args, - ) - - # async cancel the response - if isinstance(response, ResponsesAPIResponse): - cancel_result = await litellm.acancel_responses( - response_id=response.id, **base_completion_call_args - ) - assert cancel_result is not None - assert hasattr(cancel_result, "id") - # The actual response structure depends on the provider implementation - assert isinstance(cancel_result, ResponsesAPIResponse) - else: - raise ValueError("response is not a ResponsesAPIResponse") - except Exception as e: - if "Cannot cancel a completed response" in str(e): - pass - else: - raise e - - @pytest.mark.asyncio - async def test_responses_api_context_management_server_side_compaction(self): - """ - E2E test for server-side compaction (context_management) on OpenAI Responses API. - Passes context_management with compact_threshold; validates that the request is - accepted and returns a valid response. Compaction may not run for short inputs. - """ - base_completion_call_args = self.get_base_completion_call_args() - model = base_completion_call_args.get("model") or "" - # Azure does not support compaction context_management (only clear_tool_results) - if "azure/" in str(model): - pytest.skip("context_management compaction is not supported on Azure") - if "openai/" not in str(model): - pytest.skip( - "context_management server-side compaction e2e is only run for OpenAI" - ) - context_management = [{"type": "compaction", "compact_threshold": 200000}] - try: - response = await litellm.aresponses( - input="Short ping to verify context_management is accepted.", - max_output_tokens=20, - context_management=context_management, - **base_completion_call_args, - ) - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") - validate_responses_api_response(response, final_chunk=True) - assert response.get("id") is not None - assert response.get("status") is not None - - @pytest.mark.asyncio - async def test_responses_api_shell_tool(self): - """ - E2E test for Shell tool on OpenAI Responses API. - Passes tools=[{"type": "shell", "environment": {"type": "container_auto"}}]; - validates that the request is accepted and returns a valid response. - Only runs for OpenAI; offline coverage for the Azure route lives in - tests/unit/responses/test_responses_api_request_body.py. - """ - base_completion_call_args = self.get_base_completion_call_args() - model = ( - self.get_advanced_model_for_shell_tool() - or base_completion_call_args.get("model") - or "" - ) - if "openai/" not in str(model): - pytest.skip( - "Shell tool e2e is OpenAI-only; no Azure deployment supports the shell tool yet, re-enable once one exists" - ) - tools = [{"type": "shell", "environment": {"type": "container_auto"}}] - input_msg = "List files in /mnt/data and show python --version." - try: - response = await litellm.aresponses( - **{**base_completion_call_args, "model": model}, - input=input_msg, - max_output_tokens=256, - tools=tools, - tool_choice="auto", - timeout=90, - ) - except litellm.Timeout: - pytest.skip("Provider did not answer the shell tool request within 90s") - except litellm.InternalServerError: - pytest.skip("Skipping test due to litellm.InternalServerError") - except litellm.BadRequestError as e: - if "shell" in str(e).lower() and "not supported" in str(e).lower(): - pytest.skip( - "Shell tool is not supported for this model (e.g. gpt-5.5); use a model that supports shell" - ) - raise - validate_responses_api_response(response, final_chunk=True) - assert response.get("id") is not None - assert response.get("status") is not None - diff --git a/tests/llm_responses_api_testing/conftest.py b/tests/llm_responses_api_testing/conftest.py deleted file mode 100644 index 5501d99cb22..00000000000 --- a/tests/llm_responses_api_testing/conftest.py +++ /dev/null @@ -1,112 +0,0 @@ -# conftest.py - -import asyncio -import importlib - -import pytest - - -import litellm # noqa: E402 - -from tests._vcr_conftest_common import ( # noqa: E402,F401 - VerboseReporterState, - _pin_multipart_boundary, - apply_vcr_auto_marker_to_items, - emit_cassette_cache_session_banner, - emit_vcr_classification_summary, - emit_vcr_diagnostic_log, - install_live_call_probe, - record_vcr_outcome, - register_persister_if_enabled, - reset_vcr_diag_dir, - vcr_config_dict, -) - -_verbose_state = VerboseReporterState() - - -@pytest.fixture(scope="module") -def vcr_config(): - return vcr_config_dict() - - -def pytest_recording_configure(config, vcr): - register_persister_if_enabled(vcr) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_runtest_makereport(item, call): - outcome = yield - rep = outcome.get_result() - setattr(item, f"rep_{rep.when}", rep) - - -@pytest.fixture(autouse=True) -def _vcr_outcome_gate(request, vcr): - install_live_call_probe(request, vcr) - yield - record_vcr_outcome(request, vcr) - - -def pytest_configure(config): - _verbose_state.remember_pluginmanager(config) - reset_vcr_diag_dir() - - -def pytest_runtest_logreport(report): - _verbose_state.maybe_emit_verdict(report) - - -@pytest.fixture(scope="session") -def event_loop(): - try: - loop = asyncio.get_running_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - yield loop - loop.close() - - -@pytest.fixture(scope="function", autouse=True) -def setup_and_teardown(): - """ - This fixture reloads litellm before every function. To speed up testing by removing callbacks being chained. - """ - - - importlib.reload(litellm) - - try: - if hasattr(litellm, "proxy") and hasattr(litellm.proxy, "proxy_server"): - importlib.reload(litellm.proxy.proxy_server) - except Exception as e: - print(f"Error reloading litellm.proxy.proxy_server: {e}") - - loop = asyncio.get_event_loop_policy().new_event_loop() - asyncio.set_event_loop(loop) - print(litellm) - yield - - # Teardown code (executes after the yield point) - loop.close() # Close the loop created earlier - asyncio.set_event_loop(None) # Remove the reference to the loop - - -def pytest_collection_modifyitems(config, items): - apply_vcr_auto_marker_to_items(items) - - custom_logger_tests = [ - item for item in items if "custom_logger" in item.parent.name - ] - other_tests = [item for item in items if "custom_logger" not in item.parent.name] - - custom_logger_tests.sort(key=lambda x: x.name) - other_tests.sort(key=lambda x: x.name) - - items[:] = custom_logger_tests + other_tests - - -def pytest_terminal_summary(terminalreporter, exitstatus, config): - emit_cassette_cache_session_banner(terminalreporter) - emit_vcr_classification_summary(terminalreporter) - emit_vcr_diagnostic_log(terminalreporter) diff --git a/tests/llm_responses_api_testing/test_anthropic_responses_api.py b/tests/llm_responses_api_testing/test_anthropic_responses_api.py deleted file mode 100644 index 15d1a81bc68..00000000000 --- a/tests/llm_responses_api_testing/test_anthropic_responses_api.py +++ /dev/null @@ -1,90 +0,0 @@ -import litellm -from base_responses_api import BaseResponsesAPITest -from openai.types.responses.function_tool import FunctionTool - - -class TestAnthropicResponsesAPITest(BaseResponsesAPITest): - test_basic_openai_responses_delete_endpoint = None - test_basic_openai_responses_streaming_delete_endpoint = None - test_basic_openai_responses_get_endpoint = None - test_basic_openai_responses_cancel_endpoint = None - - def get_base_completion_call_args(self): - # litellm.turn_on_debug() - return { - "model": "anthropic/claude-sonnet-4-5", - } - -def test_multiturn_tool_calls(): - # Test streaming response with tools for Anthropic - litellm.turn_on_debug() - shell_tool = dict( - FunctionTool( - type="function", - name="shell", - description="Runs a shell command, and returns its output.", - parameters={ - "type": "object", - "properties": { - "command": {"type": "array", "items": {"type": "string"}}, - "workdir": { - "type": "string", - "description": "The working directory for the command.", - }, - }, - "required": ["command"], - }, - strict=True, - ) - ) - - # Step 1: Initial request with the tool - response = litellm.responses( - input=[ - { - "role": "user", - "content": [ - {"type": "input_text", "text": "make a hello world html file"} - ], - "type": "message", - } - ], - model="anthropic/claude-haiku-4-5-20251001", - instructions="You are a helpful coding assistant.", - tools=[shell_tool], - ) - - print("response=", response) - - # Step 2: Send the results of the tool call back to the model - # Get the response ID and tool call ID from the response - - response_id = response.id - tool_call_id = None - for item in response.output: - if hasattr(item, "type") and item.type == "function_call": - tool_call_id = getattr(item, "call_id", None) - if tool_call_id: - break - - # Validate that we got a tool call with a valid call_id - if not tool_call_id: - raise AssertionError( - f"Expected a function_call with a valid call_id in response.output, but got: {response.output}" - ) - - # Use await with asyncio.run for the async function - follow_up_response = litellm.responses( - model="anthropic/claude-haiku-4-5-20251001", - previous_response_id=response_id, - input=[ - { - "type": "function_call_output", - "call_id": tool_call_id, - "output": '{"output":"\\n\\n Hello Page\\n\\n\\n

    Hi

    \\n

    Welcome to this simple webpage!

    \\n\\n > index.html\\n","metadata":{"exit_code":0,"duration_seconds":0}}', - } - ], - tools=[shell_tool], - ) - - print("follow_up_response=", follow_up_response) diff --git a/tests/llm_responses_api_testing/test_azure_responses_api.py b/tests/llm_responses_api_testing/test_azure_responses_api.py deleted file mode 100644 index 3007b315859..00000000000 --- a/tests/llm_responses_api_testing/test_azure_responses_api.py +++ /dev/null @@ -1,34 +0,0 @@ -import os -import pytest - -import litellm -from base_responses_api import BaseResponsesAPITest - - -class TestAzureResponsesAPITest(BaseResponsesAPITest): - test_responses_api_with_tool_calls = None - - def get_base_completion_call_args(self): - return { - "model": "azure/gpt-4.1-mini", - "truncation": "auto", - "api_base": os.getenv("AZURE_AI_API_BASE"), - "api_key": os.getenv("AZURE_AI_API_KEY"), - "api_version": "2025-03-01-preview", - } - - -@pytest.mark.asyncio -async def test_azure_responses_api_preview_api_version(): - """ - Ensure new azure preview api version is working - """ - litellm.turn_on_debug() - response = await litellm.aresponses( - model="azure/gpt-5-mini", - truncation="auto", - api_version="preview", - api_base=os.getenv("AZURE_AI_API_BASE"), - api_key=os.getenv("AZURE_AI_API_KEY"), - input="Hello, can you tell me a short joke?", - ) diff --git a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py b/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py deleted file mode 100644 index 01bfbc3be35..00000000000 --- a/tests/llm_responses_api_testing/test_google_ai_studio_responses_api.py +++ /dev/null @@ -1,12 +0,0 @@ -from base_responses_api import BaseResponsesAPITest - - -class TestGoogleAIStudioResponsesAPITest(BaseResponsesAPITest): - test_basic_openai_responses_delete_endpoint = None - test_basic_openai_responses_streaming_delete_endpoint = None - test_basic_openai_responses_get_endpoint = None - test_basic_openai_responses_cancel_endpoint = None - - def get_base_completion_call_args(self): - # litellm.turn_on_debug() - return {"model": "gemini/gemini-2.5-flash-lite"} diff --git a/tests/llm_responses_api_testing/test_openai_responses_api.py b/tests/llm_responses_api_testing/test_openai_responses_api.py deleted file mode 100644 index bbdc7d3ad24..00000000000 --- a/tests/llm_responses_api_testing/test_openai_responses_api.py +++ /dev/null @@ -1,64 +0,0 @@ -import json - -import pytest -from base_responses_api import BaseResponsesAPITest - -import litellm - - -class TestOpenAIResponsesAPITest(BaseResponsesAPITest): - test_responses_api_with_tool_calls = None - - def get_base_completion_call_args(self): - return { - "model": "openai/gpt-5.5", - } - - def get_advanced_model_for_shell_tool(self): - return "openai/gpt-5.2" - - -@pytest.mark.asyncio -@pytest.mark.parametrize("sync_mode", [True, False]) -async def test_openai_compact_responses_api(sync_mode): - """ - Test the compact_responses API for OpenAI. - - This test verifies that the compact_responses endpoint works correctly - for compressing conversation history. - """ - litellm.turn_on_debug() - litellm.set_verbose = True - - input_messages = [ - {"role": "user", "content": "Hello, how are you?"}, - {"role": "assistant", "content": "I'm doing well, thank you for asking!"}, - {"role": "user", "content": "What is the weather like today?"}, - ] - - try: - if sync_mode: - response = litellm.compact_responses( - model="openai/gpt-5.5", - input=input_messages, - instructions="Be helpful and concise", - ) - else: - response = await litellm.acompact_responses( - model="openai/gpt-5.5", - input=input_messages, - instructions="Be helpful and concise", - ) - except litellm.InternalServerError: - pytest.skip("Skipping test due to InternalServerError") - except litellm.BadRequestError as e: - # compact_responses may not be available for all models/accounts - pytest.skip(f"Skipping test due to BadRequestError: {e}") - - print("compact_responses response=", json.dumps(response, indent=4, default=str)) - - # Validate response structure - assert response is not None - assert "id" in response, "Response should have an 'id' field" - assert "output" in response, "Response should have an 'output' field" - assert isinstance(response["output"], list), "Output should be a list" diff --git a/tests/llm_translation/Readme.md b/tests/llm_translation/Readme.md index f0a32f6c989..aef28d38782 100644 --- a/tests/llm_translation/Readme.md +++ b/tests/llm_translation/Readme.md @@ -26,7 +26,6 @@ provider APIs. The reusable conftest plumbing lives in `tests/_vcr_conftest_common.py` and is wired into: - `tests/llm_translation/` -- `tests/llm_responses_api_testing/` - `tests/audio_tests/` - `tests/batches_tests/` - `tests/guardrails_tests/` diff --git a/tests/llm_translation/base_llm_unit_tests.py b/tests/llm_translation/base_llm_unit_tests.py index e744b6275a8..e4ccb589ff4 100644 --- a/tests/llm_translation/base_llm_unit_tests.py +++ b/tests/llm_translation/base_llm_unit_tests.py @@ -1564,25 +1564,6 @@ class BaseAnthropicChatTest(ABC): print(response) - def test_completion_with_thinking_basic(self): - litellm.turn_on_debug() - base_completion_call_args = self.get_base_completion_call_args_with_thinking() - - messages = [{"role": "user", "content": "Generate 5 question + answer pairs"}] - response = self.completion_function( - **base_completion_call_args, - messages=messages, - ) - - print(f"response: {response}") - assert response.choices[0].message.reasoning_content is not None - assert isinstance(response.choices[0].message.reasoning_content, str) - assert response.choices[0].message.thinking_blocks is not None - assert isinstance(response.choices[0].message.thinking_blocks, list) - assert len(response.choices[0].message.thinking_blocks) > 0 - - assert response.choices[0].message.thinking_blocks[0]["signature"] is not None - def test_anthropic_thinking_output_stream(self): # litellm.set_verbose = True try: diff --git a/tests/llm_translation/realtime/test_openai_realtime.py b/tests/llm_translation/realtime/test_openai_realtime.py deleted file mode 100644 index f137e9df49b..00000000000 --- a/tests/llm_translation/realtime/test_openai_realtime.py +++ /dev/null @@ -1,138 +0,0 @@ -import os - -import pytest -from websockets.exceptions import ConnectionClosedError, ConnectionClosedOK - -import litellm - - -@pytest.mark.asyncio -@pytest.mark.skipif( - os.environ.get("OPENAI_API_KEY", None) is None, - reason="No OpenAI API key provided", -) -async def test_openai_realtime_direct_call_no_intent(): - """ - End-to-end test calling the actual OpenAI realtime endpoint via LiteLLM SDK - without intent parameter. This should succeed without "Invalid intent" error. - Uses real websocket connection to OpenAI. - """ - import asyncio - import json - - class RealTimeWebSocketClient: - def __init__(self): - self.messages_sent = [] - self.messages_received = [] - self.received_session_created = False - self.connection_successful = False - self._receive_called = False - self.close_code = None - self.close_reason = None - - async def accept(self): - pass - - async def send_text(self, message): - self.messages_sent.append(message) - try: - if isinstance(message, bytes): - message_str = message.decode("utf-8") - else: - message_str = message - - msg_data = json.loads(message_str) - msg_type = msg_data.get("type", "unknown") - - if msg_type == "error": - error_info = msg_data.get("error", {}) - error_code = error_info.get("code", "unknown") - error_message = error_info.get("message", "unknown") - # Don't fail on error, just record it - some errors are expected - self.messages_received.append(msg_data) - return - - if msg_type == "session.created" and not self.received_session_created: - self.messages_received.append(msg_data) - self.received_session_created = True - self.connection_successful = True - except (json.JSONDecodeError, UnicodeDecodeError): - # Non-JSON messages are acceptable - pass - - async def receive_text(self): - if not self._receive_called: - self._receive_called = True - max_wait = 60.0 - check_interval = 0.1 - waited = 0.0 - - while waited < max_wait: - if self.connection_successful: - break - await asyncio.sleep(check_interval) - waited += check_interval - - if not self.connection_successful: - await asyncio.sleep(3.0) - - raise ConnectionClosedOK(None, None) - - async def close(self, code=1000, reason=""): - self.close_code = code - self.close_reason = reason - - @property - def headers(self): - return {} - - websocket_client = RealTimeWebSocketClient() - caught_exception = None - - try: - await litellm._arealtime( - # OpenAI shut down the gpt-4o-realtime-preview family (incl. the - # undated alias) on 2026-05-07; gpt-realtime is the GA successor. - model="openai/gpt-realtime", - websocket=websocket_client, - api_key=os.environ.get("OPENAI_API_KEY"), - timeout=60, - ) - except (ConnectionClosedOK, ConnectionClosedError): - pass - except Exception as e: - caught_exception = e - if "invalid_intent" in str(e).lower(): - pytest.fail(f"Still getting invalid intent error: {e}") - # Other exceptions are recorded but don't fail immediately - - # Build detailed error message for debugging - error_details = [] - error_details.append(f"messages_sent count: {len(websocket_client.messages_sent)}") - error_details.append( - f"messages_received count: {len(websocket_client.messages_received)}" - ) - error_details.append(f"close_code: {websocket_client.close_code}") - error_details.append(f"close_reason: {websocket_client.close_reason}") - if caught_exception: - error_details.append( - f"exception: {type(caught_exception).__name__}: {caught_exception}" - ) - - assert ( - websocket_client.connection_successful - ), f"Failed to establish connection. Debug info: {'; '.join(error_details)}" - assert ( - websocket_client.received_session_created - ), "Did not receive session.created response" - assert len(websocket_client.messages_received) > 0, "No messages received" - - session_message = websocket_client.messages_received[0] - assert ( - session_message["type"] == "session.created" - ), f"Expected session.created, got {session_message.get('type')}" - assert ( - "session" in session_message - ), "session.created response missing session object" - assert "id" in session_message["session"], "Session object missing id field" - assert "model" in session_message["session"], "Session object missing model field" diff --git a/tests/llm_translation/test_anthropic_completion.py b/tests/llm_translation/test_anthropic_completion.py index 5f17b44fd3d..af7ba989b07 100644 --- a/tests/llm_translation/test_anthropic_completion.py +++ b/tests/llm_translation/test_anthropic_completion.py @@ -494,34 +494,6 @@ def test_anthropic_citations_api_streaming(): assert has_citations -@pytest.mark.parametrize( - "model", - [ - "anthropic/claude-sonnet-4-5-20250929", - "bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - ], -) -def test_anthropic_thinking_output(model): - - litellm.turn_on_debug() - - resp = completion( - model=model, - messages=[{"role": "user", "content": "What is the capital of France?"}], - thinking={"type": "enabled", "budget_tokens": 1024}, - ) - - print(resp) - assert resp.choices[0].message.reasoning_content is not None - assert isinstance(resp.choices[0].message.reasoning_content, str) - assert resp.choices[0].message.thinking_blocks is not None - assert isinstance(resp.choices[0].message.thinking_blocks, list) - assert len(resp.choices[0].message.thinking_blocks) > 0 - - assert resp.choices[0].message.thinking_blocks[0]["type"] == "thinking" - assert resp.choices[0].message.thinking_blocks[0]["signature"] is not None - - @pytest.mark.parametrize( "model", [ diff --git a/tests/llm_translation/test_bedrock_completion.py b/tests/llm_translation/test_bedrock_completion.py index 6d8e8a431db..768be9b1080 100644 --- a/tests/llm_translation/test_bedrock_completion.py +++ b/tests/llm_translation/test_bedrock_completion.py @@ -279,85 +279,6 @@ def test_bedrock_system_prompt(system, model): pytest.fail(f"Error occurred: {e}") -def test_bedrock_claude_3_tool_calling(): - try: - litellm.set_verbose = True - litellm.turn_on_debug() - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": { - "type": "string", - "enum": ["celsius", "fahrenheit"], - }, - }, - "required": ["location"], - }, - }, - } - ] - messages = [ - { - "role": "user", - "content": "What's the weather like in Boston today in fahrenheit?", - } - ] - response: ModelResponse = completion( - model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - messages=messages, - tools=tools, - tool_choice="auto", - ) # type: ignore - print(f"response: {response}") - # Add any assertions here to check the response - assert isinstance(response.choices[0].message.tool_calls[0].function.name, str) - assert isinstance( - response.choices[0].message.tool_calls[0].function.arguments, str - ) - messages.append( - response.choices[0].message.model_dump() - ) # Add assistant tool invokes - tool_result = ( - '{"location": "Boston", "temperature": "72", "unit": "fahrenheit"}' - ) - # Add user submitted tool results in the OpenAI format - messages.append( - { - "tool_call_id": response.choices[0].message.tool_calls[0].id, - "role": "tool", - "name": response.choices[0].message.tool_calls[0].function.name, - "content": tool_result, - } - ) - # In the second response, Claude should deduce answer from tool results - second_response = completion( - model="bedrock/us.anthropic.claude-sonnet-4-5-20250929-v1:0", - messages=messages, - tools=tools, - tool_choice="auto", - ) - print(f"second response: {second_response}") - assert isinstance(second_response.choices[0].message.content, str) - except RateLimitError: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - - - - - def test_completion_bedrock_mistral_completion_auth(): print("calling bedrock mistral completion params auth") diff --git a/tests/local_testing/test_caching.py b/tests/local_testing/test_caching.py index b8c7a4c189b..f4c5382b8a2 100644 --- a/tests/local_testing/test_caching.py +++ b/tests/local_testing/test_caching.py @@ -2277,56 +2277,3 @@ def test_caching_thinking_args_hit(): # test in memory cache except Exception as e: print(f"error occurred: {traceback.format_exc()}") pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.asyncio -async def test_cache_key_in_hidden_params_acompletion(): - """ - Test that cache_key is present in _hidden_params on cache hits for acompletion. - - Validates fix for missing x-litellm-cache-key header on proxy cache hits. - """ - litellm.cache = Cache( - type="redis", - host=os.environ["REDIS_HOST"], - port=os.environ["REDIS_PORT"], - password=os.environ["REDIS_PASSWORD"], - ) - - unique_content = f"test cache key hidden params {uuid.uuid4()}" - messages = [{"role": "user", "content": unique_content}] - - # First call - cache miss - response1 = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=messages, - mock_response="test response", - caching=True, - ) - - print(f"Response 1 _hidden_params: {response1._hidden_params}") - assert response1._hidden_params.get("cache_hit") is not True - - await asyncio.sleep(0.5) - - # Second call - cache hit - response2 = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=messages, - mock_response="test response", - caching=True, - ) - - print(f"Response 2 _hidden_params: {response2._hidden_params}") - - # Verify cache hit occurred - assert response2._hidden_params.get("cache_hit") is True - - # Verify cache_key is present in _hidden_params - assert "cache_key" in response2._hidden_params - assert response2._hidden_params["cache_key"] is not None - - # Verify both responses have same ID (cache hit) - assert response1.id == response2.id - - litellm.cache = None diff --git a/tests/local_testing/test_completion.py b/tests/local_testing/test_completion.py index 78e185c3b37..b5703224eb1 100644 --- a/tests/local_testing/test_completion.py +++ b/tests/local_testing/test_completion.py @@ -1413,28 +1413,6 @@ def test_replicate_custom_prompt_dict(): # test_completion_together_ai_mixtral() -def test_completion_together_ai_llama(): - litellm.set_verbose = True - model_name = "together_ai/meta-llama/Llama-3.3-70B-Instruct-Turbo" - try: - messages = [ - {"role": "user", "content": "What llm are you?"}, - ] - response = completion(model=model_name, messages=messages, max_tokens=5) - # Add any assertions here to check the response - print(response) - cost = completion_cost(completion_response=response) - assert cost > 0.0 - print( - "Cost for completion call together-computer/llama-2-70b: ", - f"${float(cost):.10f}", - ) - except litellm.Timeout as e: - pass - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - # test_completion_together_ai_yi_chat() @@ -1903,25 +1881,6 @@ def test_langfuse_completion(monkeypatch): ) - - - - -def test_deepseek_reasoning_content_completion(): - try: - litellm.set_verbose = True - litellm.turn_on_debug() - resp = litellm.completion( - timeout=5, - model="deepseek/deepseek-reasoner", - messages=[{"role": "user", "content": "Tell me a joke."}], - ) - - assert resp.choices[0].message.reasoning_content is not None - except litellm.Timeout: - pytest.skip("Model is timing out") - - def test_qwen_text_completion(): # litellm.turn_on_debug() resp = litellm.completion( diff --git a/tests/local_testing/test_custom_logger.py b/tests/local_testing/test_custom_logger.py index 192a73a68ab..b721b45a8bf 100644 --- a/tests/local_testing/test_custom_logger.py +++ b/tests/local_testing/test_custom_logger.py @@ -365,54 +365,6 @@ async def test_async_custom_handler_embedding_optional_param(): # asyncio.run(test_async_custom_handler_embedding_optional_param()) - - -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_cost_tracking_with_caching(): - """ - Important Test - This tests if that cost is 0 for cached responses - """ - from litellm import Cache - - litellm.set_verbose = True - litellm.cache = Cache( - type="redis", - host=os.environ["REDIS_HOST"], - port=os.environ["REDIS_PORT"], - password=os.environ["REDIS_PASSWORD"], - ) - customHandler_optional_params = MyCustomHandler() - litellm.callbacks = [customHandler_optional_params] - messages = [ - { - "role": "user", - "content": f"write a one sentence poem about: {time.time()}", - } - ] - response1 = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=messages, - max_tokens=40, - temperature=0.2, - caching=True, - mock_response="Hey, i'm doing well!", - ) - await asyncio.sleep(3) # success callback is async - response_cost = customHandler_optional_params.response_cost - assert response_cost > 0 - response2 = await litellm.acompletion( - model="gpt-3.5-turbo", - messages=messages, - max_tokens=40, - temperature=0.2, - caching=True, - ) - await asyncio.sleep(1) # success callback is async - response_cost_2 = customHandler_optional_params.response_cost - assert response_cost_2 == 0 - - @pytest.mark.flaky(retries=3, delay=3) def test_redis_cache_completion_stream(): # Important Test - This tests if we can add to streaming cache, when custom callbacks are set diff --git a/tests/local_testing/test_lunary.py b/tests/local_testing/test_lunary.py deleted file mode 100644 index 4312abcc0a4..00000000000 --- a/tests/local_testing/test_lunary.py +++ /dev/null @@ -1,57 +0,0 @@ -import io - - -import litellm - -litellm.failure_callback = ["lunary"] -litellm.success_callback = ["lunary"] -litellm.set_verbose = True - - - - - - - - -def test_lunary_with_tools(): - import litellm - - messages = [ - { - "role": "user", - "content": "What's the weather like in San Francisco, Tokyo, and Paris?", - } - ] - tools = [ - { - "type": "function", - "function": { - "name": "get_current_weather", - "description": "Get the current weather in a given location", - "parameters": { - "type": "object", - "properties": { - "location": { - "type": "string", - "description": "The city and state, e.g. San Francisco, CA", - }, - "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}, - }, - "required": ["location"], - }, - }, - } - ] - - response = litellm.completion( - model="gpt-6-luna", - messages=messages, - tools=tools, - tool_choice="auto", # auto is default, but we'll be explicit - ) - - response_message = response.choices[0].message - assert response.choices[0].message.tool_calls - assert all(call.function.name == "get_current_weather" for call in response.choices[0].message.tool_calls) - print("\nLLM Response:\n", response.choices[0].message) diff --git a/tests/local_testing/test_text_completion.py b/tests/local_testing/test_text_completion.py index d62b141d636..a117ca975aa 100644 --- a/tests/local_testing/test_text_completion.py +++ b/tests/local_testing/test_text_completion.py @@ -2729,19 +2729,6 @@ def test_completion_openai_engine() -> None: # test_completion_openai_engine() -def test_completion_chatgpt_prompt(): - try: - print("\n gpt3.5 test\n") - response = text_completion(model="openai/gpt-3.5-turbo", prompt="What's the weather in SF?") - print(response) - response_str = response["choices"][0]["text"] - print("\n", response.choices) - print("\n", response.choices[0]) - # print(response.choices[0].text) - except Exception as e: - pytest.fail(f"Error occurred: {e}") - - # test_completion_chatgpt_prompt() diff --git a/tests/pass_through_tests/test_gemini_with_spend.test.js b/tests/pass_through_tests/test_gemini_with_spend.test.js index f59d575c033..f184bbe14e3 100644 --- a/tests/pass_through_tests/test_gemini_with_spend.test.js +++ b/tests/pass_through_tests/test_gemini_with_spend.test.js @@ -24,57 +24,6 @@ global.fetch = async function patchedFetch(url, options) { jest.retryTimes(3); describe('Gemini AI Tests', () => { - test('should successfully generate non-streaming content with tags', async () => { - const genAI = new GoogleGenerativeAI(masterKey); - - const requestOptions = { - baseUrl: 'http://127.0.0.1:4000/gemini', - customHeaders: { - "tags": "gemini-js-sdk,pass-through-endpoint" - } - }; - - const model = genAI.getGenerativeModel({ - model: 'gemini-3.1-flash-lite' - }, requestOptions); - - const prompt = 'Say "hello test" and nothing else'; - - const result = await model.generateContent(prompt); - expect(result).toBeDefined(); - - // Use the captured callId - const callId = lastCallId; - console.log("Captured Call ID:", callId); - - // Poll for spend data with retries (DB writes can be slow in CI) - let spendData = null; - for (let attempt = 0; attempt < 6; attempt++) { - await new Promise(resolve => setTimeout(resolve, 10000)); - const spendResponse = await fetch( - `http://127.0.0.1:4000/spend/logs?request_id=${callId}`, - { headers: { 'Authorization': `Bearer ${masterKey}` } } - ); - spendData = await spendResponse.json(); - console.log(`spendData (attempt ${attempt + 1}):`, spendData); - if (spendData && spendData.length > 0 && spendData[0] && spendData[0].request_id) break; - } - - if (!spendData || !spendData.length || !spendData[0] || !spendData[0].request_id) { - console.warn('Spend data not available after polling - skipping spend assertions (DB write may be slow in CI)'); - return; - } - - expect(spendData).toBeDefined(); - expect(spendData[0].request_id).toBe(callId); - expect(spendData[0].call_type).toBe('pass_through_endpoint'); - expect(spendData[0].request_tags).toEqual(['gemini-js-sdk', 'pass-through-endpoint']); - expect(spendData[0].metadata).toHaveProperty('user_api_key'); - expect(spendData[0].model).toContain('gemini'); - expect(spendData[0].custom_llm_provider).toBe('gemini'); - expect(spendData[0].spend).toBeGreaterThan(0); - }, 90000); - test('should successfully generate streaming content with tags', async () => { const genAI = new GoogleGenerativeAI(masterKey); diff --git a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py index 001de2f7d52..f0d03ca77d6 100644 --- a/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py +++ b/tests/pass_through_unit_tests/test_anthropic_messages_passthrough.py @@ -1,13 +1,9 @@ -import json import os from datetime import datetime from typing import Dict, Any -import asyncio import unittest.mock from unittest.mock import MagicMock -import litellm -import pytest from dotenv import load_dotenv from litellm.llms.anthropic.pass_through.messages.handler import ( anthropic_messages, @@ -16,47 +12,12 @@ from litellm.llms.anthropic.pass_through.messages.handler import ( from typing import Optional from litellm.types.utils import StandardLoggingPayload from litellm.integrations.custom_logger import CustomLogger -from litellm.router import Router -import importlib from base_anthropic_unified_messages_test import BaseAnthropicMessagesTest # Load environment variables load_dotenv() -@pytest.fixture(scope="session") -def event_loop(): - """Create an instance of the default event loop for each test session.""" - loop = asyncio.get_event_loop_policy().new_event_loop() - yield loop - loop.close() - - -@pytest.fixture(scope="function", autouse=True) -def setup_and_teardown(event_loop): # Add event_loop as a dependency - curr_dir = os.getcwd() - - import litellm - from litellm import Router - - importlib.reload(litellm) - - # Set the event loop from the fixture - asyncio.set_event_loop(event_loop) - - print(litellm) - yield - - # Clean up any pending tasks - pending = asyncio.all_tasks(event_loop) - for task in pending: - task.cancel() - - # Run the event loop until all tasks are cancelled - if pending: - event_loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) - - def _validate_anthropic_response(response: Dict[str, Any]): assert "id" in response assert "content" in response @@ -117,42 +78,3 @@ class TestAnthropicOpenAIAPI(BaseAnthropicMessagesTest): This is the model name that is expected to be in the logging payload """ return "gpt-4.1-mini" - - -@pytest.mark.asyncio -async def test_anthropic_messages_litellm_router_non_streaming(): - """ - Test the anthropic_messages with non-streaming request - """ - litellm.turn_on_debug() - router = Router( - model_list=[ - { - "model_name": "claude-special-alias", - "litellm_params": { - "model": "claude-haiku-4-5-20251001", - "api_key": os.getenv("ANTHROPIC_API_KEY"), - }, - } - ] - ) - - # Set up test parameters - messages = [{"role": "user", "content": "Hello, can you tell me a short joke?"}] - - # Call the handler - response = await router.aanthropic_messages( - messages=messages, - model="claude-special-alias", - max_tokens=100, - ) - - # Verify response - assert "id" in response - assert "content" in response - assert "model" in response - assert response["role"] == "assistant" - - print(f"Non-streaming response: {json.dumps(response, indent=2)}") - return response - diff --git a/tests/test_end_users.py b/tests/test_end_users.py index 03fb4c9e86c..fab35d52e32 100644 --- a/tests/test_end_users.py +++ b/tests/test_end_users.py @@ -109,16 +109,6 @@ async def new_budget(session, i, budget_id=None): print() -@pytest.mark.asyncio -async def test_end_user_new(): - """ - Make 20 parallel calls to /user/new. Assert all worked. - """ - async with aiohttp.ClientSession() as session: - tasks = [new_end_user(session, i, str(uuid.uuid4())) for i in range(1, 11)] - await asyncio.gather(*tasks) - - @pytest.mark.asyncio async def test_enduser_tpm_limits_non_master_key(): """ diff --git a/tests/test_keys.py b/tests/test_keys.py index 835c1f12250..4e9734c60b6 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -183,33 +183,6 @@ async def chat_completion(session, key, model="gpt-4"): pass -async def chat_completion_streaming(session, key, model="gpt-4"): - client = AsyncOpenAI(api_key=key, base_url="http://0.0.0.0:4000") - messages = [ - {"role": "system", "content": "You are a helpful assistant"}, - {"role": "user", "content": "Hello!"}, - ] - prompt_tokens = litellm.token_counter(model="gpt-35-turbo", messages=messages) - data = { - "model": model, - "messages": messages, - "stream": True, - } - response = await client.chat.completions.create(**data) - - content = "" - async for chunk in response: - content += chunk.choices[0].delta.content or "" - - print(f"content: {content}") - - completion_tokens = litellm.token_counter( - model="gpt-35-turbo", text=content, count_response_tokens=True - ) - - return prompt_tokens, completion_tokens - - async def delete_key(session, get_key, auth_key=os.environ["LITELLM_MASTER_KEY"]): """ Delete key @@ -367,47 +340,6 @@ async def get_spend_logs(session, request_id): return await response.json() -@pytest.mark.asyncio -@pytest.mark.flaky(retries=6, delay=2) -@pytest.mark.skip( - reason="Temporarily skipping due to model change. Will be updated soon." -) -async def test_aaaaakey_info_spend_values_streaming(): - """ - Test to ensure spend is correctly calculated. - - create key - - make completion call - - assert cost is expected value - """ - async with aiohttp.ClientSession() as session: - ## streaming - azure - key_gen = await generate_key(session=session, i=0) - new_key = key_gen["key"] - prompt_tokens, completion_tokens = await chat_completion_streaming( - session=session, key=new_key - ) - print(f"prompt_tokens: {prompt_tokens}, completion_tokens: {completion_tokens}") - prompt_cost, completion_cost = litellm.cost_per_token( - model="azure/gpt-4o", - prompt_tokens=prompt_tokens, - completion_tokens=completion_tokens, - ) - response_cost = prompt_cost + completion_cost - await asyncio.sleep(8) # allow db log to be updated - print(f"new_key: {new_key}") - key_info = await get_key_info( - session=session, get_key=new_key, call_key=new_key - ) - print( - f"response_cost: {response_cost}; key_info spend: {key_info['info']['spend']}" - ) - rounded_response_cost = round(response_cost, 8) - rounded_key_info_spend = round(key_info["info"]["spend"], 8) - assert ( - rounded_response_cost == rounded_key_info_spend - ), f"Expected={rounded_response_cost}, Got={rounded_key_info_spend}" - - @pytest.mark.skip(reason="Frequent check on ci/cd leads to read timeout issue.") @pytest.mark.asyncio async def test_key_with_budgets(): diff --git a/tests/test_new_vector_store_endpoints.py b/tests/test_new_vector_store_endpoints.py index c44723937ac..05e1b25bc8b 100644 --- a/tests/test_new_vector_store_endpoints.py +++ b/tests/test_new_vector_store_endpoints.py @@ -13,82 +13,6 @@ import litellm from litellm.proxy._types import UserAPIKeyAuth -@pytest.mark.asyncio -async def test_vector_store_retrieve_basic(): - """Test basic vector store retrieve functionality.""" - mock_response = { - "id": "vs_test123", - "object": "vector_store", - "created_at": 1699061776, - "name": "Test Vector Store", - "file_counts": { - "in_progress": 0, - "completed": 5, - "failed": 0, - "cancelled": 0, - "total": 5, - }, - "status": "completed", - "usage_bytes": 12345, - } - - with patch( - "litellm.vector_stores.main.aretrieve", - new=AsyncMock(return_value=mock_response), - ) as mock_retrieve: - router = litellm.Router(model_list=[]) - result = await router.avector_store_retrieve( - vector_store_id="vs_test123", - custom_llm_provider="openai", - ) - - assert result["id"] == "vs_test123" - assert result["object"] == "vector_store" - assert result["status"] == "completed" - mock_retrieve.assert_called_once() - - -@pytest.mark.asyncio -async def test_vector_store_list_basic(): - """Test basic vector store list functionality.""" - mock_response = { - "object": "list", - "data": [ - { - "id": "vs_test1", - "object": "vector_store", - "created_at": 1699061776, - "name": "Store 1", - }, - { - "id": "vs_test2", - "object": "vector_store", - "created_at": 1699061777, - "name": "Store 2", - }, - ], - "first_id": "vs_test1", - "last_id": "vs_test2", - "has_more": False, - } - - with patch( - "litellm.vector_stores.main.alist", - new=AsyncMock(return_value=mock_response), - ) as mock_list: - router = litellm.Router(model_list=[]) - result = await router.avector_store_list( - limit=20, - order="desc", - custom_llm_provider="openai", - ) - - assert result["object"] == "list" - assert len(result["data"]) == 2 - assert result["data"][0]["id"] == "vs_test1" - mock_list.assert_called_once() - - @pytest.mark.asyncio async def test_vector_store_update_basic(): """Test basic vector store update functionality.""" @@ -119,76 +43,6 @@ async def test_vector_store_update_basic(): mock_update.assert_called_once() -@pytest.mark.asyncio -async def test_vector_store_delete_basic(): - """Test basic vector store delete functionality.""" - mock_response = { - "id": "vs_test123", - "object": "vector_store.deleted", - "deleted": True, - } - - with patch( - "litellm.vector_stores.main.adelete", - new=AsyncMock(return_value=mock_response), - ) as mock_delete: - router = litellm.Router(model_list=[]) - result = await router.avector_store_delete( - vector_store_id="vs_test123", - custom_llm_provider="openai", - ) - - assert result["id"] == "vs_test123" - assert result["deleted"] is True - assert result["object"] == "vector_store.deleted" - mock_delete.assert_called_once() - - -@pytest.mark.asyncio -async def test_async_vector_store_retrieve(): - """Test async vector store retrieve.""" - mock_response = { - "id": "vs_async123", - "object": "vector_store", - "name": "Async Test Store", - } - - with patch( - "litellm.vector_stores.main.aretrieve", - new=AsyncMock(return_value=mock_response), - ) as mock_aretrieve: - router = litellm.Router(model_list=[]) - result = await router.avector_store_retrieve( - vector_store_id="vs_async123", - custom_llm_provider="openai", - ) - - assert result["id"] == "vs_async123" - mock_aretrieve.assert_called_once() - - -@pytest.mark.asyncio -async def test_async_vector_store_list(): - """Test async vector store list.""" - mock_response = { - "object": "list", - "data": [{"id": "vs_1"}, {"id": "vs_2"}], - } - - with patch( - "litellm.vector_stores.main.alist", - new=AsyncMock(return_value=mock_response), - ) as mock_alist: - router = litellm.Router(model_list=[]) - result = await router.avector_store_list( - limit=10, - custom_llm_provider="openai", - ) - - assert len(result["data"]) == 2 - mock_alist.assert_called_once() - - @pytest.mark.asyncio async def test_async_vector_store_update(): """Test async vector store update.""" @@ -212,28 +66,6 @@ async def test_async_vector_store_update(): mock_aupdate.assert_called_once() -@pytest.mark.asyncio -async def test_async_vector_store_delete(): - """Test async vector store delete.""" - mock_response = { - "id": "vs_async123", - "deleted": True, - } - - with patch( - "litellm.vector_stores.main.adelete", - new=AsyncMock(return_value=mock_response), - ) as mock_adelete: - router = litellm.Router(model_list=[]) - result = await router.avector_store_delete( - vector_store_id="vs_async123", - custom_llm_provider="openai", - ) - - assert result["deleted"] is True - mock_adelete.assert_called_once() - - @pytest.mark.asyncio async def test_vector_store_list_with_pagination(): """Test vector store list with pagination parameters.""" @@ -335,18 +167,12 @@ if __name__ == "__main__": # Test basic sync operations print("✓ Testing basic sync operations...") - asyncio.run(test_vector_store_retrieve_basic()) - asyncio.run(test_vector_store_list_basic()) asyncio.run(test_vector_store_update_basic()) - asyncio.run(test_vector_store_delete_basic()) print("✓ Basic sync operations successful") # Test async operations print("✓ Testing async operations...") - asyncio.run(test_async_vector_store_retrieve()) - asyncio.run(test_async_vector_store_list()) asyncio.run(test_async_vector_store_update()) - asyncio.run(test_async_vector_store_delete()) print("✓ Async operations successful") print("\n✅ All smoke tests passed!") diff --git a/tests/test_team_members.py b/tests/test_team_members.py index 72db6bfe7ae..2bdb7db13c1 100644 --- a/tests/test_team_members.py +++ b/tests/test_team_members.py @@ -99,18 +99,6 @@ def verify_member_in_team(team_info: Dict, user_email: str) -> bool: ) -def test_team_creation(api_client): - """Test team creation""" - team_alias = f"Test Team {uuid.uuid4().hex[:6]}" - team_response, team_id = api_client.create_team(team_alias) - - # Verify team was created - team_info = api_client.get_team_info(team_id) - assert team_info["team_id"] == team_id - assert team_info["team_info"]["team_alias"] == team_alias - assert "o3-mini" in team_info["team_info"]["models"] - - def test_add_single_member(api_client, new_team): """Test adding a single member to a new team""" # Get initial team info From ce6889bbf7d71f1b8b1fbb922ca7479ff39f6c87 Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:44:11 -0700 Subject: [PATCH 42/65] fix(responses): keep tool_result next to tool_use on Anthropic previous_response_id continuations (#45322) * fix(responses): keep tool_result next to tool_use on Anthropic previous_response_id continuations Replayed history no longer re-adds each stored turn's instructions, and the current request's instructions go before the replayed history instead of between the last tool_use and its tool_result * test(responses): fake the Anthropic transport instead of acompletion in the continuation regression test * fix(responses): keep the previous turn's instructions when a continuation sends none * fix(responses): carry only the latest stored instructions when a continuation sends none Replaying each spend log's own instructions put a system message between a replayed tool_use and its tool_result, which Anthropic rejects with a 400 --- .../handler.py | 1 + .../session_handler.py | 61 +++--- .../transformation.py | 13 +- .../e2e/llm_translation/test_responses_e2e.py | 5 +- .../test_anthropic_responses_bridge.py | 207 +++++++++++++++++- 5 files changed, 252 insertions(+), 35 deletions(-) diff --git a/litellm/responses/litellm_completion_transformation/handler.py b/litellm/responses/litellm_completion_transformation/handler.py index 505b5b09433..477d3a64339 100644 --- a/litellm/responses/litellm_completion_transformation/handler.py +++ b/litellm/responses/litellm_completion_transformation/handler.py @@ -106,6 +106,7 @@ class LiteLLMCompletionTransformationHandler: litellm_completion_request = await LiteLLMCompletionResponsesConfig.async_responses_api_session_handler( previous_response_id=previous_response_id, litellm_completion_request=litellm_completion_request, + instructions=responses_api_request.get("instructions"), ) acompletion_args: Final = {} diff --git a/litellm/responses/litellm_completion_transformation/session_handler.py b/litellm/responses/litellm_completion_transformation/session_handler.py index e60a71c7494..c20eac80085 100644 --- a/litellm/responses/litellm_completion_transformation/session_handler.py +++ b/litellm/responses/litellm_completion_transformation/session_handler.py @@ -1,5 +1,6 @@ import asyncio import json +from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, cast import litellm @@ -41,6 +42,11 @@ def _normalize_redacted_tool_call_arguments(message: Message) -> None: function_call.arguments = REDACTED_TOOL_CALL_ARGUMENTS_PLACEHOLDER +def _stored_instructions(proxy_server_request: Mapping[str, object] | None) -> str | None: + instructions: Final = None if proxy_server_request is None else proxy_server_request.get("instructions") + return instructions if isinstance(instructions, str) and instructions else None + + class ResponsesSessionHandler: @staticmethod async def get_chat_completion_message_history_for_previous_response_id( @@ -70,10 +76,15 @@ class ResponsesSessionHandler: | ChatCompletionResponseMessage | Message ] = [] - for spend_log in all_spend_logs: + proxy_server_requests: Final = [ + await ResponsesSessionHandler.get_proxy_server_request_from_spend_log(spend_log=spend_log) + for spend_log in all_spend_logs + ] + for spend_log, proxy_server_request_dict in zip(all_spend_logs, proxy_server_requests): chat_completion_message_history = ( - await ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload( + ResponsesSessionHandler.extend_chat_completion_message_with_spend_log_payload( spend_log=spend_log, + proxy_server_request_dict=proxy_server_request_dict, chat_completion_message_history=chat_completion_message_history, ) ) @@ -85,11 +96,20 @@ class ResponsesSessionHandler: return ChatCompletionSession( messages=chat_completion_message_history, litellm_session_id=litellm_session_id, + instructions=next( + ( + instructions + for instructions in map(_stored_instructions, reversed(proxy_server_requests)) + if instructions + ), + None, + ), ) @staticmethod - async def extend_chat_completion_message_with_spend_log_payload( + def extend_chat_completion_message_with_spend_log_payload( spend_log: "SpendLogsPayload", + proxy_server_request_dict: Mapping[str, object] | None, chat_completion_message_history: list[ AllMessageValues | GenericChatCompletionMessage @@ -105,18 +125,15 @@ class ResponsesSessionHandler: LiteLLMCompletionResponsesConfig, ) - proxy_server_request_dict: Final = await ResponsesSessionHandler.get_proxy_server_request_from_spend_log( - spend_log=spend_log, - ) response_input_param: str | ResponseInputParam | None = None - _messages: str | ResponseInputParam | None = None ############################################################ # Add Input messages for this Spend Log ############################################################ if proxy_server_request_dict: - _response_input_param: Final = proxy_server_request_dict.get("input", None) - _messages = proxy_server_request_dict.get("messages", None) + _response_input_param: Final = proxy_server_request_dict.get("input") or proxy_server_request_dict.get( + "messages" + ) if isinstance(_response_input_param, (str, list)): response_input_param = _response_input_param elif isinstance(_response_input_param, dict): @@ -126,25 +143,13 @@ class ResponsesSessionHandler: ) if response_input_param: - chat_completion_messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( - input=response_input_param, - responses_api_request=proxy_server_request_dict or {}, - replay_reasoning=True, + chat_completion_message_history.extend( + LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( + input=response_input_param, + responses_api_request={}, + replay_reasoning=True, + ) ) - chat_completion_message_history.extend(chat_completion_messages) - - ############################################################ - # Check if `messages` field is present in the proxy server request dict - ############################################################ - elif _messages: - # ensure all messages are /chat/completions/messages - # certain requests can be stored as Responses API format - this ensures they are transformed to /chat/completions/messages - chat_completion_messages = LiteLLMCompletionResponsesConfig.transform_responses_api_input_to_messages( - input=_messages, - responses_api_request=proxy_server_request_dict or {}, - replay_reasoning=True, - ) - chat_completion_message_history.extend(chat_completion_messages) ############################################################ # Add Output messages for this Spend Log @@ -162,7 +167,7 @@ class ResponsesSessionHandler: @staticmethod async def get_proxy_server_request_from_spend_log( spend_log: "SpendLogsPayload", - ) -> dict | None: + ) -> dict[str, object] | None: """ Get the parsed proxy server request from the spend log """ diff --git a/litellm/responses/litellm_completion_transformation/transformation.py b/litellm/responses/litellm_completion_transformation/transformation.py index b1dc6b3a98a..b76dd68018f 100644 --- a/litellm/responses/litellm_completion_transformation/transformation.py +++ b/litellm/responses/litellm_completion_transformation/transformation.py @@ -239,6 +239,7 @@ class ChatCompletionSession(TypedDict, total=False): | Message ] litellm_session_id: str | None + instructions: ReadOnly[str | None] ########### End of Initialize Classes used for Responses API ########### @@ -581,6 +582,7 @@ class LiteLLMCompletionResponsesConfig: async def async_responses_api_session_handler( previous_response_id: str, litellm_completion_request: dict, + instructions: str | None, ) -> dict: """ Async hook to get the chain of previous input and output pairs and return a list of Chat Completion messages @@ -589,18 +591,25 @@ class LiteLLMCompletionResponsesConfig: if previous_response_id: chat_completion_session = ( await ResponsesSessionHandler.get_chat_completion_message_history_for_previous_response_id( - previous_response_id=previous_response_id + previous_response_id=previous_response_id, ) ) _messages: Final = litellm_completion_request.get("messages") or [] session_messages: Final = chat_completion_session.get("messages") or [] + instructions_end: Final = 1 if instructions else 0 + carried_instructions: Final = None if instructions else chat_completion_session.get("instructions") + leading_system_messages: Final = ( + [LiteLLMCompletionResponsesConfig.transform_instructions_to_system_message(carried_instructions)] + if carried_instructions + else _messages[:instructions_end] + ) # If session messages are empty (e.g., no database in test environment), # we still need to process the new input messages # Store original _messages before combining for safety check original_new_messages: Final = _messages.copy() if _messages else [] - combined_messages = session_messages + _messages + combined_messages = leading_system_messages + session_messages + _messages[instructions_end:] # Fix: Ensure tool_results have corresponding tool_calls in previous assistant message # Pass tools parameter to help reconstruct tool_calls if not in cache diff --git a/tests/e2e/llm_translation/test_responses_e2e.py b/tests/e2e/llm_translation/test_responses_e2e.py index 95d01ec71f1..5a78f77ebca 100644 --- a/tests/e2e/llm_translation/test_responses_e2e.py +++ b/tests/e2e/llm_translation/test_responses_e2e.py @@ -541,9 +541,6 @@ class TestResponses: assert arguments.locations, f"get_locations returned no locations: {function_call.arguments}" @pytest.mark.covers("llm.responses.anthropic.multi_turn.nonstream.works") - @pytest.mark.skip( - reason="stage red: product gap, Anthropic previous_response_id continuation sends invalid unmatched tool_use history" - ) @meta( Subject( domain=Domain.LLM_TRANSLATION, @@ -593,7 +590,7 @@ class TestResponses: tools=[LOCATIONS_TOOL], extra_body=NO_PROXY_CACHE, ) - assert tool_result in second.output_text, f"follow-up omitted tool result: {second.output_text!r}" + assert "47" in second.output_text, f"follow-up omitted tool result: {second.output_text!r}" @pytest.mark.covers("llm.responses.bedrock_converse.basic.nonstream.works") @meta( diff --git a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py index 42af50e39bf..677ee0cd2d9 100644 --- a/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py +++ b/tests/unit/responses/litellm_completion_transformation/test_anthropic_responses_bridge.py @@ -1,8 +1,13 @@ +from collections.abc import Mapping, Sequence +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest +from pydantic import BaseModel import litellm +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.responses.litellm_completion_transformation.handler import ( LiteLLMCompletionTransformationHandler, ) @@ -36,7 +41,9 @@ def test_response_api_handler_merges_metadata_and_service_tier_without_error(): async def test_async_response_api_handler_merges_trace_id_without_error(): handler = LiteLLMCompletionTransformationHandler() - async def fake_session_handler(previous_response_id, litellm_completion_request): + async def fake_session_handler( + previous_response_id: str, litellm_completion_request: dict[str, object], instructions: str | None = None + ) -> dict[str, object]: litellm_completion_request["litellm_trace_id"] = "session-trace" return litellm_completion_request @@ -94,3 +101,201 @@ async def test_aresponses_forwards_timeout_to_acompletion(): "this means Router(timeout=N) silently fails for providers on the " "completion transformation path." ) + + +class _FakeSpendLogsDB: + def __init__(self, spend_logs: Sequence[Mapping[str, object]]) -> None: + self._spend_logs = spend_logs + + async def query_raw(self, query: str, *args: object) -> Sequence[Mapping[str, object]]: + return self._spend_logs + + +class _FakePrismaClient: + def __init__(self, spend_logs: Sequence[Mapping[str, object]]) -> None: + self.db = _FakeSpendLogsDB(spend_logs) + + +class _AnthropicBlock(BaseModel, frozen=True): + type: str + id: str | None = None + tool_use_id: str | None = None + text: str | None = None + + +class _AnthropicMessage(BaseModel, frozen=True): + role: str + content: tuple[_AnthropicBlock, ...] + + +class _AnthropicRequest(BaseModel, frozen=True): + messages: tuple[_AnthropicMessage, ...] + system: tuple[_AnthropicBlock, ...] + + +class _RecordingAnthropicMessages: + def __init__(self) -> None: + self.request: _AnthropicRequest | None = None + + def __call__(self, request: httpx.Request) -> httpx.Response: + self.request = _AnthropicRequest.model_validate_json(request.content) + return httpx.Response( + 200, + json={ + "id": "msg_second_turn", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-5-5", + "content": [{"type": "text", "text": "Il fait 47C."}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 1, "output_tokens": 1}, + }, + request=request, + ) + + +_MODEL: Final = "anthropic/claude-sonnet-5-5" +_FIRST_TURN_INSTRUCTIONS: Final = "Be terse." +_FIRST_TURN: Final = { + "request_id": "chatcmpl-first-turn", + "call_type": "aresponses", + "session_id": "session-1", + "proxy_server_request": { + "model": _MODEL, + "input": "What is the weather in Tokyo?", + "instructions": _FIRST_TURN_INSTRUCTIONS, + }, + "response": { + "id": "chatcmpl-first-turn", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [ + { + "index": 0, + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": None, + "tool_calls": [ + { + "id": "toolu_weather", + "type": "function", + "function": {"name": "get_weather", "arguments": '{"city": "Tokyo"}'}, + } + ], + }, + } + ], + }, +} +_EXPECTED_MESSAGES: Final = [ + ("user", [("text", "What is the weather in Tokyo?")]), + ("assistant", [("tool_use", "toolu_weather")]), + ("user", [("tool_result", "toolu_weather")]), +] + + +async def _continue_first_turn_with_tool_output(instructions: str | None) -> _AnthropicRequest: + anthropic: Final = _RecordingAnthropicMessages() + with patch("litellm.proxy.proxy_server.prisma_client", _FakePrismaClient([_FIRST_TURN])): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-first-turn", + input=[{"type": "function_call_output", "call_id": "toolu_weather", "output": "47C"}], + instructions=instructions, + tools=[ + { + "type": "function", + "name": "get_weather", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}, + } + ], + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + assert anthropic.request is not None + return anthropic.request + + +def _message_shapes(request: _AnthropicRequest) -> list[tuple[str, list[tuple[str, str | None]]]]: + return [ + (message.role, [(block.type, block.id or block.tool_use_id or block.text) for block in message.content]) + for message in request.messages + ] + + +@pytest.mark.asyncio +async def test_previous_response_id_tool_output_with_new_instructions_builds_valid_anthropic_request() -> None: + """ + A continuation that resends `instructions` must not land a system message between the replayed + tool_use and its tool_result, and the previous turn's instructions do not carry over (OpenAI semantics) + """ + request: Final = await _continue_first_turn_with_tool_output(instructions="Answer in French.") + + assert _message_shapes(request) == _EXPECTED_MESSAGES + assert [block.text for block in request.system] == ["Answer in French."] + + +@pytest.mark.asyncio +async def test_previous_response_id_tool_output_without_instructions_keeps_the_previous_turns() -> None: + """ + A continuation that sends no `instructions` keeps the previous turn's instructions as the system prompt + """ + request: Final = await _continue_first_turn_with_tool_output(instructions=None) + + assert _message_shapes(request) == _EXPECTED_MESSAGES + assert [block.text for block in request.system] == [_FIRST_TURN_INSTRUCTIONS] + + +def _second_turn(instructions: str | None) -> Mapping[str, object]: + return { + "request_id": "chatcmpl-second-turn", + "call_type": "aresponses", + "session_id": "session-1", + "proxy_server_request": { + "model": _MODEL, + "input": [{"type": "function_call_output", "call_id": "toolu_weather", "output": "47C"}], + **({"instructions": instructions} if instructions else {}), + }, + "response": { + "id": "chatcmpl-second-turn", + "object": "chat.completion", + "created": 0, + "model": _MODEL, + "choices": [ + {"index": 0, "finish_reason": "stop", "message": {"role": "assistant", "content": "47C in Tokyo."}} + ], + }, + } + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("second_turn_instructions", "expected_system"), + [("Answer in French.", "Answer in French."), (None, _FIRST_TURN_INSTRUCTIONS)], +) +async def test_previous_response_id_without_instructions_carries_the_latest_ones_ahead_of_a_tool_roundtrip( + second_turn_instructions: str | None, expected_system: str +) -> None: + anthropic: Final = _RecordingAnthropicMessages() + with patch( + "litellm.proxy.proxy_server.prisma_client", + _FakePrismaClient([_FIRST_TURN, _second_turn(second_turn_instructions)]), + ): + await litellm.aresponses( + model=_MODEL, + previous_response_id="chatcmpl-second-turn", + input="What about Osaka?", + api_key="sk-ant-fake", + client=AsyncHTTPHandler(transport=httpx.MockTransport(anthropic)), + ) + + assert anthropic.request is not None + assert _message_shapes(anthropic.request) == [ + *_EXPECTED_MESSAGES, + ("assistant", [("text", "47C in Tokyo.")]), + ("user", [("text", "What about Osaka?")]), + ] + assert [block.text for block in anthropic.request.system] == [expected_system] From 6e54dced8c2950a6e9e99e965cb58df43b0dae38 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 09:44:18 -0700 Subject: [PATCH 43/65] test: realign stale decisions and lens tests with merged behaviour (#45346) Co-authored-by: yuneng --- tests/integration/database/test_lens_repository.py | 3 ++- tests/integration/providers/test_decisions_wire.py | 2 +- tests/proxy_behavior/lens/test_lifecycle.py | 6 +++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/integration/database/test_lens_repository.py b/tests/integration/database/test_lens_repository.py index a46e37668ab..ff46944b911 100644 --- a/tests/integration/database/test_lens_repository.py +++ b/tests/integration/database/test_lens_repository.py @@ -379,7 +379,8 @@ async def test_managed_registration_is_atomic_and_keeps_the_original_worker_id(l @pytest.mark.asyncio async def test_claim_pages_only_yield_work_the_worker_can_claim(lens_db: Prisma) -> None: - now: Final = datetime.now(timezone.utc) + clock: Final = datetime.now(timezone.utc) + now: Final = clock.replace(microsecond=clock.microsecond // 1000 * 1000) scope: Final = Scope(team_id=uuid4().hex) repo: Final = LensRepository(WriterDatabase(PrismaWrapper(lens_db))) prefix: Final = uuid4().hex diff --git a/tests/integration/providers/test_decisions_wire.py b/tests/integration/providers/test_decisions_wire.py index 84b31088d60..db6629a43d2 100644 --- a/tests/integration/providers/test_decisions_wire.py +++ b/tests/integration/providers/test_decisions_wire.py @@ -107,7 +107,7 @@ _PROVIDERS: Final = ( _PERPLEXITY: Final = _PROVIDERS[0] _OPENROUTER: Final = _PROVIDERS[2] _OPENROUTER_CHAT_MODEL: Final = "openrouter/openai/gpt-5-mini" -_UNSUPPORTED_PROVIDER_MODEL: Final = "openai/gpt-6-luna" +_UNSUPPORTED_PROVIDER_MODEL: Final = "anthropic/claude-opus-5-5" _CONNECTION_ERROR: Final = "litellm.APIConnectionError" _GENERIC_API_ERROR: Final = "litellm.APIError" _INVALID_BODIES: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( diff --git a/tests/proxy_behavior/lens/test_lifecycle.py b/tests/proxy_behavior/lens/test_lifecycle.py index 5f30a11cf70..b2a9628c1d3 100644 --- a/tests/proxy_behavior/lens/test_lifecycle.py +++ b/tests/proxy_behavior/lens/test_lifecycle.py @@ -361,10 +361,10 @@ async def test_scan_lifecycle_persists_results_and_revokes_worker(lens_database: await endpoints.repository().save_worker(legacy) authenticated_legacy: Final = await endpoints.worker_auth(credentials) assert authenticated_legacy.analysis_key_id is None - with pytest.raises(HTTPException) as needs_billing: + assert ( await endpoints.claim(authenticated_legacy, protocol_version=PROTOCOL_VERSION, worker_release=release_tag()) - assert needs_billing.value.status_code == 409 - assert "Assign an analysis key" in needs_billing.value.detail + is None + ) assert await endpoints.heartbeat(lens.id, claimed.job.id, authenticated_legacy) finished: Final = await endpoints.result( lens.id, claimed.job.id, Result(coverage=Coverage(screened=2)), authenticated_legacy, storage=None From 9f150f74edf2984d24ca0b3bc925d8cf34f84801 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:03:29 -0500 Subject: [PATCH 44/65] feat(prometheus): expose per-project per-model rate limit allowed and used gauges (#43561) Co-authored-by: yassin Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../grafana_dashboard.json | 127 ++++++++++++++ litellm/integrations/prometheus.py | 148 +++++++++++++--- litellm/types/integrations/prometheus.py | 15 ++ .../test_prometheus_client_ip_user_agent.py | 2 +- .../test_prometheus_rate_limit_labels.py | 160 +++++++++++++++++- 5 files changed, 419 insertions(+), 33 deletions(-) diff --git a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json index 4fb926a9658..996e440d77d 100644 --- a/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json +++ b/cookbook/litellm_proxy_server/grafana_dashboard/dashboard_all_metrics/grafana_dashboard.json @@ -6436,6 +6436,133 @@ ], "title": "litellm_zero_cost_requests rate", "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 438 + }, + "id": 112, + "panels": [], + "title": "Project model rate limits", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Configured rate limit for the Project on the requested model in the current window, by rate_limit_type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 439 + }, + "id": 113, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (project_id, project_alias, requested_model, rate_limit_type) (litellm_project_model_rate_limit_allowed_metric)", + "legendFormat": "{{project_alias}} ({{project_id}}) / {{requested_model}} / {{rate_limit_type}}", + "range": true, + "refId": "A" + } + ], + "title": "litellm_project_model_rate_limit_allowed_metric", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Requests or tokens the Project has consumed on the requested model in the current rate limit window, by rate_limit_type", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "showPoints": "never", + "spanNulls": false + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 439 + }, + "id": 114, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (project_id, project_alias, requested_model, rate_limit_type) (litellm_project_model_rate_limit_used_metric)", + "legendFormat": "{{project_alias}} ({{project_id}}) / {{requested_model}} / {{rate_limit_type}}", + "range": true, + "refId": "A" + } + ], + "title": "litellm_project_model_rate_limit_used_metric", + "type": "timeseries" } ], "preload": false, diff --git a/litellm/integrations/prometheus.py b/litellm/integrations/prometheus.py index e11ce2bf472..2d1ef62fde3 100644 --- a/litellm/integrations/prometheus.py +++ b/litellm/integrations/prometheus.py @@ -691,6 +691,24 @@ class PrometheusLogger(CustomLogger): labelnames=self.get_labels_for_metric("litellm_team_rate_limit_used_metric"), ) + self.litellm_project_model_rate_limit_allowed_metric = self._gauge_factory( + "litellm_project_model_rate_limit_allowed_metric", + ( + "Configured rate limit for the Project on the requested model in the current window " + "(model_rpm_limit / model_tpm_limit / model_itpm_limit / model_otpm_limit), by rate_limit_type" + ), + labelnames=self.get_labels_for_metric("litellm_project_model_rate_limit_allowed_metric"), + ) + + self.litellm_project_model_rate_limit_used_metric = self._gauge_factory( + "litellm_project_model_rate_limit_used_metric", + ( + "Requests or tokens the Project has consumed on the requested model in the current rate limit " + "window, by rate_limit_type" + ), + labelnames=self.get_labels_for_metric("litellm_project_model_rate_limit_used_metric"), + ) + ######################################## # LLM API Deployment Metrics / analytics ######################################## @@ -1541,6 +1559,8 @@ class PrometheusLogger(CustomLogger): model_group=standard_logging_payload["model_group"], team=user_api_team, team_alias=user_api_team_alias, + project_id=standard_logging_payload["metadata"].get("user_api_key_project_id"), + project_alias=standard_logging_payload["metadata"].get("user_api_key_project_alias"), org_id=user_api_key_org_id, org_alias=user_api_key_org_alias, user=user_id, @@ -1627,7 +1647,7 @@ class PrometheusLogger(CustomLogger): model_id=enum_values.model_id, ) - self._set_key_and_team_rate_limit_metrics( + self._set_v3_rate_limit_allowed_and_used_metrics( standard_logging_payload=standard_logging_payload, # pyright: ignore[reportArgumentType] # isinstance(dict) above narrows the TypedDict to dict[Unknown, Unknown] enum_values=enum_values, ) @@ -2232,65 +2252,139 @@ class PrometheusLogger(CustomLogger): return None return value - def _set_key_and_team_rate_limit_metrics( + def _set_v3_rate_limit_allowed_and_used_metrics( self, standard_logging_payload: StandardLoggingPayload, enum_values: UserAPIKeyLabelValues, ) -> None: - """ - Export the key-level and team-level RPM / TPM limit and current window - usage from the ``x-ratelimit-{api_key,team}-{limit,remaining}-*`` - headers the v3 rate limiter mirrors into the logging payload. The - limiter already read these counters (from Redis when configured) on - the request path, so no extra store lookup happens here. Descriptors - without a configured limit emit no header, so their series is removed - rather than left at the value from before the limit was dropped. - """ + """Export v3 rate-limit limits and window usage from mirrored logging headers.""" descriptor_gauges: Final[ - tuple[tuple[Literal["api_key", "team"], DEFINED_PROMETHEUS_METRICS, Gauge, Gauge], ...] + tuple[ + tuple[ + Literal[ + "api_key", + "team", + "model_per_project", + "model_per_project_itpm", + "model_per_project_otpm", + ], + Literal["requests", "tokens"], + Literal["requests", "tokens", "input_tokens", "output_tokens"], + DEFINED_PROMETHEUS_METRICS, + Gauge, + Gauge, + ], + ..., + ] ] = ( ( "api_key", + "requests", + "requests", + "litellm_api_key_rate_limit_allowed_metric", + self.litellm_api_key_rate_limit_allowed_metric, + self.litellm_api_key_rate_limit_used_metric, + ), + ( + "api_key", + "tokens", + "tokens", "litellm_api_key_rate_limit_allowed_metric", self.litellm_api_key_rate_limit_allowed_metric, self.litellm_api_key_rate_limit_used_metric, ), ( "team", + "requests", + "requests", "litellm_team_rate_limit_allowed_metric", self.litellm_team_rate_limit_allowed_metric, self.litellm_team_rate_limit_used_metric, ), + ( + "team", + "tokens", + "tokens", + "litellm_team_rate_limit_allowed_metric", + self.litellm_team_rate_limit_allowed_metric, + self.litellm_team_rate_limit_used_metric, + ), + ( + "model_per_project", + "requests", + "requests", + "litellm_project_model_rate_limit_allowed_metric", + self.litellm_project_model_rate_limit_allowed_metric, + self.litellm_project_model_rate_limit_used_metric, + ), + ( + "model_per_project", + "tokens", + "tokens", + "litellm_project_model_rate_limit_allowed_metric", + self.litellm_project_model_rate_limit_allowed_metric, + self.litellm_project_model_rate_limit_used_metric, + ), + ( + "model_per_project_itpm", + "tokens", + "input_tokens", + "litellm_project_model_rate_limit_allowed_metric", + self.litellm_project_model_rate_limit_allowed_metric, + self.litellm_project_model_rate_limit_used_metric, + ), + ( + "model_per_project_otpm", + "tokens", + "output_tokens", + "litellm_project_model_rate_limit_allowed_metric", + self.litellm_project_model_rate_limit_allowed_metric, + self.litellm_project_model_rate_limit_used_metric, + ), ) - for descriptor_key, metric_name, allowed_gauge, used_gauge in descriptor_gauges: - for rate_limit_type in ("requests", "tokens"): - self._set_rate_limit_allowed_and_used_gauges( - standard_logging_payload=standard_logging_payload, - enum_values=enum_values, - descriptor_key=descriptor_key, - metric_name=metric_name, - allowed_gauge=allowed_gauge, - used_gauge=used_gauge, - rate_limit_type=rate_limit_type, - ) + for ( + descriptor_key, + header_rate_limit_type, + rate_limit_type, + metric_name, + allowed_gauge, + used_gauge, + ) in descriptor_gauges: + self._set_rate_limit_allowed_and_used_gauges( + standard_logging_payload=standard_logging_payload, + enum_values=enum_values, + descriptor_key=descriptor_key, + header_rate_limit_type=header_rate_limit_type, + metric_name=metric_name, + allowed_gauge=allowed_gauge, + used_gauge=used_gauge, + rate_limit_type=rate_limit_type, + ) def _set_rate_limit_allowed_and_used_gauges( self, standard_logging_payload: StandardLoggingPayload, enum_values: UserAPIKeyLabelValues, - descriptor_key: Literal["api_key", "team"], + descriptor_key: Literal[ + "api_key", + "team", + "model_per_project", + "model_per_project_itpm", + "model_per_project_otpm", + ], + header_rate_limit_type: Literal["requests", "tokens"], metric_name: DEFINED_PROMETHEUS_METRICS, allowed_gauge: Gauge, used_gauge: Gauge, - rate_limit_type: Literal["requests", "tokens"], + rate_limit_type: Literal["requests", "tokens", "input_tokens", "output_tokens"], ) -> None: limit: Final = self._get_int_from_v3_rate_limit_headers( standard_logging_payload=standard_logging_payload, - header_name=f"x-ratelimit-{descriptor_key}-limit-{rate_limit_type}", + header_name=f"x-ratelimit-{descriptor_key}-limit-{header_rate_limit_type}", ) remaining: Final = self._get_int_from_v3_rate_limit_headers( standard_logging_payload=standard_logging_payload, - header_name=f"x-ratelimit-{descriptor_key}-remaining-{rate_limit_type}", + header_name=f"x-ratelimit-{descriptor_key}-remaining-{header_rate_limit_type}", ) labelled_values: Final = replace(enum_values, rate_limit_type=rate_limit_type) labelnames: Final = self.get_labels_for_metric(metric_name) diff --git a/litellm/types/integrations/prometheus.py b/litellm/types/integrations/prometheus.py index a951381a2c9..9f888185779 100644 --- a/litellm/types/integrations/prometheus.py +++ b/litellm/types/integrations/prometheus.py @@ -204,6 +204,8 @@ class UserAPIKeyLabelNames(Enum): API_KEY_ALIAS = "api_key_alias" TEAM = "team" TEAM_ALIAS = "team_alias" + PROJECT_ID = "project_id" + PROJECT_ALIAS = "project_alias" REQUESTED_MODEL = REQUESTED_MODEL v1_LITELLM_MODEL_NAME = "model" v2_LITELLM_MODEL_NAME = "litellm_model_name" @@ -303,6 +305,8 @@ DEFINED_PROMETHEUS_METRICS = Literal[ "litellm_api_key_rate_limit_used_metric", "litellm_team_rate_limit_allowed_metric", "litellm_team_rate_limit_used_metric", + "litellm_project_model_rate_limit_allowed_metric", + "litellm_project_model_rate_limit_used_metric", "litellm_llm_api_failed_requests_metric", "litellm_callback_logging_failures_metric", "litellm_in_flight_requests", @@ -847,6 +851,15 @@ class PrometheusMetricLabels: litellm_team_rate_limit_used_metric = litellm_team_rate_limit_allowed_metric + litellm_project_model_rate_limit_allowed_metric: ClassVar[tuple[str, ...]] = ( + UserAPIKeyLabelNames.PROJECT_ID.value, + UserAPIKeyLabelNames.PROJECT_ALIAS.value, + UserAPIKeyLabelNames.REQUESTED_MODEL.value, + UserAPIKeyLabelNames.RATE_LIMIT_TYPE.value, + ) + + litellm_project_model_rate_limit_used_metric = litellm_project_model_rate_limit_allowed_metric + litellm_llm_api_failed_requests_metric = [ UserAPIKeyLabelNames.END_USER.value, UserAPIKeyLabelNames.API_KEY_HASH.value, @@ -1058,6 +1071,8 @@ class UserAPIKeyLabelValues: api_key_alias: str | None = None team: str | None = None team_alias: str | None = None + project_id: str | None = None + project_alias: str | None = None model_group: str | None = None requested_model: str | None = None model: str | None = None diff --git a/tests/unit/integrations/test_prometheus_client_ip_user_agent.py b/tests/unit/integrations/test_prometheus_client_ip_user_agent.py index 004ac4dbffb..8f7923428e7 100644 --- a/tests/unit/integrations/test_prometheus_client_ip_user_agent.py +++ b/tests/unit/integrations/test_prometheus_client_ip_user_agent.py @@ -94,7 +94,7 @@ async def test_async_post_call_success_hook_includes_client_ip_user_agent(): logger._increment_token_metrics = MagicMock() logger._increment_remaining_budget_metrics = AsyncMock() logger._set_virtual_key_rate_limit_metrics = MagicMock() - logger._set_key_and_team_rate_limit_metrics = MagicMock() + logger._set_v3_rate_limit_allowed_and_used_metrics = MagicMock() logger._set_latency_metrics = MagicMock() logger.set_llm_deployment_success_metrics = MagicMock() logger._increment_cache_metrics = MagicMock() diff --git a/tests/unit/integrations/test_prometheus_rate_limit_labels.py b/tests/unit/integrations/test_prometheus_rate_limit_labels.py index bf1d68c7714..e4c71ff7f77 100644 --- a/tests/unit/integrations/test_prometheus_rate_limit_labels.py +++ b/tests/unit/integrations/test_prometheus_rate_limit_labels.py @@ -14,8 +14,10 @@ Covers two follow-up gaps to the unified rate-limit error work: """ from collections.abc import Mapping +from typing import Final from unittest.mock import MagicMock, patch +import litellm import pytest from litellm.exceptions import ( @@ -474,11 +476,13 @@ def test_should_ignore_non_int_v3_header_values(bad_value): ) -KEY_AND_TEAM_RATE_LIMIT_METRICS = ( +RATE_LIMIT_METRICS = ( "litellm_api_key_rate_limit_allowed_metric", "litellm_api_key_rate_limit_used_metric", "litellm_team_rate_limit_allowed_metric", "litellm_team_rate_limit_used_metric", + "litellm_project_model_rate_limit_allowed_metric", + "litellm_project_model_rate_limit_used_metric", ) @@ -534,6 +538,8 @@ def _success_kwargs_with_rate_limit_headers(additional_headers: Mapping[str, obj "user_api_key_alias": "key-alias", "user_api_key_team_id": "team-id", "user_api_key_team_alias": "team-alias", + "user_api_key_project_id": "project-id", + "user_api_key_project_alias": "project-alias", "user_api_key_user_id": "u", "user_api_key_user_email": "e@x.com", "user_api_key_org_id": None, @@ -627,6 +633,68 @@ async def test_should_emit_key_and_team_rate_limit_allowed_and_used_from_v3_head _clear_prometheus_registry() +@pytest.mark.asyncio +async def test_should_emit_project_model_rate_limit_allowed_and_used_from_v3_headers() -> None: + _clear_prometheus_registry() + try: + await _run_success_event( + { + "x-ratelimit-model_per_project-limit-requests": 100, + "x-ratelimit-model_per_project-remaining-requests": 99, + "x-ratelimit-model_per_project-limit-tokens": 10000, + "x-ratelimit-model_per_project-remaining-tokens": 9950, + "x-ratelimit-model_per_project_itpm-limit-tokens": 2000, + "x-ratelimit-model_per_project_itpm-remaining-tokens": 1900, + "x-ratelimit-model_per_project_otpm-limit-tokens": 3000, + "x-ratelimit-model_per_project_otpm-remaining-tokens": 2750, + } + ) + + project_requests: Final = ( + ("project_alias", "project-alias"), + ("project_id", "project-id"), + ("rate_limit_type", "requests"), + ("requested_model", "anthropic-haiku-4-5"), + ) + project_tokens: Final = ( + ("project_alias", "project-alias"), + ("project_id", "project-id"), + ("rate_limit_type", "tokens"), + ("requested_model", "anthropic-haiku-4-5"), + ) + project_input_tokens: Final = ( + ("project_alias", "project-alias"), + ("project_id", "project-id"), + ("rate_limit_type", "input_tokens"), + ("requested_model", "anthropic-haiku-4-5"), + ) + project_output_tokens: Final = ( + ("project_alias", "project-alias"), + ("project_id", "project-id"), + ("rate_limit_type", "output_tokens"), + ("requested_model", "anthropic-haiku-4-5"), + ) + + assert _collected_samples("litellm_project_model_rate_limit_allowed_metric") == { + project_requests: 100, + project_tokens: 10000, + project_input_tokens: 2000, + project_output_tokens: 3000, + } + assert _collected_samples("litellm_project_model_rate_limit_used_metric") == { + project_requests: 1, + project_tokens: 50, + project_input_tokens: 100, + project_output_tokens: 250, + } + assert _collected_samples("litellm_api_key_rate_limit_allowed_metric") == {} + assert _collected_samples("litellm_api_key_rate_limit_used_metric") == {} + assert _collected_samples("litellm_team_rate_limit_allowed_metric") == {} + assert _collected_samples("litellm_team_rate_limit_used_metric") == {} + finally: + _clear_prometheus_registry() + + @pytest.mark.asyncio async def test_should_emit_only_the_dimensions_the_limiter_enforced(): """ @@ -701,6 +769,79 @@ async def test_should_drop_key_and_team_series_once_the_limiter_stops_reporting_ _clear_prometheus_registry() +@pytest.mark.asyncio +async def test_should_drop_project_model_series_once_the_limiter_stops_reporting_a_limit() -> None: + _clear_prometheus_registry() + try: + logger: Final = PrometheusLogger() + await _run_success_event( + { + "x-ratelimit-model_per_project-limit-requests": 100, + "x-ratelimit-model_per_project-remaining-requests": 99, + "x-ratelimit-model_per_project-limit-tokens": 10000, + "x-ratelimit-model_per_project-remaining-tokens": 9950, + "x-ratelimit-model_per_project_itpm-limit-tokens": 2000, + "x-ratelimit-model_per_project_itpm-remaining-tokens": 1900, + "x-ratelimit-model_per_project_otpm-limit-tokens": 3000, + "x-ratelimit-model_per_project_otpm-remaining-tokens": 2750, + }, + logger=logger, + ) + await _run_success_event( + { + "x-ratelimit-model_per_project-limit-requests": 100, + "x-ratelimit-model_per_project-remaining-requests": 96, + }, + logger=logger, + ) + + project_requests: Final = ( + ("project_alias", "project-alias"), + ("project_id", "project-id"), + ("rate_limit_type", "requests"), + ("requested_model", "anthropic-haiku-4-5"), + ) + assert _collected_samples("litellm_project_model_rate_limit_allowed_metric") == { + project_requests: 100, + } + assert _collected_samples("litellm_project_model_rate_limit_used_metric") == { + project_requests: 4, + } + finally: + _clear_prometheus_registry() + + +@pytest.mark.asyncio +async def test_project_model_rate_limit_allowed_uses_same_custom_project_alias_label_as_requests() -> None: + original_custom_labels: Final = litellm.custom_prometheus_metadata_labels + litellm.custom_prometheus_metadata_labels = ["metadata.user_api_key_project_alias"] + _clear_prometheus_registry() + try: + logger: Final = PrometheusLogger() + await _run_success_event( + { + "x-ratelimit-model_per_project-limit-requests": 100, + "x-ratelimit-model_per_project-remaining-requests": 99, + }, + logger=logger, + ) + + allowed_samples: Final = _collected_samples("litellm_project_model_rate_limit_allowed_metric") + request_samples: Final = _collected_samples("litellm_proxy_total_requests_metric_total") + assert len(allowed_samples) == 1 + assert len(request_samples) == 1 + allowed_labels: Final = dict(next(iter(allowed_samples))) + request_labels: Final = dict(next(iter(request_samples))) + assert allowed_labels["metadata_user_api_key_project_alias"] == "project-alias" + assert ( + allowed_labels["metadata_user_api_key_project_alias"] + == request_labels["metadata_user_api_key_project_alias"] + ) + finally: + litellm.custom_prometheus_metadata_labels = original_custom_labels + _clear_prometheus_registry() + + @pytest.mark.asyncio @pytest.mark.parametrize( "additional_headers", @@ -710,16 +851,25 @@ async def test_should_drop_key_and_team_series_once_the_limiter_stops_reporting_ {"x-ratelimit-api_key-limit-requests": 10}, {"x-ratelimit-api_key-limit-requests": "10", "x-ratelimit-api_key-remaining-requests": "7"}, {"x-ratelimit-team-limit-tokens": True, "x-ratelimit-team-remaining-tokens": 5}, + {"x-ratelimit-model_per_project-limit-requests": 10}, + { + "x-ratelimit-model_per_project-limit-requests": "10", + "x-ratelimit-model_per_project-remaining-requests": "7", + }, + { + "x-ratelimit-model_per_project-limit-tokens": True, + "x-ratelimit-model_per_project-remaining-tokens": 5, + }, ], ) -async def test_should_emit_no_key_or_team_rate_limit_series_without_a_complete_int_pair( - additional_headers, -): +async def test_should_emit_no_rate_limit_series_without_a_complete_int_pair( + additional_headers: Mapping[str, object] | None, +) -> None: _clear_prometheus_registry() try: await _run_success_event(additional_headers) - for metric_name in KEY_AND_TEAM_RATE_LIMIT_METRICS: + for metric_name in RATE_LIMIT_METRICS: assert _collected_samples(metric_name) == {}, metric_name finally: _clear_prometheus_registry() From 49266e3b17afa89477557c46ebd86a6a3e56da38 Mon Sep 17 00:00:00 2001 From: joshua-berri Date: Thu, 8 Oct 2026 10:18:25 -0700 Subject: [PATCH 45/65] fix(mcp): preserve elicitation context and report relay failures (#45255) * fix(mcp): correlate elicitation relays and report failures * fix(mcp): preserve elicitation timeout errors on Python 3.10 * fix(mcp): keep request ID alias in type-checking imports * fix(mcp): preserve elicitation type narrowing and clarify HTTP limits * ci(mcp): publish elicitation coverage from GitHub Actions --------- Co-authored-by: Joshua Valluru <326636767+joshua-berri@users.noreply.github.com> --- .github/workflows/test-unit.yml | 10 + .../proxy/_experimental/mcp_server/README.md | 17 ++ .../mcp_server/elicitation_handler.py | 192 ++++++------------ .../mcp_server/legacy_callbacks.py | 16 +- .../mcp_server/mcp_server_manager.py | 10 +- .../test_mcp_elicitation_handler.py | 177 +++++++++++----- .../mcp_server/test_mcp_server_manager.py | 24 ++- 7 files changed, 249 insertions(+), 197 deletions(-) diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 54fdc6b43a2..ce4f5b52f4e 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -346,6 +346,16 @@ jobs: timeout-minutes: 60 job-timeout-minutes: 100 + - shard: mcp-elicitation + artifact-name: mcp-elicitation + test-path: >- + tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py + tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py + workers: 2 + reruns: 0 + timeout-minutes: 20 + job-timeout-minutes: 60 + - shard: proxy-infra artifact-name: proxy-infra test-path: >- diff --git a/litellm/proxy/_experimental/mcp_server/README.md b/litellm/proxy/_experimental/mcp_server/README.md index 1123a9779ac..7d1f58965b0 100644 --- a/litellm/proxy/_experimental/mcp_server/README.md +++ b/litellm/proxy/_experimental/mcp_server/README.md @@ -13,3 +13,20 @@ Listing failures retain per-server outcome metadata. An incomplete upstream cata For a scoped rollout, keep the previous source build serving the control pool and send only selected clients to a separate candidate pool. All candidate replicas must share configuration and salt. Verify page one on one candidate replica and continuation on another, plus a fresh listing and tool call on the control pool. Do not mirror tool calls between pools For rollback, stop sending new requests to the candidate pool, drain its in-flight operations, and return selected clients to the control pool. Clients must discard candidate cursors and start a fresh listing when crossing versions; older gateways do not validate these cursors. Keep the registry-revision migration installed when rolling back pagination. Verify a fresh listing and a tool call after switching pools + +## Elicitation + +Elicitation is disabled by default. Enable it per upstream in `config.yaml`; `allow_elicitation` is currently YAML-only and is not editable through the Admin UI or database-backed server API + +```yaml +mcp_servers: + interactive: + url: https://mcp.example.com/mcp + transport: http + allow_elicitation: true + timeout: 60 +``` + +The downstream MCP client must advertise the requested form or URL capability during initialization. Use the gateway's legacy SSE endpoint (`/mcp/sse`) for the verified interactive form and URL relay path. The current Streamable HTTP endpoint (`/mcp`) can lose initialization state before a tool call, so even a client that advertised support receives an explicit elicitation error instead of an input request. Successful interactive relay over Streamable HTTP is not currently verified. Stateless calls and LLM tool bridges with no downstream MCP client also receive explicit errors + +The relay wait uses the upstream server's existing `timeout` setting, or `LITELLM_MCP_CLIENT_TIMEOUT` (60 seconds by default). The enclosing tool call also retains its existing timeout. Unsupported modes, disconnects and relay failures return errors, never a fabricated user decline. Actual user accept, decline and cancel responses are preserved; cancellation stops the relay diff --git a/litellm/proxy/_experimental/mcp_server/elicitation_handler.py b/litellm/proxy/_experimental/mcp_server/elicitation_handler.py index 57d2d86d506..245a0612435 100644 --- a/litellm/proxy/_experimental/mcp_server/elicitation_handler.py +++ b/litellm/proxy/_experimental/mcp_server/elicitation_handler.py @@ -1,30 +1,35 @@ -""" -MCP Elicitation Handler -Handles `elicitation/create` requests from upstream MCP servers by either: -1. Relaying them to the connected downstream MCP client (if it supports elicitation) -2. Returning a decline/error response (if no downstream client or unsupported) -Supports both Form mode (structured data collection) and URL mode (external URL -navigation for sensitive interactions like OAuth). -MCP Spec Reference: - https://modelcontextprotocol.io/specification/2025-11-25/client/elicitation -""" +"""Relay upstream elicitation through the initiating downstream MCP request.""" -from typing import TYPE_CHECKING, Final, Protocol, Union +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING, Final, Protocol from litellm._logging import verbose_logger +from litellm.constants import MCP_CLIENT_TIMEOUT if TYPE_CHECKING: from mcp.types import ( + INTERNAL_ERROR, + INVALID_REQUEST, + REQUEST_TIMEOUT, + ClientCapabilities, + ElicitRequestedSchema, ElicitRequestFormParams, ElicitRequestParams, ElicitRequestURLParams, ElicitResult, ErrorData, + RequestId, ) -# Guard imports that require the mcp package try: from mcp.types import ( + INTERNAL_ERROR, + INVALID_REQUEST, + REQUEST_TIMEOUT, + ClientCapabilities, + ElicitRequestedSchema, ElicitRequestFormParams, ElicitRequestParams, ElicitRequestURLParams, @@ -38,136 +43,65 @@ except ImportError: class _DownstreamElicitSession(Protocol): - """The downstream MCP client session methods this module relays elicitation requests through.""" + async def elicit_url( + self, message: str, url: str, elicitation_id: str, related_request_id: RequestId | None = None + ) -> ElicitResult: ... - async def elicit_url(self, message: str, url: str, elicitation_id: str) -> "ElicitResult": ... - - async def elicit_form(self, message: str, requested_schema: dict[str, object]) -> "ElicitResult": ... - - async def elicit(self, message: str, requested_schema: dict[str, object]) -> "ElicitResult": ... + async def elicit_form( + self, message: str, requested_schema: ElicitRequestedSchema, related_request_id: RequestId | None = None + ) -> ElicitResult: ... async def handle_elicitation_request( context: object, - params: "ElicitRequestParams", + params: ElicitRequestParams, downstream_session: _DownstreamElicitSession | None = None, - downstream_capabilities: object = None, -) -> Union["ElicitResult", "ErrorData"]: - """ - Handle an MCP elicitation/create request from an upstream MCP server. - In Gateway mode (Mode A), we relay the elicitation request to the - connected downstream client if they declared elicitation capabilities. - In Tool Bridge mode (Mode B), there's no persistent downstream MCP - client, so we return a decline response. - Args: - context: MCP RequestContext from the upstream server connection. - params: The ElicitRequestParams (either form or URL mode). - downstream_session: The ServerSession to the downstream client, - if available (for relaying). - downstream_capabilities: The downstream client's declared - capabilities, used to check elicitation support. - Returns: - ElicitResult with the user's response, or ErrorData on failure. - """ + downstream_capabilities: ClientCapabilities | None = None, + related_request_id: RequestId | None = None, + timeout: float = MCP_CLIENT_TIMEOUT, +) -> ElicitResult | ErrorData: if not MCP_ELICITATION_AVAILABLE: - return ErrorData( - code=-1, - message="MCP elicitation is not available (mcp package not installed)", - ) + return ErrorData(code=INTERNAL_ERROR, message="MCP elicitation is not available") + if downstream_session is None: + return ErrorData(code=INVALID_REQUEST, message="MCP elicitation requires a connected downstream MCP client") try: - mode: Final = getattr(params, "mode", "form") - verbose_logger.info( - "MCP elicitation: received request mode=%s, message=%s", - mode, - getattr(params, "message", ""), + return await asyncio.wait_for( + _relay_elicitation_to_downstream(params, downstream_session, downstream_capabilities, related_request_id), + timeout=timeout, ) - # Check if we have a downstream session to relay to - if downstream_session is not None: - return await _relay_elicitation_to_downstream( - params=params, - downstream_session=downstream_session, - downstream_capabilities=downstream_capabilities, - ) - # No downstream session — we're in Tool Bridge mode - # or the client doesn't support elicitation - verbose_logger.info("MCP elicitation: no downstream session available, declining") - return ElicitResult( - action="decline", - ) - except Exception as e: - verbose_logger.exception("MCP elicitation handler failed: %s", e) + except asyncio.TimeoutError: + return ErrorData(code=REQUEST_TIMEOUT, message="MCP elicitation timed out waiting for the downstream client") + except Exception: + verbose_logger.warning("MCP elicitation: downstream relay failed") return ErrorData( - code=-1, - message=f"Elicitation failed: {e}", + code=INTERNAL_ERROR, message="MCP elicitation failed while communicating with the downstream client" ) async def _relay_elicitation_to_downstream( - params: "ElicitRequestParams", + params: ElicitRequestParams, downstream_session: _DownstreamElicitSession, - downstream_capabilities: object = None, -) -> Union["ElicitResult", "ErrorData"]: - """ - Relay an elicitation request to the downstream MCP client. - Uses the ServerSession's elicit_form() or elicit_url() methods to - send the elicitation request back to the connected client. - Args: - params: The elicitation request parameters. - downstream_session: The ServerSession connected to the downstream client. - downstream_capabilities: Client capabilities to check support. - Returns: - ElicitResult from the downstream client. - """ - mode: Final = getattr(params, "mode", "form") - # Check if the downstream client supports the requested mode - if downstream_capabilities is not None: - elicit_caps: Final[object] = getattr(downstream_capabilities, "elicitation", None) - if elicit_caps is None: - verbose_logger.info("MCP elicitation: downstream client does not support elicitation") - return ElicitResult(action="decline") - if mode == "url": - url_cap: Final[object] = getattr(elicit_caps, "url", None) - if url_cap is None: - verbose_logger.info("MCP elicitation: downstream client does not support URL mode") - return ElicitResult(action="decline") - if mode == "form": - form_cap: Final[object] = getattr(elicit_caps, "form", None) - if form_cap is None: - verbose_logger.info("MCP elicitation: downstream client does not support form mode") - return ElicitResult(action="decline") - try: - if mode == "url" and isinstance(params, ElicitRequestURLParams): - # URL mode: relay URL to client for external navigation - verbose_logger.info( - "MCP elicitation: relaying URL mode to downstream, url=%s", - getattr(params, "url", ""), - ) - result = await downstream_session.elicit_url( - message=params.message, - url=params.url, - elicitation_id=params.elicitation_id, - ) - elif isinstance(params, ElicitRequestFormParams): - # Form mode: relay structured form to client - verbose_logger.info("MCP elicitation: relaying form mode to downstream") - result = await downstream_session.elicit_form( - message=params.message, - requested_schema=params.requested_schema, - ) - else: - # Fallback for generic ElicitRequestParams — pass an empty schema - # since elicit() requires requested_schema as a positional arg. - verbose_logger.info("MCP elicitation: relaying generic elicitation to downstream") - result = await downstream_session.elicit( - message=getattr(params, "message", ""), - requested_schema=getattr(params, "requested_schema", {}), - ) - verbose_logger.info( - "MCP elicitation: downstream responded with action=%s", - getattr(result, "action", "unknown"), + downstream_capabilities: ClientCapabilities | None = None, + related_request_id: RequestId | None = None, +) -> ElicitResult | ErrorData: + capabilities: Final = downstream_capabilities.elicitation if downstream_capabilities is not None else None + if capabilities is None: + return ErrorData(code=INVALID_REQUEST, message="Downstream client has not advertised elicitation support") + if isinstance(params, ElicitRequestURLParams): + if capabilities.url is None: + return ErrorData(code=INVALID_REQUEST, message="Downstream client does not support URL elicitation") + return await downstream_session.elicit_url( + message=params.message, + url=params.url, + elicitation_id=params.elicitation_id, + related_request_id=related_request_id, ) - return result - except Exception as e: - verbose_logger.warning("MCP elicitation: failed to relay to downstream: %s", e) - # If relay fails, decline gracefully - return ElicitResult(action="decline") + if capabilities.form is None and capabilities.url is not None: + return ErrorData(code=INVALID_REQUEST, message="Downstream client does not support form elicitation") + if not isinstance(params, ElicitRequestFormParams): + return ErrorData(code=INVALID_REQUEST, message="Unsupported MCP elicitation parameters") + return await downstream_session.elicit_form( + message=params.message, + requested_schema=params.requested_schema, + related_request_id=related_request_id, + ) diff --git a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py index f52d2a006d2..13c3d265429 100644 --- a/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py +++ b/litellm/proxy/_experimental/mcp_server/legacy_callbacks.py @@ -11,7 +11,9 @@ from mcp.types import ( ErrorData, ) +from litellm.constants import MCP_CLIENT_TIMEOUT from litellm.proxy._experimental.mcp_server.contracts import OperationContext +from litellm.proxy._experimental.mcp_server.mcp_context import get_active_mcp_request_ctx from litellm.proxy._types import UserAPIKeyAuth @@ -62,11 +64,19 @@ def create_sampling_callback( return callback -def create_elicitation_callback() -> ElicitationCallback: +def create_elicitation_callback(timeout: float | None = None) -> ElicitationCallback: from litellm.proxy._experimental.mcp_server.server import get_active_mcp_session downstream_session: Final = get_active_mcp_session() - downstream_capabilities: Final = getattr(downstream_session, "capabilities", None) + request: Final = get_active_mcp_request_ctx() + client_params: Final = downstream_session.client_params if downstream_session is not None else None + downstream_capabilities: Final = ( + client_params.capabilities.model_copy(deep=True) if client_params is not None else None + ) + related_request_id: Final = ( + request.request_id if request is not None and request.session is downstream_session else None + ) + relay_timeout: Final = timeout if timeout is not None else MCP_CLIENT_TIMEOUT async def callback(context: object, params: ElicitRequestParams) -> ElicitResult | ErrorData: from litellm.proxy._experimental.mcp_server.elicitation_handler import handle_elicitation_request @@ -76,6 +86,8 @@ def create_elicitation_callback() -> ElicitationCallback: params=params, downstream_session=downstream_session, downstream_capabilities=downstream_capabilities, + related_request_id=related_request_id, + timeout=relay_timeout, ) return callback diff --git a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py index c42ab60c48b..217e373030d 100644 --- a/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py +++ b/litellm/proxy/_experimental/mcp_server/mcp_server_manager.py @@ -1735,12 +1735,12 @@ def _create_sampling_callback( return create_sampling_callback(user_api_key_auth, raw_headers, client_ip, operation_context) -def _create_elicitation_callback(): +def _create_elicitation_callback(timeout: float | None = None): if not MCP_ELICITATION_AVAILABLE: return None from litellm.proxy._experimental.mcp_server.legacy_callbacks import create_elicitation_callback - return create_elicitation_callback() + return create_elicitation_callback(timeout=timeout) def _record_mcp_guardrail_evaluations( @@ -4250,7 +4250,11 @@ class MCPServerManager: if resolved_server.allow_sampling else None ), - elicitation_callback=(_create_elicitation_callback() if resolved_server.allow_elicitation else None), + elicitation_callback=( + _create_elicitation_callback(timeout=resolved_server.timeout) + if resolved_server.allow_elicitation + else None + ), ) _create_mcp_client = create_mcp_client diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py index a59b02ec01d..dd8dddf106e 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_elicitation_handler.py @@ -7,12 +7,18 @@ well as the decline paths used in tool-bridge mode or when the downstream client lacks the requested elicitation capability. """ +import asyncio from types import SimpleNamespace from unittest.mock import AsyncMock import pytest from mcp.types import ( + ClientCapabilities, + ElicitationCapability, + FormElicitationCapability, + UrlElicitationCapability, + REQUEST_TIMEOUT, ElicitRequestFormParams, ElicitRequestURLParams, ElicitResult, @@ -43,23 +49,21 @@ def _url_params(message: str = "please authorize") -> ElicitRequestURLParams: ) -def _caps(*, url=True, form=True) -> SimpleNamespace: - elicit = SimpleNamespace( - url=object() if url else None, - form=object() if form else None, - ) - return SimpleNamespace(elicitation=elicit) +def _caps(*, url=True, form=True) -> ClientCapabilities: + return ClientCapabilities(elicitation=ElicitationCapability( + url=UrlElicitationCapability() if url else None, + form=FormElicitationCapability() if form else None, + )) class TestHandleElicitationRequest: - async def test_should_decline_when_no_downstream_session(self): + async def test_should_error_when_no_downstream_session(self): result = await handle_elicitation_request( context=SimpleNamespace(), params=_form_params(), downstream_session=None, ) - assert isinstance(result, ElicitResult) - assert result.action == "decline" + assert isinstance(result, ErrorData) async def test_should_relay_to_downstream_when_session_present(self): accepted = ElicitResult(action="accept", content={"name": "ada"}) @@ -69,7 +73,7 @@ class TestHandleElicitationRequest: context=SimpleNamespace(), params=_form_params(), downstream_session=session, - downstream_capabilities=None, + downstream_capabilities=_caps(), ) assert result is accepted @@ -85,21 +89,6 @@ class TestHandleElicitationRequest: assert isinstance(result, ErrorData) assert "not available" in result.message - async def test_should_return_error_data_on_unexpected_failure(self): - class _ExplodingParams: - mode = "form" - - @property - def message(self): - raise RuntimeError("boom") - - result = await handle_elicitation_request( - context=SimpleNamespace(), - params=_ExplodingParams(), - downstream_session=None, - ) - assert isinstance(result, ErrorData) - assert "boom" in result.message class TestRelayElicitationToDownstream: @@ -136,25 +125,17 @@ class TestRelayElicitationToDownstream: assert kwargs["url"] == "https://example.com/oauth" assert kwargs["elicitation_id"] == "elc-1" - async def test_should_use_generic_elicit_for_unknown_param_type(self): - accepted = ElicitResult(action="accept") - session = SimpleNamespace(elicit=AsyncMock(return_value=accepted)) - - # A bare params object that is neither Form nor URL params triggers - # the generic fallback path. - params = SimpleNamespace(mode="form", message="hi", requested_schema={}) - result = await _relay_elicitation_to_downstream( - params=params, - downstream_session=session, - downstream_capabilities=None, - ) - - assert result is accepted - session.elicit.assert_awaited_once() - - async def test_should_decline_when_elicitation_unsupported(self): + async def test_should_reject_invalid_elicitation_parameters(self): session = SimpleNamespace(elicit_form=AsyncMock()) - caps = SimpleNamespace(elicitation=None) + result = await _relay_elicitation_to_downstream( + params=SimpleNamespace(mode="form"), downstream_session=session, downstream_capabilities=_caps(), + ) + assert isinstance(result, ErrorData) + session.elicit_form.assert_not_awaited() + + async def test_should_error_when_elicitation_unsupported(self): + session = SimpleNamespace(elicit_form=AsyncMock()) + caps = ClientCapabilities() result = await _relay_elicitation_to_downstream( params=_form_params(), @@ -162,11 +143,10 @@ class TestRelayElicitationToDownstream: downstream_capabilities=caps, ) - assert isinstance(result, ElicitResult) - assert result.action == "decline" + assert isinstance(result, ErrorData) session.elicit_form.assert_not_awaited() - async def test_should_decline_url_mode_when_url_unsupported(self): + async def test_should_error_url_mode_when_url_unsupported(self): session = SimpleNamespace(elicit_url=AsyncMock()) result = await _relay_elicitation_to_downstream( @@ -175,11 +155,10 @@ class TestRelayElicitationToDownstream: downstream_capabilities=_caps(url=False, form=True), ) - assert isinstance(result, ElicitResult) - assert result.action == "decline" + assert isinstance(result, ErrorData) session.elicit_url.assert_not_awaited() - async def test_should_decline_form_mode_when_form_unsupported(self): + async def test_should_error_form_mode_when_form_unsupported(self): session = SimpleNamespace(elicit_form=AsyncMock()) result = await _relay_elicitation_to_downstream( @@ -188,24 +167,112 @@ class TestRelayElicitationToDownstream: downstream_capabilities=_caps(url=True, form=False), ) - assert isinstance(result, ElicitResult) - assert result.action == "decline" + assert isinstance(result, ErrorData) session.elicit_form.assert_not_awaited() - async def test_should_decline_when_downstream_relay_raises(self): + async def test_should_error_when_downstream_relay_raises(self): session = SimpleNamespace( elicit_form=AsyncMock(side_effect=RuntimeError("transport closed")) ) - result = await _relay_elicitation_to_downstream( + result = await handle_elicitation_request( + context=None, params=_form_params(), downstream_session=session, downstream_capabilities=_caps(form=True), ) - assert isinstance(result, ElicitResult) - assert result.action == "decline" + assert isinstance(result, ErrorData) if __name__ == "__main__": pytest.main([__file__, "-v"]) + + +@pytest.mark.asyncio +async def test_relay_failure_is_not_user_decline(): + session = SimpleNamespace(elicit_form=AsyncMock(side_effect=RuntimeError("private upstream credential"))) + result = await handle_elicitation_request( + context=None, params=_form_params(), downstream_session=session, downstream_capabilities=_caps(), + ) + assert isinstance(result, ErrorData), "A failed relay must not claim that the user declined" + assert "private upstream credential" not in result.message + + +@pytest.mark.asyncio +async def test_unknown_client_capabilities_prevent_relay(): + session = SimpleNamespace(elicit_form=AsyncMock(return_value=ElicitResult(action="accept"))) + result = await handle_elicitation_request( + context=None, params=_form_params(), downstream_session=session, downstream_capabilities=None, + ) + assert isinstance(result, ErrorData), "Unknown capabilities must fail explicitly" + session.elicit_form.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ["form", "url"]) +@pytest.mark.parametrize("action", ["accept", "decline", "cancel"]) +async def test_relay_preserves_response_and_request_correlation(mode, action): + response = ElicitResult(action=action) + request = AsyncMock(return_value=response) + session = SimpleNamespace(elicit_form=request, elicit_url=request) + params = _form_params() if mode == "form" else _url_params() + result = await handle_elicitation_request( + context=SimpleNamespace(request_id="upstream-id"), params=params, + downstream_session=session, downstream_capabilities=_caps(), related_request_id=0, + ) + assert result is response + assert request.await_args.kwargs["related_request_id"] == 0 + if mode == "url": + assert request.await_args.kwargs["url"] == params.url + assert request.await_args.kwargs["elicitation_id"] == params.elicitation_id + else: + assert request.await_args.kwargs["requested_schema"] == params.requested_schema + + +@pytest.mark.asyncio +async def test_expired_relay_deadline_returns_explicit_timeout(): + request = AsyncMock() + result = await handle_elicitation_request( + context=None, params=_form_params(), downstream_session=SimpleNamespace(elicit_form=request), + downstream_capabilities=_caps(), timeout=0, + ) + assert isinstance(result, ErrorData) + assert result.code == REQUEST_TIMEOUT + assert "timed out" in result.message + request.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_relay_cancellation_releases_downstream_waiter(): + started = asyncio.Event() + finished = asyncio.Event() + + async def wait_for_user(**kwargs): + started.set() + try: + await asyncio.Event().wait() + finally: + finished.set() + + task = asyncio.create_task(handle_elicitation_request( + context=None, params=_form_params(), downstream_session=SimpleNamespace(elicit_form=wait_for_user), + downstream_capabilities=_caps(), related_request_id="tool-call", + )) + await started.wait() + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert finished.is_set() + + +@pytest.mark.asyncio +async def test_legacy_empty_elicitation_capability_supports_form(): + accepted = ElicitResult(action="accept") + session = SimpleNamespace(elicit_form=AsyncMock(return_value=accepted)) + result = await handle_elicitation_request( + context=None, params=_form_params(), downstream_session=session, + downstream_capabilities=ClientCapabilities(elicitation=ElicitationCapability()), + related_request_id="legacy-call", + ) + assert result is accepted diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py index 5731738f137..c7b477a6cc6 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server_manager.py @@ -155,18 +155,26 @@ async def test_elicitation_callback_keeps_initiating_session(): from litellm.proxy._experimental.mcp_server import server as legacy_server from litellm.proxy._experimental.mcp_server.mcp_server_manager import _create_elicitation_callback - initiating = MagicMock() - replacement = MagicMock() - recorder = AsyncMock() + from types import SimpleNamespace + from mcp.types import ClientCapabilities, ElicitationCapability, FormElicitationCapability, ElicitRequestFormParams, ElicitResult + from litellm.proxy._experimental.mcp_server.mcp_context import active_mcp_request_ctx_var + + capabilities = ClientCapabilities(elicitation=ElicitationCapability(form=FormElicitationCapability())) + accepted = ElicitResult(action="accept") + request = AsyncMock(return_value=accepted) + initiating = SimpleNamespace(client_params=SimpleNamespace(capabilities=capabilities), elicit_form=request) token = legacy_server.active_mcp_session_var.set(initiating) + request_token = active_mcp_request_ctx_var.set(SimpleNamespace(session=initiating, request_id="initiating-call")) try: callback = _create_elicitation_callback() - legacy_server.active_mcp_session_var.set(replacement) - with patch("litellm.proxy._experimental.mcp_server.elicitation_handler.handle_elicitation_request", recorder): - await callback(None, None) - assert recorder.await_args.kwargs["downstream_session"] is initiating - assert recorder.await_args.kwargs["downstream_capabilities"] is initiating.capabilities + legacy_server.active_mcp_session_var.set(SimpleNamespace()) + active_mcp_request_ctx_var.set(None) + capabilities.elicitation = None + result = await callback(None, ElicitRequestFormParams(message="Confirm", requested_schema={"type":"object"})) + assert result is accepted + assert request.await_args.kwargs["related_request_id"] == "initiating-call" finally: + active_mcp_request_ctx_var.reset(request_token) legacy_server.active_mcp_session_var.reset(token) From a81f507bca3948a58337c0435f054ae4b6de8cfd Mon Sep 17 00:00:00 2001 From: nate-berri Date: Thu, 8 Oct 2026 10:21:56 -0700 Subject: [PATCH 46/65] docs(rust): add ADRs for the Rust core (#45205) * docs(rust): add ADR folder with ADR 000 and ADR 001 scaffold Co-Authored-By: Claude Opus 5.5 * docs(rust): add ADR 000 and ADR 001 on typing requests inside Rust Co-Authored-By: Claude Opus 5.5 * docs(rust): list four ADR sections in ADR 000 Co-Authored-By: Claude Opus 5.5 * docs(rust): move ADR process doc from adr_000 to AGENTS.md Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Nate Armstrong Co-authored-by: Claude Opus 5.5 Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/docs/adrs/AGENTS.md | 38 +++++++++++++++++++ ...adr_001_only_type_inference_inside_rust.md | 15 ++++++++ 2 files changed, 53 insertions(+) create mode 100644 litellm-rust/docs/adrs/AGENTS.md create mode 100644 litellm-rust/docs/adrs/adr_001_only_type_inference_inside_rust.md diff --git a/litellm-rust/docs/adrs/AGENTS.md b/litellm-rust/docs/adrs/AGENTS.md new file mode 100644 index 00000000000..040318c623d --- /dev/null +++ b/litellm-rust/docs/adrs/AGENTS.md @@ -0,0 +1,38 @@ +# ADR 000: Record architecture decisions + +Status: Accepted + +## Context + +Decisions about the Rust core get made in various places, and in synchronous conversations or on whiteboards. +Knowing the context of when and how the decision was made helps greatly to contextualize code for a reader, as well as informing agents about how things should be done. +Especially with the rust core, there are a great many decisions that need to be made in a large solution space, and the reasons for such decisions may not be obvious. + +## Decision + +We write a short ADR for any decision that is hard to reverse or that a new contributor would reasonably question. Each ADR lives in this folder as `adr_NNN_short_title.md`, numbered in order, and has four sections: Context, Decision, Alternatives Considered, and Consequences. If writing one takes more than a few minutes, it is too long. + +ADRs are never edited after they are accepted, apart from the status line. To change course, write a new ADR and mark the old one `Superseded by ADR NNN`. + +ADRs should be hand-written as much as possible, with the intention of being +extremely intention-dense. Agents have a habit of making statements stronger +than they otherwise should be, which makes it challenging to interpret +reasoning, which is the entire point of ADRs. + +ADR filenames should ideally make it clear what decision was made. Their +contents should explain (briefly) alternatives considered, as well as +consequences of the decision in pro/con format. + +## Alternatives Considered + +**Not using ADRs**: This would make us able to move a bit faster, but it makes decisionmaking less clear. Additionally, given the lack of other code documentation, there is no living document other than the code. + +## Consequences + +Benefits: + +- Reviewers and future contributors can find reasoning behind a choice easily + +Costs: + +- A few minutes of thought and writing per significant decision diff --git a/litellm-rust/docs/adrs/adr_001_only_type_inference_inside_rust.md b/litellm-rust/docs/adrs/adr_001_only_type_inference_inside_rust.md new file mode 100644 index 00000000000..0973b13369b --- /dev/null +++ b/litellm-rust/docs/adrs/adr_001_only_type_inference_inside_rust.md @@ -0,0 +1,15 @@ +# ADR 001: Only type requests inside Rust + +Status: Accepted + +## Context + +The python SDK had typed interfaces for requests living in the bridge in Python. However, there were multiple ways for requests to get there, and so the typing is very shallow and ultimately is just a shim placed on top of untyped args and kwargs. + +## Decision + +For the time being, avoid trying to restrict or type requests and their possible arguments inside python. Just pass the raw object over into rust, and have a translation layer at the boundary which handles it. + +## Consequences + +Some python interfaces are a lot less clear about what they expect as input. The typing of the rust module is entirely internal to it, and must be inferred from other channels. \ No newline at end of file From d5173e3d70d2b3915f037ff746b088d0fc836113 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:33:35 -0700 Subject: [PATCH 47/65] test: move 76 live top-level tests to offline unit and integration coverage (#45352) * test: move 76 live top-level tests to offline unit and integration coverage * test: keep the mock-param opt-in guard valid once the top-level senders are gone * test: address review on offline replacements * docs(test): drop the deleted harness test file from the harness check command * test: make the key rebind, team member delete and routes integration tests exercise the legacy paths * test: make fallback, rpm and spend integration contracts deterministic and clean up their rows, assert the rpm limit in usage-based routing * test: expect the no-deployments error at the rpm limit and cover the strategy check without pre-call checks * test: hand member cleanups to the scenario instead of growing a budget list, flatten callback kinds * test: scope the admin health check to the test's own deployment --------- Co-authored-by: yuneng --- .../test_callback_leak_contracts.py | 147 +++ .../management/test_key_route_contracts.py | 152 +++ .../test_listing_and_health_contracts.py | 138 +++ .../management/test_team_member_routes.py | 205 ++++ .../management/test_user_routes.py | 34 + .../test_fallback_retry_header_contracts.py | 291 ++++++ .../spend/test_spend_attribution_contracts.py | 102 ++ tests/rust-python-harness/AGENTS.md | 2 +- tests/rust-python-harness/cli/test_cli.py | 9 + .../shared/reporting/test_models.py | 53 + .../shared/reporting/test_ui.py | 26 + .../sdk/test_case_modules_importable.py | 23 + tests/test_anthropic_compaction_usage.py | 96 -- tests/test_budget_management.py | 102 -- tests/test_callbacks_on_proxy.py | 303 ------ tests/test_default_encoding_non_root.py | 55 -- tests/test_end_users.py | 201 ---- tests/test_fallbacks.py | 325 ------- tests/test_gpt5_azure_temperature_support.py | 102 -- tests/test_health.py | 84 -- tests/test_keys.py | 317 +----- tests/test_litellm_proxy_responses_config.py | 63 -- tests/test_models.py | 169 ---- tests/test_new_vector_store_endpoints.py | 103 +- tests/test_openai_endpoints.py | 355 +------ tests/test_otel_thread_leak.py | 91 -- tests/test_presidio_latency.py | 77 -- tests/test_ratelimit.py | 170 ---- tests/test_resource_cleanup.py | 117 --- tests/test_rust_python_harness.py | 108 --- tests/test_service_logger_otel.py | 111 +-- tests/test_spend_logs.py | 149 +-- tests/test_team.py | 912 ------------------ tests/test_team_members.py | 190 ---- tests/test_users.py | 206 ---- tests/unit/integrations/test_langfuse_otel.py | 64 +- .../test_default_encoding.py | 40 + .../test_anthropic_chat_transformation.py | 51 + .../response/test_azure_transformation.py | 25 + .../custom_httpx/test_async_client_cleanup.py | 100 ++ .../llms/litellm_proxy/responses/__init__.py | 0 .../responses/test_transformation.py | 29 + .../guardrail_hooks/test_presidio.py | 31 + tests/unit/proxy/test_route_llm_request.py | 4 +- .../router_strategy/test_lowest_tpm_rpm.py | 71 +- .../test_router_vector_store_endpoints.py | 101 ++ tests/unit/test_service_logger.py | 51 + 47 files changed, 1746 insertions(+), 4409 deletions(-) create mode 100644 tests/integration/configuration/test_callback_leak_contracts.py create mode 100644 tests/integration/management/test_key_route_contracts.py create mode 100644 tests/integration/management/test_listing_and_health_contracts.py create mode 100644 tests/integration/management/test_team_member_routes.py create mode 100644 tests/integration/management/test_user_routes.py create mode 100644 tests/integration/routing/test_fallback_retry_header_contracts.py create mode 100644 tests/integration/spend/test_spend_attribution_contracts.py create mode 100644 tests/rust-python-harness/shared/reporting/test_models.py create mode 100644 tests/rust-python-harness/shared/reporting/test_ui.py create mode 100644 tests/rust-python-harness/strategies/trace_parity/sdk/test_case_modules_importable.py delete mode 100644 tests/test_anthropic_compaction_usage.py delete mode 100644 tests/test_budget_management.py delete mode 100644 tests/test_callbacks_on_proxy.py delete mode 100644 tests/test_default_encoding_non_root.py delete mode 100644 tests/test_end_users.py delete mode 100644 tests/test_fallbacks.py delete mode 100644 tests/test_gpt5_azure_temperature_support.py delete mode 100644 tests/test_health.py delete mode 100644 tests/test_otel_thread_leak.py delete mode 100644 tests/test_presidio_latency.py delete mode 100644 tests/test_ratelimit.py delete mode 100644 tests/test_resource_cleanup.py delete mode 100644 tests/test_team.py delete mode 100644 tests/test_team_members.py create mode 100644 tests/unit/litellm_core_utils/test_default_encoding.py create mode 100644 tests/unit/llms/litellm_proxy/responses/__init__.py create mode 100644 tests/unit/llms/litellm_proxy/responses/test_transformation.py create mode 100644 tests/unit/test_router/test_router_vector_store_endpoints.py diff --git a/tests/integration/configuration/test_callback_leak_contracts.py b/tests/integration/configuration/test_callback_leak_contracts.py new file mode 100644 index 00000000000..9ae5179dad2 --- /dev/null +++ b/tests/integration/configuration/test_callback_leak_contracts.py @@ -0,0 +1,147 @@ +import json +import re +import uuid +from collections import Counter +from itertools import chain +from pathlib import Path +from typing import Final + +from pydantic import JsonValue + +from tests.integration._support.client import Gateway, object_value +from tests.integration._support.process import owned_proxy + +SAMPLES: Final = 4 +REQUESTS_PER_INTERVAL: Final = 5 +LEAK_MIN_NET_GROWTH: Final = 5 +LEAK_MIN_GROWING_INTERVALS: Final = 2 +ADDRESS: Final = re.compile(r" at 0x[0-9a-fA-F]+") +OBJECT: Final = re.compile(r"<([\w.]+) object") +BOUND_METHOD: Final = re.compile(r"bound method ([\w.]+)") + + +def _callback_type(text: str) -> str: + stripped: Final = ADDRESS.sub("", text) + if (instance := OBJECT.search(stripped)) is not None: + return instance.group(1).split(".")[-1] + if (method := BOUND_METHOD.search(stripped)) is not None: + return method.group(1) + return stripped.strip() + + +def _sample(candidate: Gateway) -> tuple[Counter[str], int]: + response: Final = candidate.request("GET", "/active/callbacks") + assert response.status_code == 200, response.text + body: Final = object_value(response.json()) + callbacks: Final = body["all_litellm_callbacks"] + alerting: Final = body["num_alerting"] + assert isinstance(callbacks, list) and isinstance(alerting, int), body + return Counter(_callback_type(str(callback)) for callback in callbacks), alerting + + +Samples = tuple[Counter[str], ...] + + +def _kinds(samples: Samples) -> frozenset[str]: + return frozenset(chain.from_iterable(samples)) + + +def _series(samples: Samples, kind: str) -> tuple[int, ...]: + return tuple(sample.get(kind, 0) for sample in samples) + + +def _deltas(series: tuple[int, ...]) -> tuple[int, ...]: + return tuple(after - before for before, after in zip(series, series[1:])) + + +def _grows(series: tuple[int, ...]) -> bool: + deltas: Final = _deltas(series) + return ( + all(delta >= 0 for delta in deltas) + and series[-1] - series[0] >= LEAK_MIN_NET_GROWTH + and sum(1 for delta in deltas if delta > 0) >= LEAK_MIN_GROWING_INTERVALS + ) + + +def _grows_only_in_last_interval(series: tuple[int, ...]) -> bool: + deltas: Final = _deltas(series) + return all(delta >= 0 for delta in deltas) and [ + index for index, delta in enumerate(deltas) if delta > 0 + ] == [len(deltas) - 1] + + +def _leaking(samples: Samples) -> dict[str, tuple[int, ...]]: + return {kind: _series(samples, kind) for kind in sorted(_kinds(samples)) if _grows(_series(samples, kind))} + + +def _config(directory: Path, upstream_url: str, model: str, extra: dict[str, JsonValue]) -> Path: + config: Final = directory / f"callback_leak_{uuid.uuid4().hex}.yaml" + general_settings: Final = extra.get("general_settings") + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": model, + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{upstream_url}/v1", + "api_key": "integration-provider-key", + }, + } + ], + "router_settings": extra.get("router_settings", {}), + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + **(general_settings if isinstance(general_settings, dict) else {}), + }, + } + ) + ) + return config + + +def _interval(candidate: Gateway, model: str, index: int) -> tuple[Counter[str], int]: + for request in range(REQUESTS_PER_INTERVAL if index else 0): + assert candidate.chat(model, text=f"leak probe {index} {request}")["model"] == model + return _sample(candidate) + + +def _sample_under_traffic(candidate: Gateway, model: str) -> tuple[Samples, tuple[int, ...]]: + taken: Final = tuple(_interval(candidate, model, index) for index in range(SAMPLES)) + callbacks: Final = tuple(sample for sample, _ in taken) + late_growth: Final = any(_grows_only_in_last_interval(_series(callbacks, kind)) for kind in _kinds(callbacks)) + confirmed: Final = taken + ((_interval(candidate, model, SAMPLES),) if late_growth else ()) + return tuple(sample for sample, _ in confirmed), tuple(alerting for _, alerting in confirmed) + + +def test_callback_registry_does_not_grow_with_traffic(gateway: Gateway, tmp_path: Path) -> None: + model: Final = f"integration-callback-leak-{uuid.uuid4().hex}" + config: Final = _config(tmp_path, gateway.upstream_url, model, {}) + with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config) as candidate: + samples, _ = _sample_under_traffic(candidate, model) + assert sum(samples[0].values()) > 0 + assert _leaking(samples) == {}, samples + + +def test_callback_registry_does_not_grow_under_latency_routing_with_alerting(gateway: Gateway, tmp_path: Path) -> None: + model: Final = f"integration-callback-leak-{uuid.uuid4().hex}" + config: Final = _config( + tmp_path, + gateway.upstream_url, + model, + { + "router_settings": {"routing_strategy": "latency-based-routing"}, + "general_settings": { + "alert_to_webhook_url": {"llm_exceptions": "http://127.0.0.1:9/integration-alerts"}, + "alert_types": ["llm_exceptions", "db_exceptions"], + }, + }, + ) + with owned_proxy(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config) as candidate: + samples, alerts = _sample_under_traffic(candidate, model) + assert sum(samples[0].values()) > 0 + assert any(kind.startswith("LowestLatencyLoggingHandler") for kind in samples[0]), samples[0] + assert _leaking(samples) == {}, samples + assert len(set(alerts)) == 1, alerts diff --git a/tests/integration/management/test_key_route_contracts.py b/tests/integration/management/test_key_route_contracts.py new file mode 100644 index 00000000000..977f60adf62 --- /dev/null +++ b/tests/integration/management/test_key_route_contracts.py @@ -0,0 +1,152 @@ +import uuid +from concurrent.futures import ThreadPoolExecutor +from hashlib import sha256 +from typing import Final, Literal + +import pytest +from pydantic import JsonValue + +from tests.integration._support.client import Gateway, Scenario, delete_key_if_present, object_value, string_value +from tests.integration._support.database import read_rows + +KEY_ALIAS: Final = "mistral-7b" +ModelAccess = Literal["all-team-models", "single-model"] +AccessLevel = Literal["key", "team"] +ModelEndpoint = Literal["/v1/models", "/model/info"] + + +def _owned_key( + gateway: Gateway, scenario: Scenario, fields: dict[str, JsonValue], *, key: str | None = None +) -> dict[str, JsonValue]: + response: Final = gateway.request("POST", "/key/generate", fields, key=key) + assert response.status_code == 200, response.text + created: Final = object_value(response.json()) + scenario.cleanups.callback(delete_key_if_present, gateway, string_value(created["key"])) + return created + + +def _data(gateway: Gateway, path: str, key: str, params: dict[str, str] | None = None) -> list[JsonValue]: + response: Final = gateway.request("GET", path, key=key, params=params) + assert response.status_code == 200, response.text + data: Final = object_value(response.json())["data"] + assert isinstance(data, list) + return data + + +def _verification_row(key: str) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT user_id FROM "LiteLLM_VerificationToken" WHERE token = %s', + (sha256(key.encode()).hexdigest(),), + ) + + +def test_concurrent_key_generation_persists_every_key(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + with ThreadPoolExecutor(max_workers=10) as pool: + keys: Final = list(pool.map(lambda _: scenario.key(models=[model]), range(10))) + assert len(set(keys)) == 10 + for key in keys: + assert len(_verification_row(key)) == 1 + + +def test_generated_key_exposes_hashed_token_and_timestamps(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + created: Final = _owned_key(gateway, scenario, {}) + key: Final = string_value(created["key"]) + assert created["token"] is not None + assert created["token"] != key + assert created["token"] == sha256(key.encode()).hexdigest() + assert created["token_id"] is not None + assert created["created_at"] is not None + assert created["updated_at"] is not None + + +def test_key_info_serves_admin_and_self_and_hides_unknown_keys(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + key: Final = scenario.key() + digest: Final = sha256(key.encode()).hexdigest() + admin: Final = gateway.request("GET", "/key/info", params={"key": key}) + assert admin.status_code == 200, admin.text + assert object_value(admin.json())["key"] == key + explicit: Final = gateway.request("GET", "/key/info", params={"key": key}, key=key) + assert explicit.status_code == 200, explicit.text + assert object_value(explicit.json())["key"] == key + implicit: Final = gateway.request("GET", "/key/info", key=key) + assert implicit.status_code == 200, implicit.text + assert object_value(implicit.json())["key"] == digest + unknown: Final = gateway.request("GET", "/key/info", params={"key": f"sk-{uuid.uuid4()}"}, key=key) + assert unknown.status_code == 404, unknown.text + + +def test_model_info_is_filtered_to_the_models_a_key_can_use(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + key: Final = scenario.key(models=[model]) + admin_models: Final = _data(gateway, "/model/info", gateway.key) + user_models: Final = _data(gateway, "/model/info", key) + assert len(admin_models) > len(user_models) + assert [object_value(entry)["model_name"] for entry in user_models] == [model] + + +def test_proxy_admin_user_key_deletes_a_key_owned_by_another_user(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + admin_user: Final = scenario.user(user_role="proxy_admin") + owner: Final = scenario.user(user_role="internal_user") + admin_key: Final = string_value(_owned_key(gateway, scenario, {"user_id": admin_user})["key"]) + victim: Final = string_value(_owned_key(gateway, scenario, {"user_id": owner})["key"]) + deleted: Final = gateway.request("POST", "/key/delete", {"keys": [victim]}, key=admin_key) + assert deleted.status_code == 200, deleted.text + assert _verification_row(victim) == [] + + +@pytest.mark.parametrize("model_endpoint", ["/v1/models", "/model/info"]) +@pytest.mark.parametrize("access_level", ["key", "team"]) +@pytest.mark.parametrize("model_access", ["all-team-models", "single-model"]) +def test_key_model_list_follows_key_or_team_access( + gateway: Gateway, model_access: ModelAccess, access_level: AccessLevel, model_endpoint: ModelEndpoint +) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + granted: Final[list[JsonValue]] = [] if model_access == "all-team-models" else [model] + team: Final = scenario.team(models=granted if access_level == "team" else []) + key: Final = scenario.key( + team_id=team, + models=granted if access_level == "key" else [], + aliases={KEY_ALIAS: model}, + ) + data: Final = _data(gateway, model_endpoint, key) + if model_access == "all-team-models": + assert len(data) > 1 + if model_endpoint == "/v1/models": + assert all(isinstance(object_value(entry)["id"], str) for entry in data) + assert model in {object_value(entry)["id"] for entry in data} + else: + assert model in {object_value(entry)["model_name"] for entry in data} + elif model_endpoint == "/v1/models": + assert {object_value(entry)["id"] for entry in data} == {model, KEY_ALIAS} + else: + assert [object_value(entry)["model_name"] for entry in data] == [model] + + +def test_internal_user_cannot_reassign_its_key_to_another_user(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + first: Final = scenario.user(user_role="internal_user") + second: Final = scenario.user(user_role="internal_user") + key: Final = string_value(_owned_key(gateway, scenario, {"user_id": first})["key"]) + own_key: Final = string_value(_owned_key(gateway, scenario, {}, key=key)["key"]) + assert _verification_row(own_key) == [{"user_id": first}] + update: Final = gateway.request("POST", "/key/update", {"key": own_key, "user_id": second}, key=key) + assert update.status_code == 403, update.text + assert _verification_row(own_key) == [{"user_id": first}] + + +def test_internal_user_cannot_delete_another_users_key(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + first: Final = scenario.user(user_role="internal_user") + second: Final = scenario.user(user_role="internal_user") + victim: Final = string_value(_owned_key(gateway, scenario, {"user_id": first})["key"]) + attacker: Final = string_value(_owned_key(gateway, scenario, {"user_id": second})["key"]) + deleted: Final = gateway.request("POST", "/key/delete", {"keys": [victim]}, key=attacker) + assert deleted.status_code == 403, deleted.text + assert _verification_row(victim) == [{"user_id": first}] diff --git a/tests/integration/management/test_listing_and_health_contracts.py b/tests/integration/management/test_listing_and_health_contracts.py new file mode 100644 index 00000000000..8492dc50adb --- /dev/null +++ b/tests/integration/management/test_listing_and_health_contracts.py @@ -0,0 +1,138 @@ +import uuid +from datetime import datetime, timezone +from typing import Final + +from pydantic import JsonValue + +from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time +from tests.integration._support.client import Gateway, Scenario, delete_key_if_present, object_value, string_value +from tests.integration._support.database import read_rows + + +def _utc(text: JsonValue) -> datetime: + parsed: Final = datetime.fromisoformat(string_value(text)) + return parsed if parsed.tzinfo is not None else parsed.replace(tzinfo=timezone.utc) + + +def _model_id(model_name: str) -> str: + rows: Final = read_rows('SELECT model_id FROM "LiteLLM_ProxyModelTable" WHERE model_name = %s', (model_name,)) + assert len(rows) == 1, rows + return string_value(rows[0]["model_id"]) + + +def _data(gateway: Gateway, path: str, key: str, params: dict[str, str] | None = None) -> list[dict[str, JsonValue]]: + response: Final = gateway.request("GET", path, key=key, params=params) + assert response.status_code == 200, response.text + data: Final = object_value(response.json())["data"] + assert isinstance(data, list) + return [object_value(entry) for entry in data] + + +def _wildcard_model(gateway: Gateway, scenario: Scenario, prefix: str) -> None: + created: Final = gateway.post( + "/model/new", + { + "model_name": f"{prefix}/*", + "litellm_params": { + "model": "anthropic/*", + "api_key": "integration-provider-key", + "api_base": gateway.upstream_url, + }, + }, + ) + scenario.cleanups.callback(scenario.delete_model, string_value(object_value(created["model_info"])["id"])) + + +def test_budget_duration_schedules_reset_at_the_next_standardized_boundary(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + budget_id: Final = scenario.budget(max_budget=10.0, budget_duration="1d") + rows: Final = read_rows( + 'SELECT created_at::text AS created_at, budget_reset_at::text AS reset_at FROM "LiteLLM_BudgetTable" ' + "WHERE budget_id = %s", + (budget_id,), + ) + assert len(rows) == 1, rows + assert rows[0]["reset_at"] is not None, rows + expected: Final = get_next_standardized_reset_time("1d", _utc(rows[0]["created_at"]), "UTC") + assert abs((_utc(rows[0]["reset_at"]) - expected).total_seconds()) <= 3, (rows, expected) + + +def test_admin_health_counts_every_deployment_and_reports_a_live_one_healthy(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + model_id: Final = _model_id(model) + report: Final = gateway.get("/health", {"model": model}) + assert (report["healthy_count"], report["unhealthy_count"]) == (1, 0), report + healthy: Final = report["healthy_endpoints"] + assert isinstance(healthy, list) and len(healthy) == 1, report + assert object_value(healthy[0]).get("model_id") == model_id, report + + +def test_routes_listing_is_served_without_credentials(gateway: Gateway) -> None: + response: Final = gateway.client.get("/routes") + assert response.status_code == 200, response.text + routes: Final = object_value(response.json())["routes"] + assert isinstance(routes, list) + assert {"/routes", "/key/generate"} <= {object_value(route)["path"] for route in routes} + + +def test_unrestricted_key_lists_models_and_none_when_only_access_groups_are_requested(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + grouped: Final = scenario.model(model_info={"access_groups": [f"integration-{uuid.uuid4().hex}"]}) + plain: Final = scenario.model() + key: Final = scenario.key() + listed: Final = {entry["id"] for entry in _data(gateway, "/models", key)} + assert {grouped, plain} <= listed + assert _data(gateway, "/models", key, {"only_model_access_groups": "True"}) == [] + + +def test_model_info_by_id_matches_the_entry_in_the_keys_listing(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + scenario.model() + model_id: Final = _model_id(model) + key: Final = scenario.key(models=[model]) + listing: Final = _data(gateway, "/model/info", key) + assert {entry["model_name"] for entry in listing} == {model} + listed: Final = [entry for entry in listing if object_value(entry["model_info"])["id"] == model_id] + assert len(listed) == 1 + by_id: Final = _data(gateway, "/model/info", key, {"litellm_model_id": model_id}) + assert by_id == listed + admin_by_id: Final = _data(gateway, "/model/info", gateway.key, {"litellm_model_id": model_id}) + assert [object_value(entry["model_info"])["id"] for entry in admin_by_id] == [model_id] + + +def test_model_group_info_for_a_personal_user_key_lists_only_the_users_model(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + scenario.model() + created: Final = gateway.post("/user/new", {"user_id": f"integration-{uuid.uuid4().hex}", "models": [model]}) + scenario.cleanups.callback(scenario.delete_user, string_value(created["user_id"])) + key: Final = string_value(created["key"]) + scenario.cleanups.callback(delete_key_if_present, gateway, key) + groups: Final = [entry["model_group"] for entry in _data(gateway, "/model_group/info", key)] + assert groups == [model] + + +def test_model_group_info_expands_a_wildcard_deployment_into_concrete_groups(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + prefix: Final = f"integration{uuid.uuid4().hex}" + _wildcard_model(gateway, scenario, prefix) + groups: Final = { + string_value(entry["model_group"]) for entry in _data(gateway, "/model_group/info", gateway.key) + } + assert f"{prefix}/*" not in groups + assert any(group.startswith(f"{prefix}/claude") for group in groups), sorted(groups)[:20] + + +def test_azure_deployment_route_denies_a_model_outside_the_keys_list(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + allowed: Final = scenario.model() + other: Final = scenario.model() + key: Final = scenario.key(models=[allowed]) + body: Final[dict[str, JsonValue]] = {"messages": [{"role": "user", "content": "integration control"}]} + served: Final = gateway.request("POST", f"/openai/deployments/{allowed}/chat/completions", body, key=key) + assert served.status_code == 200, served.text + denied: Final = gateway.request("POST", f"/openai/deployments/{other}/chat/completions", body, key=key) + assert denied.status_code == 403, denied.text + assert "is not available for this API key" in denied.text diff --git a/tests/integration/management/test_team_member_routes.py b/tests/integration/management/test_team_member_routes.py new file mode 100644 index 00000000000..3dc126b3e98 --- /dev/null +++ b/tests/integration/management/test_team_member_routes.py @@ -0,0 +1,205 @@ +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import Final, Literal + +import pytest +from pydantic import JsonValue + +from tests.integration._support.client import Gateway, Scenario, delete_key_if_present, object_value, string_value +from tests.integration._support.database import read_rows + +MemberDimension = Literal["user_id", "user_email"] +UNCHANGED_BY_ALIAS_UPDATE_SKIP: Final = frozenset( + { + "team_alias", + "members_with_roles", + "created_at", + "updated_at", + "model_spend", + "model_max_budget", + "model_id", + "litellm_organization_table", + "object_permission_id", + "object_permission", + "litellm_model_table", + "policies", + "allow_team_guardrail_config", + "projects", + } +) + + +def _team_info(gateway: Gateway, team_id: str) -> dict[str, JsonValue]: + return object_value(gateway.get("/team/info", {"team_id": team_id})["team_info"]) + + +def _member_ids(gateway: Gateway, team_id: str) -> list[str | None]: + members: Final = _team_info(gateway, team_id)["members_with_roles"] + assert isinstance(members, list) + return [ + member_id if isinstance(member_id := object_value(member).get("user_id"), str) else None for member in members + ] + + +def _user_team_ids(gateway: Gateway, user_id: str) -> list[str]: + teams: Final = gateway.get("/user/info", {"user_id": user_id})["teams"] + assert isinstance(teams, list) + return [string_value(object_value(team)["team_id"]) for team in teams] + + +def _delete_users_if_present(gateway: Gateway, user_ids: list[str]) -> None: + present: Final = [ + user_id + for user_id in user_ids + if read_rows('SELECT user_id FROM "LiteLLM_UserTable" WHERE user_id = %s', (user_id,)) + ] + if present: + response: Final = gateway.request("POST", "/user/delete", {"user_ids": list[JsonValue](present)}) + assert response.status_code == 200, response.text + + +def _member_delete(gateway: Gateway, team_id: str, user_id: str) -> int: + return gateway.request("POST", "/team/member_delete", {"team_id": team_id, "user_id": user_id}).status_code + + +def _owned_email_member(gateway: Gateway, scenario: Scenario) -> str: + user_id: Final = f"integration_{uuid.uuid4().hex}@example.com" + scenario.cleanups.callback(_delete_users_if_present, gateway, [user_id]) + return user_id + + +def test_concurrent_team_creation_records_the_named_member(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user: Final = scenario.user() + with ThreadPoolExecutor(max_workers=10) as pool: + teams: Final = list( + pool.map(lambda _: scenario.team(members_with_roles=[{"role": "user", "user_id": user}]), range(10)) + ) + assert len(set(teams)) == 10 + for team in teams: + assert user in _member_ids(gateway, team) + + +def test_team_info_serves_admin_and_team_keys_and_denies_other_keys(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + assert _team_info(gateway, team)["team_id"] == team + team_key: Final = scenario.key(team_id=team) + as_team: Final = gateway.request("GET", "/team/info", params={"team_id": team}, key=team_key) + assert as_team.status_code == 200, as_team.text + assert object_value(object_value(as_team.json())["team_info"])["team_id"] == team + outsider: Final = scenario.key() + denied: Final = gateway.request("GET", "/team/info", params={"team_id": team}, key=outsider) + assert denied.status_code in {401, 403}, denied.text + + +def test_team_alias_update_keeps_every_other_team_field(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + admin: Final = scenario.user() + created: Final = gateway.post( + "/team/new", + { + "team_alias": f"integration-{uuid.uuid4().hex}", + "members_with_roles": [{"role": "admin", "user_id": admin}], + }, + ) + team: Final = string_value(created["team_id"]) + scenario.cleanups.callback(scenario.delete_team, team) + initial_size: Final = len(_member_ids(gateway, team)) + new_members: Final = [_owned_email_member(gateway, scenario) for _ in range(10)] + gateway.post( + "/team/member_add", + {"team_id": team, "member": [{"role": "user", "user_id": member} for member in new_members]}, + ) + members: Final = _member_ids(gateway, team) + assert len(members) == initial_size + 10 + assert set(new_members) <= set(members) + new_alias: Final = f"integration-{uuid.uuid4().hex}" + updated: Final = object_value(gateway.post("/team/update", {"team_id": team, "team_alias": new_alias})["data"]) + assert updated["team_alias"] == new_alias + updated_members: Final = updated["members_with_roles"] + assert isinstance(updated_members, list) + assert len(updated_members) == len(members) + compared: Final = (set(created) | set(updated)) - UNCHANGED_BY_ALIAS_UPDATE_SKIP + for field in compared: + assert updated.get(field) == created.get(field), field + assert compared + + +def test_member_added_by_email_sees_the_team_in_user_info(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + email: Final = f"integration-{uuid.uuid4().hex}@example.com" + user: Final = scenario.user(user_email=email) + team: Final = scenario.team() + gateway.post("/team/member_add", {"team_id": team, "member": {"role": "user", "user_email": email}}) + assert team in _user_team_ids(gateway, user) + + +def test_team_delete_detaches_members_and_hides_the_team(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + first: Final = scenario.user() + second: Final = scenario.user() + created: Final = gateway.post( + "/team/new", + { + "team_alias": f"integration-{uuid.uuid4().hex}", + "members_with_roles": [{"role": "admin", "user_id": first}, {"role": "user", "user_id": second}], + }, + ) + team: Final = string_value(created["team_id"]) + scenario.cleanups.callback(gateway.request, "POST", "/team/delete", {"team_ids": [team]}) + team_key: Final = string_value(gateway.post("/key/generate", {"team_id": team, "user_id": second})["key"]) + scenario.cleanups.callback(delete_key_if_present, gateway, team_key) + assert _user_team_ids(gateway, second) == [team] + gateway.post("/team/delete", {"team_ids": [team]}) + assert _user_team_ids(gateway, second) == [] + missing: Final = gateway.request("GET", "/team/info", params={"team_id": team}) + assert missing.status_code == 404, missing.text + + +@pytest.mark.parametrize("dimension", ["user_id", "user_email"]) +def test_member_delete_removes_the_member_by_id_or_email(gateway: Gateway, dimension: MemberDimension) -> None: + with gateway.scenario() as scenario: + email: Final = f"integration-{uuid.uuid4().hex}@example.com" + user: Final = scenario.user(user_email=email) + selector: Final[dict[str, JsonValue]] = {"user_id": user} if dimension == "user_id" else {"user_email": email} + team: Final = scenario.team(members_with_roles=[{"role": "user", **selector}]) + assert user in _member_ids(gateway, team) + deleted: Final = gateway.request("POST", "/team/member_delete", {"team_id": team, **selector}) + assert deleted.status_code == 200, deleted.text + assert user not in _member_ids(gateway, team) + assert team not in _user_team_ids(gateway, user) + + +def test_member_add_with_email_shaped_user_id_grows_team_by_one(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + initial: Final = _member_ids(gateway, team) + member: Final = _owned_email_member(gateway, scenario) + gateway.post("/team/member_add", {"team_id": team, "member": {"role": "user", "user_id": member}}) + after: Final = _member_ids(gateway, team) + assert member in after + assert len(after) == len(initial) + 1 + + +def test_member_delete_removes_once_and_rejects_repeats(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + member: Final = f"integration-{uuid.uuid4().hex}" + scenario.cleanups.callback(_delete_users_if_present, gateway, [member]) + gateway.post("/team/member_add", {"team_id": team, "member": {"role": "user", "user_id": member}}) + before: Final = _member_ids(gateway, team) + assert member in before + assert _member_delete(gateway, team, member) == 200 + assert [_member_delete(gateway, team, member) for _ in range(4)] == [400, 400, 400, 400] + after: Final = _member_ids(gateway, team) + assert member not in after + assert len(after) == len(before) - 1 + + +def test_member_delete_of_a_non_member_is_a_bad_request(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + team: Final = scenario.team() + outsider: Final = f"integration-{uuid.uuid4().hex}" + assert outsider not in _member_ids(gateway, team) + assert _member_delete(gateway, team, outsider) == 400 diff --git a/tests/integration/management/test_user_routes.py b/tests/integration/management/test_user_routes.py new file mode 100644 index 00000000000..feb9ff2c3bf --- /dev/null +++ b/tests/integration/management/test_user_routes.py @@ -0,0 +1,34 @@ +import uuid +from concurrent.futures import ThreadPoolExecutor +from typing import Final + +from tests.integration._support.client import Gateway, object_value +from tests.integration._support.database import read_rows + + +def test_concurrent_user_creation_persists_models_and_aliases(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + alias: Final = f"alias-{uuid.uuid4().hex}" + with ThreadPoolExecutor(max_workers=10) as pool: + users: Final = list(pool.map(lambda _: scenario.user(models=[model], aliases={alias: model}), range(10))) + assert len(set(users)) == 10 + rows: Final = [ + read_rows('SELECT models FROM "LiteLLM_UserTable" WHERE user_id = %s', (user,)) for user in users + ] + assert rows == [[{"models": [model]}]] * len(users) + + +def test_user_info_serves_admin_and_self_and_denies_other_users(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + user: Final = scenario.user(user_role="internal_user") + own_key: Final = scenario.key(user_id=user) + other: Final = scenario.user(user_role="internal_user") + other_key: Final = scenario.key(user_id=other) + admin: Final = object_value(gateway.get("/user/info", {"user_id": user})["user_info"]) + assert admin["user_id"] == user + own: Final = gateway.request("GET", "/user/info", params={"user_id": user}, key=own_key) + assert own.status_code == 200, own.text + assert object_value(object_value(own.json())["user_info"])["user_id"] == user + denied: Final = gateway.request("GET", "/user/info", params={"user_id": user}, key=other_key) + assert denied.status_code == 403, denied.text diff --git a/tests/integration/routing/test_fallback_retry_header_contracts.py b/tests/integration/routing/test_fallback_retry_header_contracts.py new file mode 100644 index 00000000000..ebed95a683c --- /dev/null +++ b/tests/integration/routing/test_fallback_retry_header_contracts.py @@ -0,0 +1,291 @@ +import threading +import time +import uuid +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from typing import Final, Literal + +import httpx +import pytest +from pydantic import JsonValue, TypeAdapter + +from tests.integration._support.database import read_rows +from tests.integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + eventually, + object_value, + string_value, +) +from tests.integration._support.openai_wire import chat_reply +from tests.integration._support.wire import Reply, Request, wire_server + +UNREACHABLE_API_BASE: Final = "http://127.0.0.1:9/v1" +END_USER_REQUESTS: Final = 10 +CONCURRENT_REQUESTS: Final = 25 +DISTRIBUTION_REQUESTS: Final = 20 +SLOW_UPSTREAM_SECONDS: Final = 3 +HELD_REPLY_SECONDS: Final = 30 +END_USER_ROW_SECONDS: Final = 70 +CUSTOM_FALLBACK_TEXT: Final = "custom fallback prompt" +Caller = Literal["virtual-key", "master-key"] +JSON_OBJECTS: Final = TypeAdapter(list[dict[str, JsonValue]]) + + +def _messages(text: str = "integration control") -> list[JsonValue]: + return [{"role": "user", "content": text}] + + +def _chat( + gateway: Gateway, + body: dict[str, JsonValue], + *, + key: str | None = None, + headers: dict[str, str] | None = None, +) -> httpx.Response: + return gateway.request("POST", "/v1/chat/completions", {"messages": _messages(), **body}, key=key, headers=headers) + + +def _body(response: httpx.Response) -> dict[str, JsonValue]: + return JSON_OBJECT.validate_json(response.content) + + +def _content(response: httpx.Response) -> str: + choices: Final = _body(response)["choices"] + assert isinstance(choices, list) and choices, response.text + return string_value(object_value(object_value(choices[0])["message"])["content"]) + + +def _scripted_model(gateway: Gateway, scenario: Scenario, statuses: list[int], num_retries: int) -> str: + upstream_model: Final = f"integration-{uuid.uuid4().hex}" + script_url: Final = f"{gateway.upstream_url}/__scripts/{upstream_model}" + configured: Final = httpx.post(script_url, json={"statuses": statuses}) + assert configured.status_code == 200, configured.text + scenario.cleanups.callback(httpx.delete, script_url) + return scenario.model(model=f"openai/{upstream_model}", num_retries=num_retries) + + +@dataclass(frozen=True, slots=True) +class UniqueModel: + name: str + upstream: str + deployment_id: str + + +def _unique_model(scenario: Scenario) -> UniqueModel: + upstream_model: Final = f"integration-{uuid.uuid4().hex}" + deployment_id: Final = f"integration-{uuid.uuid4().hex}" + model_info: Final[dict[str, JsonValue]] = {"id": deployment_id} + name: Final = scenario.model(model=f"openai/{upstream_model}", model_info=model_info) + return UniqueModel(name, upstream_model, deployment_id) + + +def _slow_reply(_: Request) -> Reply: + time.sleep(SLOW_UPSTREAM_SECONDS) + return chat_reply("chatcmpl-slow", "gpt-4o-mini", "late", stream=False) + + +def _fallback_reply(_: Request) -> Reply: + return chat_reply("chatcmpl-fallback", "gpt-4o-mini", "served by fallback", stream=False) + + +def _held_reply(release: threading.Event) -> Callable[[Request], Reply]: + def respond(_: Request) -> Reply: + assert release.wait(timeout=HELD_REPLY_SECONDS) + return chat_reply("chatcmpl-held", "gpt-4o-mini", "held", stream=False) + + return respond + + +def _delete_auto_created_end_user(gateway: Gateway, user_id: str) -> None: + eventually( + lambda: read_rows('SELECT user_id FROM "LiteLLM_EndUserTable" WHERE user_id = %s', (user_id,)), + lambda rows: len(rows) == 1, + seconds=END_USER_ROW_SECONDS, + ) + gateway.post("/end_user/delete", {"user_ids": [user_id]}) + assert read_rows('SELECT user_id FROM "LiteLLM_EndUserTable" WHERE user_id = %s', (user_id,)) == [] + + +def _status(gateway: Gateway, model: str) -> int: + return _chat(gateway, {"model": model}).status_code + + +def _served_model_id(gateway: Gateway, model: str) -> str: + response: Final = _chat(gateway, {"model": model}) + assert response.status_code == 200, response.text + return response.headers["x-litellm-model-id"] + + +def _concurrent_round(gateway: Gateway, bad: str, good: str) -> tuple[list[int], list[int]]: + with ThreadPoolExecutor(max_workers=CONCURRENT_REQUESTS * 2) as pool: + bad_calls: Final = [pool.submit(_status, gateway, bad) for _ in range(CONCURRENT_REQUESTS)] + good_calls: Final = [pool.submit(_status, gateway, good) for _ in range(CONCURRENT_REQUESTS)] + return [call.result() for call in bad_calls], [call.result() for call in good_calls] + + +def _deployment(gateway: Gateway, scenario: Scenario, model_name: str, model: str = "openai/gpt-4o-mini") -> str: + created: Final = gateway.post( + "/model/new", + { + "model_name": model_name, + "litellm_params": { + "model": model, + "api_key": "integration-provider-key", + "api_base": f"{gateway.upstream_url}/v1", + }, + }, + ) + deployment_id: Final = string_value(object_value(created["model_info"])["id"]) + scenario.cleanups.callback(scenario.delete_model, deployment_id) + return deployment_id + + +@pytest.mark.parametrize("caller", ["virtual-key", "master-key"]) +def test_end_user_budget_tpm_limit_rate_limits_their_requests(gateway: Gateway, caller: Caller) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + budget: Final = scenario.budget(tpm_limit=2) + end_user: Final = f"integration-{uuid.uuid4().hex}" + gateway.post("/end_user/new", {"user_id": end_user, "budget_id": budget}) + scenario.cleanups.callback(gateway.post, "/end_user/delete", {"user_ids": [end_user]}) + key: Final = scenario.key(models=[model]) if caller == "virtual-key" else gateway.key + control_user: Final = f"integration-{uuid.uuid4().hex}" + scenario.cleanups.callback(_delete_auto_created_end_user, gateway, control_user) + control: Final = _chat(gateway, {"model": model, "user": control_user}, key=key) + assert control.status_code == 200, control.text + statuses: Final = [ + _chat(gateway, {"model": model, "user": end_user}, key=key).status_code for _ in range(END_USER_REQUESTS) + ] + assert statuses.count(200) < 5, statuses + assert set(statuses) <= {200, 429}, statuses + + +def test_client_fallbacks_reach_an_allowed_model_and_name_a_denied_one(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + primary: Final = scenario.model(api_base=UNREACHABLE_API_BASE) + fallback: Final = _unique_model(scenario) + body: Final[dict[str, JsonValue]] = {"model": primary, "fallbacks": [fallback.name]} + served: Final = _chat(gateway, body, key=scenario.key(models=[primary, fallback.name])) + assert served.status_code == 200, served.text + assert _body(served)["model"] == fallback.upstream + assert served.headers["x-litellm-model-id"] == fallback.deployment_id + assert _content(served) + denied: Final = _chat(gateway, body, key=scenario.key(models=[primary])) + assert denied.status_code == 403, denied.text + assert fallback.name in denied.text + + +def test_client_fallback_with_custom_messages_sends_them_to_the_fallback(gateway: Gateway) -> None: + custom: Final = _messages(CUSTOM_FALLBACK_TEXT) + with gateway.scenario() as scenario, wire_server(_fallback_reply) as wire: + primary: Final = scenario.model(api_base=UNREACHABLE_API_BASE) + fallback: Final = scenario.model(api_base=wire.url) + body: Final[dict[str, JsonValue]] = { + "model": primary, + "fallbacks": [{"model": fallback, "messages": custom}], + } + served: Final = _chat(gateway, body, key=scenario.key(models=[primary, fallback])) + assert served.status_code == 200, served.text + assert _content(served) == "served by fallback" + forwarded: Final = [JSON_OBJECT.validate_json(request.body)["messages"] for request in wire.drain()] + assert forwarded == [custom] + denied: Final = _chat(gateway, body, key=scenario.key(models=[primary])) + assert denied.status_code == 403, denied.text + assert fallback in denied.text + assert wire.drain() == () + + +def test_rate_limited_deployment_is_retried_and_reports_retry_counts(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = _scripted_model(gateway, scenario, [429, 200], 50) + response: Final = _chat(gateway, {"model": model}) + assert response.status_code == 200, response.text + assert response.headers["x-litellm-attempted-retries"] == "1" + assert response.headers["x-litellm-max-retries"] == "50" + + +def test_request_fallbacks_reroute_after_a_connection_failure(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + primary: Final = scenario.model(api_base=UNREACHABLE_API_BASE) + fallback: Final = _unique_model(scenario) + response: Final = _chat(gateway, {"model": primary, "fallbacks": [fallback.name]}) + assert response.status_code == 200, response.text + assert _body(response)["model"] == fallback.upstream + assert response.headers["x-litellm-model-id"] == fallback.deployment_id + assert response.headers["x-litellm-attempted-fallbacks"] == "1" + + +def test_model_level_timeout_is_reported_on_a_timed_out_request(gateway: Gateway) -> None: + with gateway.scenario() as scenario, wire_server(_slow_reply) as wire: + response: Final = _chat(gateway, {"model": scenario.model(api_base=wire.url, timeout=1)}) + assert response.status_code == 408, response.text + assert response.headers["x-litellm-timeout"] == "1.0" + + +def test_request_timeout_header_overrides_the_model_timeout(gateway: Gateway) -> None: + with gateway.scenario() as scenario, wire_server(_slow_reply) as wire: + response: Final = _chat( + gateway, + {"model": scenario.model(api_base=wire.url, timeout=1)}, + headers={"x-litellm-timeout": "0.001"}, + ) + assert response.status_code == 408, response.text + assert response.headers["x-litellm-timeout"] == "0.001" + + +def test_failing_model_traffic_does_not_starve_concurrent_good_requests(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + bad: Final = scenario.model(api_base=UNREACHABLE_API_BASE) + good: Final = scenario.model() + for _ in range(2): + bad_calls, good_calls = _concurrent_round(gateway, bad, good) + assert good_calls == [200] * CONCURRENT_REQUESTS + assert 200 not in bad_calls + + +def test_rpm_limited_deployment_rejects_a_second_call_while_the_first_is_in_flight(gateway: Gateway) -> None: + release: Final = threading.Event() + with gateway.scenario() as scenario, wire_server(_held_reply(release)) as wire, ThreadPoolExecutor(1) as pool: + model: Final = scenario.model(api_base=wire.url, rpm=1) + first: Final = pool.submit(_chat, gateway, {"model": model}) + try: + eventually(wire.received.qsize, lambda count: count == 1) + second: Final = _chat(gateway, {"model": model}) + assert second.status_code == 429, second.text + assert wire.received.qsize() == 1 + finally: + release.set() + assert first.result().status_code == 200 + assert len(wire.drain()) == 1 + + +def test_model_group_with_two_deployments_serves_from_both(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + first_id: Final = f"integration-{uuid.uuid4().hex}" + model: Final = scenario.model(model_info={"id": first_id}) + second_id: Final = _deployment(gateway, scenario, model) + served_by: Final = {_served_model_id(gateway, model) for _ in range(DISTRIBUTION_REQUESTS)} + assert served_by == {first_id, second_id} + + +def test_unlisted_provider_model_resolves_through_a_wildcard_deployment(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + prefix: Final = f"integration{uuid.uuid4().hex}" + _deployment(gateway, scenario, f"{prefix}/*", "openai/*") + response: Final = _chat(gateway, {"model": f"{prefix}/gpt-4o-mini"}) + assert response.status_code == 200, response.text + assert _content(response) + + +def test_comma_separated_models_fan_out_to_one_response_per_model(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + first: Final = _unique_model(scenario) + second: Final = _unique_model(scenario) + response: Final = _chat(gateway, {"model": f"{first.name},{second.name}"}) + assert response.status_code == 200, response.text + replies: Final = JSON_OBJECTS.validate_json(response.content) + assert len(replies) == 2, replies + assert {string_value(reply["model"]) for reply in replies} == {first.upstream, second.upstream} diff --git a/tests/integration/spend/test_spend_attribution_contracts.py b/tests/integration/spend/test_spend_attribution_contracts.py new file mode 100644 index 00000000000..f5036160ce1 --- /dev/null +++ b/tests/integration/spend/test_spend_attribution_contracts.py @@ -0,0 +1,102 @@ +from contextlib import ExitStack +from itertools import chain +from typing import Final + +from pydantic import JsonValue + +from tests.integration._support.client import ( + JSON_OBJECT, + Gateway, + Scenario, + delete_key_if_present, + eventually, + object_value, + string_value, +) +from tests.integration._support.database import read_rows + +MEMBER_BUDGET: Final = 0.0000001 +SPEND_LANDING_SECONDS: Final = 70 + + +def test_spend_log_of_an_org_team_key_records_org_and_team(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + organization: Final = scenario.organization(models=[model]) + team: Final = scenario.team(organization_id=organization, models=[model]) + key: Final = scenario.key(team_id=team, models=[model]) + request_id: Final = string_value(gateway.chat(model, key=key)["id"]) + rows: Final = eventually( + lambda: read_rows( + 'SELECT team_id, metadata::text AS metadata FROM "LiteLLM_SpendLogs" WHERE request_id = %s', + (request_id,), + ), + lambda found: len(found) == 1, + seconds=SPEND_LANDING_SECONDS, + ) + metadata: Final = JSON_OBJECT.validate_json(string_value(rows[0]["metadata"])) + assert metadata["user_api_key_org_id"] == organization + assert metadata["user_api_key_team_id"] == team + assert rows[0]["team_id"] == team + + +def _team_memberships(team: JsonValue, user_id: str) -> list[dict[str, JsonValue]]: + entries: Final = object_value(team).get("team_memberships") or [] + assert isinstance(entries, list) + return [object_value(entry) for entry in entries if object_value(entry).get("user_id") == user_id] + + +def _membership(gateway: Gateway, user_id: str, team_id: str) -> dict[str, JsonValue]: + teams: Final = gateway.get("/user/info", {"user_id": user_id})["teams"] + assert isinstance(teams, list) + memberships: Final = list( + chain.from_iterable( + _team_memberships(team, user_id) for team in teams if object_value(team)["team_id"] == team_id + ) + ) + assert len(memberships) == 1, teams + return memberships[0] + + +def test_team_member_budget_blocks_the_member_after_spend_lands(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model: Final = scenario.model() + team: Final = scenario.team() + created: Final = gateway.post( + "/user/new", + {"user_id": f"integration-{model}", "team_id": team, "models": [model], "max_budget": 10.0}, + ) + user: Final = string_value(created["user_id"]) + key: Final = string_value(created["key"]) + with ExitStack() as member_cleanups: + member_cleanups.callback(scenario.delete_user, user) + member_cleanups.callback(delete_key_if_present, gateway, key) + gateway.post("/team/member_update", {"team_id": team, "user_id": user, "max_budget_in_team": MEMBER_BUDGET}) + membership: Final = _membership(gateway, user, team) + scenario.cleanups.callback(scenario.delete_budget, string_value(membership["budget_id"])) + scenario.cleanups.push(member_cleanups.pop_all()) + assert object_value(membership["litellm_budget_table"])["max_budget"] == MEMBER_BUDGET + assert ( + gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "first"}]}, + key=key, + ).status_code + == 200 + ) + eventually( + lambda: read_rows( + 'SELECT spend FROM "LiteLLM_TeamMembership" WHERE user_id = %s AND team_id = %s', (user, team) + ), + lambda rows: len(rows) == 1 and isinstance(spend := rows[0]["spend"], float) and spend >= MEMBER_BUDGET, + seconds=SPEND_LANDING_SECONDS, + ) + blocked: Final = gateway.request( + "POST", + "/v1/chat/completions", + {"model": model, "messages": [{"role": "user", "content": "second"}]}, + key=key, + ) + assert blocked.status_code != 200, blocked.text + assert "Budget has been exceeded" in blocked.text diff --git a/tests/rust-python-harness/AGENTS.md b/tests/rust-python-harness/AGENTS.md index b66eaaeda9b..6b0609a1c9a 100644 --- a/tests/rust-python-harness/AGENTS.md +++ b/tests/rust-python-harness/AGENTS.md @@ -63,4 +63,4 @@ tests/rust-python-harness/ - `shared/` contains reusable parity, tracing, reporting primitives, and unit-runner machinery - Keep fixtures with their owning API and existing Python tests in their current locations - Each strategy folder carries an `AGENTS.md` one-liner stating what it should be doing -- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/trace_parity tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust tests/test_rust_python_harness.py -q` +- Run the harness's own checks with `uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/cli tests/rust-python-harness/strategies/trace_parity tests/rust-python-harness/strategies/unit_tests_parity tests/rust-python-harness/strategies/unit_tests_rust -q` diff --git a/tests/rust-python-harness/cli/test_cli.py b/tests/rust-python-harness/cli/test_cli.py index 219e1b0c6b7..ebc4e2653d9 100644 --- a/tests/rust-python-harness/cli/test_cli.py +++ b/tests/rust-python-harness/cli/test_cli.py @@ -464,3 +464,12 @@ def test_runner_interrupt_skips_the_completion_report( assert exit_code == 130 assert "Rust <-> Python parity report" not in captured.out assert captured.err == "Interrupted\n" + + +def test_strategy_subcommand_accepts_function_filter(capsys: pytest.CaptureFixture[str]) -> None: + exit_code: Final = main(["run", "unit_tests_rust", "--function", "messages"]) + + captured: Final = capsys.readouterr() + assert exit_code == 0 + assert "- messages: not_implemented" in captured.out + assert "unit_tests_rust:messages: not_implemented" not in captured.out diff --git a/tests/rust-python-harness/shared/reporting/test_models.py b/tests/rust-python-harness/shared/reporting/test_models.py new file mode 100644 index 00000000000..1dd27719972 --- /dev/null +++ b/tests/rust-python-harness/shared/reporting/test_models.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from .models import CaseResult, Coverage, HarnessCase, RunStatus +from .strategy import ModuleCaseSpec, NotImplementedCaseSpec, SkippedCaseSpec + + +def _case(spec: ModuleCaseSpec | NotImplementedCaseSpec | SkippedCaseSpec) -> HarnessCase: + return HarnessCase(strategy_id="example", strategy_label="Example", sdk_function="messages", spec=spec) + + +def _runnable_case() -> HarnessCase: + return _case(ModuleCaseSpec(coverage=Coverage.COMPLETE, module="tests.example")) + + +def test_should_mark_not_implemented_and_skipped_cases_without_running() -> None: + not_implemented: Final = CaseResult(case=_case(NotImplementedCaseSpec(reason="No case is registered."))) + skipped: Final = CaseResult(case=_case(SkippedCaseSpec(reason="The surface does not apply."))) + + not_implemented.set_initial_status() + skipped.set_initial_status() + + assert not_implemented.status is RunStatus.NOT_IMPLEMENTED + assert skipped.status is RunStatus.SKIPPED + + +def test_should_finalize_a_fully_passing_case() -> None: + result: Final = CaseResult(case=_runnable_case()) + result.set_initial_status() + result.collected.update({"one", "two"}) + result.completed.update({"one", "two"}) + result.passed = 2 + + result.finalize() + + assert result.status is RunStatus.PASSED + + +def test_should_replace_a_pass_with_a_teardown_error() -> None: + result: Final = CaseResult(case=_runnable_case()) + result.set_initial_status() + result.collected.add("one") + + result.record("one", RunStatus.PASSED, 0.1) + result.record("one", RunStatus.ERROR, 0.2) + + assert result.status is RunStatus.ERROR + assert result.passed == 0 + assert result.errors == 1 + assert result.duration == pytest.approx(0.3) diff --git a/tests/rust-python-harness/shared/reporting/test_ui.py b/tests/rust-python-harness/shared/reporting/test_ui.py new file mode 100644 index 00000000000..c0ed0bc8bd5 --- /dev/null +++ b/tests/rust-python-harness/shared/reporting/test_ui.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import Final + +from .models import Coverage, HarnessCase, HarnessRun, RunStatus +from .strategy import ModuleCaseSpec +from .ui import _format_duration, _summary + + +def test_should_format_developer_facing_run_context() -> None: + run: Final = HarnessRun.from_cases( + ( + HarnessCase( + strategy_id="example", + strategy_label="Example", + sdk_function="messages", + spec=ModuleCaseSpec(coverage=Coverage.COMPLETE, module="tests.example"), + ), + ) + ) + result: Final = next(iter(run.results.values())) + result.collected.add("tests/test_parity.py::test_one") + result.record("tests/test_parity.py::test_one", RunStatus.PASSED, 1.25) + + assert _summary(run) == (1, 0, 0, 0) + assert _format_duration(1.25) == "1.2s" diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/test_case_modules_importable.py b/tests/rust-python-harness/strategies/trace_parity/sdk/test_case_modules_importable.py new file mode 100644 index 00000000000..bc37ce3d873 --- /dev/null +++ b/tests/rust-python-harness/strategies/trace_parity/sdk/test_case_modules_importable.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from importlib import import_module +from typing import Final, cast + +import pytest + +from ..models import TraceSuite + + +@pytest.mark.parametrize( + "module", + [ + "tests.rust-python-harness.strategies.trace_parity.sdk.messages.case", + "tests.rust-python-harness.strategies.trace_parity.sdk.chat_completions.case", + "tests.rust-python-harness.strategies.trace_parity.sdk.transcription.case", + ], +) +def test_implemented_namespace_case_modules_remain_importable(module: str) -> None: + loaded: Final = import_module(module) + suite: Final = cast(object, getattr(loaded, "TRACE_SUITE")) + assert isinstance(suite, TraceSuite) + assert suite.scenarios diff --git a/tests/test_anthropic_compaction_usage.py b/tests/test_anthropic_compaction_usage.py deleted file mode 100644 index 1758a94fffc..00000000000 --- a/tests/test_anthropic_compaction_usage.py +++ /dev/null @@ -1,96 +0,0 @@ -from litellm.llms.anthropic.chat.transformation import AnthropicConfig - - -def test_anthropic_compaction_usage_calculation(): - """ - Test that calculate_usage correctly sums tokens from the iterations array - as requested in Issue #27060. - """ - anthropic_config = AnthropicConfig() - - # Mock usage object with compaction iterations - usage_object = { - "input_tokens": 100, # Top-level (excludes compaction) - "output_tokens": 50, # Top-level (excludes compaction) - "iterations": [ - { - "iteration": 1, - "type": "compaction", - "input_tokens": 1000, - "output_tokens": 500, - }, - { - "iteration": 2, - "type": "message", - "input_tokens": 100, - "output_tokens": 50, - }, - ], - } - - usage = anthropic_config.calculate_usage( - usage_object=usage_object, reasoning_content=None - ) - - # Assertions - # Total prompt tokens should be 1000 + 100 = 1100 - assert usage.prompt_tokens == 1100 - # Total completion tokens should be 500 + 50 = 550 - assert usage.completion_tokens == 550 - # Total tokens should be 1650 - assert usage.total_tokens == 1650 - - # Assert details - assert usage.prompt_tokens_details.text_tokens == 1100 - - # Assert iterations passthrough - assert usage.iterations is not None - assert len(usage.iterations) == 2 - assert usage.iterations[0]["type"] == "compaction" - - -def test_anthropic_compaction_usage_with_iteration_cache(): - """ - Test that calculate_usage correctly sums caching tokens FROM iterations. - This covers the specific case mentioned by JasonPan. - """ - anthropic_config = AnthropicConfig() - - usage_object = { - "input_tokens": 100, - "output_tokens": 50, - "iterations": [ - { - "type": "compaction", - "input_tokens": 500, - "output_tokens": 200, - "cache_creation_input_tokens": 50, - "cache_read_input_tokens": 17000, - }, - { - "type": "message", - "input_tokens": 100, - "output_tokens": 50, - "cache_creation_input_tokens": 10, - "cache_read_input_tokens": 20, - }, - ], - } - - usage = anthropic_config.calculate_usage( - usage_object=usage_object, reasoning_content=None - ) - - # input_tokens sum = 500 + 100 = 600 - # cache_creation sum = 50 + 10 = 60 - # cache_read sum = 17000 + 20 = 17020 - # Total prompt tokens = 600 + 60 + 17020 = 17680 - assert usage.prompt_tokens == 17680 - assert usage.completion_tokens == 250 - assert usage.prompt_tokens_details.cache_creation_tokens == 60 - assert usage.prompt_tokens_details.cached_tokens == 17020 - - -if __name__ == "__main__": - test_anthropic_compaction_usage_calculation() - test_anthropic_compaction_usage_with_iteration_cache() diff --git a/tests/test_budget_management.py b/tests/test_budget_management.py deleted file mode 100644 index 42d5c5d98ac..00000000000 --- a/tests/test_budget_management.py +++ /dev/null @@ -1,102 +0,0 @@ -import os -# What is this? -## Unit tests for the /budget/* endpoints -from litellm._uuid import uuid -from datetime import datetime, timezone - -import aiohttp -import pytest -import pytest_asyncio - -from litellm.litellm_core_utils.duration_parser import get_next_standardized_reset_time -from litellm.proxy.common_utils.timezone_utils import get_budget_reset_timezone - - -def _parse_budget_api_datetime(value: str) -> datetime: - """Parse ISO timestamps returned by the proxy JSON API.""" - if value.endswith("Z"): - value = value[:-1] + "+00:00" - dt = datetime.fromisoformat(value) - if dt.tzinfo is None: - dt = dt.replace(tzinfo=timezone.utc) - return dt - - -async def delete_budget(session, budget_id): - url = "http://0.0.0.0:4000/budget/delete" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = {"id": budget_id} - async with session.post(url, headers=headers, json=data) as response: - assert response.status == 200 - print(f"Deleted Budget {budget_id}") - - -async def create_budget(session, data): - url = "http://0.0.0.0:4000/budget/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - - async with session.post(url, headers=headers, json=data) as response: - assert response.status == 200 - response_data = await response.json() - budget_id = response_data["budget_id"] - print(f"Created Budget {budget_id}") - return response_data - - -@pytest_asyncio.fixture -async def budget_setup(): - """ - Fixture to create a budget for testing and clean it up afterward. - - This fixture performs the following steps: - 1. Opens an aiohttp ClientSession. - 2. Generates a random budget_id and defines the budget data (duration: 1 day, max_budget: 0.02). - 3. Calls create_budget to create the budget. - 4. Yields the budget_response (a dict) for use in the test. - 5. After the test completes, deletes the created budget by calling delete_budget. - - Returns: - dict: The JSON response from create_budget, which includes the created budget's data. - """ - - async with aiohttp.ClientSession() as session: - # Generate a unique budget_id and define the budget data. - budget_id = f"budget-{uuid.uuid4()}" - data = {"budget_id": budget_id, "budget_duration": "1d", "max_budget": 0.02} - budget_response = await create_budget(session, data) - - # Yield the response so the test can use it. - yield budget_response - - # After the test, delete the created budget to clean up. - await delete_budget(session, budget_id) - - -@pytest.mark.asyncio -async def test_create_budget_with_duration(budget_setup): - """ - Test creating a budget with a specified duration and verify that 'budget_reset_at' - matches the next standardized reset (see get_budget_reset_time / new_budget), not - necessarily created_at + wall-clock duration. - """ - - assert ( - budget_setup["budget_reset_at"] is not None - ), "The budget_reset_at field should not be None" - - created_at = _parse_budget_api_datetime(budget_setup["created_at"]) - expected_reset_at = get_next_standardized_reset_time( - duration=budget_setup["budget_duration"], - current_time=created_at, - timezone_str=get_budget_reset_timezone(), - ) - - actual_reset_at = _parse_budget_api_datetime(budget_setup["budget_reset_at"]) - - tolerance_seconds = 3 - time_difference = abs((actual_reset_at - expected_reset_at).total_seconds()) - - assert time_difference <= tolerance_seconds, ( - f"Expected budget_reset_at to be within {tolerance_seconds} seconds of {expected_reset_at}, " - f"but the difference was {time_difference} seconds." - ) diff --git a/tests/test_callbacks_on_proxy.py b/tests/test_callbacks_on_proxy.py deleted file mode 100644 index 42aa6d98cb9..00000000000 --- a/tests/test_callbacks_on_proxy.py +++ /dev/null @@ -1,303 +0,0 @@ -# What this tests ? -## Makes sure the number of callbacks on the proxy don't increase over time -## Num callbacks should be a fixed number at t=0 and t=10, t=20 -""" -PROD TEST - DO NOT Delete this Test -""" - -import pytest -import asyncio -import aiohttp -import os -import re -import dotenv -from collections import Counter -from dotenv import load_dotenv - -load_dotenv() - -# A *leak* is sustained, monotonic growth of one callback TYPE across the whole -# sampling window. A one-time bump that then plateaus is benign pollution from -# other tests sharing this proxy (this suite runs `pytest -n 4` against a single -# proxy container, so other workers legitimately add team/key-scoped callbacks -# while this test sleeps). We therefore sample N times and only flag a type -# whose normalized count never decreases, grows in >=2 distinct intervals, and -# nets >= LEAK_MIN_NET_GROWTH overall. -NUM_SAMPLES = 4 -SAMPLE_INTERVAL_SECONDS = 20 -LEAK_MIN_NET_GROWTH = 5 -LEAK_MIN_GROWING_INTERVALS = 2 -# A routing-strategy switch / alerting config is a *known, bounded, one-time* -# registration (CCI diagnostic 2026-05-16: total 85->95 on the first interval -# after switching to latency-based-routing, then flat at 95 for 2.5 min under -# load). We absorb that step by settling before the baseline sample, so only -# growth *after* the deliberate perturbation can count as a leak. -SETTLE_SECONDS = 30 - -# Strip instance-identity noise so N leaked instances of one class collapse to -# one rising counter instead of N opaque, unrelated-looking strings. -_ADDR_RE = re.compile(r" at 0x[0-9a-fA-F]+") -_OBJ_RE = re.compile(r"<([\w.]+) object") - - -def _normalize_callback(cb_str: str) -> str: - """Reduce a callback's str() to a stable type key (drops 0x… addresses).""" - s = _ADDR_RE.sub("", cb_str) - m = _OBJ_RE.search(s) - if m: - return m.group(1).split(".")[-1] - # bound methods: ">" -> "Cls.m" - bm = re.search(r"bound method ([\w.]+)", s) - if bm: - return bm.group(1) - return s.strip() - - -def _summarize(all_litellm_callbacks) -> Counter: - return Counter(_normalize_callback(str(c)) for c in all_litellm_callbacks) - - -def _detect_leaks(samples): - """ - samples: list[Counter] taken in time order. - - Returns {callback_type: [counts across samples]} for types that grew - monotonically (never decreased), in >=LEAK_MIN_GROWING_INTERVALS intervals, - and netted >=LEAK_MIN_NET_GROWTH overall — i.e. a real leak, not a one-shot - step from a parallel test. - """ - leaks = {} - all_types = set().union(*[set(s) for s in samples]) if samples else set() - for t in all_types: - series = [s.get(t, 0) for s in samples] - deltas = [b - a for a, b in zip(series, series[1:])] - net = series[-1] - series[0] - non_decreasing = all(d >= 0 for d in deltas) - growing_intervals = sum(1 for d in deltas if d > 0) - if ( - non_decreasing - and net >= LEAK_MIN_NET_GROWTH - and growing_intervals >= LEAK_MIN_GROWING_INTERVALS - ): - leaks[t] = series - return leaks - - -def _terminal_suspects(samples): - """ - Types whose net growth clears the threshold monotonically but is confined - to the *final* interval — `growing_intervals == 1` with that one growing - interval being the last. `_detect_leaks`' `>= 2` guard silently passes - these, so a real leak that accumulates entirely in the last sampled window - is indistinguishable from a one-time terminal step *without one more - sample*. Returns the set of such types so the caller can re-confirm. - """ - suspects = set() - all_types = set().union(*[set(s) for s in samples]) if samples else set() - for t in all_types: - series = [s.get(t, 0) for s in samples] - deltas = [b - a for a, b in zip(series, series[1:])] - if not deltas: - continue - net = series[-1] - series[0] - non_decreasing = all(d >= 0 for d in deltas) - growing = [i for i, d in enumerate(deltas) if d > 0] - if ( - non_decreasing - and net >= LEAK_MIN_NET_GROWTH - and growing == [len(deltas) - 1] - ): - suspects.add(t) - return suspects - - -async def _detect_leaks_confirmed(session, samples): - """ - `_detect_leaks`, plus a single confirmation sample when growth is confined - to the final interval (see `_terminal_suspects`). A genuine ongoing leak - keeps climbing -> now grows in >= 2 intervals -> flagged; a one-time - terminal registration plateaus -> still 1 growing interval -> ignored. - Returns `(leaks, samples)` (samples may have one extra entry appended). - """ - leaks = _detect_leaks(samples) - if not leaks and _terminal_suspects(samples): - await asyncio.sleep(SAMPLE_INTERVAL_SECONDS) - _, _, all_cb = await get_active_callbacks(session=session) - samples = samples + [_summarize(all_cb)] - leaks = _detect_leaks(samples) - return leaks, samples - - -def _format_report(samples, leaks) -> str: - lines = ["Callback count per type across samples (time order):"] - all_types = sorted(set().union(*[set(s) for s in samples])) - for t in all_types: - series = [s.get(t, 0) for s in samples] - marker = " <-- LEAK" if t in leaks else "" - lines.append(f" {t}: {series}{marker}") - totals = [sum(s.values()) for s in samples] - lines.append(f"TOTAL callbacks per sample: {totals}") - if leaks: - lines.append( - "Leaking callback types (sustained monotonic growth): " - + ", ".join(sorted(leaks)) - ) - return "\n".join(lines) - - -async def _sample_callbacks(session, num_samples, interval): - """Take `num_samples` callback snapshots `interval`s apart.""" - samples = [] - alerts = [] - for i in range(num_samples): - if i > 0: - await asyncio.sleep(interval) - num_cb, num_alert, all_cb = await get_active_callbacks(session=session) - samples.append(_summarize(all_cb)) - alerts.append(num_alert) - return samples, alerts - - -async def config_update(session, routing_strategy=None): - url = "http://0.0.0.0:4000/config/update" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - print("routing_strategy: ", routing_strategy) - data = { - "router_settings": { - "routing_strategy": routing_strategy, - }, - "general_settings": { - "alert_to_webhook_url": {"llm_exceptions": "example-slack-webhook-url"}, - "alert_types": ["llm_exceptions", "db_exceptions"], - }, - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - -async def get_active_callbacks(session): - url = "http://0.0.0.0:4000/active/callbacks" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - print("response from /active/callbacks") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - _json_response = await response.json() - - _num_callbacks = _json_response["num_callbacks"] - _num_alerts = _json_response["num_alerting"] - all_litellm_callbacks = _json_response["all_litellm_callbacks"] - - print("current number of callbacks: ", _num_callbacks) - print("current number of alerts: ", _num_alerts) - return _num_callbacks, _num_alerts, all_litellm_callbacks - - -async def get_current_routing_strategy(session): - url = "http://0.0.0.0:4000/get/config/callbacks" - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - _json_response = await response.json() - print("JSON response: ", _json_response) - - router_settings = _json_response["router_settings"] - print("Router settings: ", router_settings) - routing_strategy = router_settings["routing_strategy"] - return routing_strategy - - -@pytest.mark.asyncio -@pytest.mark.order1 -@pytest.mark.flaky(reruns=2, reruns_delay=5) -async def test_check_num_callbacks(): - """ - PROD invariant: no callback TYPE should grow without bound over time. - - This suite runs `pytest -n 4` against one shared proxy, so the raw count is - noisy — other workers legitimately add team/key-scoped callbacks that then - plateau. We settle first, then sample several times, and only fail on - *sustained, monotonic* per-type growth (a genuine leak), naming the type. - """ - async with aiohttp.ClientSession() as session: - # Absorb proxy warmup / in-flight parallel registration before baseline. - await asyncio.sleep(SETTLE_SECONDS) - - samples, _ = await _sample_callbacks( - session, NUM_SAMPLES, SAMPLE_INTERVAL_SECONDS - ) - - assert sum(samples[0].values()) > 0, "expected some callbacks registered" - - leaks, samples = await _detect_leaks_confirmed(session, samples) - report = _format_report(samples, leaks) - print(report) - assert not leaks, f"Callback leak detected.\n{report}" - - -@pytest.mark.asyncio -@pytest.mark.order2 -@pytest.mark.flaky(reruns=2, reruns_delay=5) -async def test_check_num_callbacks_on_lowest_latency(): - """ - Same PROD invariant as test_check_num_callbacks, but after switching the - router to latency-based-routing. That switch is a *known, bounded* one-time - registration (it adds the latency strategy handler + Slack alerting); we - settle past it before baselining so only post-switch growth counts as a - leak. Also asserts the alerting count is stable. - """ - async with aiohttp.ClientSession() as session: - await asyncio.sleep(30) - - original_routing_strategy = await get_current_routing_strategy(session=session) - await config_update(session=session, routing_strategy="latency-based-routing") - - try: - # Absorb the deliberate one-time config/update registration step. - await asyncio.sleep(SETTLE_SECONDS) - - samples, alerts = await _sample_callbacks( - session, NUM_SAMPLES, SAMPLE_INTERVAL_SECONDS - ) - - leaks, samples = await _detect_leaks_confirmed(session, samples) - report = _format_report(samples, leaks) - print(report) - assert not leaks, f"Callback leak detected.\n{report}" - assert ( - len(set(alerts)) == 1 - ), f"alerting count changed across samples: {alerts}" - finally: - await config_update( - session=session, routing_strategy=original_routing_strategy - ) diff --git a/tests/test_default_encoding_non_root.py b/tests/test_default_encoding_non_root.py deleted file mode 100644 index 06a5de51976..00000000000 --- a/tests/test_default_encoding_non_root.py +++ /dev/null @@ -1,55 +0,0 @@ -import importlib -import os -from unittest.mock import MagicMock, patch - -import litellm.litellm_core_utils.default_encoding as default_encoding - - -def _reload_default_encoding(monkeypatch, **env_overrides): - """ - Helper to reload default_encoding with a clean TIKTOKEN_CACHE_DIR and - specific environment overrides. - """ - monkeypatch.delenv("TIKTOKEN_CACHE_DIR", raising=False) - monkeypatch.delenv("CUSTOM_TIKTOKEN_CACHE_DIR", raising=False) - for key, value in env_overrides.items(): - monkeypatch.setenv(key, value) - importlib.reload(default_encoding) - - -def test_default_encoding_uses_bundled_tokenizers_by_default(monkeypatch): - """ - TIKTOKEN_CACHE_DIR should point at the bundled tokenizers directory - when no CUSTOM_TIKTOKEN_CACHE_DIR is set, even in non-root environments. - """ - _reload_default_encoding(monkeypatch, LITELLM_NON_ROOT="true") - - assert "TIKTOKEN_CACHE_DIR" in os.environ - cache_dir = os.environ["TIKTOKEN_CACHE_DIR"] - assert "tokenizers" in cache_dir - - -def test_custom_tiktoken_cache_dir_override(monkeypatch, tmp_path): - """ - CUSTOM_TIKTOKEN_CACHE_DIR must override the default bundled directory - and the directory should be created if it does not exist. - Reload with an empty custom dir would otherwise trigger tiktoken to - download the vocab; we patch get_encoding so the test is offline-safe - and does not depend on tiktoken's in-memory cache state. - """ - custom_dir = tmp_path / "tiktoken_cache" - with patch( - "litellm.litellm_core_utils.default_encoding.tiktoken.get_encoding", - return_value=MagicMock(), - ): - _reload_default_encoding(monkeypatch, CUSTOM_TIKTOKEN_CACHE_DIR=str(custom_dir)) - - cache_dir = os.environ.get("TIKTOKEN_CACHE_DIR") - assert cache_dir == str(custom_dir) - assert os.path.isdir(cache_dir) - - # Restore module to a clean state so default_encoding.encoding is a real - # tiktoken Encoding, not the MagicMock, for any test that runs after this. - monkeypatch.delenv("TIKTOKEN_CACHE_DIR", raising=False) - monkeypatch.delenv("CUSTOM_TIKTOKEN_CACHE_DIR", raising=False) - importlib.reload(default_encoding) diff --git a/tests/test_end_users.py b/tests/test_end_users.py deleted file mode 100644 index fab35d52e32..00000000000 --- a/tests/test_end_users.py +++ /dev/null @@ -1,201 +0,0 @@ -import os -# What is this? -## Unit tests for the /end_users/* endpoints -import pytest -import asyncio -import aiohttp -import time -from litellm._uuid import uuid -from openai import AsyncOpenAI -from typing import Optional - -""" -- `/end_user/new` -- `/end_user/info` -""" - - -async def generate_key( - session, - i, - budget=None, - budget_duration=None, - models=["azure-models", "gpt-4", "dall-e-3"], - max_parallel_requests: Optional[int] = None, - user_id: Optional[str] = None, - team_id: Optional[str] = None, - calling_key=os.environ["LITELLM_MASTER_KEY"], -): - url = "http://0.0.0.0:4000/key/generate" - headers = { - "Authorization": f"Bearer {calling_key}", - "Content-Type": "application/json", - } - data = { - "models": models, - "aliases": {"mistral-7b": "gpt-3.5-turbo"}, - "duration": None, - "max_budget": budget, - "budget_duration": budget_duration, - "max_parallel_requests": max_parallel_requests, - "user_id": user_id, - "team_id": team_id, - } - - print(f"data: {data}") - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -async def new_end_user( - session, - i, - user_id=str(uuid.uuid4()), - model_region=None, - default_model=None, - budget_id=None, -): - url = "http://0.0.0.0:4000/end_user/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = { - "user_id": user_id, - "allowed_model_region": model_region, - "default_model": default_model, - } - - if budget_id is not None: - data["budget_id"] = budget_id - print("end user data: {}".format(data)) - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -async def new_budget(session, i, budget_id=None): - url = "http://0.0.0.0:4000/budget/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = { - "budget_id": budget_id, - "tpm_limit": 2, - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - -@pytest.mark.asyncio -async def test_enduser_tpm_limits_non_master_key(): - """ - 1. budget_id = Create Budget with tpm_limit = 10 - 2. create end_user with budget_id - 3. Make /chat/completions calls - 4. Sleep 1 second - 4. Make /chat/completions call -> expect this to fail because rate limit hit - """ - async with aiohttp.ClientSession() as session: - # create a budget with budget_id = "free-tier" - budget_id = f"free-tier-{uuid.uuid4()}" - await new_budget(session, 0, budget_id=budget_id) - await asyncio.sleep(2) - - end_user_id = str(uuid.uuid4()) - - await new_end_user( - session=session, i=0, user_id=end_user_id, budget_id=budget_id - ) - - ## MAKE CALL ## - key_gen = await generate_key(session=session, i=0, models=[]) - - key = key_gen["key"] - - # chat completion 1 - client = AsyncOpenAI(api_key=key, base_url="http://0.0.0.0:4000", max_retries=0) - - # chat completion 2 - passed = 0 - for _ in range(10): - try: - result = await client.chat.completions.create( - model="fake-openai-endpoint", - messages=[{"role": "user", "content": "Hey!"}], - user=end_user_id, - ) - passed += 1 - except Exception: - pass - print("Passed requests=", passed) - - assert ( - passed < 5 - ), f"Sent 10 requests and end-user has tpm_limit of 2. Number requests passed: {passed}. Expected less than 5 to pass" - - -@pytest.mark.asyncio -async def test_enduser_tpm_limits_with_master_key(): - """ - 1. budget_id = Create Budget with tpm_limit = 10 - 2. create end_user with budget_id - 3. Make /chat/completions calls - 4. Sleep 1 second - 4. Make /chat/completions call -> expect this to fail because rate limit hit - """ - async with aiohttp.ClientSession() as session: - # create a budget with budget_id = "free-tier" - budget_id = f"free-tier-{uuid.uuid4()}" - await new_budget(session, 0, budget_id=budget_id) - - end_user_id = str(uuid.uuid4()) - - await new_end_user( - session=session, i=0, user_id=end_user_id, budget_id=budget_id - ) - - # chat completion 1 - client = AsyncOpenAI( - api_key=os.environ["LITELLM_MASTER_KEY"], base_url="http://0.0.0.0:4000", max_retries=0 - ) - - # chat completion 2 - passed = 0 - for _ in range(10): - try: - result = await client.chat.completions.create( - model="fake-openai-endpoint", - messages=[{"role": "user", "content": "Hey!"}], - user=end_user_id, - ) - passed += 1 - except Exception: - pass - print("Passed requests=", passed) - - assert ( - passed < 5 - ), f"Sent 10 requests and end-user has tpm_limit of 2. Number requests passed: {passed}. Expected less than 5 to pass" diff --git a/tests/test_fallbacks.py b/tests/test_fallbacks.py deleted file mode 100644 index 8418847d062..00000000000 --- a/tests/test_fallbacks.py +++ /dev/null @@ -1,325 +0,0 @@ -import os -from typing import Final - -# What is this? -## This tests if the proxy fallbacks work as expected -import pytest -import asyncio -import aiohttp -from tests.large_text import text -import time -from typing import Optional -from openai import AsyncOpenAI, PermissionDeniedError - -PROXY_BASE_URL: Final = os.environ.get("LITELLM_PROXY_BASE_URL", "http://0.0.0.0:4000") - - -async def generate_key( - session, - i, - models: list, - calling_key=os.environ["LITELLM_MASTER_KEY"], -): - url: Final = f"{PROXY_BASE_URL}/key/generate" - headers = { - "Authorization": f"Bearer {calling_key}", - "Content-Type": "application/json", - } - data = { - "models": models, - } - - print(f"data: {data}") - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -async def chat_completion( - session, - key: str, - model: str, - messages: list, - return_headers: bool = False, - extra_headers: Optional[dict] = None, - **kwargs, -): - url: Final = f"{PROXY_BASE_URL}/chat/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - if extra_headers is not None: - headers.update(extra_headers) - data = {"model": model, "messages": messages, **kwargs} - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - if return_headers: - return None, response.headers - else: - raise Exception(f"Request did not return a 200 status code: {status}") - - if return_headers: - return await response.json(), response.headers - else: - return await response.json() - - -@pytest.mark.parametrize("has_access", [True, False]) -@pytest.mark.asyncio -async def test_chat_completion_client_fallbacks(has_access: bool) -> None: - models: Final = ["gpt-3.5-turbo", "gpt-6-luna"] if has_access else ["gpt-3.5-turbo"] - async with aiohttp.ClientSession() as session: - generated_key: Final = await generate_key(session=session, i=0, models=models) - async with AsyncOpenAI(api_key=generated_key["key"], base_url=PROXY_BASE_URL, max_retries=0) as client: - request: Final = { - "model": "gpt-3.5-turbo", - "messages": [{"role": "user", "content": "Who was Alexander?"}], - "max_tokens": 32, - "temperature": 0, - "extra_body": { - "mock_testing_fallbacks": True, - "fallbacks": ["gpt-6-luna"], - }, - } - if not has_access: - with pytest.raises(PermissionDeniedError) as denied: - await client.chat.completions.create(**request) - assert denied.value.status_code == 403 - assert "gpt-6-luna" in str(denied.value) - return - response: Final = await client.chat.completions.create(**request) - assert response.model == "gpt-6-luna" - assert response.choices[0].message.content - - -@pytest.mark.asyncio -async def test_chat_completion_with_retries(): - """ - make chat completion call with prompt > context window. expect it to work with fallback - """ - async with aiohttp.ClientSession() as session: - model = "fake-openai-endpoint-4" - messages = [ - {"role": "system", "content": text}, - {"role": "user", "content": "Who was Alexander?"}, - ] - response, headers = await chat_completion( - session=session, - key=os.environ["LITELLM_MASTER_KEY"], - model=model, - messages=messages, - mock_testing_rate_limit_error=True, - return_headers=True, - ) - print(f"headers: {headers}") - assert headers["x-litellm-attempted-retries"] == "1" - assert headers["x-litellm-max-retries"] == "50" - - -@pytest.mark.asyncio -async def test_chat_completion_with_fallbacks(): - """ - make chat completion call with prompt > context window. expect it to work with fallback - """ - async with aiohttp.ClientSession() as session: - model = "badly-configured-openai-endpoint" - messages = [ - {"role": "system", "content": text}, - {"role": "user", "content": "Who was Alexander?"}, - ] - response, headers = await chat_completion( - session=session, - key=os.environ["LITELLM_MASTER_KEY"], - model=model, - messages=messages, - fallbacks=["fake-openai-endpoint-5"], - return_headers=True, - ) - print(f"headers: {headers}") - assert headers["x-litellm-attempted-fallbacks"] == "1" - - -@pytest.mark.asyncio -async def test_chat_completion_with_timeout(): - """ - make chat completion call with low timeout and `mock_timeout`: true. Expect it to fail and correct timeout to be set in headers. - """ - async with aiohttp.ClientSession() as session: - model = "fake-openai-endpoint-5" - messages = [ - {"role": "system", "content": text}, - {"role": "user", "content": "Who was Alexander?"}, - ] - start_time = time.time() - response, headers = await chat_completion( - session=session, - key=os.environ["LITELLM_MASTER_KEY"], - model=model, - messages=messages, - num_retries=0, - mock_timeout=True, - return_headers=True, - ) - end_time = time.time() - print(f"headers: {headers}") - assert ( - headers["x-litellm-timeout"] == "1.0" - ) # assert model-specific timeout used - - -@pytest.mark.asyncio -async def test_chat_completion_with_timeout_from_request(): - """ - make chat completion call with low timeout and `mock_timeout`: true. Expect it to fail and correct timeout to be set in headers. - """ - async with aiohttp.ClientSession() as session: - model = "fake-openai-endpoint-5" - messages = [ - {"role": "system", "content": text}, - {"role": "user", "content": "Who was Alexander?"}, - ] - extra_headers = { - "x-litellm-timeout": "0.001", - } - start_time = time.time() - response, headers = await chat_completion( - session=session, - key=os.environ["LITELLM_MASTER_KEY"], - model=model, - messages=messages, - num_retries=0, - mock_timeout=True, - extra_headers=extra_headers, - return_headers=True, - ) - end_time = time.time() - print(f"headers: {headers}") - assert ( - headers["x-litellm-timeout"] == "0.001" - ) # assert model-specific timeout used - - -@pytest.mark.parametrize("has_access", [True, False]) -@pytest.mark.asyncio -async def test_chat_completion_client_fallbacks_with_custom_message(has_access: bool) -> None: - original_messages: Final = [{"role": "user", "content": "Who was Alexander?"}] - custom_messages: Final = [ - { - "role": "user", - "content": ( - "Describe the weather in a coastal city during winter, including the usual temperature, rain, wind, " - "and the clothing a visitor should bring." - ), - } - ] - models: Final = ["gpt-3.5-turbo", "gpt-6-luna"] if has_access else ["gpt-3.5-turbo"] - async with aiohttp.ClientSession() as session: - generated_key: Final = await generate_key(session=session, i=0, models=models) - async with AsyncOpenAI(api_key=generated_key["key"], base_url=PROXY_BASE_URL, max_retries=0) as client: - request: Final = { - "model": "gpt-3.5-turbo", - "messages": original_messages, - "max_tokens": 32, - "temperature": 0, - "extra_body": { - "mock_testing_fallbacks": True, - "fallbacks": [ - { - "model": "gpt-6-luna", - "messages": custom_messages, - } - ], - }, - } - if not has_access: - with pytest.raises(PermissionDeniedError) as denied: - await client.chat.completions.create(**request) - assert denied.value.status_code == 403 - assert "gpt-6-luna" in str(denied.value) - return - response: Final = await client.chat.completions.create(**request) - assert response.model == "gpt-6-luna" - assert response.choices[0].message.content - custom_control: Final = await client.chat.completions.create( - model="gpt-6-luna", - messages=custom_messages, - max_tokens=32, - temperature=0, - ) - original_control: Final = await client.chat.completions.create( - model="gpt-6-luna", - messages=original_messages, - max_tokens=32, - temperature=0, - ) - assert response.usage is not None - assert custom_control.usage is not None - assert original_control.usage is not None - assert custom_control.usage.completion_tokens > 0 - assert original_control.usage.completion_tokens > 0 - assert custom_control.usage.prompt_tokens != original_control.usage.prompt_tokens - assert response.usage.prompt_tokens == custom_control.usage.prompt_tokens - - -from typing import List - - -async def make_request(client: AsyncOpenAI, model: str) -> bool: - try: - await client.chat.completions.create( - model=model, - messages=[{"role": "user", "content": "Who was Alexander?"}], - ) - return True - except Exception as e: - print(f"Error with {model}: {str(e)}") - return False - - -async def run_good_model_test(client: AsyncOpenAI, num_requests: int) -> bool: - tasks = [make_request(client, "good-model") for _ in range(num_requests)] - good_results = await asyncio.gather(*tasks) - return all(good_results) - - -@pytest.mark.asyncio -async def test_chat_completion_bad_and_good_model(): - """ - Prod test - ensure even if bad model is down, good model is still working. - """ - client = AsyncOpenAI(api_key=os.environ["LITELLM_MASTER_KEY"], base_url="http://0.0.0.0:4000") - num_requests = 100 - num_iterations = 3 - - for iteration in range(num_iterations): - print(f"\nIteration {iteration + 1}/{num_iterations}") - start_time = time.time() - - # Fire and forget bad model requests - for _ in range(num_requests): - asyncio.create_task(make_request(client, "bad-model")) - - # Wait only for good model requests - success = await run_good_model_test(client, num_requests) - print( - f"Iteration {iteration + 1}: {'✓' if success else '✗'} ({time.time() - start_time:.2f}s)" - ) - assert success, "Not all good model requests succeeded" diff --git a/tests/test_gpt5_azure_temperature_support.py b/tests/test_gpt5_azure_temperature_support.py deleted file mode 100644 index 025b921236a..00000000000 --- a/tests/test_gpt5_azure_temperature_support.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -Test that Azure GPT-5 models support temperature parameter in Responses API. -""" - -import pytest -from litellm.utils import ProviderConfigManager -from litellm.types.utils import LlmProviders - - -def test_azure_gpt5_supports_temperature(): - """Test that Azure GPT-5 uses the correct config that supports temperature.""" - config = ProviderConfigManager.get_provider_responses_api_config( - provider=LlmProviders.AZURE, model="gpt-5" - ) - - # Should use AzureOpenAIResponsesAPIConfig, not AzureOpenAIOSeriesResponsesAPIConfig - assert type(config).__name__ == "AzureOpenAIResponsesAPIConfig" - - # Should support temperature parameter - supported_params = config.get_supported_openai_params("gpt-5") - assert ( - "temperature" in supported_params - ), "Azure GPT-5 should support temperature parameter" - - -def test_azure_o_series_does_not_support_temperature(): - """Test that Azure O-series models still use the correct O-series config.""" - test_models = ["o1", "o3"] - - for model in test_models: - config = ProviderConfigManager.get_provider_responses_api_config( - provider=LlmProviders.AZURE, model=model - ) - - # Should use AzureOpenAIOSeriesResponsesAPIConfig - assert ( - type(config).__name__ == "AzureOpenAIOSeriesResponsesAPIConfig" - ), f"Azure {model} should use O-series config" - - # Should NOT support temperature parameter - supported_params = config.get_supported_openai_params(model) - assert ( - "temperature" not in supported_params - ), f"Azure {model} should NOT support temperature parameter" - - -def test_openai_gpt5_supports_temperature(): - """Test that OpenAI GPT-5 supports temperature parameter.""" - config = ProviderConfigManager.get_provider_responses_api_config( - provider=LlmProviders.OPENAI, model="gpt-5" - ) - - # Should use OpenAIResponsesAPIConfig - assert type(config).__name__ == "OpenAIResponsesAPIConfig" - - # Should support temperature parameter - supported_params = config.get_supported_openai_params("gpt-5") - assert ( - "temperature" in supported_params - ), "OpenAI GPT-5 should support temperature parameter" - - -def test_azure_gpt5_variants_support_temperature(): - """Test that various GPT-5 model name variants support temperature.""" - gpt5_variants = ["gpt-5", "gpt-5-turbo", "GPT-5", "azure/gpt-5"] - - for model in gpt5_variants: - config = ProviderConfigManager.get_provider_responses_api_config( - provider=LlmProviders.AZURE, model=model - ) - - # All GPT-5 variants should use the base config, not O-series config - assert ( - type(config).__name__ == "AzureOpenAIResponsesAPIConfig" - ), f"Model '{model}' should not use O-series config" - - # All should support temperature - supported_params = config.get_supported_openai_params(model) - assert ( - "temperature" in supported_params - ), f"Model '{model}' should support temperature parameter" - - -def test_azure_gpt_models_support_temperature(): - """Test that all GPT models (gpt-3.5, gpt-4, gpt-5, etc.) support temperature.""" - gpt_models = ["gpt-3.5-turbo", "gpt-4", "gpt-4-turbo", "gpt-4o", "gpt-5"] - - for model in gpt_models: - config = ProviderConfigManager.get_provider_responses_api_config( - provider=LlmProviders.AZURE, model=model - ) - - # All GPT models should use the base config, not O-series config - assert ( - type(config).__name__ == "AzureOpenAIResponsesAPIConfig" - ), f"Model '{model}' should not use O-series config" - - # All should support temperature - supported_params = config.get_supported_openai_params(model) - assert ( - "temperature" in supported_params - ), f"Model '{model}' should support temperature parameter" diff --git a/tests/test_health.py b/tests/test_health.py deleted file mode 100644 index a92c57314b1..00000000000 --- a/tests/test_health.py +++ /dev/null @@ -1,84 +0,0 @@ -import os - -# What this tests? -## Tests /health + /routes endpoints. - -import pytest -import asyncio -import aiohttp - - -async def health(session, call_key): - url = "http://0.0.0.0:4000/health" - headers = { - "Authorization": f"Bearer {call_key}", - "Content-Type": "application/json", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - - print(f"Response (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - return await response.json() - - -async def generate_key(session): - url = "http://0.0.0.0:4000/key/generate" - headers = { - "Authorization": "Bearer " + os.environ["LITELLM_MASTER_KEY"], - "Content-Type": "application/json", - } - data = { - "models": ["gpt-4", "text-embedding-ada-002", "gpt-image-1"], - "duration": None, - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - -@pytest.mark.asyncio -async def test_health(): - """ - - Call /health - """ - async with aiohttp.ClientSession() as session: - # as admin # - all_healthy_models = await health(session=session, call_key=os.environ["LITELLM_MASTER_KEY"]) - total_model_count = ( - all_healthy_models["healthy_count"] + all_healthy_models["unhealthy_count"] - ) - assert total_model_count > 0 - - -@pytest.mark.asyncio -async def test_routes(): - """ - Check if 200 - """ - async with aiohttp.ClientSession() as session: - url = "http://0.0.0.0:4000/routes" - async with session.get(url) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") diff --git a/tests/test_keys.py b/tests/test_keys.py index 4e9734c60b6..c1862563b0f 100644 --- a/tests/test_keys.py +++ b/tests/test_keys.py @@ -2,57 +2,11 @@ ## Tests /key endpoints. import pytest -import asyncio, uuid +import asyncio import aiohttp -from openai import AsyncOpenAI -import sys, os +import os from typing import Optional -import litellm -from litellm.proxy._types import LitellmUserRoles - - -async def generate_team( - session, models: Optional[list] = None, team_id: Optional[str] = None -): - url = "http://0.0.0.0:4000/team/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - if team_id is None: - team_id = "litellm-dashboard" - data = {"team_id": team_id, **({"models": models} if models is not None else {})} - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response (Status code: {status}):") - print(response_text) - print() - _json_response = await response.json() - return _json_response - - -async def generate_user( - session, - user_role="app_owner", -): - url = "http://0.0.0.0:4000/user/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = { - "user_role": user_role, - "team_id": "litellm-dashboard", - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response (Status code: {status}):") - print(response_text) - print() - _json_response = await response.json() - return _json_response - async def generate_key( session, @@ -99,25 +53,6 @@ async def generate_key( return await response.json() -@pytest.mark.asyncio -async def test_key_gen(): - async with aiohttp.ClientSession() as session: - tasks = [generate_key(session, i) for i in range(1, 11)] - await asyncio.gather(*tasks) - - -@pytest.mark.asyncio -async def test_simple_key_gen(): - async with aiohttp.ClientSession() as session: - key_data = await generate_key(session, i=0) - key = key_data["key"] - assert key_data["token"] is not None - assert key_data["token"] != key - assert key_data["token_id"] is not None - assert key_data["created_at"] is not None - assert key_data["updated_at"] is not None - - @pytest.mark.asyncio async def test_key_gen_bad_key(): """ @@ -147,64 +82,6 @@ async def test_key_gen_bad_key(): pass -async def chat_completion(session, key, model="gpt-4"): - url = "http://0.0.0.0:4000/chat/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - data = { - "model": model, - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - ], - } - - for i in range(3): - try: - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception( - f"Request did not return a 200 status code: {status}. Response: {response_text}" - ) - - return await response.json() - except Exception as e: - if "Request did not return a 200 status code" in str(e): - raise e - else: - pass - - -async def delete_key(session, get_key, auth_key=os.environ["LITELLM_MASTER_KEY"]): - """ - Delete key - """ - url = "http://0.0.0.0:4000/key/delete" - headers = { - "Authorization": f"Bearer {auth_key}", - "Content-Type": "application/json", - } - data = {"keys": [get_key]} - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - async def get_key_info(session, call_key, get_key=None): """ Make sure only models user has access to are returned @@ -235,111 +112,6 @@ async def get_key_info(session, call_key, get_key=None): return await response.json() -async def get_model_list(session, call_key, endpoint: str = "/v1/models"): - """ - Make sure only models user has access to are returned - """ - url = "http://0.0.0.0:4000" + endpoint - headers = { - "Authorization": f"Bearer {call_key}", - "Content-Type": "application/json", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - print(response_text) - print() - - if status != 200: - raise Exception( - f"Request did not return a 200 status code: {status}. Responses {response_text}" - ) - return await response.json() - - -async def get_model_info(session, call_key): - """ - Make sure only models user has access to are returned - """ - url = "http://0.0.0.0:4000/model/info" - headers = { - "Authorization": f"Bearer {call_key}", - "Content-Type": "application/json", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - print(response_text) - print() - - if status != 200: - raise Exception( - f"Request did not return a 200 status code: {status}. Responses {response_text}" - ) - return await response.json() - - -@pytest.mark.asyncio -async def test_key_info(): - """ - Get key info - - as admin -> 200 - - as key itself -> 200 - - as non existent key -> 404 - """ - async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session, i=0) - key = key_gen["key"] - # as admin # - await get_key_info(session=session, get_key=key, call_key=os.environ["LITELLM_MASTER_KEY"]) - # as key itself # - await get_key_info(session=session, get_key=key, call_key=key) - - # as key itself, use the auth param, and no query key needed - await get_key_info(session=session, call_key=key) - # as random key # - random_key = f"sk-{uuid.uuid4()}" - status = await get_key_info(session=session, get_key=random_key, call_key=key) - assert status == 404 - - -@pytest.mark.asyncio -async def test_model_info(): - """ - Get model info for models key has access to - """ - async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session, i=0) - key = key_gen["key"] - # as admin # - admin_models = await get_model_info(session=session, call_key=os.environ["LITELLM_MASTER_KEY"]) - admin_models = admin_models["data"] - # as key itself # - user_models = await get_model_info(session=session, call_key=key) - user_models = user_models["data"] - - assert len(admin_models) > len(user_models) - assert len(user_models) > 0 - - -async def get_spend_logs(session, request_id): - url = f"http://0.0.0.0:4000/spend/logs?request_id={request_id}" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - @pytest.mark.skip(reason="Frequent check on ci/cd leads to read timeout issue.") @pytest.mark.asyncio async def test_key_with_budgets(): @@ -387,88 +159,3 @@ async def test_key_with_budgets(): # assert rounded_response_cost == rounded_key_info_spend - - -@pytest.mark.asyncio -async def test_key_delete_ui(): - """ - Admin UI flow - DO NOT DELETE - -> Create a key with user_id = "ishaan" - -> Log on Admin UI, delete the key for user "ishaan" - -> This should work, since we're on the admin UI and role == "proxy_admin - """ - async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session, i=0, user_id="ishaan-smart") - key = key_gen["key"] - - # generate a admin UI key - team = await generate_team(session=session) - admin_ui_key = await generate_user( - session=session, user_role=LitellmUserRoles.PROXY_ADMIN.value - ) - print( - "trying to delete key=", - key, - "using key=", - admin_ui_key["key"], - " to auth in", - ) - - await delete_key( - session=session, - get_key=key, - auth_key=admin_ui_key["key"], - ) - - -@pytest.mark.parametrize("model_access", ["all-team-models", "gpt-3.5-turbo"]) -@pytest.mark.parametrize("model_access_level", ["key", "team"]) -@pytest.mark.parametrize("model_endpoint", ["/v1/models", "/model/info"]) -@pytest.mark.asyncio -async def test_key_model_list(model_access, model_access_level, model_endpoint): - """ - Test if `/v1/models` works as expected. - """ - async with aiohttp.ClientSession() as session: - _models = [] if model_access == "all-team-models" else [model_access] - team_id = "litellm_dashboard_{}".format(uuid.uuid4()) - new_team = await generate_team( - session=session, - models=_models if model_access_level == "team" else None, - team_id=team_id, - ) - assert new_team["team_id"] == team_id - key_gen = await generate_key( - session=session, - i=0, - team_id=team_id, - models=_models if model_access_level == "key" else [], - ) - key = key_gen["key"] - print(f"key: {key}") - - model_list = await get_model_list( - session=session, call_key=key, endpoint=model_endpoint - ) - print(f"model_list: {model_list}") - - if model_access == "all-team-models": - if model_endpoint == "/v1/models": - assert not isinstance(model_list["data"][0]["id"], list) - assert isinstance(model_list["data"][0]["id"], str) - elif model_endpoint == "/model/info": - assert isinstance(model_list["data"], list) - assert len(model_list["data"]) > 0 - if model_access == "gpt-3.5-turbo": - if model_endpoint == "/v1/models": - assert {entry["id"] for entry in model_list["data"]} == { - model_access, - "mistral-7b", - }, "generate_key sets alias mistral-7b -> gpt-3.5-turbo, so /v1/models lists both; model_access={}, model_access_level={}".format( - model_access, model_access_level - ) - elif model_endpoint == "/model/info": - assert isinstance(model_list["data"], list) - assert len(model_list["data"]) == 1 - - diff --git a/tests/test_litellm_proxy_responses_config.py b/tests/test_litellm_proxy_responses_config.py index 0743565874a..7bfb89a8711 100644 --- a/tests/test_litellm_proxy_responses_config.py +++ b/tests/test_litellm_proxy_responses_config.py @@ -2,61 +2,7 @@ Unit test for LiteLLM Proxy Responses API configuration. """ -import pytest - from litellm.types.utils import LlmProviders -from litellm.utils import ProviderConfigManager - - -def test_litellm_proxy_responses_api_config(): - """Test that litellm_proxy provider returns correct Responses API config""" - from litellm.llms.litellm_proxy.responses.transformation import ( - LiteLLMProxyResponsesAPIConfig, - ) - - config = ProviderConfigManager.get_provider_responses_api_config( - model="litellm_proxy/gpt-5.5", - provider=LlmProviders.LITELLM_PROXY, - ) - print(f"config: {config}") - assert config is not None, "Config should not be None for litellm_proxy provider" - assert isinstance( - config, LiteLLMProxyResponsesAPIConfig - ), f"Expected LiteLLMProxyResponsesAPIConfig, got {type(config)}" - assert ( - config.custom_llm_provider == LlmProviders.LITELLM_PROXY - ), "custom_llm_provider should be LITELLM_PROXY" - - -def test_litellm_proxy_responses_api_config_get_complete_url(): - """Test that get_complete_url works correctly""" - import os - from litellm.llms.litellm_proxy.responses.transformation import ( - LiteLLMProxyResponsesAPIConfig, - ) - - config = LiteLLMProxyResponsesAPIConfig() - - # Test with explicit api_base - url = config.get_complete_url( - api_base="https://my-proxy.example.com", - litellm_params={}, - ) - assert url == "https://my-proxy.example.com/responses" - - # Test with trailing slash - url = config.get_complete_url( - api_base="https://my-proxy.example.com/", - litellm_params={}, - ) - assert url == "https://my-proxy.example.com/responses" - - # Test that it raises error when api_base is None and env var is not set - if "LITELLM_PROXY_API_BASE" in os.environ: - del os.environ["LITELLM_PROXY_API_BASE"] - - with pytest.raises(ValueError, match="api_base not set"): - config.get_complete_url(api_base=None, litellm_params={}) def test_litellm_proxy_responses_api_config_inherits_from_openai(): @@ -70,15 +16,6 @@ def test_litellm_proxy_responses_api_config_inherits_from_openai(): config = LiteLLMProxyResponsesAPIConfig() - # Should inherit from OpenAI config assert isinstance(config, OpenAIResponsesAPIConfig) - # Should have the correct provider set assert config.custom_llm_provider == LlmProviders.LITELLM_PROXY - - -if __name__ == "__main__": - test_litellm_proxy_responses_api_config() - test_litellm_proxy_responses_api_config_get_complete_url() - test_litellm_proxy_responses_api_config_inherits_from_openai() - print("All tests passed!") diff --git a/tests/test_models.py b/tests/test_models.py index 14706098f27..8ae9d8edcd7 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -5,8 +5,6 @@ import pytest import asyncio import aiohttp import os -import dotenv -from typing import Final from dotenv import load_dotenv load_dotenv() @@ -32,46 +30,6 @@ async def generate_key(session, models=[]): return await response.json() -async def get_models(session, key, only_model_access_groups=False): - url = "http://0.0.0.0:4000/models" - if only_model_access_groups: - url += "?only_model_access_groups=True" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - print("response from /models") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - -@pytest.mark.asyncio -async def test_get_models_multiple_tests(): - async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session) - key = key_gen["key"] - models = await get_models(session=session, key=key) - print(f"\n\nmodels: {models}") - assert len(models["data"]) > 0 - - ## Test only_model_access_groups - new_response = await get_models( - session=session, key=key, only_model_access_groups=True - ) - print(f"\n\nnew_response: {new_response}") - assert ( - len(new_response["data"]) == 0 - ) # no model access groups set on config.yaml - - async def add_models( session, model_id="123", model_name="azure-gpt-3.5", key=os.environ["LITELLM_MASTER_KEY"], team_id=None ): @@ -106,48 +64,6 @@ async def add_models( return response_json -async def get_model_info(session, key, litellm_model_id=None): - """ - Make sure only models user has access to are returned - """ - if litellm_model_id: - url = f"http://0.0.0.0:4000/model/info?litellm_model_id={litellm_model_id}" - else: - url = "http://0.0.0.0:4000/model/info" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - -async def get_model_group_info(session, key): - url = "http://0.0.0.0:4000/model_group/info" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - async def chat_completion(session, key, model="azure-gpt-3.5"): url = "http://0.0.0.0:4000/chat/completions" headers = { @@ -173,49 +89,6 @@ async def chat_completion(session, key, model="azure-gpt-3.5"): raise Exception(f"Request did not return a 200 status code: {status}") -@pytest.mark.asyncio -async def test_get_models(): - """ - Get models user has access to - """ - async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session, models=["gpt-4"]) - key = key_gen["key"] - response = await get_model_info(session=session, key=key) - models = [m["model_name"] for m in response["data"]] - for m in models: - assert m == "gpt-4" - - -@pytest.mark.asyncio -async def test_get_specific_model(): - """ - Return specific model info - - Ensure value of model_info is same as on `/model/info` (no id set) - """ - async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session, models=["gpt-4"]) - key = key_gen["key"] - response = await get_model_info(session=session, key=key) - models = [m["model_name"] for m in response["data"]] - model_specific_info = None - for idx, m in enumerate(models): - assert m == "gpt-4" - litellm_model_id = response["data"][idx]["model_info"]["id"] - model_specific_info = response["data"][idx] - assert litellm_model_id is not None - response = await get_model_info( - session=session, key=key, litellm_model_id=litellm_model_id - ) - assert response["data"][0]["model_info"]["id"] == litellm_model_id - assert ( - response["data"][0] == model_specific_info - ), "Model info is not the same. Got={}, Expected={}".format( - response["data"][0], model_specific_info - ) - - async def delete_model(session, model_id="123", key=os.environ["LITELLM_MASTER_KEY"]): """ Make sure only models user has access to are returned @@ -270,45 +143,3 @@ async def test_add_and_delete_models(): pass -@pytest.mark.asyncio -async def test_get_personal_models_for_user(): - """ - Test /models endpoint with team - """ - from tests.test_users import new_user - - async with aiohttp.ClientSession() as session: - # Creat a user - user_data = await new_user(session=session, i=0, models=["gpt-3.5-turbo"]) - user_id = user_data["user_id"] - user_api_key = user_data["key"] - - model_group_info = await get_model_group_info(session=session, key=user_api_key) - print(model_group_info) - - assert len(model_group_info["data"]) == 1 - assert model_group_info["data"][0]["model_group"] == "gpt-3.5-turbo" - - -@pytest.mark.asyncio -async def test_model_group_info_e2e(): - """ - Test /model/group/info endpoint - """ - async with aiohttp.ClientSession() as session: - models = await get_models(session=session, key=os.environ["LITELLM_MASTER_KEY"]) - print(models) - - model_group_info = await get_model_group_info(session=session, key=os.environ["LITELLM_MASTER_KEY"]) - print(model_group_info) - - model_groups: Final = [m["model_group"] for m in model_group_info["data"]] - - assert "anthropic/*" not in model_groups, ( - f"Expected 'anthropic/*' to be expanded, but it was returned verbatim: {model_groups}" - ) - assert any(m.startswith("anthropic/") for m in model_groups), ( - f"Expected concrete anthropic models from the 'anthropic/*' config entry, got: {model_groups}" - ) - - diff --git a/tests/test_new_vector_store_endpoints.py b/tests/test_new_vector_store_endpoints.py index 05e1b25bc8b..8cd866a8e3c 100644 --- a/tests/test_new_vector_store_endpoints.py +++ b/tests/test_new_vector_store_endpoints.py @@ -4,43 +4,12 @@ Tests both basic functionality and complex scenarios including target_model_name """ import asyncio -from unittest.mock import AsyncMock, MagicMock, patch +from unittest.mock import AsyncMock, patch import pytest import litellm -from litellm.proxy._types import UserAPIKeyAuth - - -@pytest.mark.asyncio -async def test_vector_store_update_basic(): - """Test basic vector store update functionality.""" - mock_response = { - "id": "vs_test123", - "object": "vector_store", - "created_at": 1699061776, - "name": "Updated Name", - "metadata": {"key": "value"}, - "status": "completed", - } - - with patch( - "litellm.vector_stores.main.aupdate", - new=AsyncMock(return_value=mock_response), - ) as mock_update: - router = litellm.Router(model_list=[]) - result = await router.avector_store_update( - vector_store_id="vs_test123", - name="Updated Name", - metadata={"key": "value"}, - custom_llm_provider="openai", - ) - - assert result["id"] == "vs_test123" - assert result["name"] == "Updated Name" - assert result["metadata"]["key"] == "value" - mock_update.assert_called_once() @pytest.mark.asyncio @@ -66,71 +35,6 @@ async def test_async_vector_store_update(): mock_aupdate.assert_called_once() -@pytest.mark.asyncio -async def test_vector_store_list_with_pagination(): - """Test vector store list with pagination parameters.""" - mock_response = { - "object": "list", - "data": [{"id": f"vs_{i}"} for i in range(5)], - "has_more": True, - "first_id": "vs_0", - "last_id": "vs_4", - } - - with patch( - "litellm.vector_stores.main.list", - return_value=mock_response, - ) as mock_list: - router = litellm.Router(model_list=[]) - result = router.vector_store_list( - limit=5, - after="vs_previous", - order="asc", - custom_llm_provider="openai", - ) - - assert result["has_more"] is True - assert len(result["data"]) == 5 - - # Verify pagination params were passed - call_kwargs = mock_list.call_args.kwargs - assert call_kwargs["limit"] == 5 - assert call_kwargs["after"] == "vs_previous" - assert call_kwargs["order"] == "asc" - - -@pytest.mark.asyncio -async def test_vector_store_update_with_expires_after(): - """Test vector store update with expiration policy.""" - expires_after = { - "anchor": "last_active_at", - "days": 7, - } - - mock_response = { - "id": "vs_test123", - "expires_after": expires_after, - "expires_at": 1699668576, - } - - with patch( - "litellm.vector_stores.main.update", - return_value=mock_response, - ) as mock_update: - router = litellm.Router(model_list=[]) - result = router.vector_store_update( - vector_store_id="vs_test123", - expires_after=expires_after, - custom_llm_provider="openai", - ) - - assert result["expires_after"]["days"] == 7 - assert result["expires_at"] is not None - - call_kwargs = mock_update.call_args.kwargs - assert call_kwargs["expires_after"] == expires_after - - def test_router_initializes_new_endpoints(): """Test that router properly initializes the new vector store endpoints.""" router = litellm.Router(model_list=[]) @@ -165,11 +69,6 @@ if __name__ == "__main__": test_router_initializes_new_endpoints() print("✓ Router initialization successful") - # Test basic sync operations - print("✓ Testing basic sync operations...") - asyncio.run(test_vector_store_update_basic()) - print("✓ Basic sync operations successful") - # Test async operations print("✓ Testing async operations...") asyncio.run(test_async_vector_store_update()) diff --git a/tests/test_openai_endpoints.py b/tests/test_openai_endpoints.py index 6b92373a98c..691ed37ac9f 100644 --- a/tests/test_openai_endpoints.py +++ b/tests/test_openai_endpoints.py @@ -3,10 +3,7 @@ from typing import Final # What this tests ? ## Tests /chat/completions by generating a key and then making a chat completions-request import pytest -import asyncio -import aiohttp, openai -from openai import OpenAI, AsyncOpenAI, AzureOpenAI, AsyncAzureOpenAI -from typing import Optional, List, Union +from openai import AsyncOpenAI LITELLM_MASTER_KEY = os.environ["LITELLM_MASTER_KEY"] @@ -19,64 +16,6 @@ def response_header_check(response): assert headers_size < 4096, "Response headers exceed the 4kb limit" -async def generate_key( - session, - models=[ - "gpt-4", - "text-embedding-ada-002", - "gpt-image-1", - "fake-openai-endpoint-2", - "mistral-embed", - ], -): - url = "http://0.0.0.0:4000/key/generate" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = { - "models": models, - "duration": None, - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - response_header_check( - response - ) # calling the function to check response headers - - return await response.json() - - -async def new_user(session): - url = "http://0.0.0.0:4000/user/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = { - "models": ["gpt-4", "text-embedding-ada-002", "gpt-image-1"], - "duration": None, - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - response_header_check( - response - ) # calling the function to check response headers - return await response.json() - - async def moderation(session, key): url = "http://0.0.0.0:4000/moderations" headers = { @@ -98,141 +37,6 @@ async def moderation(session, key): return await response.json() -async def chat_completion(session, key, model: Union[str, List] = "gpt-4"): - url = "http://0.0.0.0:4000/chat/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - data = { - "model": model, - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - ], - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception( - f"Request did not return a 200 status code: {status}, response text={response_text}" - ) - - response_header_check( - response - ) # calling the function to check response headers - - return await response.json() - - -async def queue_chat_completion( - session, key, priority: int, model: Union[str, List] = "gpt-4" -): - url = "http://0.0.0.0:4000/queue/chat/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - data = { - "model": model, - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - ], - "priority": priority, - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - return response.raw_headers - - -async def chat_completion_with_headers(session, key, model="gpt-4"): - url = "http://0.0.0.0:4000/chat/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - data = { - "model": model, - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - ], - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - response_header_check( - response - ) # calling the function to check response headers - - raw_headers = response.raw_headers - raw_headers_json = {} - - for ( - item - ) in ( - response.raw_headers - ): # ((b'date', b'Fri, 19 Apr 2024 21:17:29 GMT'), (), ) - raw_headers_json[item[0].decode("utf-8")] = item[1].decode("utf-8") - - return raw_headers_json - - -async def chat_completion_with_model_from_route(session, key, route): - url = "http://0.0.0.0:4000/chat/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - - -async def completion(session, key): - url = "http://0.0.0.0:4000/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - data = {"model": "gpt-4", "prompt": "Hello!"} - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - response_header_check( - response - ) # calling the function to check response headers - - response = await response.json() - - return response - - async def embeddings(session, key, model="text-embedding-ada-002"): url = "http://0.0.0.0:4000/embeddings" headers = { @@ -258,120 +62,6 @@ async def embeddings(session, key, model="text-embedding-ada-002"): ) # calling the function to check response headers -async def image_generation(session, key): - url = "http://0.0.0.0:4000/images/generations" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - data = { - "model": "gpt-image-1", - "prompt": "A cute baby sea otter", - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - if ( - "Connection error" in response_text - ): # OpenAI endpoint returns a connection error - return - raise Exception(f"Request did not return a 200 status code: {status}") - - response_header_check( - response - ) # calling the function to check response headers - - -@pytest.mark.asyncio -async def test_chat_completion(): - """ - - Create key - Make chat completion call - - Create user - make chat completion call - """ - async with aiohttp.ClientSession() as session: - key_gen = await generate_key(session=session, models=["gpt-3.5-turbo"]) - azure_client = AsyncAzureOpenAI( - azure_endpoint="http://0.0.0.0:4000", - azure_deployment="random-model", - api_key=key_gen["key"], - api_version="2024-02-15-preview", - ) - with pytest.raises(openai.PermissionDeniedError) as e: - response = await azure_client.chat.completions.create( - model="gpt-4", - messages=[{"role": "user", "content": "Hello!"}], - ) - assert "is not available for this API key" in str(e.value) - - -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -@pytest.mark.skip(reason="Flaky test, this works locally but not on CI") -async def test_chat_completion_ratelimit(): - """ - - call model with rpm 1 - - make 2 parallel calls - - make sure 1 fails - """ - async with aiohttp.ClientSession() as session: - # key_gen = await generate_key(session=session) - key = os.environ["LITELLM_MASTER_KEY"] - tasks = [] - tasks.append( - chat_completion(session=session, key=key, model="fake-openai-endpoint-2") - ) - tasks.append( - chat_completion(session=session, key=key, model="fake-openai-endpoint-2") - ) - try: - await asyncio.gather(*tasks) - pytest.fail("Expected at least 1 call to fail") - except Exception as e: - if "Request did not return a 200 status code: 429" in str(e): - pass - else: - pytest.fail(f"Wrong error received - {str(e)}") - - -@pytest.mark.asyncio -@pytest.mark.skip(reason="Flaky test") -async def test_chat_completion_different_deployments(): - """ - - call model group with 2 deployments - - make 5 calls - - expect 2 unique deployments - """ - async with aiohttp.ClientSession() as session: - # key_gen = await generate_key(session=session) - key = os.environ["LITELLM_MASTER_KEY"] - results = [] - for _ in range(20): - results.append( - await chat_completion_with_headers( - session=session, key=key, model="fake-openai-endpoint-3" - ) - ) - try: - print(f"results: {results}") - init_model_id = results[0]["x-litellm-model-id"] - deployments_shuffled = False - for result in results[1:]: - if init_model_id != result["x-litellm-model-id"]: - deployments_shuffled = True - if deployments_shuffled == False: - pytest.fail("Expected at least 1 shuffled call") - except Exception as e: - pass - - @pytest.mark.asyncio async def test_chat_completion_streaming(): """ @@ -425,46 +115,3 @@ async def test_completion_streaming_usage_metrics(): assert last_chunk.usage.total_tokens > 0, "Total tokens should be greater than 0" -@pytest.mark.asyncio -async def test_proxy_all_models(): - """ - - proxy_server_config.yaml has model = * / * - - Make chat completion call - - groq is NOT defined on /models - - - """ - async with aiohttp.ClientSession() as session: - # call chat/completions with a model that the key was not created for + the model is not on the config.yaml - await chat_completion( - session=session, key=LITELLM_MASTER_KEY, model="groq/openai/gpt-oss-120b" - ) - - await chat_completion( - session=session, - key=LITELLM_MASTER_KEY, - model="anthropic/claude-sonnet-4-5-20250929", - ) - - -@pytest.mark.asyncio -async def test_batch_chat_completions(): - """ - - Make chat completion call using - - """ - async with aiohttp.ClientSession() as session: - - # call chat/completions with a model that the key was not created for + the model is not on the config.yaml - response = await chat_completion( - session=session, - key=os.environ["LITELLM_MASTER_KEY"], - model="gpt-3.5-turbo,fake-openai-endpoint", - ) - - print(f"response: {response}") - - assert len(response) == 2 - assert isinstance(response, list) - - diff --git a/tests/test_otel_thread_leak.py b/tests/test_otel_thread_leak.py deleted file mode 100644 index cb5f54eefa4..00000000000 --- a/tests/test_otel_thread_leak.py +++ /dev/null @@ -1,91 +0,0 @@ -import sys -import os -import threading -import time -import pytest - -# Add the project root to the path -sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) - -from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig -from litellm.types.utils import StandardCallbackDynamicParams - - -def get_thread_count() -> int: - """Helper to get active thread count""" - return threading.active_count() - - -@pytest.fixture -def otel_logger(): - """Fixture to provide a clean OTEL logger for each test""" - config = OpenTelemetryConfig( - exporter="console", enable_metrics=False, service_name="litellm-unit-test" - ) - return OpenTelemetry(config=config) - - -def test_otel_thread_leak_dynamic_headers(otel_logger): - """ - Unit test to verify that calling get_tracer_to_use_for_request with - dynamic headers doesn't cause a linear thread leak. - - This test reproduces the issue where each unique team/key credential - set causes a new TracerProvider (and its background threads) to be - spawned but never closed. - """ - - # 1. Setup dynamic header simulation (monkey-patch) - # This simulates what LangfuseOtelLogger does for per-team keys - def mock_construct_dynamic_headers(standard_callback_dynamic_params): - if standard_callback_dynamic_params: - return {"Authorization": "Bearer fake_token"} - return None - - otel_logger.construct_dynamic_otel_headers = mock_construct_dynamic_headers - - # 2. Establish Baseline - initial_threads = get_thread_count() - - # 3. Simulate requests - num_requests = 10 - latencies = [] - - print("\n🚀 Simulating requests with dynamic headers:") - for i in range(num_requests): - kwargs = { - "standard_callback_dynamic_params": StandardCallbackDynamicParams( - langfuse_public_key=f"key_{i}", - langfuse_secret_key=f"secret_{i}", - ) - } - - # Measure latency - start_time = time.perf_counter() - tracer = otel_logger.get_tracer_to_use_for_request(kwargs) - end_time = time.perf_counter() - - latency_ms = (end_time - start_time) * 1000 - latencies.append(latency_ms) - print(f" Request {i+1:2d}: Latency = {latency_ms:6.2f} ms") - - # Verify a tracer was actually returned - assert tracer is not None - - avg_latency = sum(latencies) / len(latencies) - print(f"\n📊 Average Latency: {avg_latency:.2f} ms") - - # 4. Check for leaks - # Allow for a small constant increase (OTEL might start a few shared threads) - # but a linear leak would result in +10 or more threads here. - final_threads = get_thread_count() - thread_delta = final_threads - initial_threads - - print(f"\nThread growth: {thread_delta} threads across {num_requests} requests") - - # ASSERTION: The growth should be significantly less than 1 thread per request. - # If the bug exists, thread_delta will be >= num_requests. - assert thread_delta < (num_requests / 2), ( - f"Thread leak detected! Threads grew by {thread_delta} over {num_requests} requests. " - "Each request with dynamic headers appears to be leaking background threads." - ) diff --git a/tests/test_presidio_latency.py b/tests/test_presidio_latency.py deleted file mode 100644 index bb676ca051e..00000000000 --- a/tests/test_presidio_latency.py +++ /dev/null @@ -1,77 +0,0 @@ -import asyncio -import aiohttp -import pytest -from unittest.mock import MagicMock, patch -from litellm.proxy.guardrails.guardrail_hooks.presidio import ( - OPTIONAL_PresidioPIIMasking, -) - - -@pytest.mark.asyncio -async def test_sanity_presidio_session_reuse_main_thread(): - """ - SANITY CHECK: - Verify that Presidio guardrail reuses sessions in the main thread. - This ensures we don't break existing session pooling functionality. - """ - presidio = OPTIONAL_PresidioPIIMasking( - mock_testing=True, - presidio_analyzer_api_base="http://mock-analyzer", - presidio_anonymizer_api_base="http://mock-anonymizer", - ) - - session_creations = 0 - original_init = aiohttp.ClientSession.__init__ - - def mocked_init(self, *args, **kwargs): - nonlocal session_creations - session_creations += 1 - original_init(self, *args, **kwargs) - - with patch.object(aiohttp.ClientSession, "__init__", side_effect=mocked_init, autospec=True): - for _ in range(10): - async with presidio._get_session_iterator() as session: - pass - - # Expected: Only 1 session created for all 10 calls. - assert session_creations == 1 - - await presidio._close_http_session() - - -@pytest.mark.asyncio -async def test_bug_presidio_session_explosion_background_thread_causes_latency(): - """ - BUG REPRODUCTION: - Verify that background threads (like logging hooks) REUSE sessions. - Previously, each call in a background loop created a NEW ephemeral session, - leading to socket exhaustion and the reported 97s latency spike. - """ - import threading - - presidio = OPTIONAL_PresidioPIIMasking( - mock_testing=True, - presidio_analyzer_api_base="http://mock-analyzer", - presidio_anonymizer_api_base="http://mock-anonymizer", - ) - - # Force the code to think it's in a background thread - presidio._main_thread_id = threading.get_ident() + 1 - - session_creations = 0 - original_init = aiohttp.ClientSession.__init__ - - def mocked_init(self, *args, **kwargs): - nonlocal session_creations - session_creations += 1 - original_init(self, *args, **kwargs) - - with patch.object(aiohttp.ClientSession, "__init__", side_effect=mocked_init, autospec=True): - for _ in range(10): - async with presidio._get_session_iterator() as session: - pass - - # FIX VERIFICATION: Should now be 1 session (reused) instead of 10. - assert session_creations == 1 - - await presidio._close_http_session() diff --git a/tests/test_ratelimit.py b/tests/test_ratelimit.py deleted file mode 100644 index 94d48f0accf..00000000000 --- a/tests/test_ratelimit.py +++ /dev/null @@ -1,170 +0,0 @@ -# %% -import asyncio -import os -import pytest -import random -from typing import Any -from dotenv import load_dotenv - -load_dotenv() - - -import litellm -from pydantic import BaseModel -from litellm import utils, Router - -COMPLETION_TOKENS = 5 -base_model_list = [ - { - "model_name": "gpt-5-mini", - "litellm_params": { - "model": "gpt-5-mini", - "api_key": os.getenv("OPENAI_API_KEY"), - "max_tokens": COMPLETION_TOKENS, - }, - } -] - - -class RouterConfig(BaseModel): - rpm: int - tpm: int - - -@pytest.fixture(scope="function") -def router_factory(): - def create_router(rpm, tpm, routing_strategy): - model_list = base_model_list.copy() - model_list[0]["rpm"] = rpm - model_list[0]["tpm"] = tpm - return Router( - model_list=model_list, - routing_strategy=routing_strategy, - enable_pre_call_checks=True, - debug_level="DEBUG", - ) - - return create_router - - -def generate_list_of_messages(num_messages): - """ - create num_messages new chat conversations - """ - return [ - [{"role": "user", "content": f"{i}. Hey, how's it going? {random.random()}"}] - for i in range(num_messages) - ] - - -def calculate_limits(list_of_messages): - """ - Return the min rpm and tpm level that would let all messages in list_of_messages be sent this minute - """ - rpm = len(list_of_messages) - tpm = sum( - (utils.token_counter(messages=m) + COMPLETION_TOKENS for m in list_of_messages) - ) - return rpm, tpm - - -async def async_call(router: Router, list_of_messages) -> Any: - tasks = [ - router.acompletion(model="gpt-5-mini", messages=m) for m in list_of_messages - ] - return await asyncio.gather(*tasks) - - -def sync_call(router: Router, list_of_messages) -> Any: - return [ - router.completion(model="gpt-5-mini", messages=m) for m in list_of_messages - ] - - -class ExpectNoException(Exception): - pass - - -@pytest.mark.parametrize( - "num_try_send, num_allowed_send", - [ - (2, 3), # sending as many as allowed, ExpectNoException - # (10, 10), # sending as many as allowed, ExpectNoException - (3, 2), # Sending more than allowed, ValueError - # (10, 9), # Sending more than allowed, ValueError - ], -) -@pytest.mark.parametrize( - "sync_mode", [True, False] -) # Use parametrization for sync/async -@pytest.mark.parametrize( - "routing_strategy", - [ - "usage-based-routing", - # "simple-shuffle", # dont expect to rate limit - # "least-busy", # dont expect to rate limit - # "latency-based-routing", - ], -) -def test_async_rate_limit( - router_factory, num_try_send, num_allowed_send, sync_mode, routing_strategy -): - """ - Check if router.completion and router.acompletion can send more messages than they've been limited to. - Args: - router_factory: makes new router object, without any shared Global state - num_try_send (int): number of messages to try to send - num_allowed_send (int): max number of messages allowed to be sent in 1 minute - sync_mode (bool): if making sync (router.completion) or async (router.acompletion) - Raises: - ValueError: Error router throws when it hits rate limits - ExpectNoException: Signfies that no other error has happened. A NOP - """ - # Can send more messages then we're going to; so don't expect a rate limit error - litellm.logging_callback_manager._reset_all_callbacks() - args = locals() - print(f"args: {args}") - expected_exception = ( - ExpectNoException if num_try_send <= num_allowed_send else ValueError - ) - - # usage-based-routing tracks RPM in log_success_event which runs in a - # background ThreadPoolExecutor. The cache update races with the next - # call's routing check, so over-limit detection is non-deterministic in - # both sync tight-loops and async concurrent gathers. - if num_try_send > num_allowed_send: - pytest.skip( - "RPM tracking via background thread is racy; " - "RPM over-limit rejection is tested for usage-based-routing-v2 in " - "tests/unit/router_strategy/test_router_routing_groups.py" - ) - - list_of_messages = generate_list_of_messages(max(num_try_send, num_allowed_send)) - rpm, tpm = calculate_limits(list_of_messages[:num_allowed_send]) - list_of_messages = list_of_messages[:num_try_send] - router: Router = router_factory(rpm, tpm, routing_strategy) - - print(f"router: {router.model_list}") - received = [] - - def _send_and_check(): - results = ( - sync_call(router, list_of_messages) - if sync_mode - else asyncio.run(async_call(router, list_of_messages)) - ) - received.extend(results) - print(results) - if len([i for i in results if i is not None]) != num_try_send: - # since not all results got returned, raise rate limit error - raise ValueError("No deployments available for selected model") - raise ExpectNoException - - with pytest.raises(expected_exception) as excinfo: # asserts correct type raised - _send_and_check() - - print(expected_exception, excinfo) - if expected_exception is ValueError: - assert "No deployments available for selected model" in str(excinfo.value) - else: - assert len([i for i in received if i is not None]) == num_try_send diff --git a/tests/test_resource_cleanup.py b/tests/test_resource_cleanup.py deleted file mode 100644 index d205b739915..00000000000 --- a/tests/test_resource_cleanup.py +++ /dev/null @@ -1,117 +0,0 @@ -""" -Test that async HTTP clients are properly cleaned up to prevent resource leaks. -Issue: https://github.com/BerriAI/litellm/issues/12107 -""" - -import asyncio -import os -import warnings - -import pytest - -import litellm - - -@pytest.mark.asyncio -async def test_acompletion_resource_cleanup(): - """Test that acompletion doesn't leave unclosed client sessions.""" - # Suppress warnings to check for them later - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - - # Make an async completion call - response = await litellm.acompletion( - model="gemini/gemini-2.0-flash-lite-001", - messages=[{"role": "user", "content": "Hello"}], - mock_response="Hi there! How can I help you today?", - ) - - # Check that response was received - assert ( - response.choices[0].message.content == "Hi there! How can I help you today?" - ) - - # Manually close async clients - await litellm.close_litellm_async_clients() - - # Give a small delay for any warnings to appear - await asyncio.sleep(0.1) - - # Check for resource warnings - resource_warnings = [ - warning - for warning in w - if "Unclosed" in str(warning.message) - and ( - "client session" in str(warning.message) - or "connector" in str(warning.message) - ) - ] - - # Should be no unclosed resource warnings - assert ( - len(resource_warnings) == 0 - ), f"Found unclosed resources: {[str(w.message) for w in resource_warnings]}" - - -@pytest.mark.asyncio -async def test_multiple_acompletion_calls_cleanup(): - """Test that multiple acompletion calls reuse clients and don't leak resources.""" - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always") - - # Make multiple async completion calls - for i in range(3): - response = await litellm.acompletion( - model="gemini/gemini-2.0-flash-lite-001", - messages=[{"role": "user", "content": f"Hello {i}"}], - mock_response=f"Response {i}", - ) - assert response.choices[0].message.content == f"Response {i}" - - # Clean up - await litellm.close_litellm_async_clients() - - # Give a small delay for any warnings to appear - await asyncio.sleep(0.1) - - # Check for resource warnings - resource_warnings = [ - warning - for warning in w - if "Unclosed" in str(warning.message) - and ( - "client session" in str(warning.message) - or "connector" in str(warning.message) - ) - ] - - assert ( - len(resource_warnings) == 0 - ), f"Found unclosed resources: {[str(w.message) for w in resource_warnings]}" - - -@pytest.mark.asyncio -async def test_cleanup_function_is_safe_to_call_multiple_times(): - """Test that the cleanup function can be called multiple times safely.""" - # This should not raise any errors - await litellm.close_litellm_async_clients() - await litellm.close_litellm_async_clients() - await litellm.close_litellm_async_clients() - - # Should still work after multiple cleanups - response = await litellm.acompletion( - model="gemini/gemini-2.0-flash-lite-001", - messages=[{"role": "user", "content": "Hello"}], - mock_response="Hi!", - ) - assert response.choices[0].message.content == "Hi!" - - # Clean up again - await litellm.close_litellm_async_clients() - - -if __name__ == "__main__": - # Run the test - asyncio.run(test_acompletion_resource_cleanup()) - print("✅ All tests passed!") diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py index 9af38941684..e115cd4839c 100644 --- a/tests/test_rust_python_harness.py +++ b/tests/test_rust_python_harness.py @@ -1,119 +1,11 @@ from __future__ import annotations import importlib -from typing import Final -import pytest - -models = importlib.import_module("tests.rust-python-harness.shared.reporting.models") -strategy_module = importlib.import_module("tests.rust-python-harness.shared.reporting.strategy") -ui = importlib.import_module("tests.rust-python-harness.shared.reporting.ui") contracts = importlib.import_module("tests.rust-python-harness.shared.unit_runners.contracts") -cli = importlib.import_module("tests.rust-python-harness.cli") UNIT_TEST_CONTRACTS = contracts.UNIT_TEST_CONTRACTS -CaseResult = models.CaseResult -Coverage = models.Coverage -HarnessCase = models.HarnessCase -HarnessRun = models.HarnessRun -RunStatus = models.RunStatus -ModuleCaseSpec = strategy_module.ModuleCaseSpec -NotImplementedCaseSpec = strategy_module.NotImplementedCaseSpec -SkippedCaseSpec = strategy_module.SkippedCaseSpec -_format_duration = ui._format_duration -_summary = ui._summary - - -def _case(module: str = "tests.example") -> HarnessCase: - return HarnessCase( - strategy_id="example", - strategy_label="Example", - sdk_function="messages", - spec=ModuleCaseSpec(coverage=Coverage.COMPLETE, module=module), - ) - - -@pytest.mark.parametrize( - "module", - [ - "tests.rust-python-harness.strategies.trace_parity.sdk.messages.case", - "tests.rust-python-harness.strategies.trace_parity.sdk.chat_completions.case", - "tests.rust-python-harness.strategies.trace_parity.sdk.transcription.case", - ], -) -def test_implemented_namespace_case_modules_remain_importable(module: str) -> None: - assert importlib.import_module(module) - - -def test_should_mark_not_implemented_and_skipped_cases_without_running() -> None: - not_implemented: Final = CaseResult( - case=HarnessCase( - strategy_id="example", - strategy_label="Example", - sdk_function="messages", - spec=NotImplementedCaseSpec(reason="No case is registered."), - ) - ) - skipped: Final = CaseResult( - case=HarnessCase( - strategy_id="example", - strategy_label="Example", - sdk_function="messages", - spec=SkippedCaseSpec(reason="The surface does not apply."), - ) - ) - - not_implemented.set_initial_status() - skipped.set_initial_status() - - assert not_implemented.status is RunStatus.NOT_IMPLEMENTED - assert skipped.status is RunStatus.SKIPPED - - -def test_should_finalize_a_fully_passing_case() -> None: - result = CaseResult(case=_case()) - result.set_initial_status() - result.collected.update({"one", "two"}) - result.completed.update({"one", "two"}) - result.passed = 2 - - result.finalize() - - assert result.status is RunStatus.PASSED - - -def test_should_replace_a_pass_with_a_teardown_error() -> None: - result = CaseResult(case=_case()) - result.set_initial_status() - result.collected.add("one") - - result.record("one", RunStatus.PASSED, 0.1) - result.record("one", RunStatus.ERROR, 0.2) - - assert result.status is RunStatus.ERROR - assert result.passed == 0 - assert result.errors == 1 - assert result.duration == pytest.approx(0.3) - - -def test_should_format_developer_facing_run_context() -> None: - run = HarnessRun.from_cases((_case(),)) - result = next(iter(run.results.values())) - result.collected.add("tests/test_parity.py::test_one") - result.record("tests/test_parity.py::test_one", RunStatus.PASSED, 1.25) - - assert _summary(run) == (1, 0, 0, 0) - assert _format_duration(1.25) == "1.2s" def test_should_leave_functions_without_unit_test_contracts_unimplemented() -> None: assert "messages" not in UNIT_TEST_CONTRACTS - - -def test_strategy_subcommand_accepts_function_filter(capsys: pytest.CaptureFixture[str]) -> None: - exit_code: Final = cli.main(["run", "unit_tests_rust", "--function", "messages"]) - - captured: Final = capsys.readouterr() - assert exit_code == 0 - assert "- messages: not_implemented" in captured.out - assert "unit_tests_rust:messages: not_implemented" not in captured.out diff --git a/tests/test_service_logger_otel.py b/tests/test_service_logger_otel.py index 044d37d6781..8152a45f945 100644 --- a/tests/test_service_logger_otel.py +++ b/tests/test_service_logger_otel.py @@ -1,18 +1,13 @@ import os import sys import unittest -from datetime import datetime -from unittest.mock import patch, AsyncMock, MagicMock +from unittest.mock import patch # Add the project root to sys.path sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "../.."))) import litellm from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger -from litellm.integrations.opentelemetry import OpenTelemetry -from litellm.types.services import ServiceTypes -from litellm._service_logger import ServiceLogging -from litellm.types.utils import StandardCallbackDynamicParams class TestServiceLoggerOTEL(unittest.IsolatedAsyncioTestCase): @@ -43,110 +38,6 @@ class TestServiceLoggerOTEL(unittest.IsolatedAsyncioTestCase): "LangfuseOtelLogger.async_service_failure_hook", ) - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing") - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics") - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs") - async def test_langfuse_otel_does_not_create_proxy_request_span( - self, mock_logs, mock_metrics, mock_tracing - ): - """ - Test that LangfuseOtelLogger returns None for create_litellm_proxy_request_started_span. - - This prevents empty proxy request spans from being sent to Langfuse when - requests don't result in actual LLM calls (e.g., auth failures, health checks). - """ - logger = LangfuseOtelLogger() - - # Verify the method is overridden - self.assertEqual( - logger.create_litellm_proxy_request_started_span.__qualname__, - "LangfuseOtelLogger.create_litellm_proxy_request_started_span", - ) - - # Verify it returns None - result = logger.create_litellm_proxy_request_started_span( - start_time=datetime.now(), - headers={"Authorization": "Bearer test"}, - ) - self.assertIsNone(result) - - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing") - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics") - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs") - async def test_service_logging_shadowing_fix( - self, mock_logs, mock_metrics, mock_tracing - ): - """ - Test the architectural fix: multiple OTEL loggers should receive logs independently. - """ - # 1. Initialize two loggers - langfuse_logger = LangfuseOtelLogger() - otel_logger = OpenTelemetry() - - # 2. Setup service_callback list - litellm.service_callback = [langfuse_logger, otel_logger] - - service_logging = ServiceLogging() - - # 3. Mock the base OpenTelemetry hook - with patch.object( - OpenTelemetry, "async_service_success_hook", new_callable=AsyncMock - ) as mock_base_hook: - # Trigger a service event - await service_logging.async_service_success_hook( - service=ServiceTypes.DB, - call_type="success", - duration=0.1, - parent_otel_span=MagicMock(), - start_time=0.0, - end_time=1.0, - ) - - # The architectural fix ensures we call each correctly. - self.assertEqual( - mock_base_hook.call_count, - 1, - "Generic OTEL logger should have received the log exactly once.", - ) - - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing") - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics") - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs") - async def test_langfuse_otel_env_config_includes_v4_ingestion_header( - self, mock_logs, mock_metrics, mock_tracing - ): - logger = LangfuseOtelLogger() - - headers = OpenTelemetry._get_headers_dictionary(logger.config.headers) - - self.assertEqual( - headers["x-langfuse-ingestion-version"], - "4", - ) - self.assertTrue(headers["Authorization"].startswith("Basic ")) - - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_tracing") - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_metrics") - @patch("litellm.integrations.opentelemetry.OpenTelemetry._init_logs") - async def test_langfuse_otel_dynamic_headers_include_v4_ingestion_header( - self, mock_logs, mock_metrics, mock_tracing - ): - logger = LangfuseOtelLogger() - - headers = logger.construct_dynamic_otel_headers( - StandardCallbackDynamicParams( - langfuse_public_key="pk-lf-dynamic", - langfuse_secret_key="sk-lf-dynamic", - ) - ) - - self.assertIsNotNone(headers) - self.assertEqual( - headers["x-langfuse-ingestion-version"], - "4", - ) - self.assertTrue(headers["Authorization"].startswith("Basic ")) - if __name__ == "__main__": unittest.main() diff --git a/tests/test_spend_logs.py b/tests/test_spend_logs.py index f239fbbb53f..7e641f1f301 100644 --- a/tests/test_spend_logs.py +++ b/tests/test_spend_logs.py @@ -2,157 +2,10 @@ import os # What this tests? ## Tests /spend endpoints. -import pytest, uuid, json -import asyncio +import pytest import aiohttp -async def generate_key(session, models=[], team_id=None): - url = "http://0.0.0.0:4000/key/generate" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = { - "models": models, - "duration": None, - } - if team_id is not None: - data["team_id"] = team_id - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - -async def chat_completion(session, key, model="gpt-3.5-turbo"): - url = "http://0.0.0.0:4000/chat/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - data = { - "model": model, - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": f"Hello! {uuid.uuid4()}"}, - ], - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - - return await response.json() - - -async def get_spend_logs(session, request_id=None, api_key=None): - if api_key is not None: - url = f"http://0.0.0.0:4000/spend/logs?api_key={api_key}" - else: - url = f"http://0.0.0.0:4000/spend/logs?request_id={request_id}" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - -async def generate_org(session: aiohttp.ClientSession) -> dict: - """ - Generate a new organization using the API. - - Args: - session: aiohttp client session - - Returns: - dict: Response containing org_id - """ - url = "http://0.0.0.0:4000/organization/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - - request_body = { - "organization_alias": f"test-org-{uuid.uuid4()}", - } - - async with session.post(url, headers=headers, json=request_body) as response: - return await response.json() - - -async def generate_team(session: aiohttp.ClientSession, org_id: str) -> dict: - """ - Generate a new team within an organization using the API. - - Args: - session: aiohttp client session - org_id: Organization ID to create the team in - - Returns: - dict: Response containing team_id - """ - url = "http://0.0.0.0:4000/team/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = {"organization_id": org_id} - - async with session.post(url, headers=headers, json=data) as response: - return await response.json() - - -@pytest.mark.skip( - reason="Flaky in CI: /spend/logs?request_id=... returns 500 even after a 20s wait for the spend log to be written. Spend-log accuracy is covered by tests/unit/proxy/spend_tracking/ and tests/integration/spend/." -) -@pytest.mark.asyncio -async def test_spend_logs_with_org_id(): - """ - - Create Organization - - Create Team in organization - - Create Key in organization - - Make call (makes sure it's in spend logs) - - Get request id from logs - - Assert spend logs have correct org_id and team_id - """ - async with aiohttp.ClientSession() as session: - org_gen = await generate_org(session=session) - print("org_gen: ", json.dumps(org_gen, indent=4, default=str)) - org_id = org_gen["organization_id"] - team_gen = await generate_team(session=session, org_id=org_id) - print("team_gen: ", json.dumps(team_gen, indent=4, default=str)) - team_id = team_gen["team_id"] - key_gen = await generate_key(session=session, team_id=team_id) - print("key_gen: ", json.dumps(key_gen, indent=4, default=str)) - key = key_gen["key"] - response = await chat_completion(session=session, key=key) - await asyncio.sleep(20) - spend_logs_response = await get_spend_logs( - session=session, request_id=response["id"] - ) - print( - "spend_logs_response: ", - json.dumps(spend_logs_response, indent=4, default=str), - ) - spend_logs_response = spend_logs_response[0] - assert spend_logs_response["metadata"]["user_api_key_org_id"] == org_id - assert spend_logs_response["metadata"]["user_api_key_team_id"] == team_id - assert spend_logs_response["team_id"] == team_id - - async def get_predict_spend_logs(session): url = "http://0.0.0.0:4000/global/predict/spend/logs" headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} diff --git a/tests/test_team.py b/tests/test_team.py deleted file mode 100644 index e7b5d0cac6d..00000000000 --- a/tests/test_team.py +++ /dev/null @@ -1,912 +0,0 @@ -import os -# What this tests ? -## Tests /team endpoints. -import pytest -import asyncio -import aiohttp -import time, uuid -from openai import AsyncOpenAI -from typing import Optional -import openai -from unittest.mock import MagicMock, patch - - -async def get_user_info(session, get_user, call_user, view_all: Optional[bool] = None): - """ - Make sure only models user has access to are returned - """ - if view_all is True: - url = "http://localhost:4000/user/info" - else: - url = f"http://localhost:4000/user/info?user_id={get_user}" - headers = { - "Authorization": f"Bearer {call_user}", - "Content-Type": "application/json", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - print(response_text) - print() - - if status != 200: - if call_user != get_user: - return status - else: - print(f"call_user: {call_user}; get_user: {get_user}") - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - -async def wait_for_team_member_spend_update( - session, user_id, team_id, expected_min_spend, max_wait=10 -): - """ - Wait for the team member spend update to be committed to the database. - Polls the user info endpoint until the spend is updated. - This is needed because spend updates are queued asynchronously and committed periodically. - """ - start_time = time.time() - initial_spend = None - while time.time() - start_time < max_wait: - try: - user_info = await get_user_info(session, user_id, call_user=os.environ["LITELLM_MASTER_KEY"]) - if user_info.get("teams"): - for team in user_info["teams"]: - if team.get("team_id") == team_id: - for membership in team.get("team_memberships", []): - spend = membership.get("spend", 0.0) - if initial_spend is None: - initial_spend = spend - print(f"Initial team member spend: {spend}") - - if spend >= expected_min_spend: - print( - f"[OK] Team member spend updated: {spend} >= {expected_min_spend}" - ) - return True - - print( - f"[WAITING] Team member spend: {spend}, expected >= {expected_min_spend}, elapsed: {time.time() - start_time:.1f}s" - ) - await asyncio.sleep(0.5) - except Exception as e: - print(f"Error checking team member spend: {e}") - await asyncio.sleep(0.5) - print( - f"[TIMEOUT] Timeout waiting for team member spend update (expected >= {expected_min_spend})" - ) - return False - - -async def new_user( - session, - i, - user_id=None, - budget=None, - budget_duration=None, - models=["azure-models"], - team_id=None, - user_email=None, -): - url = "http://localhost:4000/user/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = { - "models": models, - "aliases": {"mistral-7b": "gpt-3.5-turbo"}, - "duration": None, - "max_budget": budget, - "budget_duration": budget_duration, - "user_email": user_email, - } - - if user_id is not None: - data["user_id"] = user_id - - if team_id is not None: - data["team_id"] = team_id - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception( - f"Request {i} did not return a 200 status code: {status}, response: {response_text}" - ) - - return await response.json() - - -async def add_member( - session, i, team_id, user_id=None, user_email=None, max_budget=None, members=None -): - url = "http://localhost:4000/team/member_add" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = {"team_id": team_id, "member": {"role": "user"}} - if user_email is not None: - data["member"]["user_email"] = user_email - elif user_id is not None: - data["member"]["user_id"] = user_id - elif members is not None: - data["member"] = members - - if max_budget is not None: - data["max_budget_in_team"] = max_budget - - print("sent data: {}".format(data)) - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"ADD MEMBER Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -async def update_member( - session, - i, - team_id, - user_id=None, - user_email=None, - max_budget=None, -): - url = "http://localhost:4000/team/member_update" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = {"team_id": team_id} - if user_id is not None: - data["user_id"] = user_id - elif user_email is not None: - data["user_email"] = user_email - - if max_budget is not None: - data["max_budget_in_team"] = max_budget - - print("sent data: {}".format(data)) - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"ADD MEMBER Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception( - f"Request {i} did not return a 200 status code: {status}, response: {response_text}" - ) - - return await response.json() - - -async def delete_member(session, i, team_id, user_id=None, user_email=None): - url = "http://localhost:4000/team/member_delete" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = {"team_id": team_id} - if user_id is not None: - data["user_id"] = user_id - elif user_email is not None: - data["user_email"] = user_email - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -async def generate_key( - session, - i, - budget=None, - budget_duration=None, - models=["azure-models", "gpt-4", "dall-e-3"], - team_id=None, -): - url = "http://localhost:4000/key/generate" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = { - "models": models, - "duration": None, - "max_budget": budget, - "budget_duration": budget_duration, - } - if team_id is not None: - data["team_id"] = team_id - - print(f"data: {data}") - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -async def chat_completion(session, key, model="gpt-4"): - url = "http://localhost:4000/chat/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - data = { - "model": model, - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - ], - } - - for i in range(3): - try: - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(response_text) - print() - - if status != 200: - raise Exception( - f"Request did not return a 200 status code: {status}. Response: {response_text}" - ) - - return await response.json() - except Exception as e: - if "Request did not return a 200 status code" in str(e): - raise e - else: - pass - - -async def new_team(session, i, user_id=None, member_list=None, model_aliases=None): - import json - - url = "http://localhost:4000/team/new" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = {"team_alias": "my-new-team"} - if user_id is not None: - data["members_with_roles"] = [{"role": "user", "user_id": user_id}] - elif member_list is not None: - data["members_with_roles"] = member_list - - if model_aliases is not None: - data["model_aliases"] = model_aliases - - print(f"data: {data}") - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -async def update_team(session, i, team_id, user_id=None, member_list=None, **kwargs): - url = "http://localhost:4000/team/update" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = {"team_id": team_id, **kwargs} - if user_id is not None: - data["members_with_roles"] = [{"role": "user", "user_id": user_id}] - elif member_list is not None: - data["members_with_roles"] = member_list - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -async def delete_team( - session, - i, - team_id, -): - url = "http://localhost:4000/team/delete" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - data = { - "team_ids": [team_id], - } - - async with session.post(url, headers=headers, json=data) as response: - status = response.status - response_text = await response.text() - - print(f"Response {i} (Status code: {status}):") - print(response_text) - print() - - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -async def list_teams( - session, - i, -): - url = "http://localhost:4000/team/list" - headers = {"Authorization": f"Bearer {os.environ['LITELLM_MASTER_KEY']}", "Content-Type": "application/json"} - - async with session.get(url, headers=headers) as response: - status = response.status - if status != 200: - raise Exception(f"Request {i} did not return a 200 status code: {status}") - - return await response.json() - - -@pytest.mark.asyncio -async def test_team_new(): - """ - Make 20 parallel calls to /user/new. Assert all worked. - """ - user_id = f"{uuid.uuid4()}" - async with aiohttp.ClientSession() as session: - new_user(session=session, i=0, user_id=user_id) - tasks = [new_team(session, i, user_id=user_id) for i in range(1, 11)] - await asyncio.gather(*tasks) - - -async def get_team_info(session, get_team, call_key): - url = f"http://localhost:4000/team/info?team_id={get_team}" - headers = { - "Authorization": f"Bearer {call_key}", - "Content-Type": "application/json", - } - - async with session.get(url, headers=headers) as response: - status = response.status - response_text = await response.text() - print(response_text) - print() - - if status == 404: - raise openai.NotFoundError( - message="404 received", response=MagicMock(), body=None - ) - - if status != 200: - raise Exception(f"Request did not return a 200 status code: {status}") - return await response.json() - - -@pytest.mark.asyncio -async def test_team_info(): - """ - Scenario 1: - - test with admin key -> expect to work - Scenario 2: - - test with team key -> expect to work - Scenario 3: - - test with non-team key -> expect to fail - """ - async with aiohttp.ClientSession() as session: - """ - Scenario 1 - as admin - """ - new_team_data = await new_team( - session, - 0, - ) - team_id = new_team_data["team_id"] - ## as admin ## - await get_team_info(session=session, get_team=team_id, call_key=os.environ["LITELLM_MASTER_KEY"]) - """ - Scenario 2 - as team key - """ - key_gen = await generate_key(session=session, i=0, team_id=team_id) - key = key_gen["key"] - - await get_team_info(session=session, get_team=team_id, call_key=key) - - """ - Scenario 3 - as non-team key - """ - key_gen = await generate_key(session=session, i=0) - key = key_gen["key"] - - try: - await get_team_info(session=session, get_team=team_id, call_key=key) - pytest.fail("Expected call to fail") - except Exception as e: - pass - - -""" -- Create team -- Add user (user exists in db) -- Update team -- Check if it works -""" - -""" -- Create team -- Add user (user doesn't exist in db) -- Update team -- Check if it works -""" - - -@pytest.mark.asyncio -async def test_team_update_sc_2(): - """ - - Create team - - Add 3 users (doesn't exist in db) - - Change team alias - - Check if it works - - Assert team object unchanged besides team alias - """ - async with aiohttp.ClientSession() as session: - ## Create admin - admin_user = f"{uuid.uuid4()}" - await new_user(session=session, i=0, user_id=admin_user) - ## Create team with 1 admin and 1 user - member_list = [ - {"role": "admin", "user_id": admin_user}, - ] - team_data = await new_team(session=session, i=0, member_list=member_list) - ## Create 10 normal users - members = [ - {"role": "user", "user_id": f"krrish_{uuid.uuid4()}@berri.ai"} - for _ in range(10) - ] - await add_member( - session=session, i=0, team_id=team_data["team_id"], members=members - ) - ## ASSERT TEAM SIZE - team_info = await get_team_info( - session=session, get_team=team_data["team_id"], call_key=os.environ["LITELLM_MASTER_KEY"] - ) - - assert len(team_info["team_info"]["members_with_roles"]) == 12 - - ## CHANGE TEAM ALIAS - - new_team_data = await update_team( - session=session, i=0, team_id=team_data["team_id"], team_alias="my-new-team" - ) - - assert new_team_data["data"]["team_alias"] == "my-new-team" - print(f"team_data: {team_data}") - ## assert rest of object is the same - for k, v in new_team_data["data"].items(): - if k == "members_with_roles": - assert len(new_team_data["data"][k]) == len( - team_info["team_info"]["members_with_roles"] - ) - elif ( - k == "created_at" - or k == "updated_at" - or k == "model_spend" - or k == "model_max_budget" - or k == "model_id" - or k == "litellm_organization_table" - or k == "object_permission_id" - or k == "object_permission" - or k == "litellm_model_table" - or k == "policies" - or k == "allow_team_guardrail_config" - or k == "projects" - ): - pass - else: - assert new_team_data["data"][k] == team_data[k] - - -@pytest.mark.asyncio -async def test_team_member_add_email(): - from tests.test_users import get_user_info - - async with aiohttp.ClientSession() as session: - ## Create admin - admin_user = f"{uuid.uuid4()}" - await new_user(session=session, i=0, user_id=admin_user) - ## Create team with 1 admin and 1 user - member_list = [ - {"role": "admin", "user_id": admin_user}, - ] - team_data = await new_team(session=session, i=0, member_list=member_list) - ## Add 1 user via email - user_email = "krrish{}@berri.ai".format(uuid.uuid4()) - new_user_info = await new_user(session=session, i=0, user_email=user_email) - new_member = {"role": "user", "user_email": user_email} - await add_member( - session=session, i=0, team_id=team_data["team_id"], members=[new_member] - ) - - ## check user info to confirm user is in team - updated_user_info = await get_user_info( - session=session, get_user=new_user_info["user_id"], call_user=os.environ["LITELLM_MASTER_KEY"] - ) - - print(updated_user_info) - - ## check if team in user table - is_team_in_list: bool = False - for team in updated_user_info["teams"]: - if team_data["team_id"] == team["team_id"]: - is_team_in_list = True - assert is_team_in_list - - -@pytest.mark.asyncio -async def test_team_delete(): - """ - - Create team - - Create key for team - - Check if key works - - Delete team - """ - async with aiohttp.ClientSession() as session: - ## Create admin - admin_user = f"{uuid.uuid4()}" - await new_user(session=session, i=0, user_id=admin_user) - ## Create normal user - normal_user = f"{uuid.uuid4()}" - await new_user(session=session, i=0, user_id=normal_user) - ## Create team with 1 admin and 1 user - member_list = [ - {"role": "admin", "user_id": admin_user}, - {"role": "user", "user_id": normal_user}, - ] - team_data = await new_team(session=session, i=0, member_list=member_list) - - ## ASSERT USER MEMBERSHIP IS CREATED - user_info = await get_user_info( - session=session, get_user=normal_user, call_user=os.environ["LITELLM_MASTER_KEY"] - ) - assert len(user_info["teams"]) == 1 - - ## Create key - key_gen = await generate_key(session=session, i=0, team_id=team_data["team_id"]) - key = key_gen["key"] - ## Test key - # response = await chat_completion(session=session, key=key) - ## Delete team - await delete_team(session=session, i=0, team_id=team_data["team_id"]) - - ## ASSERT USER MEMBERSHIP IS DELETED - user_info = await get_user_info( - session=session, get_user=normal_user, call_user=os.environ["LITELLM_MASTER_KEY"] - ) - assert len(user_info["teams"]) == 0 - - ## ASSERT TEAM INFO NOW RETURNS A 404 - with pytest.raises(openai.NotFoundError): - await get_team_info( - session=session, get_team=team_data["team_id"], call_key=os.environ["LITELLM_MASTER_KEY"] - ) - - -@pytest.mark.parametrize("dimension", ["user_id", "user_email"]) -@pytest.mark.asyncio -async def test_member_delete(dimension): - """ - - Create team - - Add member - - Get team info (check if member in team) - - Delete member - - Get team info (check if member in team) - """ - async with aiohttp.ClientSession() as session: - # Create Team - ## Create admin - admin_user = f"{uuid.uuid4()}" - await new_user(session=session, i=0, user_id=admin_user) - ## Create normal user - normal_user = f"{uuid.uuid4()}" - normal_user_email = "{}@berri.ai".format(normal_user) - print(f"normal_user: {normal_user}") - await new_user( - session=session, i=0, user_id=normal_user, user_email=normal_user_email - ) - ## Create team with 1 admin and 1 user - member_list = [ - {"role": "admin", "user_id": admin_user}, - ] - if dimension == "user_id": - member_list.append({"role": "user", "user_id": normal_user}) - elif dimension == "user_email": - member_list.append({"role": "user", "user_email": normal_user_email}) - team_data = await new_team(session=session, i=0, member_list=member_list) - - user_in_team = False - for member in team_data["members_with_roles"]: - if dimension == "user_id" and member["user_id"] == normal_user: - user_in_team = True - elif ( - dimension == "user_email" and member["user_email"] == normal_user_email - ): - user_in_team = True - - assert ( - user_in_team is True - ), "User not in team. Team list={}, User details - id={}, email={}. Dimension={}".format( - team_data["members_with_roles"], normal_user, normal_user_email, dimension - ) - # Delete member - if dimension == "user_id": - updated_team_data = await delete_member( - session=session, i=0, team_id=team_data["team_id"], user_id=normal_user - ) - elif dimension == "user_email": - updated_team_data = await delete_member( - session=session, - i=0, - team_id=team_data["team_id"], - user_email=normal_user_email, - ) - print(f"updated_team_data: {updated_team_data}") - user_in_team = False - for member in team_data["members_with_roles"]: - if dimension == "user_id" and member["user_id"] == normal_user: - user_in_team = True - elif ( - dimension == "user_email" and member["user_email"] == normal_user_email - ): - user_in_team = True - - assert user_in_team is True - - -@pytest.mark.asyncio -async def test_users_in_team_budget(): - """ - - Create User - - Create Team with User - - Add User to team with budget = 0.0000001 - - Make Call 1 -> pass - - Make Call 2 -> fail - """ - get_user = f"krrish_{time.time()}@berri.ai" - async with aiohttp.ClientSession() as session: - # IMPORTANT: Create team first, then create user with team_id. - # This order is critical for the test to work correctly: - # - When a user is created with team_id, the API key gets team_id set from the start - # - This ensures spend tracking and budget enforcement work correctly - # - If we create the user first (without team_id) and then add them to a team, - # the key's team_id remains None, breaking team budget tracking - # DO NOT change this order - it's testing the intended flow where keys are - # associated with teams at creation time. - team = await new_team(session, 0, user_id=None) - print(f"[DEBUG] Created team: {team['team_id']}") - print(f"[DEBUG] Full team data: {team}") - - # Create user with team_id so the key is associated with the team from the start - key_gen = await new_user( - session, - 0, - user_id=get_user, - budget=10, - budget_duration="5s", - models=["fake-openai-endpoint"], - team_id=team["team_id"], - ) - key = key_gen["key"] - print(f"[DEBUG] Created user '{get_user}' with key: {key}") - print(f"[DEBUG] User budget: 10, budget_duration: 5s") - print(f"[DEBUG] Key team_id: {team['team_id']}") - - # Check user info BEFORE updating member budget - user_info_before = await get_user_info(session, get_user, call_user=os.environ["LITELLM_MASTER_KEY"]) - print(f"[DEBUG] User info BEFORE update_member:") - print(f" - User budget: {user_info_before.get('max_budget')}") - print(f" - User spend: {user_info_before.get('spend')}") - if user_info_before.get("teams"): - for team_info in user_info_before["teams"]: - if team_info.get("team_id") == team["team_id"]: - print(f" - Team memberships: {team_info.get('team_memberships')}") - - # update user to have budget = 0.0000001 - update_result = await update_member( - session, 0, team_id=team["team_id"], user_id=get_user, max_budget=0.0000001 - ) - print(f"[DEBUG] Updated member budget to 0.0000001") - print(f"[DEBUG] Update result: {update_result}") - - # Check user info AFTER updating member budget - user_info_after = await get_user_info(session, get_user, call_user=os.environ["LITELLM_MASTER_KEY"]) - print(f"[DEBUG] User info AFTER update_member:") - print(f" - User budget: {user_info_after.get('max_budget')}") - print(f" - User spend: {user_info_after.get('spend')}") - if user_info_after.get("teams"): - for team_info in user_info_after["teams"]: - if team_info.get("team_id") == team["team_id"]: - print(f" - Team: {team_info.get('team_id')}") - for membership in team_info.get("team_memberships", []): - print(f" - Membership: {membership}") - if "litellm_budget_table" in membership: - budget_table = membership["litellm_budget_table"] - print(f" - Max budget: {budget_table.get('max_budget')}") - print(f" - Current spend: {membership.get('spend', 0)}") - - # Call 1 - print("\n[DEBUG] ===== Making Call 1 =====") - result = await chat_completion(session, key, model="fake-openai-endpoint") - print(f"[DEBUG] Call 1 PASSED (expected)") - print(f"[DEBUG] Call 1 result: {result}") - # Extract cost from result if available - if isinstance(result, dict): - usage = result.get("usage", {}) - print(f"[DEBUG] Call 1 usage: {usage}") - - # Wait for spend to be committed to database before checking budget - # Spend updates are queued asynchronously and committed periodically (every minute), - # so we need to wait for the spend from Call 1 to be persisted - print("\n[DEBUG] ===== Waiting for spend to be committed =====") - print("Waiting for team member spend to be committed to database...") - print( - "Note: Spend updates are flushed periodically, this may take up to 90 seconds..." - ) - spend_updated = await wait_for_team_member_spend_update( - session, get_user, team["team_id"], 0.0000001, max_wait=90 - ) - if not spend_updated: - pytest.fail( - "Team member spend was not updated within 90s. " - "The spend update queue may not have flushed, or the model may have 0 cost." - ) - - # Check user info BEFORE Call 2 - user_info_before_call2 = await get_user_info( - session, get_user, call_user=os.environ["LITELLM_MASTER_KEY"] - ) - print(f"\n[DEBUG] User info BEFORE Call 2:") - print(f" - User budget: {user_info_before_call2.get('max_budget')}") - print(f" - User spend: {user_info_before_call2.get('spend')}") - if user_info_before_call2.get("teams"): - for team_info in user_info_before_call2["teams"]: - if team_info.get("team_id") == team["team_id"]: - print(f" - Team: {team_info.get('team_id')}") - for membership in team_info.get("team_memberships", []): - if "litellm_budget_table" in membership: - budget_table = membership["litellm_budget_table"] - current_spend = membership.get("spend", 0) - max_budget = budget_table.get("max_budget") - print(f" - Max budget in team: {max_budget}") - print(f" - Current spend in team: {current_spend}") - print( - f" - Budget remaining: {max_budget - current_spend}" - ) - print(f" - Should fail?: {current_spend >= max_budget}") - - # Call 2 - print("\n[DEBUG] ===== Making Call 2 =====") - call2_failed = False - call2_error = None - call2_status = None - try: - # Capture the response to check status code - url = "http://localhost:4000/chat/completions" - headers = { - "Authorization": f"Bearer {key}", - "Content-Type": "application/json", - } - data = { - "model": "fake-openai-endpoint", - "messages": [ - {"role": "system", "content": "You are a helpful assistant."}, - {"role": "user", "content": "Hello!"}, - ], - } - async with session.post(url, headers=headers, json=data) as response: - call2_status = response.status - response_text = await response.text() - print(f"[DEBUG] Call 2 status code: {call2_status}") - print(f"[DEBUG] Call 2 response: {response_text}") - - if call2_status != 200: - call2_failed = True - call2_error = f"Status {call2_status}: {response_text}" - raise Exception(call2_error) - else: - # Call succeeded when it should have failed - print(f"[ERROR] Call 2 PASSED when it should have FAILED!") - print(f"[ERROR] Response was 200 OK") - - except Exception as e: - if call2_failed: - print(f"[DEBUG] Call 2 FAILED (expected): {e}") - print(f"[DEBUG] Checking if error message indicates budget exceeded...") - else: - call2_error = str(e) - print(f"[DEBUG] Call 2 raised exception: {e}") - - # Check user info AFTER Call 2 - user_info_after_call2 = await get_user_info( - session, get_user, call_user=os.environ["LITELLM_MASTER_KEY"] - ) - print(f"\n[DEBUG] User info AFTER Call 2:") - print(f" - User budget: {user_info_after_call2.get('max_budget')}") - print(f" - User spend: {user_info_after_call2.get('spend')}") - if user_info_after_call2.get("teams"): - for team_info in user_info_after_call2["teams"]: - if team_info.get("team_id") == team["team_id"]: - print(f" - Team: {team_info.get('team_id')}") - for membership in team_info.get("team_memberships", []): - if "litellm_budget_table" in membership: - budget_table = membership["litellm_budget_table"] - print(f" - Max budget: {budget_table.get('max_budget')}") - print(f" - Current spend: {membership.get('spend', 0)}") - - # Assert Call 2 failed - if not call2_failed: - error_msg = ( - f"\n[FAILURE] Call 2 should have failed but it passed!\n" - f"Expected: Budget enforcement to block the call\n" - f"Actual: Call returned status {call2_status}\n" - f"Team member budget: 0.0000001\n" - f"User budget: {user_info_before_call2.get('max_budget')}\n" - f"User spend before call: {user_info_before_call2.get('spend')}\n" - ) - # Add team member info if available - if user_info_before_call2.get("teams"): - for team_info in user_info_before_call2["teams"]: - if team_info.get("team_id") == team["team_id"]: - for membership in team_info.get("team_memberships", []): - if "litellm_budget_table" in membership: - error_msg += f"Team member spend before call: {membership.get('spend', 0)}\n" - error_msg += f"Team member max budget: {membership['litellm_budget_table'].get('max_budget')}\n" - pytest.fail(error_msg) - - # Check the error message contains budget exceeded - if call2_error and "Budget has been exceeded" not in call2_error: - pytest.fail( - f"Call 2 failed but not with expected error message.\n" - f"Expected error to contain: 'Budget has been exceeded'\n" - f"Actual error: {call2_error}" - ) - - print("[DEBUG] Call 2 failed as expected with budget exceeded error") - - ## Check user info - user_info = await get_user_info(session, get_user, call_user=os.environ["LITELLM_MASTER_KEY"]) - - assert ( - user_info["teams"][0]["team_memberships"][0]["litellm_budget_table"][ - "max_budget" - ] - == 0.0000001 - ) diff --git a/tests/test_team_members.py b/tests/test_team_members.py deleted file mode 100644 index 2bdb7db13c1..00000000000 --- a/tests/test_team_members.py +++ /dev/null @@ -1,190 +0,0 @@ -import os -import pytest -import requests -import time -from typing import Dict, List -import logging -from litellm._uuid import uuid - -# Configure logging -logging.basicConfig(level=logging.INFO) -logger = logging.getLogger(__name__) - - -class TeamAPI: - def __init__(self, base_url: str, auth_token: str): - self.base_url = base_url - self.headers = { - "Authorization": f"Bearer {auth_token}", - "Content-Type": "application/json", - } - - def create_team(self, team_alias: str, models: List[str] = None) -> Dict: - """Create a new team""" - # Generate a unique team_id using uuid - team_id = f"test_team_{uuid.uuid4().hex[:8]}" - - data = { - "team_id": team_id, - "team_alias": team_alias, - "models": models or ["o3-mini"], - } - - response = requests.post( - f"{self.base_url}/team/new", headers=self.headers, json=data - ) - response.raise_for_status() - logger.info(f"Created new team: {team_id}") - return response.json(), team_id - - def get_team_info(self, team_id: str) -> Dict: - """Get current team information""" - response = requests.get( - f"{self.base_url}/team/info", - headers=self.headers, - params={"team_id": team_id}, - ) - response.raise_for_status() - return response.json() - - def add_team_member(self, team_id: str, user_email: str, role: str) -> Dict: - """Add a single team member""" - data = {"team_id": team_id, "member": [{"role": role, "user_id": user_email}]} - response = requests.post( - f"{self.base_url}/team/member_add", headers=self.headers, json=data - ) - response.raise_for_status() - return response.json() - - def delete_team_member(self, team_id: str, user_id: str) -> Dict: - """Delete a team member - - Args: - team_id (str): ID of the team - user_id (str): User ID to remove from team - - Returns: - Dict: Response from the API - """ - data = {"team_id": team_id, "user_id": user_id} - response = requests.post( - f"{self.base_url}/team/member_delete", headers=self.headers, json=data - ) - response.raise_for_status() - return response.json() - - -@pytest.fixture -def api_client(): - """Fixture for TeamAPI client""" - base_url = "http://localhost:4000" - auth_token = os.environ["LITELLM_MASTER_KEY"] # Replace with your token - return TeamAPI(base_url, auth_token) - - -@pytest.fixture -def new_team(api_client): - """Fixture that creates a new team for each test""" - team_alias = f"Test Team {uuid.uuid4().hex[:6]}" - team_response, team_id = api_client.create_team(team_alias) - logger.info(f"Created test team: {team_id} ({team_alias})") - return team_id - - -def verify_member_in_team(team_info: Dict, user_email: str) -> bool: - """Verify if a member exists in team""" - return any( - member["user_id"] == user_email - for member in team_info["team_info"]["members_with_roles"] - ) - - -def test_add_single_member(api_client, new_team): - """Test adding a single member to a new team""" - # Get initial team info - initial_info = api_client.get_team_info(new_team) - initial_size = len(initial_info["team_info"]["members_with_roles"]) - - # Add new member - test_email = f"pytest_user_{uuid.uuid4().hex[:6]}@mycompany.com" - api_client.add_team_member(new_team, test_email, "user") - - # Allow time for system to process - time.sleep(1) - - # Verify addition - updated_info = api_client.get_team_info(new_team) - updated_size = len(updated_info["team_info"]["members_with_roles"]) - - # Assertions - assert verify_member_in_team( - updated_info, test_email - ), f"Member {test_email} not found in team" - assert ( - updated_size == initial_size + 1 - ), f"Team size did not increase by 1 (was {initial_size}, now {updated_size})" - - -def test_member_deletion(api_client, new_team): - """Test that member deletion works correctly and removes all instances of a user""" - # Add a test user - user_id = f"pytest_user_{uuid.uuid4().hex[:6]}" - api_client.add_team_member(new_team, user_id, "user") - time.sleep(1) - - # Verify user was added - team_info_before = api_client.get_team_info(new_team) - assert verify_member_in_team( - team_info_before, user_id - ), "User was not added successfully" - - initial_size = len(team_info_before["team_info"]["members_with_roles"]) - - # Attempt to delete the same user multiple times (5 times) - for i in range(5): - logger.info(f"Attempting deletion {i+1}/5") - if i == 0: - # First deletion should succeed - api_client.delete_team_member(new_team, user_id) - time.sleep(1) - else: - # Subsequent deletions should raise an error - try: - api_client.delete_team_member(new_team, user_id) - pytest.fail("Expected HTTPError for duplicate deletion") - except requests.exceptions.HTTPError as e: - logger.info( - f"Expected error received on deletion attempt {i+1}: {str(e)}" - ) - - # Verify final state - final_info = api_client.get_team_info(new_team) - final_size = len(final_info["team_info"]["members_with_roles"]) - - # Verify user is completely removed - assert not verify_member_in_team( - final_info, user_id - ), "User still exists in team after deletion" - - # Verify only one member was removed - assert ( - final_size == initial_size - 1 - ), f"Team size changed unexpectedly (was {initial_size}, now {final_size})" - - -def test_delete_nonexistent_member(api_client, new_team): - """Test that attempting to delete a nonexistent member raises appropriate error""" - nonexistent_user = f"nonexistent_{uuid.uuid4().hex[:6]}" - - # Verify user doesn't exist first - team_info = api_client.get_team_info(new_team) - assert not verify_member_in_team( - team_info, nonexistent_user - ), "Test setup error: nonexistent user somehow exists" - - # Attempt to delete nonexistent user - with pytest.raises(requests.exceptions.HTTPError) as exc_info: - api_client.delete_team_member(new_team, nonexistent_user) - e = exc_info.value - logger.info(f"Expected error received: {str(e)}") - assert e.response.status_code == 400 diff --git a/tests/test_users.py b/tests/test_users.py index 55caeb592c8..9b19848694a 100644 --- a/tests/test_users.py +++ b/tests/test_users.py @@ -5,10 +5,7 @@ import pytest import asyncio import aiohttp import time -from openai import AsyncOpenAI -from tests.test_team import list_teams from typing import Optional -from fastapi import HTTPException async def new_user( @@ -41,16 +38,6 @@ async def new_user( return await response.json() -@pytest.mark.asyncio -async def test_user_new(): - """ - Make 20 parallel calls to /user/new. Assert all worked. - """ - async with aiohttp.ClientSession() as session: - tasks = [new_user(session, i) for i in range(1, 11)] - await asyncio.gather(*tasks) - - async def get_user_info(session, get_user, call_user, view_all: Optional[bool] = None): """ Make sure only models user has access to are returned @@ -79,37 +66,6 @@ async def get_user_info(session, get_user, call_user, view_all: Optional[bool] = return await response.json() -@pytest.mark.asyncio -async def test_user_info(): - """ - Get user info - - as admin - - as user themself - - as random - """ - get_user = f"krrish_{time.time()}@berri.ai" - async with aiohttp.ClientSession() as session: - key_gen = await new_user(session, 0, user_id=get_user) - key = key_gen["key"] - ## as admin ## - resp = await get_user_info( - session=session, get_user=get_user, call_user=os.environ["LITELLM_MASTER_KEY"] - ) - assert isinstance(resp["user_info"], dict) - assert len(resp["user_info"]) > 0 - ## as user themself ## - resp = await get_user_info(session=session, get_user=get_user, call_user=key) - assert isinstance(resp["user_info"], dict) - assert len(resp["user_info"]) > 0 - # as random user # - key_gen = await new_user(session=session, i=0) - random_key = key_gen["key"] - status = await get_user_info( - session=session, get_user=get_user, call_user=random_key - ) - assert status == 403 - - @pytest.mark.skip(reason="Frequent check on ci/cd leads to read timeout issue.") @pytest.mark.asyncio async def test_users_budgets_reset(): @@ -145,168 +101,6 @@ async def test_users_budgets_reset(): assert reset_at_init_value != reset_at_new_value -async def chat_completion(session, key, model="gpt-4"): - client = AsyncOpenAI(api_key=key, base_url="http://0.0.0.0:4000") - messages = [ - {"role": "system", "content": "You are a helpful assistant"}, - {"role": "user", "content": f"Hello! {time.time()}"}, - ] - - data = { - "model": model, - "messages": messages, - } - response = await client.chat.completions.create(**data) - - -async def chat_completion_streaming(session, key, model="gpt-4"): - client = AsyncOpenAI(api_key=key, base_url="http://0.0.0.0:4000") - messages = [ - {"role": "system", "content": "You are a helpful assistant"}, - {"role": "user", "content": f"Hello! {time.time()}"}, - ] - - data = {"model": model, "messages": messages, "stream": True} - response = await client.chat.completions.create(**data) - async for chunk in response: - continue - - - - -import json -from litellm._uuid import uuid import pytest -from typing import Dict, Tuple -async def setup_test_users(session: aiohttp.ClientSession) -> Tuple[Dict, Dict]: - """ - Create two test users and an additional key for the first user. - Returns tuple of (user1_data, user2_data) where each contains user info and keys. - """ - # Create two test users - user1 = await new_user( - session=session, - i=0, - budget=100, - budget_duration="30d", - models=["anthropic.claude-haiku-4-5-20251001-v1:0"], - ) - - user2 = await new_user( - session=session, - i=1, - budget=100, - budget_duration="30d", - models=["anthropic.claude-haiku-4-5-20251001-v1:0"], - ) - - print("\nCreated two test users:") - print(f"User 1 ID: {user1['user_id']}") - print(f"User 2 ID: {user2['user_id']}") - - # Create an additional key for user1 - headers = { - "Content-Type": "application/json", - "Authorization": f"Bearer {user1['key']}", - } - - key_payload = { - "user_id": user1["user_id"], - "duration": "7d", - "key_alias": f"test_key_{uuid.uuid4()}", - "models": ["anthropic.claude-haiku-4-5-20251001-v1:0"], - } - - print("\nGenerating additional key for user1...") - key_response = await session.post( - f"http://0.0.0.0:4000/key/generate", headers=headers, json=key_payload - ) - - assert key_response.status == 200, "Failed to generate additional key for user1" - user1_additional_key = await key_response.json() - - print(f"\nGenerated key details:") - print(json.dumps(user1_additional_key, indent=2)) - - # Return both users' data including the additional key - return { - "user_data": user1, - "additional_key": user1_additional_key, - "headers": headers, - }, { - "user_data": user2, - "headers": { - "Content-Type": "application/json", - "Authorization": f"Bearer {user2['key']}", - }, - } - - -async def print_response_details(response: aiohttp.ClientResponse) -> None: - """Helper function to print response details""" - print("\nResponse Details:") - print(f"Status Code: {response.status}") - print("\nResponse Content:") - try: - formatted_json = json.dumps(await response.json(), indent=2) - print(formatted_json) - except json.JSONDecodeError: - print(await response.text()) - - -@pytest.mark.asyncio -async def test_key_update_user_isolation(): - """Test that a user cannot update a key that belongs to another user""" - async with aiohttp.ClientSession() as session: - user1_data, user2_data = await setup_test_users(session) - - # Try to update the key to belong to user2 - update_payload = { - "key": user1_data["additional_key"]["key"], - "user_id": user2_data["user_data"][ - "user_id" - ], # Attempting to change ownership - "metadata": {"purpose": "testing_user_isolation", "environment": "test"}, - } - - print("\nAttempting to update key ownership to user2...") - update_response = await session.post( - f"http://0.0.0.0:4000/key/update", - headers=user1_data["headers"], # Using user1's headers - json=update_payload, - ) - - await print_response_details(update_response) - - # Verify update attempt was rejected - assert ( - update_response.status == 403 - ), "Request should have been rejected with 403 status code" - - -@pytest.mark.asyncio -async def test_key_delete_user_isolation(): - """Test that a user cannot delete a key that belongs to another user""" - async with aiohttp.ClientSession() as session: - user1_data, user2_data = await setup_test_users(session) - - # Try to delete user1's additional key using user2's credentials - delete_payload = { - "keys": [user1_data["additional_key"]["key"]], - } - - print("\nAttempting to delete user1's key using user2's credentials...") - delete_response = await session.post( - f"http://0.0.0.0:4000/key/delete", - headers=user2_data["headers"], - json=delete_payload, - ) - - await print_response_details(delete_response) - - # Verify delete attempt was rejected - assert ( - delete_response.status == 403 - ), "Request should have been rejected with 403 status code" diff --git a/tests/unit/integrations/test_langfuse_otel.py b/tests/unit/integrations/test_langfuse_otel.py index 0a9ce55fe16..13a49f40d7e 100644 --- a/tests/unit/integrations/test_langfuse_otel.py +++ b/tests/unit/integrations/test_langfuse_otel.py @@ -1,12 +1,17 @@ +import base64 import json import os +from datetime import datetime, timezone +from typing import Final from unittest.mock import MagicMock, patch import pytest +from opentelemetry.sdk.trace import TracerProvider from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger -from litellm.integrations.opentelemetry import OpenTelemetryConfig +from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.utils import StandardCallbackDynamicParams class TestLangfuseOtelIntegration: @@ -1006,5 +1011,58 @@ class TestLangfuseOtelResponsesAPI: assert output_data[0]["arguments"] == {} -if __name__ == "__main__": - pytest.main([__file__]) +LANGFUSE_ENV_ONLY: Final = { + key: value + for key, value in os.environ.items() + if not key.startswith(("LANGFUSE_", "OTEL_")) +} + + +def _decoded_basic_auth(header: str) -> str: + assert header.startswith("Basic ") + return base64.b64decode(header.removeprefix("Basic ")).decode() + + +def test_langfuse_otel_env_config_headers_carry_v4_ingestion_and_basic_auth() -> None: + with patch.dict( + os.environ, {**LANGFUSE_ENV_ONLY, "LANGFUSE_PUBLIC_KEY": "pk-lf-123", "LANGFUSE_SECRET_KEY": "sk-lf-123"}, clear=True + ): + logger: Final = LangfuseOtelLogger() + headers: Final = OpenTelemetry._get_headers_dictionary(logger.config.headers) + assert headers["x-langfuse-ingestion-version"] == "4" + assert _decoded_basic_auth(headers["Authorization"]) == "pk-lf-123:sk-lf-123" + + +def test_langfuse_otel_dynamic_headers_carry_v4_ingestion_and_basic_auth() -> None: + with patch.dict(os.environ, LANGFUSE_ENV_ONLY, clear=True): + logger: Final = LangfuseOtelLogger() + headers: Final = logger.construct_dynamic_otel_headers( + StandardCallbackDynamicParams(langfuse_public_key="pk-lf-dynamic", langfuse_secret_key="sk-lf-dynamic") + ) + assert headers is not None + assert headers["x-langfuse-ingestion-version"] == "4" + assert _decoded_basic_auth(headers["Authorization"]) == "pk-lf-dynamic:sk-lf-dynamic" + + +def test_langfuse_otel_does_not_start_proxy_request_span() -> None: + langfuse_provider: Final = TracerProvider() + generic_provider: Final = TracerProvider() + with patch.dict(os.environ, LANGFUSE_ENV_ONLY, clear=True): + langfuse_logger: Final = LangfuseOtelLogger(tracer_provider=langfuse_provider) + generic_logger: Final = OpenTelemetry( + config=OpenTelemetryConfig(exporter="console", skip_set_global=True), tracer_provider=generic_provider + ) + started_at: Final = datetime(2026, 1, 1, tzinfo=timezone.utc) + request_headers: Final = {"Authorization": "Bearer test"} + try: + assert ( + langfuse_logger.create_litellm_proxy_request_started_span(start_time=started_at, headers=request_headers) + is None + ) + assert ( + generic_logger.create_litellm_proxy_request_started_span(start_time=started_at, headers=request_headers) + is not None + ) + finally: + langfuse_provider.shutdown() + generic_provider.shutdown() diff --git a/tests/unit/litellm_core_utils/test_default_encoding.py b/tests/unit/litellm_core_utils/test_default_encoding.py new file mode 100644 index 00000000000..fd00a4b013a --- /dev/null +++ b/tests/unit/litellm_core_utils/test_default_encoding.py @@ -0,0 +1,40 @@ +import importlib +import os +from collections.abc import Iterator +from pathlib import Path +from typing import Final + +import pytest + +import litellm.litellm_core_utils.default_encoding as default_encoding + +BUNDLED_TOKENIZERS: Final = Path(default_encoding.filename) + + +@pytest.fixture +def reload_default_encoding(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.delenv("TIKTOKEN_CACHE_DIR", raising=False) + monkeypatch.delenv("CUSTOM_TIKTOKEN_CACHE_DIR", raising=False) + yield + monkeypatch.delenv("TIKTOKEN_CACHE_DIR", raising=False) + monkeypatch.delenv("CUSTOM_TIKTOKEN_CACHE_DIR", raising=False) + importlib.reload(default_encoding) + + +@pytest.mark.usefixtures("reload_default_encoding") +def test_tiktoken_cache_dir_defaults_to_bundled_tokenizers_for_non_root(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("LITELLM_NON_ROOT", "true") + importlib.reload(default_encoding) + assert Path(os.environ["TIKTOKEN_CACHE_DIR"]) == BUNDLED_TOKENIZERS + assert BUNDLED_TOKENIZERS.name == "tokenizers" + assert default_encoding.encoding.name == "cl100k_base" + assert default_encoding.encoding.decode(default_encoding.encoding.encode("hello world")) == "hello world" + + +@pytest.mark.usefixtures("reload_default_encoding") +def test_custom_tiktoken_cache_dir_overrides_and_is_created(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + custom_dir: Final = tmp_path / "tiktoken_cache" + monkeypatch.setenv("CUSTOM_TIKTOKEN_CACHE_DIR", str(custom_dir)) + importlib.reload(default_encoding) + assert os.environ["TIKTOKEN_CACHE_DIR"] == str(custom_dir) + assert custom_dir.is_dir() diff --git a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py index fe0c4c84923..74d68d76813 100644 --- a/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py +++ b/tests/unit/llms/anthropic/chat/test_anthropic_chat_transformation.py @@ -7917,3 +7917,54 @@ def test_is_prompt_caching_enabled(anthropic_messages): custom_llm_provider="anthropic", model="anthropic/claude-sonnet-4-5-20250929", ) + + +def test_calculate_usage_sums_compaction_and_message_iterations(): + usage: Final = AnthropicConfig().calculate_usage( + usage_object={ + "input_tokens": 100, + "output_tokens": 50, + "iterations": [ + {"iteration": 1, "type": "compaction", "input_tokens": 1000, "output_tokens": 500}, + {"iteration": 2, "type": "message", "input_tokens": 100, "output_tokens": 50}, + ], + }, + reasoning_content=None, + ) + assert usage.prompt_tokens == 1100 + assert usage.completion_tokens == 550 + assert usage.total_tokens == 1650 + assert usage.prompt_tokens_details.text_tokens == 1100 + assert usage.iterations is not None + assert len(usage.iterations) == 2 + assert usage.iterations[0]["type"] == "compaction" + + +def test_calculate_usage_sums_cache_tokens_across_compaction_iterations(): + usage: Final = AnthropicConfig().calculate_usage( + usage_object={ + "input_tokens": 100, + "output_tokens": 50, + "iterations": [ + { + "type": "compaction", + "input_tokens": 500, + "output_tokens": 200, + "cache_creation_input_tokens": 50, + "cache_read_input_tokens": 17000, + }, + { + "type": "message", + "input_tokens": 100, + "output_tokens": 50, + "cache_creation_input_tokens": 10, + "cache_read_input_tokens": 20, + }, + ], + }, + reasoning_content=None, + ) + assert usage.prompt_tokens == 17680 + assert usage.completion_tokens == 250 + assert usage.prompt_tokens_details.cache_creation_tokens == 60 + assert usage.prompt_tokens_details.cached_tokens == 17020 diff --git a/tests/unit/llms/azure/response/test_azure_transformation.py b/tests/unit/llms/azure/response/test_azure_transformation.py index 8db2ba4835e..fea5304a689 100644 --- a/tests/unit/llms/azure/response/test_azure_transformation.py +++ b/tests/unit/llms/azure/response/test_azure_transformation.py @@ -12,8 +12,11 @@ from litellm.llms.azure.responses.o_series_transformation import ( AzureOpenAIOSeriesResponsesAPIConfig, ) from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager @pytest.mark.serial @@ -920,3 +923,25 @@ async def test_azure_responses_api_headers_with_llm_provider_prefix(): # Also verify openai-compatible headers are included assert "x-ratelimit-limit-tokens" in headers assert "x-ratelimit-remaining-tokens" in headers + + +@pytest.mark.parametrize( + "model", ["gpt-5", "gpt-5-turbo", "GPT-5", "azure/gpt-5", "gpt-3.5-turbo", "gpt-4", "gpt-4-turbo", "gpt-4o"] +) +def test_azure_gpt_models_resolve_to_responses_config_with_temperature(model: str) -> None: + config: Final = ProviderConfigManager.get_provider_responses_api_config(provider=LlmProviders.AZURE, model=model) + assert type(config) is AzureOpenAIResponsesAPIConfig + assert "temperature" in config.get_supported_openai_params(model) + + +@pytest.mark.parametrize("model", ["o1", "o3"]) +def test_azure_o_series_resolves_to_o_series_config_without_temperature(model: str) -> None: + config: Final = ProviderConfigManager.get_provider_responses_api_config(provider=LlmProviders.AZURE, model=model) + assert type(config) is AzureOpenAIOSeriesResponsesAPIConfig + assert "temperature" not in config.get_supported_openai_params(model) + + +def test_openai_gpt5_resolves_to_responses_config_with_temperature() -> None: + config: Final = ProviderConfigManager.get_provider_responses_api_config(provider=LlmProviders.OPENAI, model="gpt-5") + assert type(config) is OpenAIResponsesAPIConfig + assert "temperature" in config.get_supported_openai_params("gpt-5") diff --git a/tests/unit/llms/custom_httpx/test_async_client_cleanup.py b/tests/unit/llms/custom_httpx/test_async_client_cleanup.py index e8fb0808019..6b7697be945 100644 --- a/tests/unit/llms/custom_httpx/test_async_client_cleanup.py +++ b/tests/unit/llms/custom_httpx/test_async_client_cleanup.py @@ -1,4 +1,10 @@ +import re +from collections.abc import Iterator +from typing import Final + +import httpx import pytest +import respx import litellm from litellm.llms.custom_httpx.async_client_cleanup import close_litellm_async_clients @@ -19,3 +25,97 @@ async def test_second_cleanup_pass_does_not_resurrect_owned_client(): litellm.in_memory_llm_clients_cache.cache_dict.pop(cache_key, None) assert handler._client is original_client + + +GEMINI_GENERATE_URL: Final = re.compile(r"https://generativelanguage\.googleapis\.com/.*:generateContent.*") + + +def _gemini_reply(text: str) -> httpx.Response: + return httpx.Response( + 200, + json={ + "candidates": [ + {"content": {"parts": [{"text": text}], "role": "model"}, "finishReason": "STOP", "index": 0} + ], + "usageMetadata": {"promptTokenCount": 1, "candidatesTokenCount": 1, "totalTokenCount": 2}, + }, + ) + + +def _cached_async_handlers() -> list[AsyncHTTPHandler]: + return [ + handler + for handler in litellm.in_memory_llm_clients_cache.cache_dict.values() + if isinstance(handler, AsyncHTTPHandler) + ] + + +@pytest.fixture +def gemini_httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("GEMINI_API_KEY", "gemini-cleanup-test") + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("gemini_httpx_transport") +async def test_acompletion_client_is_closed_by_cleanup() -> None: + with respx.mock(assert_all_called=True) as mock: + mock.post(GEMINI_GENERATE_URL).mock(return_value=_gemini_reply("Hi there!")) + response: Final = await litellm.acompletion( + model="gemini/gemini-2.0-flash-lite-001", + messages=[{"role": "user", "content": "Hello"}], + ) + assert response.choices[0].message.content == "Hi there!" + clients: Final = [handler._client for handler in _cached_async_handlers()] + assert clients + assert not any(client.is_closed for client in clients) + + await close_litellm_async_clients() + + assert all(client.is_closed for client in clients) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("gemini_httpx_transport") +async def test_repeated_acompletion_calls_reuse_one_client_that_cleanup_closes() -> None: + with respx.mock(assert_all_called=True) as mock: + route: Final = mock.post(GEMINI_GENERATE_URL).mock( + side_effect=[_gemini_reply(f"Response {index}") for index in range(3)] + ) + for index in range(3): + response = await litellm.acompletion( + model="gemini/gemini-2.0-flash-lite-001", + messages=[{"role": "user", "content": f"Hello {index}"}], + ) + assert response.choices[0].message.content == f"Response {index}" + assert route.call_count == 3 + handlers: Final = _cached_async_handlers() + assert len(handlers) == 1 + client: Final = handlers[0]._client + + await close_litellm_async_clients() + + assert client.is_closed + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("gemini_httpx_transport") +async def test_cleanup_is_idempotent_and_acompletion_works_afterwards() -> None: + with respx.mock(assert_all_called=True) as mock: + route: Final = mock.post(GEMINI_GENERATE_URL).mock(side_effect=[_gemini_reply("Hello!"), _gemini_reply("Hi!")]) + await litellm.acompletion( + model="gemini/gemini-2.0-flash-lite-001", + messages=[{"role": "user", "content": "Hello"}], + ) + for _ in range(3): + await close_litellm_async_clients() + response: Final = await litellm.acompletion( + model="gemini/gemini-2.0-flash-lite-001", + messages=[{"role": "user", "content": "Hello"}], + ) + assert response.choices[0].message.content == "Hi!" + assert route.call_count == 2 + await close_litellm_async_clients() diff --git a/tests/unit/llms/litellm_proxy/responses/__init__.py b/tests/unit/llms/litellm_proxy/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/litellm_proxy/responses/test_transformation.py b/tests/unit/llms/litellm_proxy/responses/test_transformation.py new file mode 100644 index 00000000000..e2742d6a40b --- /dev/null +++ b/tests/unit/llms/litellm_proxy/responses/test_transformation.py @@ -0,0 +1,29 @@ +from typing import Final + +import pytest + +from litellm.llms.litellm_proxy.responses.transformation import LiteLLMProxyResponsesAPIConfig +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager + + +def test_provider_config_manager_returns_litellm_proxy_responses_config() -> None: + config: Final = ProviderConfigManager.get_provider_responses_api_config( + model="litellm_proxy/gpt-5.5", provider=LlmProviders.LITELLM_PROXY + ) + assert isinstance(config, LiteLLMProxyResponsesAPIConfig) + assert config.custom_llm_provider == LlmProviders.LITELLM_PROXY + + +@pytest.mark.parametrize("api_base", ["https://my-proxy.example.com", "https://my-proxy.example.com/"]) +def test_get_complete_url_appends_responses_path(api_base: str) -> None: + assert ( + LiteLLMProxyResponsesAPIConfig().get_complete_url(api_base=api_base, litellm_params={}) + == "https://my-proxy.example.com/responses" + ) + + +def test_get_complete_url_requires_api_base(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("LITELLM_PROXY_API_BASE", raising=False) + with pytest.raises(ValueError, match="api_base not set"): + LiteLLMProxyResponsesAPIConfig().get_complete_url(api_base=None, litellm_params={}) diff --git a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py index bd760ed27b1..7d0c81af4c2 100644 --- a/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py +++ b/tests/unit/proxy/guardrails/guardrail_hooks/test_presidio.py @@ -9,6 +9,7 @@ import json import os import re from collections.abc import Iterable, Sequence +from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager from typing import Final, Literal from unittest.mock import MagicMock, patch @@ -4799,3 +4800,33 @@ async def test_legacy_pii_masking_config_registers_logging_only_guardrail(monkey assert pii_masking_obj.should_run_guardrail( data={}, event_type=GuardrailEventHooks.logging_only ) + + +async def _one_session(guardrail: OPTIONAL_PresidioPIIMasking) -> aiohttp.ClientSession: + async with guardrail._get_session_iterator() as session: + return session + + +@pytest.mark.asyncio +async def test_get_session_iterator_reuses_one_session_on_main_thread( + presidio_guardrail: OPTIONAL_PresidioPIIMasking, +) -> None: + sessions: Final = tuple([await _one_session(presidio_guardrail) for _ in range(10)]) + assert all(session is sessions[0] for session in sessions) + assert sessions[0] is presidio_guardrail._http_session + await presidio_guardrail._close_http_session() + + +def test_get_session_iterator_reuses_one_session_per_background_loop( + presidio_guardrail: OPTIONAL_PresidioPIIMasking, +) -> None: + async def collect_and_close() -> tuple[aiohttp.ClientSession, ...]: + collected: Final = tuple([await _one_session(presidio_guardrail) for _ in range(10)]) + await collected[0].close() + return collected + + with ThreadPoolExecutor(max_workers=1) as pool: + sessions: Final = pool.submit(asyncio.run, collect_and_close()).result() + assert len(sessions) == 10 + assert all(session is sessions[0] for session in sessions) + assert presidio_guardrail._http_session is None diff --git a/tests/unit/proxy/test_route_llm_request.py b/tests/unit/proxy/test_route_llm_request.py index d5546da024b..a4b34825d95 100644 --- a/tests/unit/proxy/test_route_llm_request.py +++ b/tests/unit/proxy/test_route_llm_request.py @@ -516,12 +516,10 @@ def test_e2e_proxy_config_opts_in_to_the_mock_params_its_suite_sends(): for param in GATED_MOCK_PARAM_NAMES if f"{param}=" in source or f'"{param}"' in source ) - assert senders, "expected the E2E suite to still exercise the gated mock testing params" - config = yaml.safe_load((repo_root / "proxy_server_config.yaml").read_text(encoding="utf-8")) general_settings = config.get("general_settings") or {} - assert general_settings.get(MOCK_TESTING_CONFIG_KEY) is True, ( + assert not senders or general_settings.get(MOCK_TESTING_CONFIG_KEY) is True, ( f"proxy_server_config.yaml must set general_settings.{MOCK_TESTING_CONFIG_KEY}: true — " f"the E2E suite sends gated mock testing params ({', '.join(sorted(senders))}) " "and the proxy rejects them with a 400 otherwise" diff --git a/tests/unit/router_strategy/test_lowest_tpm_rpm.py b/tests/unit/router_strategy/test_lowest_tpm_rpm.py index 625f648bec4..1c659f47279 100644 --- a/tests/unit/router_strategy/test_lowest_tpm_rpm.py +++ b/tests/unit/router_strategy/test_lowest_tpm_rpm.py @@ -1,13 +1,14 @@ +import asyncio from datetime import datetime, timedelta from typing import Final from unittest.mock import AsyncMock import pytest -from litellm import Router +from litellm import Router, token_counter from litellm.caching.dual_cache import DualCache from litellm.router_strategy.lowest_tpm_rpm_v2 import LowestTPMLoggingHandler_v2, PrefetchedUsage -from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict +from litellm.types.router import DeploymentTypedDict, LiteLLMParamsTypedDict, RouterErrors MODEL_GROUP: Final = "lowest-tpm-router" HIGH_USAGE_DEPLOYMENT_ID: Final = "highest-usage" @@ -117,3 +118,69 @@ async def test_v2_subclass_overriding_async_get_available_deployments_with_the_o f"from {HIGH_USAGE_DEPLOYMENT_ID}", f"from {LOW_USAGE_DEPLOYMENT_ID}", } + + +def _rate_limited_router(num_allowed_send: int) -> tuple[Router, tuple[list[dict[str, str]], ...]]: + conversations: Final = tuple( + [{"role": "user", "content": f"{index}. Hey, how's it going?"}] for index in range(num_allowed_send) + ) + tpm: Final = sum(token_counter(model="gpt-4o", messages=messages) + 5 for messages in conversations) + deployment: Final = _deployment(LOW_USAGE_DEPLOYMENT_ID) + router: Final = Router( + model_list=[{**deployment, "rpm": num_allowed_send, "tpm": tpm}], + routing_strategy="usage-based-routing", + enable_pre_call_checks=True, + num_retries=0, + ) + return router, conversations + + +def test_usage_based_routing_v1_serves_sync_calls_within_rpm_and_tpm() -> None: + router, conversations = _rate_limited_router(num_allowed_send=3) + responses: Final = [router.completion(model=MODEL_GROUP, messages=messages) for messages in conversations[:2]] + assert [response.choices[0].message.content for response in responses] == [f"from {LOW_USAGE_DEPLOYMENT_ID}"] * 2 + + +@pytest.mark.asyncio +async def test_usage_based_routing_v1_serves_async_calls_within_rpm_and_tpm() -> None: + router, conversations = _rate_limited_router(num_allowed_send=3) + responses: Final = await asyncio.gather( + *(router.acompletion(model=MODEL_GROUP, messages=messages) for messages in conversations[:2]) + ) + assert [response.choices[0].message.content for response in responses] == [f"from {LOW_USAGE_DEPLOYMENT_ID}"] * 2 + + +RPM_LIMIT: Final = 3 + + +def _router_with_recorded_rpm(recorded: int, enable_pre_call_checks: bool) -> Router: + router: Final = Router( + model_list=[{**_deployment(LOW_USAGE_DEPLOYMENT_ID), "rpm": RPM_LIMIT}], + routing_strategy="usage-based-routing", + enable_pre_call_checks=enable_pre_call_checks, + num_retries=0, + ) + now: Final = datetime.now() + for offset in range(-1, 2): + router.cache.set_cache( + key=f"{MODEL_GROUP}:rpm:{(now + timedelta(minutes=offset)).strftime('%H-%M')}", + value={LOW_USAGE_DEPLOYMENT_ID: recorded}, + ttl=float("inf"), + ) + return router + + +@pytest.mark.parametrize("enable_pre_call_checks", [True, False]) +def test_usage_based_routing_v1_serves_a_deployment_below_its_recorded_rpm_limit(enable_pre_call_checks: bool) -> None: + router: Final = _router_with_recorded_rpm(recorded=1, enable_pre_call_checks=enable_pre_call_checks) + response: Final = router.completion(model=MODEL_GROUP, messages=[{"role": "user", "content": "hello"}]) + assert response.choices[0].message.content == f"from {LOW_USAGE_DEPLOYMENT_ID}" + + +@pytest.mark.parametrize("enable_pre_call_checks", [True, False]) +def test_usage_based_routing_v1_rejects_a_deployment_that_reached_its_recorded_rpm_limit( + enable_pre_call_checks: bool, +) -> None: + router: Final = _router_with_recorded_rpm(recorded=RPM_LIMIT, enable_pre_call_checks=enable_pre_call_checks) + with pytest.raises(ValueError, match=RouterErrors.no_deployments_available.value): + router.completion(model=MODEL_GROUP, messages=[{"role": "user", "content": "hello"}]) diff --git a/tests/unit/test_router/test_router_vector_store_endpoints.py b/tests/unit/test_router/test_router_vector_store_endpoints.py new file mode 100644 index 00000000000..ad431c656f9 --- /dev/null +++ b/tests/unit/test_router/test_router_vector_store_endpoints.py @@ -0,0 +1,101 @@ +import json +from collections.abc import Iterator +from typing import Final + +import httpx +import litellm +import pytest +import respx +from litellm import Router + +VECTOR_STORES_URL: Final = "https://api.openai.com/v1/vector_stores" + + +@pytest.fixture(autouse=True) +def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setenv("OPENAI_API_KEY", "sk-vector-store-test") + monkeypatch.delenv("OPENAI_BASE_URL", raising=False) + monkeypatch.delenv("OPENAI_API_BASE", raising=False) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.mark.asyncio +async def test_avector_store_update_sends_name_and_metadata_to_provider() -> None: + with respx.mock(assert_all_called=True) as mock: + route: Final = mock.post(f"{VECTOR_STORES_URL}/vs_test123").mock( + return_value=httpx.Response( + 200, + json={ + "id": "vs_test123", + "object": "vector_store", + "created_at": 1699061776, + "name": "Updated Name", + "metadata": {"key": "value"}, + "status": "completed", + }, + ) + ) + result: Final = await Router(model_list=[]).avector_store_update( + vector_store_id="vs_test123", + name="Updated Name", + metadata={"key": "value"}, + custom_llm_provider="openai", + ) + sent: Final = json.loads(route.calls.last.request.content) + assert route.call_count == 1 + assert sent["name"] == "Updated Name" + assert sent["metadata"] == {"key": "value"} + assert result["id"] == "vs_test123" + assert result["name"] == "Updated Name" + assert result["metadata"]["key"] == "value" + + +def test_vector_store_list_forwards_pagination_params_to_provider() -> None: + with respx.mock(assert_all_called=True) as mock: + route: Final = mock.get(VECTOR_STORES_URL).mock( + return_value=httpx.Response( + 200, + json={ + "object": "list", + "data": [{"id": f"vs_{index}", "object": "vector_store"} for index in range(5)], + "has_more": True, + "first_id": "vs_0", + "last_id": "vs_4", + }, + ) + ) + result: Final = Router(model_list=[]).vector_store_list( + limit=5, after="vs_previous", order="asc", custom_llm_provider="openai" + ) + params: Final = route.calls.last.request.url.params + assert params["limit"] == "5" + assert params["after"] == "vs_previous" + assert params["order"] == "asc" + assert result["has_more"] is True + assert len(result["data"]) == 5 + + +def test_vector_store_update_forwards_expires_after_to_provider() -> None: + expires_after: Final = {"anchor": "last_active_at", "days": 7} + with respx.mock(assert_all_called=True) as mock: + route: Final = mock.post(f"{VECTOR_STORES_URL}/vs_test123").mock( + return_value=httpx.Response( + 200, + json={ + "id": "vs_test123", + "object": "vector_store", + "expires_after": expires_after, + "expires_at": 1699668576, + }, + ) + ) + result: Final = Router(model_list=[]).vector_store_update( + vector_store_id="vs_test123", expires_after=expires_after, custom_llm_provider="openai" + ) + sent: Final = json.loads(route.calls.last.request.content) + assert sent["expires_after"] == expires_after + assert result["expires_after"]["days"] == 7 + assert result["expires_at"] == 1699668576 diff --git a/tests/unit/test_service_logger.py b/tests/unit/test_service_logger.py index 3d74642a03e..99ba32ed955 100644 --- a/tests/unit/test_service_logger.py +++ b/tests/unit/test_service_logger.py @@ -7,10 +7,17 @@ is called without call_type in kwargs (e.g. from batch polling callbacks). import pytest from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, patch +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter + import litellm from litellm._service_logger import ServiceLogging +from litellm.integrations.langfuse.langfuse_otel import LangfuseOtelLogger +from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig from litellm.types.services import ServiceTypes @@ -316,3 +323,47 @@ async def test_only_redis_service_spans_carry_the_ambient_key_family(monkeypatch "redis.get router_session_pins": "router_session_pins", "batch_write_to_db _PROXY_track_cost_callback": None, } + + +def _in_memory_provider(exporter: InMemorySpanExporter) -> TracerProvider: + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(exporter)) + return provider + + +@pytest.mark.asyncio +async def test_generic_otel_logger_receives_service_event_once_beside_langfuse_otel( + monkeypatch: pytest.MonkeyPatch, +) -> None: + langfuse_exporter: Final = InMemorySpanExporter() + generic_exporter: Final = InMemorySpanExporter() + langfuse_provider: Final = _in_memory_provider(langfuse_exporter) + generic_provider: Final = _in_memory_provider(generic_exporter) + langfuse_logger: Final = LangfuseOtelLogger( + config=OpenTelemetryConfig(exporter="console", skip_set_global=True), tracer_provider=langfuse_provider + ) + generic_logger: Final = OpenTelemetry( + config=OpenTelemetryConfig(exporter="console", skip_set_global=True), tracer_provider=generic_provider + ) + monkeypatch.setattr(litellm, "service_callback", [langfuse_logger, generic_logger]) + parent: Final = generic_logger.tracer.start_span("parent") + + try: + await ServiceLogging().async_service_success_hook( + service=ServiceTypes.DB, + call_type="success", + duration=0.1, + parent_otel_span=parent, + start_time=0.0, + end_time=1.0, + ) + finally: + langfuse_provider.shutdown() + generic_provider.shutdown() + + service_spans: Final = [ + span for span in generic_exporter.get_finished_spans() if span.attributes.get("service") == ServiceTypes.DB.value + ] + assert len(service_spans) == 1 + assert service_spans[0].attributes.get("call_type") == "success" + assert langfuse_exporter.get_finished_spans() == () From b7ecb0534060972a99715d02cc34762bda82abaa Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 10:35:52 -0700 Subject: [PATCH 48/65] fix(proxy): mark background responses stale_expired when provider returns 404 (#41724) * fix(proxy): mark background responses stale_expired when provider returns 404 Responses deleted upstream (store=false / ZDR rows dropped after provider retention) now move to stale_expired on the poll instead of being retried every cycle until the 7 day stale cleanup. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): drop inline comment in check_responses_cost Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): drive _get_response seam in 404 stale_expired tests Assigning AsyncMock on the instance keeps the new tests inside the TQ002/TQ008 test-quality ceilings Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): only expire 404 responses polled through a resolved deployment A NotFoundError from the bare SDK fallback can mean a missing or misconfigured deployment, which a config fix inside the staleness window can still recover. Rows whose poll went through their router deployment are the ones the provider actually dropped, so only those move to stale_expired. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): resolve the deployment before fetching so via_router binds once Drops the reinitialised loop flag flagged by review and modernizes the moved annotation. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): drive the real response fetch in the poll 404 regression tests * fix(enterprise): expire background response rows on any provider 404 status The GET path maps the provider's 404 body to litellm.BadRequestError that still carries status_code 404, so an except on NotFoundError never fired and the poll job kept retrying the row every cycle. Key the terminal decision on the status code instead of the class * fix(enterprise): expire a background response only on a 404 naming it and skip a bad deployment entry per job * chore(enterprise): record the poll-cycle bound on the managed-object IN lists * test(integration): cover background response poll retirement on the real proxy --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- .../common_utils/check_responses_cost.py | 57 ++- .../unbounded_in_baseline.txt | 1 - ...est_background_response_poll_retirement.py | 370 ++++++++++++++++++ .../common_utils/test_check_responses_cost.py | 273 +++++++++++++ 4 files changed, 691 insertions(+), 10 deletions(-) create mode 100644 tests/integration/spend/test_background_response_poll_retirement.py diff --git a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py index 738b9ebe1e0..7dd1fd7a1ee 100644 --- a/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py +++ b/enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py @@ -6,7 +6,7 @@ same route are non-inference and free. """ from datetime import datetime, timedelta, timezone -from typing import TYPE_CHECKING, Dict, Final, Optional, Protocol, cast +from typing import TYPE_CHECKING, Dict, Final, Protocol, cast import litellm from litellm._logging import verbose_proxy_logger @@ -47,6 +47,10 @@ def _managed_object_table(prisma_client: "PrismaClient") -> "TableActions[_Manag return ManagedObjectRepository(prisma_client).table +def _is_response_gone_at_provider(error: Exception, provider_response_id: str) -> bool: + return getattr(error, "status_code", None) == 404 and provider_response_id in str(error) + + class CheckResponsesCost: def __init__( self, @@ -61,10 +65,15 @@ class CheckResponsesCost: self.prisma_client: PrismaClient = prisma_client self.llm_router: Router = llm_router + def _resolve_deployment(self, response_id: str) -> bool: + model_id: str | None = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id) + return model_id is not None and self.llm_router.get_deployment(model_id=model_id) is not None + async def _get_response( self, response_id: str, litellm_metadata: Dict[str, str], + via_router: bool, ) -> ResponsesAPIResponse: """Fetch the upstream response, using deployment credentials when available. @@ -75,8 +84,7 @@ class CheckResponsesCost: sees provider env vars, so it fails for every deployment whose credentials live in the config; the row then never leaves ``queued``. """ - model_id: Optional[str] = ResponsesAPIRequestUtils.get_model_id_from_response_id(response_id) - if model_id is None or self.llm_router.get_deployment(model_id=model_id) is None: + if not via_router: return await litellm.aget_responses(response_id=response_id, litellm_metadata=litellm_metadata) router_response = await self.llm_router.aget_responses( response_id=response_id, litellm_metadata=litellm_metadata @@ -140,6 +148,8 @@ class CheckResponsesCost: - Cost is tracked by the get-responses call, billed because the poll is stamped with BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN - Mark responses in a terminal state as complete in the database + - Mark responses the provider no longer has (404 through a resolved + deployment) as stale_expired """ try: await self._cleanup_stale_managed_objects() @@ -159,6 +169,7 @@ class CheckResponsesCost: verbose_proxy_logger.debug(f"Found {len(jobs)} response jobs to check") completed_jobs: Final[list[_ManagedObjectRow]] = [] + expired_jobs: Final[list[_ManagedObjectRow]] = [] for job in jobs: unified_object_id = job.unified_object_id @@ -171,31 +182,48 @@ class CheckResponsesCost: # Get the stored response object to extract model information stored_response = job.file_object model_name = stored_response.get("model", None) - + # Decrypt the response ID responses_id_security, _, _ = ResponsesIDSecurity()._decrypt_response_id(unified_object_id) - + # Prepare metadata with model information for cost tracking litellm_metadata = { "user_api_key_user_id": job.created_by or "default-user-id", INTERNAL_CALL_ORIGIN_METADATA_KEY: BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN, } - + # Add model information if available if model_name: litellm_metadata["model"] = model_name litellm_metadata["model_group"] = model_name # Use same value for model_group - + via_router = self._resolve_deployment(responses_id_security) + except Exception as e: + verbose_proxy_logger.warning( + f"Skipping job {unified_object_id} due to error: {e}" + ) + continue + + provider_response_id = ResponsesAPIRequestUtils.decode_responses_api_response_id( + responses_id_security + ).get("response_id", responses_id_security) + try: response = await self._get_response( response_id=responses_id_security, litellm_metadata=litellm_metadata, + via_router=via_router, ) - + verbose_proxy_logger.debug( f"Response {unified_object_id} status: {response.status}, model: {model_name}" ) - + except Exception as e: + if via_router and _is_response_gone_at_provider(e, provider_response_id): + verbose_proxy_logger.info( + f"Response {unified_object_id} no longer available at provider (404), marking stale_expired: {e}" + ) + expired_jobs.append(job) + continue verbose_proxy_logger.warning( f"Skipping job {unified_object_id} due to error: {e}" ) @@ -210,6 +238,7 @@ class CheckResponsesCost: # Mark completed jobs in the database if len(completed_jobs) > 0: await _managed_object_table(self.prisma_client).update_many( + # bounded-ok: at most MAX_OBJECTS_PER_POLL_CYCLE rows per cycle, the find_many take above where={"id": {"in": [job.id for job in completed_jobs]}}, data={"status": "completed"}, ) @@ -217,3 +246,13 @@ class CheckResponsesCost: f"Marked {len(completed_jobs)} response jobs as completed" ) + if len(expired_jobs) > 0: + await _managed_object_table(self.prisma_client).update_many( + # bounded-ok: at most MAX_OBJECTS_PER_POLL_CYCLE rows per cycle, the find_many take above + where={"id": {"in": [job.id for job in expired_jobs]}}, + data={"status": "stale_expired"}, + ) + verbose_proxy_logger.info( + f"Marked {len(expired_jobs)} response jobs as stale_expired" + ) + diff --git a/tests/code_coverage_tests/unbounded_in_baseline.txt b/tests/code_coverage_tests/unbounded_in_baseline.txt index 77c7c362f39..da56a25542d 100644 --- a/tests/code_coverage_tests/unbounded_in_baseline.txt +++ b/tests/code_coverage_tests/unbounded_in_baseline.txt @@ -1,6 +1,5 @@ # Grandfathered findings of check_unbounded_in_lists.py: path::scope::kind::subject::occurrence. # Fix a site and delete its line; regenerate with `check_unbounded_in_lists.py --update-baseline`. -enterprise/litellm_enterprise/proxy/common_utils/check_responses_cost.py CheckResponsesCost.check_responses_cost prisma id.in `[job.id for job in completed_jobs]` 0 litellm/integrations/shadow_eval_logger.py ShadowEvalLogger._active_jobs prisma job_id.in `[str(record.id) for record in records]` 0 litellm/llms/litellm_proxy/skills/handler.py LiteLLMSkillsHandler.list_skills prisma created_by.in `owner_scopes` 0 litellm/proxy/_experimental/mcp_server/db.py get_mcp_servers prisma server_id.in `server_ids` 0 diff --git a/tests/integration/spend/test_background_response_poll_retirement.py b/tests/integration/spend/test_background_response_poll_retirement.py new file mode 100644 index 00000000000..f1b1e96901c --- /dev/null +++ b/tests/integration/spend/test_background_response_poll_retirement.py @@ -0,0 +1,370 @@ +import itertools +import os +import uuid +from collections.abc import Iterator, Sequence +from hashlib import sha256 +from pathlib import Path +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from integration._support.database import read_rows, scratch_database, write_rows +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.upstream import ScenarioHandle, delete_scenario, register_scenario +from integration.cost_calculation.cost_tracking_case import JsonResponse, RoutedResponse +from pydantic import JsonValue + +from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE +from litellm.responses.utils import ResponsesAPIRequestUtils +from litellm.types.utils import BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN + +INPUT_COST_PER_TOKEN: Final = 0.001 +OUTPUT_COST_PER_TOKEN: Final = 0.002 +INPUT_TOKENS: Final = 19 +OUTPUT_TOKENS: Final = 7 +SCHEDULER_PERIOD_CEILING_SECONDS: Final = 31 +ONE_POLL_SECONDS: Final = 2 * SCHEDULER_PERIOD_CEILING_SECONDS + 8 +TWO_POLLS_SECONDS: Final = 4 * SCHEDULER_PERIOD_CEILING_SECONDS + 8 +OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 240 +PROVIDER_ID: Final = "resp_$REQUEST_ID" + + +def _response_body(status: str) -> dict[str, JsonValue]: + completed: Final = status == "completed" + return { + "id": PROVIDER_ID, + "object": "response", + "created_at": 1, + "status": status, + "background": True, + "store": False, + "error": None, + "incomplete_details": None, + "model": "gpt-4o-mini", + "output": ( + [ + { + "id": "msg_$REQUEST_ID", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "pong", "annotations": []}], + } + ] + if completed + else [] + ), + "parallel_tool_calls": True, + "temperature": 1.0, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "truncation": "disabled", + "usage": ( + { + "input_tokens": INPUT_TOKENS, + "output_tokens": OUTPUT_TOKENS, + "total_tokens": INPUT_TOKENS + OUTPUT_TOKENS, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens_details": {"reasoning_tokens": 0}, + } + if completed + else None + ), + "metadata": {}, + } + + +def _json(body: dict[str, JsonValue], status: int = 200) -> JsonResponse: + return JsonResponse(content_type="application/json", body=body, status=status) + + +def _provider_404(message: str) -> JsonResponse: + return _json({"error": {"message": message, "type": "invalid_request_error", "param": None, "code": None}}, 404) + + +def _routes(submission_status: str, retrieve: JsonResponse) -> RoutedResponse: + return RoutedResponse( + content_type="application/x-routed", + routes={"POST /responses": _json(_response_body(submission_status)), f"GET /responses/{PROVIDER_ID}": retrieve}, + ) + + +def _gone_routes(submission_status: str = "queued") -> RoutedResponse: + return _routes(submission_status, _provider_404(f"Response with id '{PROVIDER_ID}' not found.")) + + +def _scenario_id(marker: str) -> str: + return f"bg-{marker}-{uuid.uuid4().hex[:12]}" + + +def _scripted_deployment(scenario: Scenario, marker: str, routes: RoutedResponse) -> tuple[str, ScenarioHandle]: + return _deployment_on(scenario, _scenario_id(marker), routes) + + +def _register_deployment( + scenario: Scenario, scenario_id: str, routes: RoutedResponse +) -> tuple[str, str, ScenarioHandle]: + handle: Final = register_scenario(scenario_id, routes) + scenario.cleanups.callback(delete_scenario, handle) + created: Final = scenario.gateway.post( + "/model/new", + { + "model_name": f"bg-{sha256(scenario_id.encode()).hexdigest()[:12]}", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-scripted-provider", + "api_base": handle.api_base(), + "input_cost_per_token": INPUT_COST_PER_TOKEN, + "output_cost_per_token": OUTPUT_COST_PER_TOKEN, + }, + }, + ) + return string_value(created["model_name"]), string_value(object_value(created["model_info"])["id"]), handle + + +def _deployment_on(scenario: Scenario, scenario_id: str, routes: RoutedResponse) -> tuple[str, ScenarioHandle]: + model_name, model_id, handle = _register_deployment(scenario, scenario_id, routes) + scenario.cleanups.callback(scenario.delete_model, model_id) + return model_name, handle + + +def _forget_row(unified_id: str, database_url: str | None = None) -> None: + write_rows( + 'DELETE FROM "LiteLLM_ManagedObjectTable" WHERE unified_object_id = %s', + (unified_id,), + database_url=database_url, + ) + + +def _submit_background_response(scenario: Scenario, key: str, model: str, database_url: str | None = None) -> str: + response: Final = scenario.gateway.request( + "POST", "/v1/responses", {"model": model, "input": "poll me", "background": True, "store": False}, key=key + ) + assert response.status_code == 200, response.text + unified_id: Final = string_value(object_value(response.json())["id"]) + scenario.cleanups.callback(_forget_row, unified_id, database_url) + return unified_id + + +def _row_status(unified_id: str, database_url: str | None = None) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT status FROM "LiteLLM_ManagedObjectTable" WHERE unified_object_id = %s', + (unified_id,), + database_url=database_url, + ) + + +def _await_status(unified_id: str, status: str, seconds: float, database_url: str | None = None) -> None: + assert eventually( + lambda: _row_status(unified_id, database_url), lambda rows: rows == [{"status": status}], seconds=seconds + ) == [{"status": status}] + + +def _observed_requests(gateway: Gateway) -> tuple[dict[str, JsonValue], ...]: + with httpx.Client(base_url=gateway.upstream_url, timeout=5, trust_env=False) as upstream: + return tuple(map(object_value, upstream.get("/__observations").json()["requests"])) + + +def _calls_to(requests: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> tuple[dict[str, JsonValue], ...]: + return tuple(request for request in requests if string_value(request["path"]).startswith(f"/{handle.scenario_id}/")) + + +def _upstream_log(gateway: Gateway, handle: ScenarioHandle) -> Iterator[tuple[dict[str, JsonValue], ...]]: + fresh: Final = (_calls_to(_observed_requests(gateway), handle) for _ in itertools.count()) + return itertools.accumulate(fresh, lambda seen, calls: (*seen, *calls)) + + +def _polls(log: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> tuple[dict[str, JsonValue], ...]: + poll_path: Final = f"/{handle.scenario_id}/responses/resp_{handle.scenario_id}" + return tuple(call for call in log if call["method"] == "GET" and call["path"] == poll_path) + + +def _await_polls(gateway: Gateway, handle: ScenarioHandle, at_least: int, seconds: float) -> None: + log: Final = _upstream_log(gateway, handle) + polls: Final = _polls( + eventually(lambda: next(log), lambda seen: len(_polls(seen, handle)) >= at_least, seconds), handle + ) + assert len(polls) >= at_least, polls + + +def _submissions(log: Sequence[dict[str, JsonValue]], handle: ScenarioHandle) -> tuple[dict[str, JsonValue], ...]: + return tuple( + call for call in log if call["method"] == "POST" and call["path"] == f"/{handle.scenario_id}/responses" + ) + + +@pytest.mark.timeout(240) +@pytest.mark.parametrize("submission_status", ["queued", "in_progress"]) +def test_a_response_gone_at_the_provider_is_retired_from_polling(gateway: Gateway, submission_status: str) -> None: + with gateway.scenario() as scenario: + key: Final = scenario.key() + model, handle = _scripted_deployment(scenario, "gone", _gone_routes(submission_status)) + log: Final = _upstream_log(gateway, handle) + unified_id: Final = _submit_background_response(scenario, key, model) + _await_status(unified_id, "stale_expired", ONE_POLL_SECONDS) + seen: Final = next(log) + assert [call["body"] for call in _submissions(seen, handle)] == [ + {"model": "gpt-4o-mini", "input": "poll me", "background": True, "store": False} + ] + assert len(_polls(seen, handle)) >= 1, seen + caller_view: Final = gateway.request("GET", f"/v1/responses/{unified_id}", key=key) + assert caller_view.status_code == 404, caller_view.text + assert f"Response with id 'resp_{handle.scenario_id}' not found." in caller_view.text + first_clock, _ = _scripted_deployment(scenario, "clock1", _gone_routes()) + _await_status(_submit_background_response(scenario, key, first_clock), "stale_expired", ONE_POLL_SECONDS) + polls_after_first_clock: Final = len(_polls(next(log), handle)) + second_clock, _ = _scripted_deployment(scenario, "clock2", _gone_routes()) + _await_status(_submit_background_response(scenario, key, second_clock), "stale_expired", ONE_POLL_SECONDS) + assert len(_polls(next(log), handle)) == polls_after_first_clock + assert _row_status(unified_id) == [{"status": "stale_expired"}] + + +@pytest.mark.timeout(180) +def test_a_404_that_does_not_name_the_response_keeps_the_row_queued_for_retry(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model, handle = _scripted_deployment(scenario, "vague404", _routes("queued", _provider_404("Not found."))) + unified_id: Final = _submit_background_response(scenario, scenario.key(), model) + _await_polls(gateway, handle, 2, TWO_POLLS_SECONDS) + assert _row_status(unified_id) == [{"status": "queued"}] + + +@pytest.mark.timeout(180) +def test_a_provider_error_other_than_404_keeps_the_row_queued_for_retry(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + outage: Final = _json({"error": {"message": "The server had an error.", "type": "server_error"}}, 500) + model, handle = _scripted_deployment(scenario, "outage", _routes("queued", outage)) + unified_id: Final = _submit_background_response(scenario, scenario.key(), model) + _await_polls(gateway, handle, 2, TWO_POLLS_SECONDS) + assert _row_status(unified_id) == [{"status": "queued"}] + + +def _polled_provider_id(row: dict[str, JsonValue]) -> str: + return ResponsesAPIRequestUtils.decode_responses_api_response_id(string_value(row["request_id"]))["response_id"] + + +def _poll_spend_rows(handle: ScenarioHandle) -> list[dict[str, JsonValue]]: + billed: Final = read_rows( + 'SELECT request_id, call_type, status, prompt_tokens, completion_tokens, spend FROM "LiteLLM_SpendLogs" ' + "WHERE metadata->>'internal_call_origin' = %s", + (BACKGROUND_RESPONSE_COST_POLL_CALL_ORIGIN,), + ) + return [ + {column: value for column, value in row.items() if column != "request_id"} + for row in billed + if _polled_provider_id(row) == f"resp_{handle.scenario_id}" + ] + + +@pytest.mark.timeout(180) +def test_a_completed_response_is_marked_completed_and_its_poll_is_billed(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + model, handle = _scripted_deployment(scenario, "done", _routes("queued", _json(_response_body("completed")))) + unified_id: Final = _submit_background_response(scenario, scenario.key(), model) + _await_status(unified_id, "completed", ONE_POLL_SECONDS) + spend_rows: Final = eventually(lambda: _poll_spend_rows(handle), lambda rows: len(rows) >= 1, seconds=70) + assert spend_rows[0] == { + "call_type": "aget_responses", + "status": "success", + "prompt_tokens": INPUT_TOKENS, + "completion_tokens": OUTPUT_TOKENS, + "spend": pytest.approx(INPUT_TOKENS * INPUT_COST_PER_TOKEN + OUTPUT_TOKENS * OUTPUT_COST_PER_TOKEN), + } + + +def _insert_unreadable_row(scenario: Scenario) -> str: + unified_id: Final = f"resp_unreadable-{uuid.uuid4().hex[:12]}" + write_rows( + 'INSERT INTO "LiteLLM_ManagedObjectTable" ' + '("id", "unified_object_id", "model_object_id", "file_object", "file_purpose", "status", "created_at", "updated_at") ' + "VALUES (%s, %s, %s, '[]'::jsonb, 'response', 'queued', NOW() - INTERVAL '1 hour', NOW())", + (str(uuid.uuid4()), unified_id, unified_id), + ) + scenario.cleanups.callback(_forget_row, unified_id) + return unified_id + + +@pytest.mark.timeout(180) +def test_a_row_the_poll_cannot_prepare_skips_only_that_row(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + unreadable_id: Final = _insert_unreadable_row(scenario) + model, _ = _scripted_deployment(scenario, "gone", _gone_routes()) + unified_id: Final = _submit_background_response(scenario, scenario.key(), model) + _await_status(unified_id, "stale_expired", ONE_POLL_SECONDS) + assert _row_status(unreadable_id) == [{"status": "queued"}] + + +@pytest.mark.timeout(300) +def test_responses_gone_at_the_provider_do_not_starve_a_newer_response_out_of_cost_polling(gateway: Gateway) -> None: + with gateway.scenario() as scenario: + key: Final = scenario.key() + gone_ids: Final = tuple( + _submit_background_response( + scenario, key, _scripted_deployment(scenario, f"gone{index}", _gone_routes())[0] + ) + for index in range(MAX_OBJECTS_PER_POLL_CYCLE) + ) + completable_model, completable = _scripted_deployment( + scenario, "done", _routes("queued", _json(_response_body("completed"))) + ) + completable_id: Final = _submit_background_response(scenario, key, completable_model) + _await_status(completable_id, "completed", TWO_POLLS_SECONDS) + assert [_row_status(gone_id) for gone_id in gone_ids] == [ + [{"status": "stale_expired"}] + ] * MAX_OBJECTS_PER_POLL_CYCLE + spend_rows: Final = eventually(lambda: _poll_spend_rows(completable), lambda rows: len(rows) >= 1, seconds=70) + assert spend_rows[0]["spend"] == pytest.approx( + INPUT_TOKENS * INPUT_COST_PER_TOKEN + OUTPUT_TOKENS * OUTPUT_COST_PER_TOKEN + ) + + +@pytest.mark.timeout(OWNED_PROXY_CELL_SECONDS) +def test_a_404_on_a_response_whose_deployment_left_the_router_keeps_the_row_queued_for_retry( + gateway: Gateway, tmp_path: Path +) -> None: + deployment_scenario_id: Final = _scenario_id("left") + provider_id: Final = f"resp_{deployment_scenario_id}" + env_handle: Final = register_scenario( + _scenario_id("envbase"), + RoutedResponse( + content_type="application/x-routed", + routes={f"GET /responses/{provider_id}": _provider_404(f"Response with id '{provider_id}' not found.")}, + ), + ) + try: + with scratch_database() as database_url: + overrides: Final = { + "DATABASE_URL": database_url, + "OPENAI_API_BASE": env_handle.api_base(), + "OPENAI_API_KEY": "sk-scripted-provider", + } + with owned_proxy_process( + gateway, tmp_path, overrides, remove_environment=("DATABASE_URL_READ_REPLICA",) + ) as owned: + with owned.gateway.scenario() as scenario: + outage: Final = _json( + {"error": {"message": "The server had an error.", "type": "server_error"}}, 500 + ) + model, model_id, _ = _register_deployment( + scenario, deployment_scenario_id, _routes("queued", outage) + ) + unified_id: Final = _submit_background_response(scenario, scenario.key(), model, database_url) + owned.gateway.post("/model/delete", {"id": model_id}) + log: Final = _upstream_log(gateway, env_handle) + fallback_polls: Final = eventually( + lambda: next(log), + lambda seen: len(_fallback_polls(seen, env_handle, provider_id)) >= 2, + seconds=TWO_POLLS_SECONDS, + ) + assert len(_fallback_polls(fallback_polls, env_handle, provider_id)) >= 2, fallback_polls + assert _row_status(unified_id, database_url) == [{"status": "queued"}] + finally: + delete_scenario(env_handle) + + +def _fallback_polls( + log: Sequence[dict[str, JsonValue]], env_handle: ScenarioHandle, provider_id: str +) -> tuple[dict[str, JsonValue], ...]: + poll_path: Final = f"/{env_handle.scenario_id}/responses/{provider_id}" + return tuple(call for call in log if call["method"] == "GET" and call["path"] == poll_path) diff --git a/tests/unit/proxy/common_utils/test_check_responses_cost.py b/tests/unit/proxy/common_utils/test_check_responses_cost.py index e806e9a3394..8ba3f0b000a 100644 --- a/tests/unit/proxy/common_utils/test_check_responses_cost.py +++ b/tests/unit/proxy/common_utils/test_check_responses_cost.py @@ -3,7 +3,9 @@ Unit tests for CheckResponsesCost class """ import asyncio +from collections.abc import Mapping from datetime import datetime +from typing import Final from unittest.mock import AsyncMock, MagicMock, Mock, patch import pytest @@ -12,6 +14,19 @@ from litellm.constants import MAX_OBJECTS_PER_POLL_CYCLE from litellm.types.llms.openai import ResponseAPIUsage, ResponsesAPIResponse +class _RecordingManagedObjectTable: + def __init__(self, rows: tuple[object, ...]) -> None: + self.rows: Final = rows + self.updates: tuple[tuple[Mapping[str, object], Mapping[str, object]], ...] = () + + async def find_many(self, **query: object) -> tuple[object, ...]: + return self.rows + + async def update_many(self, where: Mapping[str, object], data: Mapping[str, object]) -> int: + self.updates = (*self.updates, (where, data)) + return len(self.rows) + + class TestCheckResponsesCost: """Test suite for CheckResponsesCost class""" @@ -364,6 +379,264 @@ class TestCheckResponsesCost: # Stale cleanup still ran via _expire_stale_rows check_responses_cost_instance._expire_stale_rows.assert_called_once() + @pytest.mark.asyncio + async def test_check_responses_cost_marks_404_response_stale_expired( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router + ): + """A provider 404 on a response polled through its deployment marks the row stale_expired.""" + import litellm + from litellm.responses.utils import ResponsesAPIRequestUtils + + encoded_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", + model_id="deployment-404", + response_id="resp_upstream_404", + ) + + mock_job = MagicMock() + mock_job.unified_object_id = encoded_response_id + mock_job.created_by = "test-user" + mock_job.id = "job-404" + mock_job.file_object = {"model": "gpt-4o", "id": encoded_response_id} + + mock_llm_router.get_deployment.return_value = {"model_id": "deployment-404"} + mock_llm_router.aget_responses = AsyncMock( + side_effect=litellm.NotFoundError( + message="Response with id 'resp_upstream_404' not found.", model="gpt-5", llm_provider="openai" + ) + ) + + table = _RecordingManagedObjectTable(rows=(mock_job,)) + mock_prisma_client.db.litellm_managedobjecttable = table + + with patch("litellm.aget_responses", new_callable=AsyncMock) as mock_sdk_aget: + await check_responses_cost_instance.check_responses_cost() + + mock_sdk_aget.assert_not_called() + mock_llm_router.get_deployment.assert_called_once_with(model_id="deployment-404") + assert mock_llm_router.aget_responses.call_args.kwargs["response_id"] == encoded_response_id + assert table.updates == (({"id": {"in": ["job-404"]}}, {"status": "stale_expired"}),) + + @pytest.mark.asyncio + async def test_check_responses_cost_marks_mapped_provider_404_stale_expired( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router + ): + """The provider's own 404 body, mapped the way the GET path maps it, still expires the row.""" + import json + + import openai + + import litellm + from litellm.llms.openai.common_utils import OpenAIError + from litellm.responses.utils import ResponsesAPIRequestUtils + + encoded_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", + model_id="deployment-404-mapped", + response_id="resp_upstream_404_mapped", + ) + provider_body = json.dumps( + { + "error": { + "message": "Response with id 'resp_upstream_404_mapped' not found.", + "type": "invalid_request_error", + "param": None, + "code": None, + } + } + ) + with pytest.raises(openai.APIStatusError) as mapped: + raise litellm.exception_type( + model="gpt-5.5", + custom_llm_provider="openai", + original_exception=OpenAIError(message=provider_body, status_code=404), + completion_kwargs={}, + extra_kwargs={}, + ) + assert mapped.value.status_code == 404 + + mock_job = MagicMock() + mock_job.unified_object_id = encoded_response_id + mock_job.created_by = "test-user" + mock_job.id = "job-404-mapped" + mock_job.file_object = {"model": "gpt-5.5", "id": encoded_response_id} + + mock_llm_router.get_deployment.return_value = {"model_id": "deployment-404-mapped"} + mock_llm_router.aget_responses = AsyncMock(side_effect=mapped.value) + + table = _RecordingManagedObjectTable(rows=(mock_job,)) + mock_prisma_client.db.litellm_managedobjecttable = table + + with patch("litellm.aget_responses", new_callable=AsyncMock) as mock_sdk_aget: + await check_responses_cost_instance.check_responses_cost() + + mock_sdk_aget.assert_not_called() + assert table.updates == (({"id": {"in": ["job-404-mapped"]}}, {"status": "stale_expired"}),) + + @pytest.mark.asyncio + async def test_check_responses_cost_404_without_router_deployment_keeps_row_for_retry( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router + ): + """A 404 while polling without a resolved deployment skips the row for retry.""" + import litellm + from litellm.responses.utils import ResponsesAPIRequestUtils + + encoded_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", + model_id="deployment-gone", + response_id="resp_upstream_gone", + ) + + mock_job = MagicMock() + mock_job.unified_object_id = encoded_response_id + mock_job.created_by = "test-user" + mock_job.id = "job-404-fallback" + mock_job.file_object = {"model": "gpt-4o", "id": encoded_response_id} + + mock_llm_router.get_deployment.return_value = None + + table = _RecordingManagedObjectTable(rows=(mock_job,)) + mock_prisma_client.db.litellm_managedobjecttable = table + + with patch( + "litellm.aget_responses", + new_callable=AsyncMock, + side_effect=litellm.NotFoundError( + message="Response not found", model="gpt-5", llm_provider="openai" + ), + ) as mock_sdk_aget: + await check_responses_cost_instance.check_responses_cost() + + mock_sdk_aget.assert_awaited_once() + mock_llm_router.aget_responses.assert_not_called() + assert table.updates == () + + @pytest.mark.asyncio + async def test_check_responses_cost_404_not_naming_the_response_keeps_row_for_retry( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router + ): + """A 404 that does not name the response (a gateway or a reconfigured deployment) is retried, not expired.""" + import litellm + from litellm.responses.utils import ResponsesAPIRequestUtils + + encoded_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="azure", + model_id="deployment-renamed", + response_id="resp_upstream_still_there", + ) + + mock_job = MagicMock() + mock_job.unified_object_id = encoded_response_id + mock_job.created_by = "test-user" + mock_job.id = "job-404-other" + mock_job.file_object = {"model": "azure-gpt-5", "id": encoded_response_id} + + mock_llm_router.get_deployment.return_value = {"model_id": "deployment-renamed"} + mock_llm_router.aget_responses = AsyncMock( + side_effect=litellm.NotFoundError( + message="Resource not found", model="gpt-5", llm_provider="azure" + ) + ) + + table = _RecordingManagedObjectTable(rows=(mock_job,)) + mock_prisma_client.db.litellm_managedobjecttable = table + + with patch("litellm.aget_responses", new_callable=AsyncMock) as mock_sdk_aget: + await check_responses_cost_instance.check_responses_cost() + + mock_sdk_aget.assert_not_called() + assert mock_llm_router.aget_responses.call_args.kwargs["response_id"] == encoded_response_id + assert table.updates == () + + @pytest.mark.asyncio + async def test_check_responses_cost_deployment_lookup_error_skips_only_that_job( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router + ): + """A deployment lookup that raises skips its own job and the cycle still records the next job.""" + from litellm.responses.utils import ResponsesAPIRequestUtils + + broken_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="deployment-broken", response_id="resp_upstream_broken" + ) + healthy_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", model_id="deployment-healthy", response_id="resp_upstream_healthy" + ) + + broken_job = MagicMock() + broken_job.unified_object_id = broken_response_id + broken_job.created_by = "test-user" + broken_job.id = "job-broken" + broken_job.file_object = {"model": "gpt-5.5", "id": broken_response_id} + + healthy_job = MagicMock() + healthy_job.unified_object_id = healthy_response_id + healthy_job.created_by = "test-user" + healthy_job.id = "job-healthy" + healthy_job.file_object = {"model": "gpt-5.5", "id": healthy_response_id} + + mock_llm_router.get_deployment.side_effect = [ + Exception("Model invalid format - "), + {"model_id": "deployment-healthy"}, + ] + mock_llm_router.aget_responses = AsyncMock( + return_value=ResponsesAPIResponse( + id=healthy_response_id, + object="response", + status="completed", + created_at=int(datetime.now().timestamp()), + output=[], + usage=ResponseAPIUsage(input_tokens=100, output_tokens=50, total_tokens=150), + ) + ) + + table = _RecordingManagedObjectTable(rows=(broken_job, healthy_job)) + mock_prisma_client.db.litellm_managedobjecttable = table + + with patch("litellm.aget_responses", new_callable=AsyncMock) as mock_sdk_aget: + await check_responses_cost_instance.check_responses_cost() + + mock_sdk_aget.assert_not_called() + mock_llm_router.aget_responses.assert_awaited_once() + assert mock_llm_router.aget_responses.call_args.kwargs["response_id"] == healthy_response_id + assert table.updates == (({"id": {"in": ["job-healthy"]}}, {"status": "completed"}),) + + @pytest.mark.asyncio + async def test_check_responses_cost_non_404_error_keeps_row_for_retry( + self, check_responses_cost_instance, mock_prisma_client, mock_llm_router + ): + """A non-404 provider error through a resolved deployment skips the job so it is retried next cycle.""" + import litellm + from litellm.responses.utils import ResponsesAPIRequestUtils + + encoded_response_id = ResponsesAPIRequestUtils._build_responses_api_response_id( + custom_llm_provider="openai", + model_id="deployment-500", + response_id="resp_upstream_500", + ) + + mock_job = MagicMock() + mock_job.unified_object_id = encoded_response_id + mock_job.created_by = "test-user" + mock_job.id = "job-500" + mock_job.file_object = {"model": "gpt-4o", "id": encoded_response_id} + + mock_llm_router.get_deployment.return_value = {"model_id": "deployment-500"} + mock_llm_router.aget_responses = AsyncMock( + side_effect=litellm.InternalServerError( + message="boom", model="gpt-5", llm_provider="openai" + ) + ) + + table = _RecordingManagedObjectTable(rows=(mock_job,)) + mock_prisma_client.db.litellm_managedobjecttable = table + + with patch("litellm.aget_responses", new_callable=AsyncMock) as mock_sdk_aget: + await check_responses_cost_instance.check_responses_cost() + + mock_sdk_aget.assert_not_called() + mock_llm_router.aget_responses.assert_awaited_once() + assert table.updates == () + @pytest.mark.asyncio async def test_check_responses_cost_multiple_jobs( self, check_responses_cost_instance, mock_prisma_client From 26b9bf2e6cccaa9ca5d3253bc4b51784367d7965 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:23:29 -0700 Subject: [PATCH 49/65] fix(bedrock): take context, output limits and EOL dates from the Bedrock model cards (#45409) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 338 ++++++++++-------- model_prices_and_context_window.json | 338 ++++++++++-------- 2 files changed, 396 insertions(+), 280 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 97760a4f0b9..e67fb7b9f31 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -13078,8 +13078,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13089,9 +13089,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13105,16 +13106,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13125,13 +13126,15 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -13139,8 +13142,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13149,7 +13152,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13168,13 +13171,15 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" }, "bedrock/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -13182,11 +13187,11 @@ "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, - "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13218,8 +13223,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13229,9 +13234,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13245,16 +13251,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13265,13 +13271,15 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.94e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -13279,8 +13287,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13289,7 +13297,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13310,11 +13318,11 @@ "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13328,8 +13336,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13339,9 +13347,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13355,16 +13364,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13378,8 +13387,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13388,7 +13397,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13428,8 +13437,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13439,9 +13448,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13455,16 +13465,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13478,8 +13488,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13488,7 +13498,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.01635, @@ -13578,6 +13588,7 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13591,16 +13602,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13645,6 +13656,7 @@ "output_cost_per_token": 6.5e-07 }, "bedrock/eu-west-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13658,16 +13670,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13712,6 +13724,7 @@ "output_cost_per_token": 7.8e-07 }, "bedrock/eu-west-2/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 4.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13725,16 +13738,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13809,6 +13822,7 @@ "supports_tool_choice": true }, "bedrock/eu-south-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13822,16 +13836,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13897,8 +13911,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13908,9 +13922,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13924,16 +13939,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13944,13 +13959,15 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -13958,8 +13975,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13968,7 +13985,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14127,8 +14144,8 @@ "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "supports_function_calling": true, @@ -14138,9 +14155,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/us-east-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -14154,13 +14172,13 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", @@ -14170,16 +14188,18 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -14187,8 +14207,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -14197,7 +14217,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -14219,8 +14239,8 @@ "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "supports_function_calling": true, @@ -14230,9 +14250,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/us-east-2/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -14246,16 +14267,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14266,13 +14287,15 @@ "output_cost_per_token": 1.2e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -14280,8 +14303,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -14290,7 +14313,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -14757,8 +14780,8 @@ "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "supports_function_calling": true, @@ -14768,9 +14791,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/us-west-2/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -14784,13 +14808,13 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", @@ -14800,16 +14824,18 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -14817,8 +14843,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -14827,7 +14853,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -22487,14 +22513,15 @@ "supports_tool_choice": true }, "deepseek.v3-v1:0": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 5.8e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 163840, - "max_output_tokens": 81920, - "max_tokens": 81920, + "max_input_tokens": 128000, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.68e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -32129,6 +32156,7 @@ "output_cost_per_token": 2e-06 }, "google.gemma-3-12b-it": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -32136,7 +32164,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.9e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-12b-it.html", "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, @@ -32144,6 +32172,7 @@ "supports_vision": true }, "google.gemma-3-27b-it": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -32151,7 +32180,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 3.8e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-27b-pt.html", "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, @@ -32159,6 +32188,7 @@ "supports_vision": true }, "google.gemma-3-4b-it": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -32166,7 +32196,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-08, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-4b-it.html", "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, @@ -38434,6 +38464,7 @@ "supports_tool_choice": true }, "minimax.minimax-m2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, @@ -38441,7 +38472,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2.html", "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, @@ -38450,6 +38481,7 @@ "supports_vision": false }, "minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, @@ -38460,7 +38492,7 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false @@ -38607,6 +38639,7 @@ "max_output_tokens": 128000 }, "mistral.devstral-2-123b": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -38617,7 +38650,7 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-devstral-2-123b.html", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false @@ -38761,27 +38794,29 @@ "supports_tool_choice": true }, "mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-08, "litellm_provider": "bedrock_converse", - "max_input_tokens": 128000, + "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-voxtral-mini-3b-2507.html", "supports_audio_input": true, "supports_system_messages": true, "supports_native_structured_output": true }, "mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 1e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 128000, + "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-voxtral-small-24b-2507.html", "supports_audio_input": true, "supports_system_messages": true, "supports_native_structured_output": true @@ -39590,6 +39625,7 @@ "supports_vision": true }, "moonshot.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -39597,7 +39633,7 @@ "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, @@ -40708,6 +40744,7 @@ "source": "https://tokenfactory.nebius.com/models/catalog/embedding/Qwen%2FQwen3-Embedding-8B" }, "nvidia.nemotron-nano-12b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -40715,7 +40752,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, @@ -40723,6 +40760,7 @@ "supports_vision": true }, "nvidia.nemotron-nano-9b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -40730,7 +40768,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.3e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, @@ -42098,11 +42136,11 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42116,11 +42154,11 @@ "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -47781,6 +47819,7 @@ "supports_vision": false }, "us-gov.nvidia.nemotron-nano-12b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -47788,6 +47827,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -47795,6 +47835,7 @@ "supports_response_schema": true }, "us-gov.nvidia.nemotron-nano-9b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.2e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -47802,6 +47843,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -47829,10 +47871,11 @@ "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -47846,10 +47889,11 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -52898,6 +52942,7 @@ "supports_web_search": true }, "zai.glm-4.7": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, @@ -52909,7 +52954,7 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7.html", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false @@ -52933,6 +52978,7 @@ "supports_vision": false }, "zai.glm-4.7-flash": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, @@ -52944,7 +52990,7 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7-flash.html", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false @@ -65587,6 +65633,7 @@ "supports_vision": false }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -65594,6 +65641,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -65601,6 +65649,7 @@ "supports_response_schema": true }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.2e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -65608,6 +65657,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -65635,10 +65685,11 @@ "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -65652,10 +65703,11 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -65849,6 +65901,7 @@ "supports_vision": false }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -65856,6 +65909,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -65863,6 +65917,7 @@ "supports_response_schema": true }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.2e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -65870,6 +65925,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -65897,10 +65953,11 @@ "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -65914,10 +65971,11 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions" ], diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 97760a4f0b9..e67fb7b9f31 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -13078,8 +13078,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13089,9 +13089,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/ap-northeast-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13105,16 +13106,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13125,13 +13126,15 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -13139,8 +13142,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13149,7 +13152,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13168,13 +13171,15 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-coder-next.html" }, "bedrock/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -13182,11 +13187,11 @@ "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, - "source": "https://platform.moonshot.ai/docs/guide/kimi-k2-5-quickstart", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13218,8 +13223,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13229,9 +13234,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/ap-south-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13245,16 +13251,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13265,13 +13271,15 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.94e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -13279,8 +13287,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13289,7 +13297,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13310,11 +13318,11 @@ "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://b0.p.awsstatic.com/pricing/2.0/meteredUnitMaps/bedrock/USD/current/bedrock.json", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13328,8 +13336,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13339,9 +13347,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/ap-southeast-3/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13355,16 +13364,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13378,8 +13387,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13388,7 +13397,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -13428,8 +13437,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13439,9 +13448,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/eu-north-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13455,16 +13465,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13478,8 +13488,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13488,7 +13498,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/eu-central-1/1-month-commitment/anthropic.claude-instant-v1": { "cost_per_second": 0.01635, @@ -13578,6 +13588,7 @@ "supports_tool_choice": true }, "bedrock/eu-central-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13591,16 +13602,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13645,6 +13656,7 @@ "output_cost_per_token": 6.5e-07 }, "bedrock/eu-west-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13658,16 +13670,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13712,6 +13724,7 @@ "output_cost_per_token": 7.8e-07 }, "bedrock/eu-west-2/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 4.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13725,16 +13738,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13809,6 +13822,7 @@ "supports_tool_choice": true }, "bedrock/eu-south-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13822,16 +13836,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13897,8 +13911,8 @@ "input_cost_per_token": 7.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, "supports_function_calling": true, @@ -13908,9 +13922,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/sa-east-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -13924,16 +13939,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -13944,13 +13959,15 @@ "output_cost_per_token": 1.44e-06 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -13958,8 +13975,8 @@ "input_cost_per_token": 7.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, "supports_function_calling": true, @@ -13968,7 +13985,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, @@ -14127,8 +14144,8 @@ "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "supports_function_calling": true, @@ -14138,9 +14155,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/us-east-1/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -14154,13 +14172,13 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", @@ -14170,16 +14188,18 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" }, "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -14187,8 +14207,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -14197,7 +14217,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -14219,8 +14239,8 @@ "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "supports_function_calling": true, @@ -14230,9 +14250,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/us-east-2/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -14246,16 +14267,16 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html", "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -14266,13 +14287,15 @@ "output_cost_per_token": 1.2e-06 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -14280,8 +14303,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -14290,7 +14313,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -14757,8 +14780,8 @@ "input_cost_per_token": 6.2e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, - "max_output_tokens": 163840, - "max_tokens": 163840, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, "supports_function_calling": true, @@ -14768,9 +14791,10 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-2.html" }, "bedrock/us-west-2/minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, @@ -14784,13 +14808,13 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html" }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, "output_cost_per_token": 1.2e-06, "litellm_provider": "bedrock", - "max_input_tokens": 1000000, + "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", @@ -14800,16 +14824,18 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-5.html" }, "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, @@ -14817,8 +14843,8 @@ "input_cost_per_token": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, - "max_output_tokens": 262144, - "max_tokens": 262144, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, "supports_function_calling": true, @@ -14827,7 +14853,7 @@ "supports_vision": true, "supports_audio_input": false, "supports_response_schema": true, - "source": "https://aws.amazon.com/bedrock/pricing/" + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-5.html" }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, @@ -22487,14 +22513,15 @@ "supports_tool_choice": true }, "deepseek.v3-v1:0": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 5.8e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 163840, - "max_output_tokens": 81920, - "max_tokens": 81920, + "max_input_tokens": 128000, + "max_output_tokens": 8000, + "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.68e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html", "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -32129,6 +32156,7 @@ "output_cost_per_token": 2e-06 }, "google.gemma-3-12b-it": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 9e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -32136,7 +32164,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.9e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-12b-it.html", "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, @@ -32144,6 +32172,7 @@ "supports_vision": true }, "google.gemma-3-27b-it": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -32151,7 +32180,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 3.8e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-27b-pt.html", "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, @@ -32159,6 +32188,7 @@ "supports_vision": true }, "google.gemma-3-4b-it": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -32166,7 +32196,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-08, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-4b-it.html", "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, @@ -38434,6 +38464,7 @@ "supports_tool_choice": true }, "minimax.minimax-m2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, @@ -38441,7 +38472,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2.html", "supports_audio_input": false, "supports_function_calling": true, "supports_system_messages": true, @@ -38450,6 +38481,7 @@ "supports_vision": false }, "minimax.minimax-m2.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, @@ -38460,7 +38492,7 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2-1.html", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false @@ -38607,6 +38639,7 @@ "max_output_tokens": 128000 }, "mistral.devstral-2-123b": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -38617,7 +38650,7 @@ "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-devstral-2-123b.html", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false @@ -38761,27 +38794,29 @@ "supports_tool_choice": true }, "mistral.voxtral-mini-3b-2507": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-08, "litellm_provider": "bedrock_converse", - "max_input_tokens": 128000, + "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-voxtral-mini-3b-2507.html", "supports_audio_input": true, "supports_system_messages": true, "supports_native_structured_output": true }, "mistral.voxtral-small-24b-2507": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 1e-07, "litellm_provider": "bedrock_converse", - "max_input_tokens": 128000, + "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-voxtral-small-24b-2507.html", "supports_audio_input": true, "supports_system_messages": true, "supports_native_structured_output": true @@ -39590,6 +39625,7 @@ "supports_vision": true }, "moonshot.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, @@ -39597,7 +39633,7 @@ "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_audio_input": false, "supports_function_calling": true, "supports_reasoning": true, @@ -40708,6 +40744,7 @@ "source": "https://tokenfactory.nebius.com/models/catalog/embedding/Qwen%2FQwen3-Embedding-8B" }, "nvidia.nemotron-nano-12b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -40715,7 +40752,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, @@ -40723,6 +40760,7 @@ "supports_vision": true }, "nvidia.nemotron-nano-9b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -40730,7 +40768,7 @@ "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.3e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_audio_input": false, "supports_function_calling": true, "supports_response_schema": true, @@ -42098,11 +42136,11 @@ "input_cost_per_token": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 6e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -42116,11 +42154,11 @@ "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3e-07, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supports_function_calling": true, "supports_reasoning": true, "supports_response_schema": true, @@ -47781,6 +47819,7 @@ "supports_vision": false }, "us-gov.nvidia.nemotron-nano-12b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -47788,6 +47827,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -47795,6 +47835,7 @@ "supports_response_schema": true }, "us-gov.nvidia.nemotron-nano-9b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.2e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, @@ -47802,6 +47843,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -47829,10 +47871,11 @@ "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -47846,10 +47889,11 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -52898,6 +52942,7 @@ "supports_web_search": true }, "zai.glm-4.7": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, @@ -52909,7 +52954,7 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7.html", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false @@ -52933,6 +52978,7 @@ "supports_vision": false }, "zai.glm-4.7-flash": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, @@ -52944,7 +52990,7 @@ "supports_reasoning": true, "supports_system_messages": true, "supports_tool_choice": true, - "source": "https://aws.amazon.com/bedrock/pricing/", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-zai-glm-4-7-flash.html", "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false @@ -65587,6 +65633,7 @@ "supports_vision": false }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-12b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -65594,6 +65641,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -65601,6 +65649,7 @@ "supports_response_schema": true }, "bedrock/us-gov-west-1/nvidia.nemotron-nano-9b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.2e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -65608,6 +65657,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -65635,10 +65685,11 @@ "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -65652,10 +65703,11 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -65849,6 +65901,7 @@ "supports_vision": false }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-12b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 2.4e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -65856,6 +65909,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_system_messages": true, "supports_vision": true, "supports_audio_input": false, @@ -65863,6 +65917,7 @@ "supports_response_schema": true }, "bedrock/us-gov-east-1/nvidia.nemotron-nano-9b-v2": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 7.2e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, @@ -65870,6 +65925,7 @@ "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 2.76e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_system_messages": true, "supports_audio_input": false, "supports_function_calling": true, @@ -65897,10 +65953,11 @@ "input_cost_per_token": 8.4e-08, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 3.6e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -65914,10 +65971,11 @@ "input_cost_per_token": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 128000, - "max_output_tokens": 128000, - "max_tokens": 128000, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 7.2e-07, + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions" ], From d8d8364ec7f5ac44db2eed0878fce6654d31bf2a Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:27:23 -0700 Subject: [PATCH 50/65] chore(cost-map): add openai gpt-6.1-sol ultrafast tier prices from the pricing page (#45408) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- litellm/model_prices_and_context_window_backup.json | 8 ++++++++ model_prices_and_context_window.json | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index e67fb7b9f31..cba7e934ae9 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -80242,25 +80242,31 @@ "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": 3e-05, "cache_creation_input_token_cost_batches": 1.25e-06, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, + "cache_creation_input_token_cost_ultrafast": 1.5e-05, "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_above_272k_tokens": 2e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 1e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": 1.2e-06, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "cache_read_input_token_cost_priority": 2e-07, + "cache_read_input_token_cost_ultrafast": 6e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "input_cost_per_token_above_272k_tokens_ultrafast": 2.4e-05, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, + "input_cost_per_token_ultrafast": 1.2e-05, "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -80271,9 +80277,11 @@ "output_cost_per_token_above_272k_tokens_batches": 7.5e-06, "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, "output_cost_per_token_above_272k_tokens_priority": 3e-05, + "output_cost_per_token_above_272k_tokens_ultrafast": 9e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05, + "output_cost_per_token_ultrafast": 6e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index e67fb7b9f31..cba7e934ae9 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -80242,25 +80242,31 @@ "cache_creation_input_token_cost_above_272k_tokens_batches": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_flex": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens_priority": 1e-05, + "cache_creation_input_token_cost_above_272k_tokens_ultrafast": 3e-05, "cache_creation_input_token_cost_batches": 1.25e-06, "cache_creation_input_token_cost_flex": 1.25e-06, "cache_creation_input_token_cost_priority": 5e-06, + "cache_creation_input_token_cost_ultrafast": 1.5e-05, "cache_read_input_token_cost": 1e-07, "cache_read_input_token_cost_above_272k_tokens": 2e-07, "cache_read_input_token_cost_above_272k_tokens_batches": 1e-07, "cache_read_input_token_cost_above_272k_tokens_flex": 1e-07, "cache_read_input_token_cost_above_272k_tokens_priority": 4e-07, + "cache_read_input_token_cost_above_272k_tokens_ultrafast": 1.2e-06, "cache_read_input_token_cost_batches": 5e-08, "cache_read_input_token_cost_flex": 5e-08, "cache_read_input_token_cost_priority": 2e-07, + "cache_read_input_token_cost_ultrafast": 6e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "input_cost_per_token_above_272k_tokens_batches": 2e-06, "input_cost_per_token_above_272k_tokens_flex": 2e-06, "input_cost_per_token_above_272k_tokens_priority": 8e-06, + "input_cost_per_token_above_272k_tokens_ultrafast": 2.4e-05, "input_cost_per_token_batches": 1e-06, "input_cost_per_token_flex": 1e-06, "input_cost_per_token_priority": 4e-06, + "input_cost_per_token_ultrafast": 1.2e-05, "litellm_provider": "openai", "max_input_tokens": 922000, "max_output_tokens": 128000, @@ -80271,9 +80277,11 @@ "output_cost_per_token_above_272k_tokens_batches": 7.5e-06, "output_cost_per_token_above_272k_tokens_flex": 7.5e-06, "output_cost_per_token_above_272k_tokens_priority": 3e-05, + "output_cost_per_token_above_272k_tokens_ultrafast": 9e-05, "output_cost_per_token_batches": 5e-06, "output_cost_per_token_flex": 5e-06, "output_cost_per_token_priority": 2e-05, + "output_cost_per_token_ultrafast": 6e-05, "regional_processing_uplift_multiplier_eu": 1.1, "regional_processing_uplift_multiplier_us": 1.1, "search_context_cost_per_query": { From b0e0a305f4eca8435605cfcbbcc92ccfbe9768b3 Mon Sep 17 00:00:00 2001 From: "berriai-litellm-provider-info-sync[bot]" <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 11:33:28 -0700 Subject: [PATCH 51/65] fix(bedrock_mantle): take gpt-oss output limits and EOL dates from the Bedrock model cards (#45417) Price-Sync: litellm-providers Co-authored-by: berriai-litellm-provider-info-sync[bot] <328147090+berriai-litellm-provider-info-sync[bot]@users.noreply.github.com> --- ...odel_prices_and_context_window_backup.json | 42 ++++++++++++------- model_prices_and_context_window.json | 42 ++++++++++++------- 2 files changed, 52 insertions(+), 32 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cba7e934ae9..5b9df8cafde 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -58292,9 +58292,10 @@ "output_cost_per_token": 6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -58310,9 +58311,10 @@ "output_cost_per_token": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -58328,9 +58330,10 @@ "output_cost_per_token": 6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-safeguard-120b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -58344,9 +58347,10 @@ "output_cost_per_token": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-safeguard-20b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -66357,9 +66361,10 @@ "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -66375,9 +66380,10 @@ "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -66514,9 +66520,10 @@ "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -66532,9 +66539,10 @@ "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -66546,6 +66554,7 @@ "supports_tool_choice": true }, "bedrock_mantle/deepseek.v3.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 5.8e-07, "output_cost_per_token": 1.68e-06, "litellm_provider": "bedrock_mantle", @@ -66561,6 +66570,7 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html" }, "bedrock_mantle/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "output_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_mantle", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cba7e934ae9..5b9df8cafde 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -58292,9 +58292,10 @@ "output_cost_per_token": 6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -58310,9 +58311,10 @@ "output_cost_per_token": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -58328,9 +58330,10 @@ "output_cost_per_token": 6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-safeguard-120b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -58344,9 +58347,10 @@ "output_cost_per_token": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 65536, - "max_tokens": 65536, + "max_output_tokens": 16000, + "max_tokens": 16000, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-safeguard-20b.html", "supported_endpoints": [ "/v1/chat/completions" ], @@ -66357,9 +66361,10 @@ "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -66375,9 +66380,10 @@ "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -66514,9 +66520,10 @@ "output_cost_per_token": 3.6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-20b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -66532,9 +66539,10 @@ "output_cost_per_token": 7.2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, - "max_output_tokens": 32768, - "max_tokens": 32768, + "max_output_tokens": 16384, + "max_tokens": 16384, "mode": "chat", + "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-openai-gpt-oss-120b.html", "supported_endpoints": [ "/v1/chat/completions", "/v1/responses" @@ -66546,6 +66554,7 @@ "supports_tool_choice": true }, "bedrock_mantle/deepseek.v3.1": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 5.8e-07, "output_cost_per_token": 1.68e-06, "litellm_provider": "bedrock_mantle", @@ -66561,6 +66570,7 @@ "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-deepseek-deepseek-v3-1.html" }, "bedrock_mantle/moonshotai.kimi-k2-thinking": { + "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, "output_cost_per_token": 2.5e-06, "litellm_provider": "bedrock_mantle", From e843cb7aca68a973bc8741156cb1a52d6659025d Mon Sep 17 00:00:00 2001 From: nate-berri Date: Thu, 8 Oct 2026 11:51:58 -0700 Subject: [PATCH 52/65] fix(rust): path-qualify attribute aliases so rust-analyzer resolves them (#45416) rust-analyzer cannot resolve the bare alias names emitted by macro_rules_attribute::apply, so every aliased type was invisible to it (no go-to-definition or find-references). Referencing the aliases through crate:: resolves them via the re-export that attribute_alias! generates. Co-authored-by: Nate Armstrong --- .../src/formats/audio_transcription.rs | 2 +- .../crates/llms-types/src/formats/batches.rs | 6 +- .../src/formats/chat_completions.rs | 28 ++-- .../src/formats/chat_completions/content.rs | 22 +-- .../src/formats/messages/content.rs | 126 +++++++++--------- .../src/formats/messages/metadata.rs | 42 +++--- .../src/formats/messages/request.rs | 38 +++--- .../src/formats/messages/response.rs | 2 +- .../src/formats/messages/streaming.rs | 14 +- .../llms-types/src/formats/messages/tools.rs | 82 ++++++------ .../llms-types/src/formats/messages/usage.rs | 12 +- .../crates/llms-types/src/formats/ocr.rs | 16 +-- .../src/formats/responses/output.rs | 106 +++++++-------- .../src/formats/responses/response.rs | 2 +- .../formats/responses/streaming_websocket.rs | 8 +- litellm-rust/crates/llms-types/src/headers.rs | 4 +- .../crates/llms-types/src/json_schema.rs | 8 +- .../llms-types/src/providers/minimax.rs | 10 +- .../crates/llms-types/src/recognized.rs | 2 +- .../crates/traces-clickhouse/src/query.rs | 30 ++--- .../traces-clickhouse/src/query/lens.rs | 42 +++--- .../crates/traces-clickhouse/src/table.rs | 2 +- .../crates/traces/src/normalize/mod.rs | 2 +- litellm-rust/crates/traces/src/query/guide.rs | 2 +- litellm-rust/crates/traces/src/query/named.rs | 2 +- .../crates/traces/src/query_access.rs | 2 +- litellm-rust/crates/traces/src/request.rs | 10 +- litellm-rust/crates/traces/src/response.rs | 2 +- litellm-rust/crates/traces/src/tenant.rs | 2 +- litellm-rust/crates/traces/src/ui.rs | 10 +- litellm-rust/crates/traces/src/view.rs | 22 +-- 31 files changed, 329 insertions(+), 329 deletions(-) diff --git a/litellm-rust/crates/llms-types/src/formats/audio_transcription.rs b/litellm-rust/crates/llms-types/src/formats/audio_transcription.rs index e00ecb0b5fb..624eb7be4e4 100644 --- a/litellm-rust/crates/llms-types/src/formats/audio_transcription.rs +++ b/litellm-rust/crates/llms-types/src/formats/audio_transcription.rs @@ -1,6 +1,6 @@ use serde_json::Value; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct AudioTranscriptionResponseData { pub text: String, } diff --git a/litellm-rust/crates/llms-types/src/formats/batches.rs b/litellm-rust/crates/llms-types/src/formats/batches.rs index 9749b042a36..0390be04dec 100644 --- a/litellm-rust/crates/llms-types/src/formats/batches.rs +++ b/litellm-rust/crates/llms-types/src/formats/batches.rs @@ -1,4 +1,4 @@ -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum BatchStatus { @@ -7,7 +7,7 @@ pub enum BatchStatus { Completed, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Eq)] pub struct BatchRequestCounts { pub total: u64, @@ -15,7 +15,7 @@ pub struct BatchRequestCounts { pub failed: u64, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Eq)] pub struct BatchResponse { pub id: String, diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions.rs index b6bcd614ab4..ca4ffa3c718 100644 --- a/litellm-rust/crates/llms-types/src/formats/chat_completions.rs +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions.rs @@ -8,7 +8,7 @@ pub use content::{ }; /// Reasoning effort level accepted or applied by the model. -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq, IntoStaticStr)] #[serde(rename_all = "snake_case")] #[strum(serialize_all = "snake_case")] @@ -44,14 +44,14 @@ impl ReasoningEffort { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum ChatMessageContent { Text(String), Parts(Vec), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatMessage { pub role: String, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -62,7 +62,7 @@ pub struct ChatMessage { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatCompletionToolCallFunctionChunk { #[serde(default, skip_serializing_if = "Option::is_none")] pub name: Option, @@ -71,7 +71,7 @@ pub struct ChatCompletionToolCallFunctionChunk { pub provider_specific_fields: Option>, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatCompletionToolCallChunk { #[serde(default, skip_serializing_if = "Option::is_none")] pub id: Option, @@ -81,7 +81,7 @@ pub struct ChatCompletionToolCallChunk { pub index: i64, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ChatCompletionThinkingBlock { Thinking { @@ -102,7 +102,7 @@ pub enum ChatCompletionThinkingBlock { /// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python /// path reports so cost tracking sees the same numbers on either path. -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct PromptTokensDetails { pub cached_tokens: u64, @@ -110,7 +110,7 @@ pub struct PromptTokensDetails { pub text_tokens: u64, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ChatCompletionsUsage { pub prompt_tokens: u64, @@ -119,7 +119,7 @@ pub struct ChatCompletionsUsage { pub prompt_tokens_details: PromptTokensDetails, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatCompletionsChoiceMessage { pub role: String, // Whether an empty turn is `None` or `""` is the provider's choice, not a @@ -129,7 +129,7 @@ pub struct ChatCompletionsChoiceMessage { pub content: Option, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatCompletionsChoice { pub index: u64, pub message: ChatCompletionsChoiceMessage, @@ -141,7 +141,7 @@ pub struct ChatCompletionsChoice { /// There is deliberately no `id`: Python mints the `chatcmpl-…` id on the /// `ModelResponse` it already created, and echoing the provider's own id here /// would change it. Pinned by `response_carries_no_id` in the Anthropic chat transformation tests. -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatCompletionsResponse { pub created: u64, pub model: String, @@ -149,7 +149,7 @@ pub struct ChatCompletionsResponse { pub usage: ChatCompletionsUsage, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ChatCompletionDelta { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -168,7 +168,7 @@ pub struct ChatCompletionDelta { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatCompletionStreamingChoice { pub index: u64, pub delta: ChatCompletionDelta, @@ -178,7 +178,7 @@ pub struct ChatCompletionStreamingChoice { pub logprobs: Option, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatCompletionChunk { pub id: String, pub created: u64, diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs index bc0d63fab97..d508bb672ab 100644 --- a/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/content.rs @@ -2,7 +2,7 @@ use serde_json::{Map, Value}; use crate::formats::messages::{CacheControl, CitationsConfig, ContentSource}; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ChatContentPart { Text { @@ -58,21 +58,21 @@ pub enum ChatContentPart { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct PromptCacheBreakpoint { pub mode: PromptCacheMode, #[serde(flatten)] pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum PromptCacheMode { Explicit, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum ChatMediaUrl { Url(String), @@ -80,7 +80,7 @@ pub enum ChatMediaUrl { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatMediaUrlParameters { pub url: String, pub detail: Option, @@ -90,7 +90,7 @@ pub struct ChatMediaUrlParameters { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatInputAudio { pub data: String, pub format: String, @@ -99,7 +99,7 @@ pub struct ChatInputAudio { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ChatFile { pub file_data: Option, @@ -113,7 +113,7 @@ pub struct ChatFile { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ChatVideoMetadata { pub fps: Option, @@ -124,7 +124,7 @@ pub struct ChatVideoMetadata { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ChatLogprobs { pub content: Option>, @@ -134,7 +134,7 @@ pub struct ChatLogprobs { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatTokenLogprob { pub token: String, pub logprob: serde_json::Number, @@ -145,7 +145,7 @@ pub struct ChatTokenLogprob { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ChatTopLogprob { pub token: String, pub logprob: serde_json::Number, diff --git a/litellm-rust/crates/llms-types/src/formats/messages/content.rs b/litellm-rust/crates/llms-types/src/formats/messages/content.rs index 53d0ebdcafb..595ba68c103 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/content.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/content.rs @@ -3,7 +3,7 @@ use serde_json::{Map, Value}; use super::{CacheControl, MessagesToolParam}; use crate::json_schema::JsonSchema; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ContentSource { Base64 { @@ -35,14 +35,14 @@ pub enum ContentSource { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum BlockContent { Text(String), Blocks(Vec), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ToolCaller { Direct { @@ -64,7 +64,7 @@ pub enum ToolCaller { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct CitationsConfig { pub enabled: Option, @@ -73,7 +73,7 @@ pub struct CitationsConfig { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct PageCitation { pub cited_text: String, pub document_index: u64, @@ -86,7 +86,7 @@ pub struct PageCitation { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct CharCitation { pub cited_text: String, pub document_index: u64, @@ -99,7 +99,7 @@ pub struct CharCitation { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ContentBlockCitation { pub cited_text: String, pub document_index: u64, @@ -112,7 +112,7 @@ pub struct ContentBlockCitation { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebSearchCitation { pub cited_text: String, pub url: String, @@ -123,7 +123,7 @@ pub struct WebSearchCitation { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct SearchResultCitation { pub cited_text: String, pub search_result_index: u64, @@ -135,7 +135,7 @@ pub struct SearchResultCitation { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum Citation { PageLocation(PageCitation), @@ -145,7 +145,7 @@ pub enum Citation { SearchResultLocation(SearchResultCitation), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum MessagesContentPart { Text(TextBlock), @@ -179,7 +179,7 @@ pub enum MessagesContentPart { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct TextBlock { pub text: String, pub citations: Option>, @@ -189,7 +189,7 @@ pub struct TextBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ImageBlock { pub source: ContentSource, pub cache_control: Option, @@ -198,7 +198,7 @@ pub struct ImageBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct DocumentBlock { pub source: ContentSource, pub title: Option, @@ -210,7 +210,7 @@ pub struct DocumentBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct SearchResultBlock { pub source: String, pub title: String, @@ -221,7 +221,7 @@ pub struct SearchResultBlock { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ThinkingBlock { pub thinking: String, pub signature: String, @@ -229,7 +229,7 @@ pub struct ThinkingBlock { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct RedactedThinkingBlock { pub data: String, #[serde(flatten)] @@ -237,7 +237,7 @@ pub struct RedactedThinkingBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ToolUseBlock { pub id: String, pub name: String, @@ -250,7 +250,7 @@ pub struct ToolUseBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ToolResultBlock { pub tool_use_id: String, pub content: Option, @@ -262,7 +262,7 @@ pub struct ToolResultBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ToolReferenceBlock { pub tool_name: String, pub cache_control: Option, @@ -271,7 +271,7 @@ pub struct ToolReferenceBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct BrowserStateBlock { pub tabs: Vec, pub state_changes: Option>, @@ -281,7 +281,7 @@ pub struct BrowserStateBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct BrowserTab { pub tab_id: String, pub title: String, @@ -291,7 +291,7 @@ pub struct BrowserTab { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum BrowserStateChange { TabOpened { @@ -326,7 +326,7 @@ pub enum BrowserStateChange { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ServerToolUseBlock { pub id: String, pub name: String, @@ -338,7 +338,7 @@ pub struct ServerToolUseBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ServerToolResultBlock { pub tool_use_id: String, pub content: C, @@ -349,7 +349,7 @@ pub struct ServerToolResultBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ServerToolError { pub error_code: String, pub error_message: Option, @@ -357,14 +357,14 @@ pub struct ServerToolError { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum WebSearchToolResultContent { Error(WebSearchResultError), Results(Vec), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebSearchResultError { #[serde(rename = "type")] pub error_type: WebSearchResultErrorType, @@ -373,14 +373,14 @@ pub struct WebSearchResultError { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum WebSearchResultErrorType { WebSearchToolResultError, } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebSearchResult { #[serde(rename = "type")] pub result_type: WebSearchResultType, @@ -392,13 +392,13 @@ pub struct WebSearchResult { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum WebSearchResultType { WebSearchResult, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum WebFetchToolResultContent { WebFetchToolResultError(ServerToolError), @@ -406,7 +406,7 @@ pub enum WebFetchToolResultContent { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebFetchResult { pub url: String, pub content: WebFetchDocument, @@ -415,13 +415,13 @@ pub struct WebFetchResult { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum WebFetchDocument { Document(Box), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum CodeExecutionToolResultContent { CodeExecutionToolResultError(ServerToolError), @@ -429,14 +429,14 @@ pub enum CodeExecutionToolResultContent { EncryptedCodeExecutionResult(EncryptedCodeExecutionResult), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum BashCodeExecutionToolResultContent { BashCodeExecutionToolResultError(ServerToolError), BashCodeExecutionResult(CodeExecutionResult), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct CodeExecutionResult { pub stdout: String, pub stderr: String, @@ -446,7 +446,7 @@ pub struct CodeExecutionResult { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct EncryptedCodeExecutionResult { pub encrypted_stdout: String, pub stderr: String, @@ -456,7 +456,7 @@ pub struct EncryptedCodeExecutionResult { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum CodeExecutionOutput { CodeExecutionOutput { @@ -466,7 +466,7 @@ pub enum CodeExecutionOutput { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum BashCodeExecutionOutput { BashCodeExecutionOutput { @@ -476,7 +476,7 @@ pub enum BashCodeExecutionOutput { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum TextEditorCodeExecutionToolResultContent { TextEditorCodeExecutionToolResultError(ServerToolError), @@ -486,7 +486,7 @@ pub enum TextEditorCodeExecutionToolResultContent { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct TextEditorViewResult { pub content: String, pub file_type: TextEditorFileType, @@ -497,7 +497,7 @@ pub struct TextEditorViewResult { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum TextEditorFileType { Text, @@ -505,7 +505,7 @@ pub enum TextEditorFileType { Pdf, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct TextEditorCreateResult { pub is_file_update: bool, #[serde(flatten)] @@ -513,7 +513,7 @@ pub struct TextEditorCreateResult { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct TextEditorStrReplaceResult { pub lines: Option>, @@ -525,27 +525,27 @@ pub struct TextEditorStrReplaceResult { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ToolSearchToolResultContent { ToolSearchToolResultError(ServerToolError), ToolSearchToolSearchResult(ToolSearchResult), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ToolSearchResult { pub tool_references: Vec, #[serde(flatten)] pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ToolSearchReference { ToolReference(ToolReferenceBlock), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum AdvisorToolResultContent { AdvisorToolResultError(ServerToolError), @@ -566,7 +566,7 @@ pub enum AdvisorToolResultContent { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ContainerUploadBlock { pub file_id: String, pub cache_control: Option, @@ -575,7 +575,7 @@ pub struct ContainerUploadBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct McpToolUseBlock { pub id: String, pub name: String, @@ -587,7 +587,7 @@ pub struct McpToolUseBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct McpToolResultBlock { pub tool_use_id: String, pub content: Option, @@ -597,20 +597,20 @@ pub struct McpToolResultBlock { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum McpToolResultContent { Text(String), Blocks(Vec), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum McpToolResultText { Text(TextBlock), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct McpToolListingBlock { pub mcp_server_name: String, pub tools: Vec, @@ -619,7 +619,7 @@ pub struct McpToolListingBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct McpListedTool { pub name: String, pub description: Option, @@ -629,7 +629,7 @@ pub struct McpListedTool { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct CompactionBlock { pub content: Option, @@ -641,7 +641,7 @@ pub struct CompactionBlock { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ToolChange { ToolAddition(ToolChangeBlock), @@ -649,7 +649,7 @@ pub enum ToolChange { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ToolChangeBlock { pub tool: ToolChangeTarget, pub cache_control: Option, @@ -657,7 +657,7 @@ pub struct ToolChangeBlock { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ToolChangeTarget { ToolReference { @@ -684,7 +684,7 @@ pub enum ToolChangeTarget { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct FallbackBlock { pub from: FallbackModel, pub to: FallbackModel, @@ -693,14 +693,14 @@ pub struct FallbackBlock { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct FallbackModel { pub model: String, #[serde(flatten)] pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum FallbackTrigger { Refusal { diff --git a/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs b/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs index 28a5aaa149a..9db13cab498 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/metadata.rs @@ -4,7 +4,7 @@ use crate::json_schema::JsonSchema; use crate::recognized::Recognized; #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MessagesMetadata { pub user_id: Option, @@ -13,7 +13,7 @@ pub struct MessagesMetadata { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct OutputFormat { #[serde(rename = "type")] pub format_type: OutputFormatType, @@ -23,14 +23,14 @@ pub struct OutputFormat { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum OutputFormatType { JsonSchema, } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct MessagesCompaction { #[serde(rename = "type")] pub compaction_type: CompactionType, @@ -39,14 +39,14 @@ pub struct MessagesCompaction { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum CompactionType { Summarize, } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MessagesContainer { pub id: Option, @@ -57,7 +57,7 @@ pub struct MessagesContainer { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ContainerSkill { #[serde(rename = "type")] pub skill_type: SkillType, @@ -67,7 +67,7 @@ pub struct ContainerSkill { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum SkillType { Anthropic, @@ -75,7 +75,7 @@ pub enum SkillType { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct McpServer { #[serde(rename = "type")] pub server_type: McpServerType, @@ -87,14 +87,14 @@ pub struct McpServer { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum McpServerType { Url, } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct McpToolConfiguration { pub allowed_tools: Option>, @@ -104,7 +104,7 @@ pub struct McpToolConfiguration { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct StopDetails { #[serde(rename = "type")] pub detail_type: Recognized, @@ -114,14 +114,14 @@ pub struct StopDetails { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum StopDetailsType { Refusal, } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ContextManagementResponse { pub applied_edits: Option>, @@ -130,7 +130,7 @@ pub struct ContextManagementResponse { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct AppliedEdit { #[serde(rename = "type")] @@ -143,7 +143,7 @@ pub struct AppliedEdit { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct Safeguard { #[serde(rename = "type")] pub safeguard_type: String, @@ -275,7 +275,7 @@ impl From for String { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum ContainerReference { Id(String), @@ -283,7 +283,7 @@ pub enum ContainerReference { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MessagesDiagnosticsParam { #[serde( @@ -298,7 +298,7 @@ pub struct MessagesDiagnosticsParam { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MessagesDiagnostics { pub cache_miss_reason: Option>, @@ -306,7 +306,7 @@ pub struct MessagesDiagnostics { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum CacheMissReason { ModelChanged(CacheMissedTokens), @@ -323,7 +323,7 @@ pub enum CacheMissReason { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct CacheMissedTokens { pub cache_missed_input_tokens: u64, #[serde(flatten)] diff --git a/litellm-rust/crates/llms-types/src/formats/messages/request.rs b/litellm-rust/crates/llms-types/src/formats/messages/request.rs index 1474dcf2428..c4a31142fb6 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/request.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/request.rs @@ -4,14 +4,14 @@ use strum::IntoStaticStr; use crate::formats::chat_completions::ReasoningEffort; use crate::recognized::Recognized; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum SystemPrompt { Text(String), Blocks(Vec), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum MessageContent { Text(String), @@ -57,7 +57,7 @@ impl From for String { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ContentBlock { #[serde(rename = "type", default, skip_serializing_if = "Option::is_none")] @@ -102,7 +102,7 @@ impl ContentBlock { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct CacheControl { #[serde(rename = "type", skip_serializing_if = "Option::is_none")] @@ -115,7 +115,7 @@ pub struct CacheControl { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct Message { pub role: String, pub content: MessageContent, @@ -123,7 +123,7 @@ pub struct Message { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Hash, IntoStaticStr, Eq)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] @@ -153,7 +153,7 @@ impl From for ReasoningEffort { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, IntoStaticStr, Eq)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] @@ -170,7 +170,7 @@ impl Speed { /// The tools whose presence changes how the request is sent. Every other tool, custom or /// server, deserializes as `Recognized::Unrecognized` and passes through verbatim. -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type")] pub enum MessagesTool { #[serde(rename = "advisor_20260301")] @@ -190,7 +190,7 @@ pub enum MessagesTool { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ContextTrigger { InputTokens { @@ -205,7 +205,7 @@ pub enum ContextTrigger { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type")] pub enum ContextEdit { #[serde(rename = "compact_20260112")] @@ -225,7 +225,7 @@ pub enum ContextEdit { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ContextManagement { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -234,7 +234,7 @@ pub struct ContextManagement { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct OutputConfig { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -251,7 +251,7 @@ impl OutputConfig { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "lowercase")] pub enum ThinkingDisplay { @@ -260,7 +260,7 @@ pub enum ThinkingDisplay { Updates, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct EnabledThinking { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -271,7 +271,7 @@ pub struct EnabledThinking { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct AdaptiveThinking { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -280,14 +280,14 @@ pub struct AdaptiveThinking { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct DisabledThinking { #[serde(flatten)] pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "lowercase")] pub enum ThinkingConfig { Enabled(EnabledThinking), @@ -311,7 +311,7 @@ impl ThinkingConfig { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct MessagesRequest { pub model: String, pub messages: Vec, @@ -319,7 +319,7 @@ pub struct MessagesRequest { pub params: MessagesOptionalParams, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MessagesOptionalParams { #[serde(skip_serializing_if = "Option::is_none")] diff --git a/litellm-rust/crates/llms-types/src/formats/messages/response.rs b/litellm-rust/crates/llms-types/src/formats/messages/response.rs index 2d8e1c054fa..84966aa2a0a 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/response.rs @@ -1,6 +1,6 @@ use serde_json::{Map, Value}; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct MessagesResponse { pub id: String, #[serde(rename = "type")] diff --git a/litellm-rust/crates/llms-types/src/formats/messages/streaming.rs b/litellm-rust/crates/llms-types/src/formats/messages/streaming.rs index abdcfa26a8c..88f05edd6ed 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/streaming.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/streaming.rs @@ -1,6 +1,6 @@ use serde_json::{Map, Value}; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MessagesStreamUsage { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -17,7 +17,7 @@ pub struct MessagesStreamUsage { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct MessagesStreamMessage { pub id: String, #[serde(rename = "type")] @@ -32,7 +32,7 @@ pub struct MessagesStreamMessage { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum MessagesContentBlockDelta { TextDelta { @@ -56,7 +56,7 @@ pub enum MessagesContentBlockDelta { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct MessagesContentBlock { #[serde(rename = "type")] pub block_type: String, @@ -82,7 +82,7 @@ pub struct MessagesContentBlock { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MessagesDelta { #[serde(default, skip_serializing_if = "Option::is_none")] @@ -97,7 +97,7 @@ pub struct MessagesDelta { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct MessagesStreamError { #[serde(rename = "type")] pub error_type: String, @@ -108,7 +108,7 @@ pub struct MessagesStreamError { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum MessagesStreamEvent { MessageStart { diff --git a/litellm-rust/crates/llms-types/src/formats/messages/tools.rs b/litellm-rust/crates/llms-types/src/formats/messages/tools.rs index 7d32c7123da..e6c5d6dc345 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/tools.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/tools.rs @@ -4,7 +4,7 @@ use serde_json::{Map, Value}; use super::{CacheControl, CitationsConfig, McpListedTool}; use crate::json_schema::JsonSchema; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] pub enum AllowedCaller { #[serde(rename = "direct")] @@ -18,7 +18,7 @@ pub enum AllowedCaller { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebSearchUserLocation { #[serde(rename = "type")] pub location_type: UserLocationType, @@ -30,14 +30,14 @@ pub struct WebSearchUserLocation { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum UserLocationType { Approximate, } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ToolChoice { #[serde(rename = "type")] pub choice_type: ToolChoiceType, @@ -47,7 +47,7 @@ pub struct ToolChoice { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum ToolChoiceType { Auto, @@ -57,7 +57,7 @@ pub enum ToolChoiceType { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct CustomTool { #[serde(rename = "type")] pub tool_type: Option, @@ -74,83 +74,83 @@ pub struct CustomTool { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum CustomToolType { Custom, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum BashToolName { Bash, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum StrReplaceEditorName { StrReplaceEditor, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum StrReplaceBasedEditToolName { StrReplaceBasedEditTool, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum MemoryToolName { Memory, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum ComputerToolName { Computer, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum CodeExecutionToolName { CodeExecution, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum ToolSearchRegexToolName { ToolSearchToolRegex, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum ToolSearchBm25ToolName { ToolSearchToolBm25, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum WebSearchToolName { WebSearch, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum WebFetchToolName { WebFetch, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum AdvisorToolName { @@ -158,7 +158,7 @@ pub enum AdvisorToolName { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ServerTool { pub name: N, pub allowed_callers: Option>, @@ -170,7 +170,7 @@ pub struct ServerTool { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ClientTool { pub name: N, pub allowed_callers: Option>, @@ -183,7 +183,7 @@ pub struct ClientTool { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct TextEditorTool20250728 { pub name: StrReplaceBasedEditToolName, pub allowed_callers: Option>, @@ -197,7 +197,7 @@ pub struct TextEditorTool20250728 { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ComputerTool { pub name: ComputerToolName, pub display_width_px: u64, @@ -213,7 +213,7 @@ pub struct ComputerTool { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ComputerTool20251124 { pub name: ComputerToolName, pub display_width_px: u64, @@ -229,7 +229,7 @@ pub struct ComputerTool20251124 { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Eq)] #[serde(rename_all = "snake_case")] pub enum ResponseInclusion { @@ -238,7 +238,7 @@ pub enum ResponseInclusion { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebSearchTool { pub name: WebSearchToolName, pub allowed_callers: Option>, @@ -254,7 +254,7 @@ pub struct WebSearchTool { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebSearchTool20260318 { pub name: WebSearchToolName, pub allowed_callers: Option>, @@ -270,7 +270,7 @@ pub struct WebSearchTool20260318 { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum UrlSourceToolReference { ToolReference { @@ -280,7 +280,7 @@ pub enum UrlSourceToolReference { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ToolResultUrlSource { All { @@ -303,7 +303,7 @@ pub enum ToolResultUrlSource { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum UserInputUrlSource { All { @@ -317,7 +317,7 @@ pub enum UserInputUrlSource { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebFetchUrlSources { pub client_tool_results: Option, pub server_tool_results: Option, @@ -327,7 +327,7 @@ pub struct WebFetchUrlSources { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebFetchTool { pub name: WebFetchToolName, pub allowed_callers: Option>, @@ -345,7 +345,7 @@ pub struct WebFetchTool { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebFetchTool20260309 { pub name: WebFetchToolName, pub allowed_callers: Option>, @@ -364,7 +364,7 @@ pub struct WebFetchTool20260309 { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct WebFetchTool20260318 { pub name: WebFetchToolName, pub allowed_callers: Option>, @@ -384,7 +384,7 @@ pub struct WebFetchTool20260318 { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct AdvisorTool { pub name: AdvisorToolName, pub model: String, @@ -400,7 +400,7 @@ pub struct AdvisorTool { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ToolsetToolConfig { pub defer_loading: Option, pub enabled: Option, @@ -409,7 +409,7 @@ pub struct ToolsetToolConfig { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct McpToolset { pub mcp_server_name: String, pub cache_control: Option, @@ -421,7 +421,7 @@ pub struct McpToolset { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct BrowserToolsetConfigs { #[serde(rename = "type")] pub type_text: Option, @@ -460,7 +460,7 @@ pub struct BrowserToolsetConfigs { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ComputerToolsetConfigs { #[serde(rename = "type")] pub type_text: Option, @@ -485,7 +485,7 @@ pub struct ComputerToolsetConfigs { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct Toolset { pub cache_control: Option, pub configs: Option>, @@ -493,7 +493,7 @@ pub struct Toolset { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type")] pub enum BuiltinMessagesTool { #[serde(rename = "bash_20241022")] @@ -556,7 +556,7 @@ pub enum BuiltinMessagesTool { McpToolset(McpToolset), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum MessagesToolParam { Builtin(Box), diff --git a/litellm-rust/crates/llms-types/src/formats/messages/usage.rs b/litellm-rust/crates/llms-types/src/formats/messages/usage.rs index aa235c437e8..4ce0ef1d5c4 100644 --- a/litellm-rust/crates/llms-types/src/formats/messages/usage.rs +++ b/litellm-rust/crates/llms-types/src/formats/messages/usage.rs @@ -3,7 +3,7 @@ use serde_json::{Map, Value}; use crate::recognized::Recognized; #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ServerToolUsage { pub web_search_requests: Option, @@ -13,7 +13,7 @@ pub struct ServerToolUsage { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct UsageIteration { #[serde(rename = "type")] pub iteration_type: Recognized, @@ -27,7 +27,7 @@ pub struct UsageIteration { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum UsageIterationType { Compaction, @@ -37,7 +37,7 @@ pub enum UsageIterationType { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MessagesUsage { pub input_tokens: Option, @@ -56,7 +56,7 @@ pub struct MessagesUsage { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct CacheCreationUsage { pub ephemeral_1h_input_tokens: Option, @@ -66,7 +66,7 @@ pub struct CacheCreationUsage { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MessagesOutputTokensDetails { pub thinking_tokens: Option, diff --git a/litellm-rust/crates/llms-types/src/formats/ocr.rs b/litellm-rust/crates/llms-types/src/formats/ocr.rs index eaecf4feda7..68c991a6e8b 100644 --- a/litellm-rust/crates/llms-types/src/formats/ocr.rs +++ b/litellm-rust/crates/llms-types/src/formats/ocr.rs @@ -5,7 +5,7 @@ use serde_with::serde_as; use crate::serde_compat::{FiniteF64, LaxI64}; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type")] pub enum OcrDocument { #[serde(rename = "document_url")] @@ -49,7 +49,7 @@ impl OcrDocument { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Copy, Default, Eq)] #[serde(rename_all = "lowercase")] pub enum OcrResponseFormat { @@ -59,7 +59,7 @@ pub enum OcrResponseFormat { } #[serde_as] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct OcrPageDimensions { #[serde_as(deserialize_as = "Option")] @@ -70,7 +70,7 @@ pub struct OcrPageDimensions { pub width: Option, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct OcrPageImage { pub image_base64: Option, @@ -80,7 +80,7 @@ pub struct OcrPageImage { } #[serde_as] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct OcrPage { #[serde_as(deserialize_as = "LaxI64")] @@ -93,7 +93,7 @@ pub struct OcrPage { } #[serde_as] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct OcrUsageInfo { #[serde_as(deserialize_as = "Option")] @@ -108,7 +108,7 @@ pub struct OcrUsageInfo { pub extra_fields: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct LiteLLMOcrResponse { pub pages: Vec, pub model: String, @@ -152,7 +152,7 @@ fn ocr_object() -> String { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct OcrBoundingBox { pub top_left_x: Option, diff --git a/litellm-rust/crates/llms-types/src/formats/responses/output.rs b/litellm-rust/crates/llms-types/src/formats/responses/output.rs index a5c79ff1fdc..ba4c011d458 100644 --- a/litellm-rust/crates/llms-types/src/formats/responses/output.rs +++ b/litellm-rust/crates/llms-types/src/formats/responses/output.rs @@ -2,7 +2,7 @@ use crate::formats::chat_completions::PromptCacheBreakpoint; use serde_json::{Map, Value}; use std::collections::BTreeMap; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesOutputItem { Message(ResponsesMessage), @@ -35,7 +35,7 @@ pub enum ResponsesOutputItem { CustomToolCallOutput(ResponsesCustomToolCallOutput), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesContentPart { OutputText { @@ -64,7 +64,7 @@ pub enum ResponsesContentPart { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesAnnotation { UrlCitation(ResponsesUrlCitation), @@ -73,7 +73,7 @@ pub enum ResponsesAnnotation { ContainerFileCitation(ResponsesFileCitation), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesCodeOutput { Logs { @@ -89,7 +89,7 @@ pub enum ResponsesCodeOutput { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesMessage { pub id: Option, @@ -102,7 +102,7 @@ pub struct ResponsesMessage { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesFunctionCall { pub id: Option, @@ -118,7 +118,7 @@ pub struct ResponsesFunctionCall { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesCustomToolCall { pub id: Option, @@ -134,7 +134,7 @@ pub struct ResponsesCustomToolCall { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesReasoning { pub id: Option, @@ -147,7 +147,7 @@ pub struct ResponsesReasoning { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesWebSearchCall { pub id: Option, @@ -158,7 +158,7 @@ pub struct ResponsesWebSearchCall { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesFileSearchCall { pub id: Option, @@ -170,7 +170,7 @@ pub struct ResponsesFileSearchCall { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesFileSearchResult { pub file_id: Option, @@ -183,7 +183,7 @@ pub struct ResponsesFileSearchResult { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesImageGenerationCall { pub id: Option, @@ -200,7 +200,7 @@ pub struct ResponsesImageGenerationCall { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesCodeInterpreterCall { pub id: Option, @@ -213,7 +213,7 @@ pub struct ResponsesCodeInterpreterCall { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesMcpCall { pub id: Option, @@ -229,7 +229,7 @@ pub struct ResponsesMcpCall { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesUrlCitation { pub url: Option, @@ -241,7 +241,7 @@ pub struct ResponsesUrlCitation { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesFileCitation { pub file_id: Option, @@ -254,7 +254,7 @@ pub struct ResponsesFileCitation { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesWebSearchAction { Search { @@ -283,7 +283,7 @@ pub enum ResponsesWebSearchAction { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesWebSearchSource { #[serde(rename = "type")] @@ -294,7 +294,7 @@ pub struct ResponsesWebSearchSource { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesMcpListTools { pub id: Option, @@ -306,7 +306,7 @@ pub struct ResponsesMcpListTools { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesMcpTool { pub name: Option, @@ -317,14 +317,14 @@ pub struct ResponsesMcpTool { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum ResponsesMcpError { Message(String), Detail(ResponsesMcpErrorDetail), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesMcpErrorDetail { McpProtocolError { @@ -346,7 +346,7 @@ pub enum ResponsesMcpErrorDetail { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesToolCaller { Direct { @@ -360,14 +360,14 @@ pub enum ResponsesToolCaller { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum ResponsesToolOutput { Text(String), Content(Vec), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesInputContent { InputText { @@ -407,7 +407,7 @@ pub enum ResponsesInputContent { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesFunctionCallOutput { pub id: Option, @@ -423,7 +423,7 @@ pub struct ResponsesFunctionCallOutput { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesCustomToolCallOutput { pub id: Option, @@ -437,7 +437,7 @@ pub struct ResponsesCustomToolCallOutput { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesSafetyCheck { pub id: Option, @@ -448,7 +448,7 @@ pub struct ResponsesSafetyCheck { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesComputerCall { pub id: Option, @@ -461,7 +461,7 @@ pub struct ResponsesComputerCall { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesComputerAction { Click { @@ -526,7 +526,7 @@ pub enum ResponsesComputerAction { }, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ResponsesCoordinate { pub x: i64, pub y: i64, @@ -535,7 +535,7 @@ pub struct ResponsesCoordinate { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesComputerCallOutput { pub id: Option, @@ -548,7 +548,7 @@ pub struct ResponsesComputerCallOutput { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesComputerOutput { ComputerScreenshot { @@ -562,7 +562,7 @@ pub enum ResponsesComputerOutput { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesProgram { pub id: Option, @@ -574,7 +574,7 @@ pub struct ResponsesProgram { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesProgramOutput { pub id: Option, @@ -586,7 +586,7 @@ pub struct ResponsesProgramOutput { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesToolSearchCall { pub id: Option, @@ -600,7 +600,7 @@ pub struct ResponsesToolSearchCall { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesToolSearchOutput { pub id: Option, @@ -614,7 +614,7 @@ pub struct ResponsesToolSearchOutput { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesAdditionalTools { pub id: Option, @@ -625,7 +625,7 @@ pub struct ResponsesAdditionalTools { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesCompaction { pub id: Option, @@ -636,7 +636,7 @@ pub struct ResponsesCompaction { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesLocalShellCall { pub id: Option, @@ -647,7 +647,7 @@ pub struct ResponsesLocalShellCall { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesLocalShellAction { Exec { @@ -665,7 +665,7 @@ pub enum ResponsesLocalShellAction { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesLocalShellCallOutput { pub id: Option, @@ -676,7 +676,7 @@ pub struct ResponsesLocalShellCallOutput { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesShellCall { pub id: Option, @@ -691,7 +691,7 @@ pub struct ResponsesShellCall { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesShellAction { pub commands: Option>, @@ -701,7 +701,7 @@ pub struct ResponsesShellAction { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesShellEnvironment { Local { @@ -716,7 +716,7 @@ pub enum ResponsesShellEnvironment { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesShellCallOutput { pub id: Option, @@ -731,7 +731,7 @@ pub struct ResponsesShellCallOutput { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesShellOutputChunk { pub outcome: Option, @@ -742,7 +742,7 @@ pub struct ResponsesShellOutputChunk { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesShellOutcome { Timeout { @@ -757,7 +757,7 @@ pub enum ResponsesShellOutcome { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesApplyPatchCall { pub id: Option, @@ -770,7 +770,7 @@ pub struct ResponsesApplyPatchCall { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum ResponsesApplyPatchOperation { CreateFile { @@ -793,7 +793,7 @@ pub enum ResponsesApplyPatchOperation { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesApplyPatchCallOutput { pub id: Option, @@ -807,7 +807,7 @@ pub struct ResponsesApplyPatchCallOutput { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesMcpApprovalRequest { pub id: Option, @@ -819,7 +819,7 @@ pub struct ResponsesMcpApprovalRequest { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesMcpApprovalResponse { pub id: Option, diff --git a/litellm-rust/crates/llms-types/src/formats/responses/response.rs b/litellm-rust/crates/llms-types/src/formats/responses/response.rs index 7017d0fa4e4..813294b0f1f 100644 --- a/litellm-rust/crates/llms-types/src/formats/responses/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/responses/response.rs @@ -1,6 +1,6 @@ use serde_json::{Map, Value}; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ResponsesApiResponse { pub id: String, pub model: String, diff --git a/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs b/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs index b3c204d7383..6053eea3e14 100644 --- a/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs +++ b/litellm-rust/crates/llms-types/src/formats/responses/streaming_websocket.rs @@ -38,7 +38,7 @@ impl ResponsesWsEventType { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct ResponsesWsEvent { #[serde(rename = "type")] pub event_type: ResponsesWsEventType, @@ -64,7 +64,7 @@ impl ResponsesWsEvent { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Eq)] pub struct ResponsesErrorFrame { #[serde(rename = "type")] @@ -84,7 +84,7 @@ impl ResponsesErrorFrame { } } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Eq)] pub struct ResponsesErrorBody { #[serde(rename = "type")] @@ -93,7 +93,7 @@ pub struct ResponsesErrorBody { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ResponsesEventResponse { pub id: Option, diff --git a/litellm-rust/crates/llms-types/src/headers.rs b/litellm-rust/crates/llms-types/src/headers.rs index bf4f42b493d..eefb76280f7 100644 --- a/litellm-rust/crates/llms-types/src/headers.rs +++ b/litellm-rust/crates/llms-types/src/headers.rs @@ -1,6 +1,6 @@ use serde_json::{Map, Value}; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct ProviderSpecificHeader { #[serde(default)] @@ -9,7 +9,7 @@ pub struct ProviderSpecificHeader { pub extra_headers: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum ProviderSpecificHeaders { One(ProviderSpecificHeader), diff --git a/litellm-rust/crates/llms-types/src/json_schema.rs b/litellm-rust/crates/llms-types/src/json_schema.rs index 702e1bb8a80..50a96ebef09 100644 --- a/litellm-rust/crates/llms-types/src/json_schema.rs +++ b/litellm-rust/crates/llms-types/src/json_schema.rs @@ -1,21 +1,21 @@ use indexmap::IndexMap; use serde_json::{Map, Value}; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum JsonSchema { Boolean(bool), Object(Box), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum JsonSchemaType { Name(String), Names(Vec), } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum JsonSchemaItems { Schema(Box), @@ -23,7 +23,7 @@ pub enum JsonSchemaItems { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct JsonSchemaObject { #[serde(rename = "type")] diff --git a/litellm-rust/crates/llms-types/src/providers/minimax.rs b/litellm-rust/crates/llms-types/src/providers/minimax.rs index efc4133e1af..7ddbf32af2c 100644 --- a/litellm-rust/crates/llms-types/src/providers/minimax.rs +++ b/litellm-rust/crates/llms-types/src/providers/minimax.rs @@ -2,7 +2,7 @@ use serde_json::{Map, Value}; use crate::formats::messages::CacheControl; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum MinimaxMessagesContentBlock { Image(MinimaxMediaBlock), @@ -15,7 +15,7 @@ pub enum MinimaxMessagesContentBlock { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] pub struct MinimaxMediaBlock { pub source: MinimaxMediaSource, pub cache_control: Option, @@ -23,7 +23,7 @@ pub struct MinimaxMediaBlock { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(tag = "type", rename_all = "snake_case")] pub enum MinimaxMediaSource { Base64 { @@ -40,7 +40,7 @@ pub enum MinimaxMediaSource { } #[serde_with::skip_serializing_none] -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Default)] pub struct MinimaxMediaOptions { pub detail: Option, @@ -50,7 +50,7 @@ pub struct MinimaxMediaOptions { pub extra: Map, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(rename_all = "snake_case")] pub enum MinimaxMediaDetail { Low, diff --git a/litellm-rust/crates/llms-types/src/recognized.rs b/litellm-rust/crates/llms-types/src/recognized.rs index 148d65381a5..4fe5f66b31b 100644 --- a/litellm-rust/crates/llms-types/src/recognized.rs +++ b/litellm-rust/crates/llms-types/src/recognized.rs @@ -1,6 +1,6 @@ use serde_json::Value; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[serde(untagged)] pub enum Recognized { Known(T), diff --git a/litellm-rust/crates/traces-clickhouse/src/query.rs b/litellm-rust/crates/traces-clickhouse/src/query.rs index e564dcd31d2..a08affb0cad 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query.rs @@ -40,12 +40,12 @@ struct MetadataRow { metadata: String, } -#[macro_rules_attribute::apply(request_type)] +#[macro_rules_attribute::apply(crate::request_type)] struct AttributeRow { key: String, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] #[serde(untagged)] enum PathPart { @@ -53,7 +53,7 @@ enum PathPart { Index(usize), } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd, strum::Display)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase")] @@ -82,13 +82,13 @@ impl JsonKind { } } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Copy, Debug, strum::Display)] enum MapValueType { String, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryMetadataField"))] struct MetadataField { @@ -97,7 +97,7 @@ struct MetadataField { expression: String, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryColumn"))] struct ColumnSchema { name: String, @@ -107,7 +107,7 @@ struct ColumnSchema { details: BTreeMap, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryTable"))] struct TableSchema { @@ -175,7 +175,7 @@ impl Serialize for Discovery { } } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] struct MetadataSample { fields: Vec, sampled_rows: usize, @@ -194,7 +194,7 @@ impl Unobserved for MetadataSample { } } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryMetadata"))] struct MetadataCatalog { @@ -206,7 +206,7 @@ struct MetadataCatalog { scope: &'static str, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryAttributeField"))] struct AttributeField { @@ -216,7 +216,7 @@ struct AttributeField { expression: String, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] struct AttributeSample { fields: Vec, truncated: bool, @@ -231,7 +231,7 @@ impl Unobserved for AttributeSample { } } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryAttributes"))] struct AttributeCatalog { @@ -243,7 +243,7 @@ struct AttributeCatalog { scope: &'static str, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryNormalizedField"))] struct NormalizedField { @@ -267,7 +267,7 @@ impl From<&NormalizedFieldDefinition> for NormalizedField { } } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryRelationship"))] struct Relationship { @@ -284,7 +284,7 @@ const RELATIONSHIPS: [Relationship; 1] = [Relationship { meaning: "LiteLLMRequestId contains the first normalized request or provider response ID. This relationship matches response IDs only; CallKeys retains all typed identifiers. Cached requests can share response_id; joins may return multiple spend rows", }]; -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryHelp"))] pub struct QueryHelp { diff --git a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs index 97b74de4c20..2af254488bd 100644 --- a/litellm-rust/crates/traces-clickhouse/src/query/lens.rs +++ b/litellm-rust/crates/traces-clickhouse/src/query/lens.rs @@ -18,7 +18,7 @@ pub const LENS_QUERIES: [litellm_traces::ReadQuery; 9] = [ litellm_traces::ReadQuery::FeedbackSummary, ]; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(rename_all = "lowercase")] pub enum ExecutionSource { @@ -27,7 +27,7 @@ pub enum ExecutionSource { Both, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(rename_all = "lowercase")] pub enum ContentSource { @@ -35,7 +35,7 @@ pub enum ContentSource { Requests, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] pub struct LensAccessParams { @@ -54,7 +54,7 @@ pub struct LensAccessParams { pub struct LensAvailability; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(deny_unknown_fields)] pub struct LensAvailabilityParams { @@ -62,7 +62,7 @@ pub struct LensAvailabilityParams { pub access: LensAccessParams, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(rename = "ActivityAvailability"))] pub struct LensAvailabilityRow { @@ -89,7 +89,7 @@ impl Query for LensAvailability { pub struct LensAgents; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(deny_unknown_fields)] pub struct LensAgentsParams { @@ -97,7 +97,7 @@ pub struct LensAgentsParams { pub access: LensAccessParams, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(rename = "AgentRow"))] pub struct LensAgentsRow { @@ -114,7 +114,7 @@ impl Query for LensAgents { pub struct TraceAgents; /// Same access shape as `list_traces`: every team, the caller's own traces, or their teams' traces. -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(deny_unknown_fields)] #[cfg_attr(feature = "schema", schemars(deny_unknown_fields))] @@ -138,7 +138,7 @@ pub struct TraceAgentsParams { pub limit: u32, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(rename = "TraceAgentRow"))] pub struct TraceAgentsRow { @@ -174,7 +174,7 @@ impl Query for TraceAgents { pub struct LensSample; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(deny_unknown_fields)] pub struct LensSampleParams { @@ -209,7 +209,7 @@ pub struct LensSampleParams { pub offset: u64, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(rename = "ExecutionRow"))] pub struct LensSampleRow { @@ -265,7 +265,7 @@ impl Query for LensSample { pub struct LensContent; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(deny_unknown_fields)] pub struct LensContentParams { @@ -281,7 +281,7 @@ pub struct LensContentParams { pub offset: u32, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(rename = "PartRow"))] pub struct LensContentRow { @@ -309,7 +309,7 @@ impl Query for LensContent { pub struct LensEvidence; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(deny_unknown_fields)] pub struct LensEvidenceParams { @@ -324,7 +324,7 @@ pub struct LensEvidenceParams { pub quote: String, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(rename = "CountRow"))] pub struct LensEvidenceRow { @@ -345,7 +345,7 @@ impl Query for LensEvidence { pub struct LensFeedbackTarget; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(deny_unknown_fields)] pub struct LensFeedbackTargetParams { @@ -355,7 +355,7 @@ pub struct LensFeedbackTargetParams { pub trace_ref: String, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(rename = "FeedbackTargetRow"))] pub struct LensFeedbackTargetRow { @@ -373,7 +373,7 @@ impl Query for LensFeedbackTarget { pub struct LensFeedback; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(deny_unknown_fields)] pub struct LensFeedbackParams { @@ -383,7 +383,7 @@ pub struct LensFeedbackParams { pub trace_ref: String, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(rename = "FeedbackRow"))] pub struct LensFeedbackRow { @@ -410,7 +410,7 @@ impl Query for LensFeedback { pub struct LensFeedbackSummary; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[serde(deny_unknown_fields)] pub struct LensFeedbackSummaryParams { @@ -419,7 +419,7 @@ pub struct LensFeedbackSummaryParams { pub trace_ids: Vec, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Debug)] #[cfg_attr(feature = "schema", schemars(rename = "FeedbackSummaryRow"))] pub struct LensFeedbackSummaryRow { diff --git a/litellm-rust/crates/traces-clickhouse/src/table.rs b/litellm-rust/crates/traces-clickhouse/src/table.rs index c74cf6d4de1..60ab4943f5e 100644 --- a/litellm-rust/crates/traces-clickhouse/src/table.rs +++ b/litellm-rust/crates/traces-clickhouse/src/table.rs @@ -1,4 +1,4 @@ -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(rename = "TraceTableName"))] #[derive( Clone, Copy, Debug, strum::Display, strum::AsRefStr, strum::EnumIter, strum::IntoStaticStr, diff --git a/litellm-rust/crates/traces/src/normalize/mod.rs b/litellm-rust/crates/traces/src/normalize/mod.rs index 53d5e521799..4d11e76e19b 100644 --- a/litellm-rust/crates/traces/src/normalize/mod.rs +++ b/litellm-rust/crates/traces/src/normalize/mod.rs @@ -29,7 +29,7 @@ use instrumentation::Instrumentation; pub(crate) use messages::{HIDDEN_BLOCK_TYPES, MessagePayload, encode}; pub use metadata::{AgentMetadata, AgentType, Integration}; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Clone, Copy, Debug, Eq, PartialEq, strum::EnumString)] #[serde(rename_all = "lowercase")] #[strum(serialize_all = "lowercase", ascii_case_insensitive)] diff --git a/litellm-rust/crates/traces/src/query/guide.rs b/litellm-rust/crates/traces/src/query/guide.rs index b97f518a7ae..8fbde481df0 100644 --- a/litellm-rust/crates/traces/src/query/guide.rs +++ b/litellm-rust/crates/traces/src/query/guide.rs @@ -1,6 +1,6 @@ use askama::Template; -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[cfg_attr(feature = "schema", schemars(rename = "TraceQueryExample"))] pub struct Example { pub name: String, diff --git a/litellm-rust/crates/traces/src/query/named.rs b/litellm-rust/crates/traces/src/query/named.rs index dfa7ac2f2cd..cd8caf4da48 100644 --- a/litellm-rust/crates/traces/src/query/named.rs +++ b/litellm-rust/crates/traces/src/query/named.rs @@ -1,7 +1,7 @@ use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Clone, Debug)] #[cfg_attr(feature = "schema", schemars(rename = "TraceScope"))] pub struct ReadAccessParams { diff --git a/litellm-rust/crates/traces/src/query_access.rs b/litellm-rust/crates/traces/src/query_access.rs index c543ddb5808..12cd551b09d 100644 --- a/litellm-rust/crates/traces/src/query_access.rs +++ b/litellm-rust/crates/traces/src/query_access.rs @@ -1,6 +1,6 @@ use crate::InvalidScope; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Clone, Debug)] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryScope { diff --git a/litellm-rust/crates/traces/src/request.rs b/litellm-rust/crates/traces/src/request.rs index 0ade6112f02..e9ac54972a6 100644 --- a/litellm-rust/crates/traces/src/request.rs +++ b/litellm-rust/crates/traces/src/request.rs @@ -1,7 +1,7 @@ pub const TRACE_PAGE_SIZE_MIN: u16 = 1; pub const TRACE_PAGE_SIZE_MAX: u16 = 500; -#[macro_rules_attribute::apply(request_type)] +#[macro_rules_attribute::apply(crate::request_type)] #[derive(Clone, Debug)] pub struct TraceListRequest { /// Window start, unix ms. Default: 24h ago @@ -15,7 +15,7 @@ pub struct TraceListRequest { pub cursor: Option, } -#[macro_rules_attribute::apply(request_type)] +#[macro_rules_attribute::apply(crate::request_type)] #[derive(Clone, Debug)] pub struct TraceDetailRequest { #[serde(default)] @@ -31,14 +31,14 @@ pub struct TraceDetailRequest { pub page_size: Option, } -#[macro_rules_attribute::apply(request_type)] +#[macro_rules_attribute::apply(crate::request_type)] #[derive(Clone, Debug)] pub struct TraceSpanRequest { #[serde(default)] pub trace_ref: String, } -#[macro_rules_attribute::apply(request_type)] +#[macro_rules_attribute::apply(crate::request_type)] #[derive(Clone, Debug)] pub struct TraceErrorPageRequest { #[serde(default)] @@ -48,7 +48,7 @@ pub struct TraceErrorPageRequest { pub cursor: Option, } -#[macro_rules_attribute::apply(request_type)] +#[macro_rules_attribute::apply(crate::request_type)] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct TraceQueryRequest { diff --git a/litellm-rust/crates/traces/src/response.rs b/litellm-rust/crates/traces/src/response.rs index cd8538ed602..af5173d6ee9 100644 --- a/litellm-rust/crates/traces/src/response.rs +++ b/litellm-rust/crates/traces/src/response.rs @@ -1,4 +1,4 @@ -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Debug)] #[serde(deny_unknown_fields)] pub struct TraceSQLResponse { diff --git a/litellm-rust/crates/traces/src/tenant.rs b/litellm-rust/crates/traces/src/tenant.rs index a097dd947c1..bd519150a24 100644 --- a/litellm-rust/crates/traces/src/tenant.rs +++ b/litellm-rust/crates/traces/src/tenant.rs @@ -1,6 +1,6 @@ /// Who sent a batch of spans. Always taken from the caller's authentication, never from span /// attributes. -#[macro_rules_attribute::apply(request_type)] +#[macro_rules_attribute::apply(crate::request_type)] #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct Tenant { pub team_id: String, diff --git a/litellm-rust/crates/traces/src/ui.rs b/litellm-rust/crates/traces/src/ui.rs index ef849d2b440..03f1b7e7e9c 100644 --- a/litellm-rust/crates/traces/src/ui.rs +++ b/litellm-rust/crates/traces/src/ui.rs @@ -6,7 +6,7 @@ use serde_json::Value; use crate::normalize::{HIDDEN_BLOCK_TYPES, MessagePayload, encode}; -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Copy, Debug, PartialEq)] #[serde(rename_all = "lowercase")] pub enum ChatRole { @@ -16,7 +16,7 @@ pub enum ChatRole { Tool, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Debug, PartialEq)] #[serde(tag = "kind", rename_all = "snake_case")] #[cfg_attr(feature = "schema", schemars(rename = "UIContent"))] @@ -29,7 +29,7 @@ pub enum UiContent { Text { text: String }, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Debug, PartialEq)] #[cfg_attr(feature = "schema", schemars(rename = "UIMessage"))] pub struct UiMessage { @@ -41,7 +41,7 @@ pub struct UiMessage { pub tool_calls: Option>, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Debug, PartialEq)] #[cfg_attr(feature = "schema", schemars(rename = "UIToolCall"))] pub struct UiToolCall { @@ -49,7 +49,7 @@ pub struct UiToolCall { pub arguments: String, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Debug, PartialEq)] #[cfg_attr(feature = "schema", schemars(rename = "UIField"))] pub struct UiField { diff --git a/litellm-rust/crates/traces/src/view.rs b/litellm-rust/crates/traces/src/view.rs index 345e1454855..34bf6da07cd 100644 --- a/litellm-rust/crates/traces/src/view.rs +++ b/litellm-rust/crates/traces/src/view.rs @@ -4,7 +4,7 @@ use std::collections::BTreeMap; use crate::ui::UiContent; -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[serde(rename_all = "lowercase")] pub enum SpanStatus { @@ -16,7 +16,7 @@ pub enum SpanStatus { Unset, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Debug, PartialEq)] pub struct Span { pub span_id: String, @@ -41,7 +41,7 @@ pub struct Span { pub spend_match: Option, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[serde(rename_all = "snake_case")] pub enum SpendMatch { @@ -53,7 +53,7 @@ pub enum SpendMatch { } /// One distinct agent in a trace: 200 invocations of `researcher` are one node. -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Debug, PartialEq)] pub struct AgentNode { pub name: String, @@ -66,7 +66,7 @@ pub struct AgentNode { pub priced_calls: u64, } -#[macro_rules_attribute::apply(wire_type)] +#[macro_rules_attribute::apply(crate::wire_type)] #[derive(Clone, Copy, Debug, Eq, PartialEq)] #[serde(rename_all = "snake_case")] pub enum RunSourceType { @@ -81,7 +81,7 @@ pub enum RunSourceType { } /// The conversation that started the run, from the `agent.source.*` span attributes. -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Debug, PartialEq)] pub struct RunSource { #[serde(rename = "type")] @@ -94,7 +94,7 @@ pub struct RunSource { pub user: String, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Debug, PartialEq)] pub struct TraceSummary { #[cfg_attr(feature = "schema", schemars(extend("x-python-optional" = true)))] @@ -128,7 +128,7 @@ pub struct TraceSummary { pub source: Option, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Clone, Debug, PartialEq)] pub struct Trace { /// Read-cache metadata from gateway resolution; display estimates must not clear it. @@ -141,14 +141,14 @@ pub struct Trace { pub next_cursor: Option, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Debug, PartialEq)] pub struct TracePage { pub data: Vec, pub next_cursor: Option, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Debug, PartialEq)] pub struct SpanDetail { pub span_id: String, @@ -159,7 +159,7 @@ pub struct SpanDetail { pub attributes: BTreeMap, } -#[macro_rules_attribute::apply(response_type)] +#[macro_rules_attribute::apply(crate::response_type)] #[derive(Debug, PartialEq)] pub struct SpanErrorPage { pub span_id: String, From 0e455ae7a92acaad76141db63b55fb5d69605b13 Mon Sep 17 00:00:00 2001 From: Solomon Mithra Date: Thu, 8 Oct 2026 12:02:40 -0700 Subject: [PATCH 53/65] fix(proxy): use budget reset window for projected spend alerts (#31942) * fix(proxy): use budget reset window for projected spend alerts * style(proxy): use PEP 585 tuple annotations in projection helpers * fix(proxy): derive projection date from budget reset timezone * fix(proxy): project spend in real time within the budget reset window * fix(proxy): derive the projection window start from the reset schedule 30d and monthly keys reset on the 1st of the month and unrecognized spellings like 1hr reset at the next midnight, but the window start stepped back the literal duration, so now could land before the window and the elapsed floor inflated the projection into a false alert. The window start now mirrors get_next_standardized_reset_time, and a reset more than one window away no longer projects at all --------- Co-authored-by: ryan-crabbe-berri --- litellm/litellm_core_utils/duration_parser.py | 31 +++ litellm/proxy/proxy_server.py | 23 +-- litellm/proxy/utils.py | 127 +++++++----- .../test_duration_parser.py | 120 ++++++------ .../helpers/test_month_end_projection.py | 181 +++++++++++++++++- 5 files changed, 353 insertions(+), 129 deletions(-) diff --git a/litellm/litellm_core_utils/duration_parser.py b/litellm/litellm_core_utils/duration_parser.py index 7ac2299b7d8..fe8b71e7623 100644 --- a/litellm/litellm_core_utils/duration_parser.py +++ b/litellm/litellm_core_utils/duration_parser.py @@ -166,6 +166,37 @@ def get_next_standardized_reset_time( return base_midnight + timedelta(days=1) +def _subtract_months(moment: datetime, months: int) -> datetime: + total_months: Final = moment.year * 12 + moment.month - 1 - months + year, month_index = divmod(total_months, 12) + month: Final = month_index + 1 + return moment.replace(year=year, month=month, day=min(moment.day, get_last_day_of_month(year, month))) + + +def get_budget_window_start(duration: str, reset_at: datetime) -> datetime: + """Start of the budget period that ends at `reset_at`, under the same rules + `get_next_standardized_reset_time` used to pick it: `30d` and `Nmo` span calendar + months and a duration it does not recognize resets at the next midnight.""" + value, unit = _parse_duration(_normalize_duration(duration)) + if value is None: + return reset_at - timedelta(days=1) + match unit: + case "mo": + return _subtract_months(reset_at, value) + case "d": + return _subtract_months(reset_at, 1) if value == 30 else reset_at - timedelta(days=value) + case "w": + return reset_at - timedelta(weeks=value) + case "h": + return reset_at - timedelta(hours=value) + case "m": + return reset_at - timedelta(minutes=value) + case "s": + return reset_at - timedelta(seconds=value) + case _: + return reset_at - timedelta(days=1) + + def _setup_timezone(current_time: datetime, timezone_str: str = "UTC") -> tuple[datetime, tzinfo]: """Set up timezone and normalize current time to that timezone.""" try: diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 000ffa9c305..90e760093b4 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -887,7 +887,7 @@ from litellm.proxy.utils import ( # noqa: F401, RUF100 # legacy module exports hash_password, hash_token, invalidate_config_param, - is_projected_spend_over_limit, + is_projected_spend_over_limit, # pyright: ignore[reportUnusedImport] # backwards-compatible package export is_valid_team_configs, litellm_config_cache, migrate_passwords_to_scrypt_async, @@ -4165,21 +4165,18 @@ async def update_cache( new_spend: Final = existing_spend + response_cost ## CHECK IF USER PROJECTED SPEND > SOFT LIMIT - if ( - existing_spend_obj.soft_budget_cooldown is False - and existing_spend_obj.soft_budget is not None - and ( - is_projected_spend_over_limit( - current_spend=new_spend, - soft_budget_limit=existing_spend_obj.soft_budget, - ) - is True - ) - ): - projected_spend, projected_exceeded_date = get_projected_spend_over_limit( + projection: Final = ( + get_projected_spend_over_limit( current_spend=new_spend, soft_budget_limit=existing_spend_obj.soft_budget, + budget_duration=existing_spend_obj.budget_duration, + budget_reset_at=existing_spend_obj.budget_reset_at, ) + if existing_spend_obj.soft_budget_cooldown is False and existing_spend_obj.soft_budget is not None + else None + ) + if projection is not None: + projected_spend, projected_exceeded_date = projection soft_limit: Final = existing_spend_obj.soft_budget call_info: Final = CallInfo( token=existing_spend_obj.token or "", diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py index 3e73e2dee40..8f796f065a6 100644 --- a/litellm/proxy/utils.py +++ b/litellm/proxy/utils.py @@ -147,6 +147,7 @@ from litellm.litellm_core_utils.core_helpers import ( independent_snapshot, is_expected_client_error, ) +from litellm.litellm_core_utils.duration_parser import get_budget_window_start from litellm.litellm_core_utils.litellm_logging import Logging from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.litellm_core_utils.safe_json_loads import safe_json_loads @@ -8101,66 +8102,94 @@ def _get_month_end_date(today: date) -> date: return date(today.year, today.month + 1, 1) - timedelta(days=1) -def is_projected_spend_over_limit(current_spend: float, soft_budget_limit: float | None) -> bool: - if soft_budget_limit is None: - # If there's no limit, we can't exceed it. - return False +MIN_ELAPSED_WINDOW_FRACTION: Final = 1 / 24 + +def _as_aware(moment: datetime) -> datetime: + return moment if moment.tzinfo is not None else moment.replace(tzinfo=timezone.utc) + + +def _reset_window(budget_duration: str, budget_reset_at: datetime) -> tuple[datetime, datetime] | None: + window_end: Final = _as_aware(budget_reset_at) + window_start: Final = get_budget_window_start(budget_duration, window_end) + return (window_start, window_end) if window_start < window_end else None + + +def _project_within_window( + current_spend: float, + soft_budget_limit: float, + window: tuple[datetime, datetime], + now: datetime | None, +) -> tuple[float, date] | None: + window_start, window_end = window + moment: Final = (_as_aware(now) if now is not None else datetime.now(timezone.utc)).astimezone(window_end.tzinfo) + if moment < window_start: + return None + elapsed: Final = max(moment - window_start, (window_end - window_start) * MIN_ELAPSED_WINDOW_FRACTION) + remaining: Final = max(window_end - moment, timedelta(0)) + spend_per_second: Final = current_spend / elapsed.total_seconds() + projected_spend: Final = current_spend + spend_per_second * remaining.total_seconds() + if projected_spend <= soft_budget_limit: + return None + remaining_budget: Final = soft_budget_limit - current_spend + if spend_per_second <= 0 or remaining_budget <= 0: + return projected_spend, moment.date() + exceed_at: Final = min(moment + timedelta(seconds=remaining_budget / spend_per_second), window_end) + return projected_spend, exceed_at.date() + + +def _project_to_month_end(current_spend: float, soft_budget_limit: float) -> tuple[float, date] | None: today: Final = date.today() + remaining_days: Final = (_get_month_end_date(today) - today).days + daily_spend: Final = current_spend / max(today.day - 1, 1) + projected_spend: Final = current_spend + daily_spend * remaining_days + if projected_spend <= soft_budget_limit: + return None + remaining_budget: Final = soft_budget_limit - current_spend + if daily_spend <= 0 or remaining_budget <= 0: + return projected_spend, today + return projected_spend, today + timedelta(days=remaining_budget / daily_spend) - # Finding the first day of the next month, then subtracting one day to get the end of the current month. - end_month: Final = _get_month_end_date(today) - remaining_days: Final = (end_month - today).days - - # Check for the start of the month to avoid division by zero - if today.day == 1: - daily_spend_estimate = current_spend - else: - daily_spend_estimate = current_spend / (today.day - 1) - - # Total projected spend for the month - projected_spend: Final = current_spend + (daily_spend_estimate * remaining_days) - - if projected_spend > soft_budget_limit: - print_verbose("Projected spend exceeds soft budget limit!") - return True - return False +def is_projected_spend_over_limit( + current_spend: float, + soft_budget_limit: float | None, + budget_duration: str | None = None, + budget_reset_at: datetime | None = None, + now: datetime | None = None, +) -> bool: + return ( + get_projected_spend_over_limit( + current_spend=current_spend, + soft_budget_limit=soft_budget_limit, + budget_duration=budget_duration, + budget_reset_at=budget_reset_at, + now=now, + ) + is not None + ) _is_projected_spend_over_limit: Final = is_projected_spend_over_limit -def get_projected_spend_over_limit(current_spend: float, soft_budget_limit: float | None) -> tuple | None: +def get_projected_spend_over_limit( + current_spend: float, + soft_budget_limit: float | None, + budget_duration: str | None = None, + budget_reset_at: datetime | None = None, + now: datetime | None = None, +) -> tuple[float, date] | None: if soft_budget_limit is None: return None - - today: Final = date.today() - end_month: Final = _get_month_end_date(today) - remaining_days: Final = (end_month - today).days - - # assuming the current spend till today (not including today) - if today.day == 1: - daily_spend = current_spend - else: - daily_spend = current_spend / (today.day - 1) - projected_spend: Final = current_spend + (daily_spend * remaining_days) - - if projected_spend > soft_budget_limit: - if daily_spend <= 0: - limit_exceed_date = today - else: - remaining_budget: Final = soft_budget_limit - current_spend - if remaining_budget <= 0: - limit_exceed_date = today - else: - approx_days: Final = remaining_budget / daily_spend - limit_exceed_date = today + timedelta(days=approx_days) - - # return the projected spend and the date it will exceeded - return projected_spend, limit_exceed_date - - return None + window: Final = ( + _reset_window(budget_duration, budget_reset_at) + if budget_duration is not None and budget_reset_at is not None + else None + ) + if window is None: + return _project_to_month_end(current_spend, soft_budget_limit) + return _project_within_window(current_spend, soft_budget_limit, window, now) _get_projected_spend_over_limit: Final = get_projected_spend_over_limit diff --git a/tests/unit/litellm_core_utils/test_duration_parser.py b/tests/unit/litellm_core_utils/test_duration_parser.py index cb9f273a0a7..f5e50d67e71 100644 --- a/tests/unit/litellm_core_utils/test_duration_parser.py +++ b/tests/unit/litellm_core_utils/test_duration_parser.py @@ -1,11 +1,13 @@ import unittest from datetime import datetime, time, timezone +from typing import Final from unittest.mock import patch from zoneinfo import ZoneInfo import litellm.litellm_core_utils.duration_parser as duration_parser from litellm.litellm_core_utils.duration_parser import ( duration_in_seconds, + get_budget_window_start, get_next_standardized_reset_time, ) @@ -23,9 +25,7 @@ class TestStandardizedResetTime(unittest.TestCase): # Weekly reset (7d) - should reset on next Monday wednesday = datetime(2023, 5, 17, 15, 45, 0, tzinfo=timezone.utc) # A Wednesday - weekly_expected = datetime( - 2023, 5, 22, 0, 0, 0, tzinfo=timezone.utc - ) # Next Monday + weekly_expected = datetime(2023, 5, 22, 0, 0, 0, tzinfo=timezone.utc) # Next Monday weekly_result = get_next_standardized_reset_time("7d", wednesday, "UTC") self.assertEqual(weekly_result, weekly_expected) @@ -105,17 +105,13 @@ class TestStandardizedResetTime(unittest.TestCase): # Europe/London (UTC+1): 11:30 PM, so next 15m reset is 11:45 PM london = ZoneInfo("Europe/London") london_expected = datetime(2023, 5, 15, 23, 45, 0, tzinfo=london) - london_result = get_next_standardized_reset_time( - "15m", base_time, "Europe/London" - ) + london_result = get_next_standardized_reset_time("15m", base_time, "Europe/London") self.assertEqual(london_result, london_expected) # Test Bangkok timezone (UTC+7): 5:30 AM next day, so next reset is midnight the day after bangkok = ZoneInfo("Asia/Bangkok") bangkok_expected = datetime(2023, 5, 17, 0, 0, 0, tzinfo=bangkok) - bangkok_result = get_next_standardized_reset_time( - "1d", base_time, "Asia/Bangkok" - ) + bangkok_result = get_next_standardized_reset_time("1d", base_time, "Asia/Bangkok") self.assertEqual(bangkok_result, bangkok_expected) def test_edge_cases(self): @@ -137,16 +133,12 @@ class TestStandardizedResetTime(unittest.TestCase): # 30m near midnight - should roll over to next day midnight_minute_expected = datetime(2023, 5, 16, 0, 0, 0, tzinfo=timezone.utc) - midnight_minute_result = get_next_standardized_reset_time( - "30m", near_midnight, "UTC" - ) + midnight_minute_result = get_next_standardized_reset_time("30m", near_midnight, "UTC") self.assertEqual(midnight_minute_result, midnight_minute_expected) # Invalid timezone - should fall back to UTC invalid_tz_expected = datetime(2023, 5, 16, 0, 0, 0, tzinfo=timezone.utc) - invalid_tz_result = get_next_standardized_reset_time( - "1d", on_hour, "NonExistentTimeZone" - ) + invalid_tz_result = get_next_standardized_reset_time("1d", on_hour, "NonExistentTimeZone") self.assertEqual(invalid_tz_result, invalid_tz_expected) def test_iana_timezones_previously_unsupported(self): @@ -164,17 +156,13 @@ class TestStandardizedResetTime(unittest.TestCase): sydney = ZoneInfo("Australia/Sydney") # At 15:00 UTC it's 01:00 AEST May 16 → next midnight is May 17 00:00 AEST sydney_expected = datetime(2023, 5, 17, 0, 0, 0, tzinfo=sydney) - sydney_result = get_next_standardized_reset_time( - "1d", base_time, "Australia/Sydney" - ) + sydney_result = get_next_standardized_reset_time("1d", base_time, "Australia/Sydney") self.assertEqual(sydney_result, sydney_expected) # America/Chicago (UTC-5): at 15:00 UTC it's 10:00 CDT → next midnight is May 16 00:00 CDT chicago = ZoneInfo("America/Chicago") chicago_expected = datetime(2023, 5, 16, 0, 0, 0, tzinfo=chicago) - chicago_result = get_next_standardized_reset_time( - "1d", base_time, "America/Chicago" - ) + chicago_result = get_next_standardized_reset_time("1d", base_time, "America/Chicago") self.assertEqual(chicago_result, chicago_expected) def test_dst_fall_back(self): @@ -209,107 +197,77 @@ class TestResetTimeOfDay(unittest.TestCase): def test_daily_reset_before_offset_is_today(self): now = datetime(2023, 5, 15, 8, 0, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "1d", now, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("1d", now, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2023, 5, 15, 12, 0, 0, tzinfo=timezone.utc)) def test_daily_reset_after_offset_is_tomorrow(self): now = datetime(2023, 5, 15, 14, 0, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "1d", now, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("1d", now, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2023, 5, 16, 12, 0, 0, tzinfo=timezone.utc)) def test_daily_reset_exactly_at_offset_rolls_forward(self): now = datetime(2023, 5, 15, 12, 0, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "1d", now, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("1d", now, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2023, 5, 16, 12, 0, 0, tzinfo=timezone.utc)) def test_daily_reset_with_seconds_offset(self): now = datetime(2023, 5, 15, 8, 0, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "1d", now, "UTC", reset_time_of_day=time(9, 30, 15) - ) + result = get_next_standardized_reset_time("1d", now, "UTC", reset_time_of_day=time(9, 30, 15)) self.assertEqual(result, datetime(2023, 5, 15, 9, 30, 15, tzinfo=timezone.utc)) def test_offset_applies_in_configured_timezone(self): # 2023-05-15 22:30 UTC == 2023-05-16 01:30 in Jerusalem (IDT, UTC+3), # so the next noon-Jerusalem reset is 2023-05-16 12:00 IDT. now = datetime(2023, 5, 15, 22, 30, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "1d", now, "Asia/Jerusalem", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("1d", now, "Asia/Jerusalem", reset_time_of_day=time(12, 0)) jerusalem = result.astimezone(ZoneInfo("Asia/Jerusalem")) - self.assertEqual( - (jerusalem.year, jerusalem.month, jerusalem.day), (2023, 5, 16) - ) + self.assertEqual((jerusalem.year, jerusalem.month, jerusalem.day), (2023, 5, 16)) self.assertEqual(jerusalem.hour, 12) self.assertEqual(jerusalem.minute, 0) def test_weekly_reset_lands_on_monday_at_offset(self): wednesday = datetime(2023, 5, 17, 15, 45, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "7d", wednesday, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("7d", wednesday, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2023, 5, 22, 12, 0, 0, tzinfo=timezone.utc)) def test_weekly_reset_today_is_monday_before_offset_is_today(self): monday_morning = datetime(2023, 5, 22, 9, 0, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "7d", monday_morning, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("7d", monday_morning, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2023, 5, 22, 12, 0, 0, tzinfo=timezone.utc)) def test_weekly_reset_today_is_monday_after_offset_is_next_week(self): monday_afternoon = datetime(2023, 5, 22, 15, 0, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "7d", monday_afternoon, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("7d", monday_afternoon, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2023, 5, 29, 12, 0, 0, tzinfo=timezone.utc)) def test_monthly_30d_lands_on_first_at_offset(self): now = datetime(2023, 5, 15, 10, 30, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "30d", now, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("30d", now, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2023, 6, 1, 12, 0, 0, tzinfo=timezone.utc)) def test_monthly_1mo_today_is_first_before_offset_is_today(self): now = datetime(2023, 5, 1, 9, 0, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "1mo", now, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("1mo", now, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2023, 5, 1, 12, 0, 0, tzinfo=timezone.utc)) def test_monthly_year_rollover_at_offset(self): now = datetime(2023, 12, 15, 9, 0, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "1mo", now, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("1mo", now, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2024, 1, 1, 12, 0, 0, tzinfo=timezone.utc)) def test_custom_day_reset_applies_offset(self): now = datetime(2023, 5, 15, 10, 30, 0, tzinfo=timezone.utc) - result = get_next_standardized_reset_time( - "3d", now, "UTC", reset_time_of_day=time(12, 0) - ) + result = get_next_standardized_reset_time("3d", now, "UTC", reset_time_of_day=time(12, 0)) self.assertEqual(result, datetime(2023, 5, 18, 12, 0, 0, tzinfo=timezone.utc)) def test_sub_day_durations_ignore_offset(self): base = datetime(2023, 5, 15, 15, 20, 30, tzinfo=timezone.utc) self.assertEqual( - get_next_standardized_reset_time( - "2h", base, "UTC", reset_time_of_day=time(12, 0) - ), + get_next_standardized_reset_time("2h", base, "UTC", reset_time_of_day=time(12, 0)), datetime(2023, 5, 15, 16, 0, 0, tzinfo=timezone.utc), ) self.assertEqual( - get_next_standardized_reset_time( - "30m", base, "UTC", reset_time_of_day=time(12, 0) - ), + get_next_standardized_reset_time("30m", base, "UTC", reset_time_of_day=time(12, 0)), datetime(2023, 5, 15, 15, 30, 0, tzinfo=timezone.utc), ) @@ -385,5 +343,35 @@ class TestWordFormBudgetDurations(unittest.TestCase): self.assertIn("garbage", mock_warning.call_args.args) +class TestGetBudgetWindowStart(unittest.TestCase): + def test_window_start_is_the_previous_reset_on_the_same_schedule(self): + created_at: Final = datetime(2024, 10, 1, 0, 30, tzinfo=timezone.utc) + durations: Final = "1d 24h daily 7d weekly 2w 10d 30d monthly 1mo 4h 5h 30m 45s 1hr fortnightly".split() + for duration in durations: + with self.subTest(duration=duration): + reset_at = get_next_standardized_reset_time(duration, created_at, "UTC") + window_start = get_budget_window_start(duration, reset_at) + self.assertLessEqual(window_start, created_at) + self.assertEqual(get_next_standardized_reset_time(duration, window_start, "UTC"), reset_at) + + def test_thirty_days_spans_the_calendar_month_it_resets_on(self): + self.assertEqual( + get_budget_window_start("30d", datetime(2024, 3, 1, tzinfo=timezone.utc)), + datetime(2024, 2, 1, tzinfo=timezone.utc), + ) + + def test_months_clamp_to_the_shorter_month(self): + self.assertEqual( + get_budget_window_start("1mo", datetime(2024, 3, 31, tzinfo=timezone.utc)), + datetime(2024, 2, 29, tzinfo=timezone.utc), + ) + + def test_unrecognized_duration_is_the_one_day_window_the_scheduler_falls_back_to(self): + self.assertEqual( + get_budget_window_start("1hr", datetime(2024, 1, 16, tzinfo=timezone.utc)), + datetime(2024, 1, 15, tzinfo=timezone.utc), + ) + + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/proxy/utils/helpers/test_month_end_projection.py b/tests/unit/proxy/utils/helpers/test_month_end_projection.py index 8c1a4ae746d..1f2a88b209d 100644 --- a/tests/unit/proxy/utils/helpers/test_month_end_projection.py +++ b/tests/unit/proxy/utils/helpers/test_month_end_projection.py @@ -1,4 +1,5 @@ -from datetime import date, timedelta +from datetime import date, datetime, timedelta, timezone +from typing import Final import pytest @@ -211,3 +212,181 @@ def test_get_projected_spend_over_limit_raises_when_today_missing(monkeypatch): monkeypatch.setattr("litellm.proxy.utils.date", _Broken) with pytest.raises(RuntimeError): get_projected_spend_over_limit(current_spend=1.0, soft_budget_limit=1.0) + + +UTC_RESET_JAN_16: Final = datetime(2024, 1, 16, tzinfo=timezone.utc) + + +def test_daily_budget_projects_pace_to_the_reset_not_month_end(): + result: Final = get_projected_spend_over_limit( + current_spend=8.0, + soft_budget_limit=10.0, + budget_duration="1d", + budget_reset_at=UTC_RESET_JAN_16, + now=datetime(2024, 1, 15, 12, tzinfo=timezone.utc), + ) + assert result == (16.0, date(2024, 1, 15)) + + +def test_daily_budget_late_in_the_day_does_not_double_todays_spend(): + assert ( + get_projected_spend_over_limit( + current_spend=6.0, + soft_budget_limit=10.0, + budget_duration="1d", + budget_reset_at=UTC_RESET_JAN_16, + now=datetime(2024, 1, 15, 23, tzinfo=timezone.utc), + ) + is None + ) + + +def test_naive_reset_time_is_read_as_utc(): + result: Final = get_projected_spend_over_limit( + current_spend=8.0, + soft_budget_limit=10.0, + budget_duration="1d", + budget_reset_at=datetime(2024, 1, 16), + now=datetime(2024, 1, 15, 12), + ) + assert result == (16.0, date(2024, 1, 15)) + + +def test_weekly_budget_measures_pace_from_the_previous_reset(): + result: Final = get_projected_spend_over_limit( + current_spend=8.0, + soft_budget_limit=10.0, + budget_duration="7d", + budget_reset_at=datetime(2024, 1, 18, tzinfo=timezone.utc), + now=datetime(2024, 1, 15, tzinfo=timezone.utc), + ) + assert result == (14.0, date(2024, 1, 16)) + + +def test_monthly_budget_window_starts_on_the_previous_reset_day(): + result: Final = get_projected_spend_over_limit( + current_spend=8.0, + soft_budget_limit=10.0, + budget_duration="1mo", + budget_reset_at=datetime(2024, 3, 1, tzinfo=timezone.utc), + now=datetime(2024, 2, 5, tzinfo=timezone.utc), + ) + assert result == (pytest.approx(58.0), date(2024, 2, 6)) + + +def test_sub_day_budget_is_measured_in_hours_not_days(): + result: Final = get_projected_spend_over_limit( + current_spend=3.0, + soft_budget_limit=5.0, + budget_duration="4h", + budget_reset_at=datetime(2024, 1, 15, 12, tzinfo=timezone.utc), + now=datetime(2024, 1, 15, 10, tzinfo=timezone.utc), + ) + assert result == (pytest.approx(6.0), date(2024, 1, 15)) + + +def test_first_minutes_after_a_reset_do_not_project_a_burst_across_the_window(): + assert ( + get_projected_spend_over_limit( + current_spend=0.10, + soft_budget_limit=10.0, + budget_duration="1d", + budget_reset_at=UTC_RESET_JAN_16, + now=datetime(2024, 1, 15, 0, 1, tzinfo=timezone.utc), + ) + is None + ) + + +def test_exceed_date_is_reported_in_the_reset_timezone(): + result: Final = get_projected_spend_over_limit( + current_spend=9.5, + soft_budget_limit=10.0, + budget_duration="1d", + budget_reset_at=datetime(2024, 1, 17, tzinfo=timezone(timedelta(hours=2))), + now=datetime(2024, 1, 15, 22, 10, tzinfo=timezone.utc), + ) + assert result is not None + assert result[1] == date(2024, 1, 16) + + +@pytest.mark.parametrize("soft_budget_limit", [9.99, 13.9, 13.99]) +def test_exceed_date_never_lands_after_the_reset(soft_budget_limit): + result: Final = get_projected_spend_over_limit( + current_spend=8.0, + soft_budget_limit=soft_budget_limit, + budget_duration="7d", + budget_reset_at=datetime(2024, 1, 18, tzinfo=timezone.utc), + now=datetime(2024, 1, 15, tzinfo=timezone.utc), + ) + assert result is not None + assert result[1] <= date(2024, 1, 18) + + +@pytest.mark.parametrize("current_spend, expected", [(4.0, False), (8.0, True)]) +def test_is_projected_spend_over_limit_follows_the_reset_window(current_spend, expected): + assert ( + is_projected_spend_over_limit( + current_spend=current_spend, + soft_budget_limit=10.0, + budget_duration="1d", + budget_reset_at=UTC_RESET_JAN_16, + now=datetime(2024, 1, 15, 12, tzinfo=timezone.utc), + ) + is expected + ) + + +def test_without_duration_keeps_month_end_behavior(monkeypatch): + _freeze_today(monkeypatch, date(2024, 1, 15)) + result: Final = get_projected_spend_over_limit(current_spend=8.0, soft_budget_limit=10.0) + assert result is not None + assert result[0] == pytest.approx(8.0 + (8.0 / 14) * 16) + + +def test_thirty_day_budget_measures_pace_from_the_first_of_the_month(): + result: Final = get_projected_spend_over_limit( + current_spend=1.0, + soft_budget_limit=10.0, + budget_duration="30d", + budget_reset_at=datetime(2024, 11, 1, tzinfo=timezone.utc), + now=datetime(2024, 10, 4, tzinfo=timezone.utc), + ) + assert result == (pytest.approx(1.0 + 672 / 72), date(2024, 10, 31)) + + +def test_hour_spelling_the_scheduler_resets_at_midnight_does_not_alert_before_midnight(): + assert ( + get_projected_spend_over_limit( + current_spend=1.0, + soft_budget_limit=10.0, + budget_duration="1hr", + budget_reset_at=UTC_RESET_JAN_16, + now=datetime(2024, 1, 15, 22, tzinfo=timezone.utc), + ) + is None + ) + + +def test_unrecognized_duration_projects_within_the_daily_window_it_resets_on(): + result: Final = get_projected_spend_over_limit( + current_spend=8.0, + soft_budget_limit=10.0, + budget_duration="fortnightly", + budget_reset_at=UTC_RESET_JAN_16, + now=datetime(2024, 1, 15, 12, tzinfo=timezone.utc), + ) + assert result == (16.0, date(2024, 1, 15)) + + +def test_reset_more_than_one_window_ahead_does_not_project(): + assert ( + get_projected_spend_over_limit( + current_spend=9.0, + soft_budget_limit=10.0, + budget_duration="1d", + budget_reset_at=UTC_RESET_JAN_16, + now=datetime(2024, 1, 14, 12, tzinfo=timezone.utc), + ) + is None + ) From 487164f762d5d32b96034a32a21317f3e1db3a9d Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:04:23 -0700 Subject: [PATCH 54/65] test(proxy): audit the websocket rejection log on every live route (#45293) * test(proxy): audit the websocket rejection log on every live route * test(proxy): scan the stopped proxy log for the websocket crash and drop the uvicorn wording pin --- .../test_websocket_rejection_logging.py | 317 ++++++++++++++++++ 1 file changed, 317 insertions(+) create mode 100644 tests/integration/authorization/test_websocket_rejection_logging.py diff --git a/tests/integration/authorization/test_websocket_rejection_logging.py b/tests/integration/authorization/test_websocket_rejection_logging.py new file mode 100644 index 00000000000..ea0a841a2b5 --- /dev/null +++ b/tests/integration/authorization/test_websocket_rejection_logging.py @@ -0,0 +1,317 @@ +import asyncio +import uuid +from collections.abc import Iterator, Mapping +from dataclasses import dataclass +from itertools import product +from types import MappingProxyType +from typing import Final + +import psutil +import pytest +import websockets +from integration._support.client import eventually, gateway_from_environment +from integration._support.process import OwnedProxy, graceful_stop_seconds, owned_proxy_process +from pydantic import JsonValue +from websockets.exceptions import InvalidStatus + +pytestmark: Final = pytest.mark.timeout(2 * graceful_stop_seconds() + 120) + +WEBSOCKET_ROUTES: Final = ( + "/v1/responses", + "/responses", + "/v1/realtime", + "/openai/v1/realtime", + "/realtime", + "/openai/v1/responses", + "/openai_passthrough/v1/responses", + "/deepgram/v1/listen", + "/deepgram/listen", + "/vertex_ai/live", +) +HTTP_ONLY_ROUTES: Final = ("/v1/traces", "/v1/logs", "/v1/chat/completions", "/nope") +HANDSHAKE_SHAPES: Final = ( + "no_header", + "empty_bearer", + "basic_scheme", + "lowercase_bearer", + "unknown_key", + "huge_key", + "duplicate_header", + "api_key_header", + "subprotocol_key", + "malformed_key", +) +BURST_SHAPES: Final = ("no_header", "unknown_key", "denied_model") +CRASH_MARKERS: Final = ( + "Exception in ASGI application", + "AttributeError: 'WebSocket' object has no attribute 'method'", + "ERROR: user_api_key_auth.py", +) +OTLP_LENS_MESSAGE: Final = "Send traces and logs directly to the Lens endpoint shown in Lens setup." +REALTIME_TRANSCRIPTION_MODEL: Final = "gpt-realtime-whisper" + +Headers = tuple[tuple[str, str], ...] + + +@dataclass(frozen=True, slots=True) +class Keys: + restricted: str + allowed_model: str + + +@dataclass(frozen=True, slots=True) +class Refusal: + status: int + window: str + + +@pytest.fixture(scope="module") +def owned(tmp_path_factory: pytest.TempPathFactory) -> Iterator[OwnedProxy]: + with ( + gateway_from_environment() as base, + owned_proxy_process( + base, + tmp_path_factory.mktemp("websocket-rejection"), + {"LITELLM_DISABLE_NO_REDIS_WARNING": "true"}, + workers=2, + ) as proxy, + ): + yield proxy + stopped_log: Final = proxy.log.read_text(errors="replace") + assert _crash_lines(stopped_log) == (), _marker_lines(stopped_log) + + +@pytest.fixture(scope="module") +def keys(owned: OwnedProxy) -> Iterator[Keys]: + with owned.gateway.scenario() as scenario: + allowed: Final = scenario.model() + yield Keys(restricted=scenario.key(models=[allowed]), allowed_model=allowed) + + +def _tag(label: str) -> str: + return f"integration-{label}-{uuid.uuid4().hex[:12]}" + + +def _ws_url(owned: OwnedProxy, path: str, query: str) -> str: + base: Final = str(owned.gateway.client.base_url).rstrip("/").replace("http://", "ws://") + return f"{base}{path}?{query}" + + +def _log_after(owned: OwnedProxy, offset: int) -> str: + return owned.log.read_bytes()[offset:].decode(errors="replace") + + +def _crash_lines(window: str) -> tuple[str, ...]: + return tuple(marker for marker in CRASH_MARKERS if marker in window) + + +def _marker_lines(text: str) -> tuple[str, ...]: + return tuple(line for line in text.splitlines() if any(marker in line for marker in CRASH_MARKERS)) + + +def _shapes(keys: Keys) -> Mapping[str, Headers]: + unknown: Final = f"sk-{uuid.uuid4().hex}" + return MappingProxyType( + { + "no_header": (), + "empty_bearer": (("Authorization", "Bearer "),), + "basic_scheme": (("Authorization", "Basic aW50ZWdyYXRpb246eA=="),), + "lowercase_bearer": (("Authorization", f"bearer {unknown}"),), + "unknown_key": (("Authorization", f"Bearer {unknown}"),), + "huge_key": (("Authorization", f"Bearer sk-{'a' * 5120}"),), + "duplicate_header": (("Authorization", f"Bearer {unknown}"), ("Authorization", f"Bearer {unknown}")), + "api_key_header": (("api-key", unknown),), + "subprotocol_key": (("Sec-WebSocket-Protocol", f"openai-insecure-api-key.{unknown}"),), + "malformed_key": (("Authorization", "Bearer integration-not-a-virtual-key"),), + "denied_model": (("Authorization", f"Bearer {keys.restricted}"),), + } + ) + + +async def _refused_status(owned: OwnedProxy, path: str, headers: Headers, query: str) -> int: + with pytest.raises(InvalidStatus) as refused: + async with websockets.connect(_ws_url(owned, path, query), additional_headers=headers): + pass + return refused.value.response.status_code + + +async def _refused(owned: OwnedProxy, path: str, headers: Headers, query: str) -> Refusal: + offset: Final = owned.log.stat().st_size + status: Final = await _refused_status(owned, path, headers, query) + window: Final = eventually( + lambda: _log_after(owned, offset), + lambda text: f'"WebSocket {path}?{query}" 403' in text, + seconds=30, + ) + return Refusal(status, window) + + +@pytest.mark.parametrize("shape", HANDSHAKE_SHAPES) +@pytest.mark.parametrize("path", WEBSOCKET_ROUTES) +async def test_a_refused_handshake_answers_403_and_logs_no_asgi_crash( + owned: OwnedProxy, keys: Keys, path: str, shape: str +) -> None: + refusal: Final = await _refused(owned, path, _shapes(keys)[shape], f"model={_tag('ws')}") + assert refusal.status == 403, refusal.window + assert _crash_lines(refusal.window) == (), refusal.window + + +@pytest.mark.parametrize("path", WEBSOCKET_ROUTES) +async def test_a_denied_model_handshake_logs_one_warning_line(owned: OwnedProxy, keys: Keys, path: str) -> None: + tag: Final = _tag("denied") + refusal: Final = await _refused(owned, path, _shapes(keys)["denied_model"], f"model={tag}") + assert refusal.status == 403, refusal.window + assert refusal.window.count(f"Tried to access {tag}") == 1, refusal.window + assert _crash_lines(refusal.window) == (), refusal.window + + +async def test_a_realtime_denial_on_the_resolved_transcription_model_logs_one_warning_line( + owned: OwnedProxy, keys: Keys +) -> None: + tag: Final = _tag("transcription") + refusal: Final = await _refused( + owned, "/v1/realtime", _shapes(keys)["denied_model"], f"intent=transcription&tag={tag}" + ) + assert refusal.status == 403, refusal.window + assert refusal.window.count(f"Tried to access {REALTIME_TRANSCRIPTION_MODEL}") == 1, refusal.window + assert _crash_lines(refusal.window) == (), refusal.window + + +async def test_a_realtime_handshake_without_a_model_is_refused_without_a_denial_line( + owned: OwnedProxy, keys: Keys +) -> None: + refusal: Final = await _refused(owned, "/v1/realtime", _shapes(keys)["denied_model"], f"tag={_tag('nomodel')}") + assert refusal.status == 403, refusal.window + assert "Tried to access" not in refusal.window, refusal.window + assert _crash_lines(refusal.window) == (), refusal.window + + +@pytest.mark.parametrize("path", HTTP_ONLY_ROUTES) +async def test_a_websocket_handshake_on_an_http_only_route_is_refused_with_403(owned: OwnedProxy, path: str) -> None: + refusal: Final = await _refused(owned, path, (), f"model={_tag('httponly')}") + assert refusal.status == 403, refusal.window + assert _crash_lines(refusal.window) == (), refusal.window + + +def _http_window(owned: OwnedProxy, offset: int, access_line: str) -> str: + return eventually(lambda: _log_after(owned, offset), lambda text: access_line in text, seconds=30) + + +@pytest.mark.parametrize("content_type", ("application/json", "application/x-protobuf")) +@pytest.mark.parametrize("path", ("/v1/traces", "/v1/logs")) +def test_an_otlp_ingest_post_still_answers_410_in_the_caller_s_encoding( + owned: OwnedProxy, path: str, content_type: str +) -> None: + tag: Final = _tag("otlp") + offset: Final = owned.log.stat().st_size + response: Final = owned.gateway.client.post( + f"{path}?tag={tag}", content=b"", headers={"content-type": content_type} + ) + assert response.status_code == 410, response.text + assert response.headers["content-type"].split(";", 1)[0] == content_type, response.headers + assert OTLP_LENS_MESSAGE.encode() in response.content, response.content + window: Final = _http_window(owned, offset, f'"POST {path}?tag={tag} HTTP/1.1" 410') + assert _crash_lines(window) == (), window + + +def test_a_get_on_the_otlp_path_is_not_otlp_ingest_and_answers_a_plain_401(owned: OwnedProxy) -> None: + tag: Final = _tag("otlpget") + offset: Final = owned.log.stat().st_size + response: Final = owned.gateway.client.get( + f"/v1/traces?tag={tag}", headers={"content-type": "application/x-protobuf"} + ) + assert response.status_code == 401, response.text + assert response.headers["content-type"].split(";", 1)[0] == "application/json", response.headers + assert response.json()["error"]["code"] == "401", response.text + window: Final = _http_window(owned, offset, f'"GET /v1/traces?tag={tag} HTTP/1.1" 401') + assert _crash_lines(window) == (), window + + +def test_an_unknown_http_route_answers_a_plain_404(owned: OwnedProxy) -> None: + tag: Final = _tag("nope") + offset: Final = owned.log.stat().st_size + response: Final = owned.gateway.request("POST", f"/nope?tag={tag}", {}) + assert response.status_code == 404, response.text + assert response.json() == {"detail": "Not Found"}, response.text + window: Final = _http_window(owned, offset, f'"POST /nope?tag={tag} HTTP/1.1" 404') + assert _crash_lines(window) == (), window + + +def test_a_bad_request_on_a_plain_route_keeps_its_detail(owned: OwnedProxy, keys: Keys) -> None: + tag: Final = _tag("tokens") + offset: Final = owned.log.stat().st_size + response: Final = owned.gateway.request("POST", f"/utils/token_counter?tag={tag}", {"model": keys.allowed_model}) + assert response.status_code == 400, response.text + assert response.json() == {"detail": "prompt or messages or contents must be provided"}, response.text + window: Final = _http_window(owned, offset, f'"POST /utils/token_counter?tag={tag} HTTP/1.1" 400') + assert _crash_lines(window) == (), window + + +def _denied_body(path: str, model: str) -> Mapping[str, JsonValue]: + message: Final[dict[str, JsonValue]] = {"role": "user", "content": "integration denial"} + match path: + case "/v1/messages": + return {"model": model, "max_tokens": 4, "messages": [message]} + case "/v1/responses": + return {"model": model, "input": "integration denial"} + case _: + return {"model": model, "messages": [message]} + + +@pytest.mark.parametrize("path", ("/v1/chat/completions", "/v1/messages", "/v1/responses")) +def test_an_http_model_denial_logs_one_warning_line_and_no_traceback(owned: OwnedProxy, keys: Keys, path: str) -> None: + tag: Final = _tag("httpdenied") + offset: Final = owned.log.stat().st_size + response: Final = owned.gateway.request("POST", path, _denied_body(path, tag), key=keys.restricted) + assert response.status_code == 403, response.text + window: Final = eventually( + lambda: _log_after(owned, offset), lambda text: f"Tried to access {tag}" in text, seconds=30 + ) + assert window.count(f"Tried to access {tag}") == 1, window + assert _crash_lines(window) == (), window + + +def _workers(owned: OwnedProxy) -> tuple[psutil.Process, ...]: + return tuple(child for child in psutil.Process(owned.process.pid).children() if _is_worker(child)) + + +def _is_worker(child: psutil.Process) -> bool: + try: + return "spawn_main" in " ".join(child.cmdline()) and child.status() != psutil.STATUS_ZOMBIE + except psutil.Error: + return False + + +async def _burst(owned: OwnedProxy, shapes: Mapping[str, Headers], label: str) -> str: + plan: Final = tuple((path, shape, _tag(label)) for path, shape in product(WEBSOCKET_ROUTES, BURST_SHAPES)) + offset: Final = owned.log.stat().st_size + statuses: Final = await asyncio.gather( + *(_refused_status(owned, path, shapes[shape], f"model={tag}") for path, shape, tag in plan) + ) + assert tuple(statuses) == (403,) * len(plan), statuses + access_lines: Final = tuple(f'"WebSocket {path}?model={tag}" 403' for path, _, tag in plan) + window: Final = eventually( + lambda: _log_after(owned, offset), + lambda text: all(line in text for line in access_lines), + seconds=60, + ) + assert window.count(f"Tried to access integration-{label}-") == len(WEBSOCKET_ROUTES), window + return window + + +async def test_a_rejection_burst_survives_a_killed_worker_without_an_asgi_crash(owned: OwnedProxy, keys: Keys) -> None: + shapes: Final = _shapes(keys) + before: Final = await _burst(owned, shapes, "burst") + assert _crash_lines(before) == (), before + victim: Final = eventually(lambda: _workers(owned), lambda workers: len(workers) == 2)[0] + victim.kill() + during: Final = await _refused(owned, "/v1/responses", shapes["unknown_key"], f"model={_tag('during')}") + assert during.status == 403, during.window + assert _crash_lines(during.window) == (), during.window + eventually( + lambda: frozenset(worker.pid for worker in _workers(owned)), + lambda pids: len(pids) == 2 and victim.pid not in pids, + seconds=graceful_stop_seconds(), + ) + after: Final = await _burst(owned, shapes, "respawned") + assert _crash_lines(after) == (), after From b9ff2d66a4da3df6a2730011dd6d3d9ab61a6c2b Mon Sep 17 00:00:00 2001 From: Mateo Wang <277851410+mateo-berri@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:05:47 -0700 Subject: [PATCH 55/65] fix(proxy): retry lock-timed-out daily spend batches in place so the shutdown flush keeps them (#45273) * fix(proxy): retry lock-timed-out daily spend batches in place so the shutdown flush keeps them Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): annotate the lock-timeout retry test and carry its context in assert messages Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(proxy): mark the remaining lock-timeout test local Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/db/db_spend_update_writer.py | 8 ++-- .../proxy/db/test_db_spend_update_writer.py | 48 +++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/litellm/proxy/db/db_spend_update_writer.py b/litellm/proxy/db/db_spend_update_writer.py index 158a6c1a3c4..9fb8cc20981 100644 --- a/litellm/proxy/db/db_spend_update_writer.py +++ b/litellm/proxy/db/db_spend_update_writer.py @@ -2589,9 +2589,11 @@ class DBSpendUpdateWriter: PrismaDBExceptionHandler, ) - is_retryable = isinstance( - e, DB_RETRY_SAFE_ERROR_TYPES - ) or PrismaDBExceptionHandler.is_deadlock_error(e) + is_retryable = ( + isinstance(e, DB_RETRY_SAFE_ERROR_TYPES) + or PrismaDBExceptionHandler.is_deadlock_error(e) + or PrismaDBExceptionHandler.is_lock_timeout_error(e) + ) if not is_retryable: raise if i >= n_retry_times: diff --git a/tests/unit/proxy/db/test_db_spend_update_writer.py b/tests/unit/proxy/db/test_db_spend_update_writer.py index dc9b8fd395b..4a508fcd272 100644 --- a/tests/unit/proxy/db/test_db_spend_update_writer.py +++ b/tests/unit/proxy/db/test_db_spend_update_writer.py @@ -537,6 +537,54 @@ async def test_update_daily_spend_retries_connect_errors(monkeypatch): assert len(prisma_client.db.statements) == 2 +@pytest.mark.asyncio +async def test_update_daily_spend_retries_lock_timeout_errors(monkeypatch: pytest.MonkeyPatch) -> None: + def _lock_timeout_error() -> PrismaDataError: + return PrismaDataError( + data={ + "user_facing_error": { + "is_panic": False, + "message": "Error querying the database: canceling statement due to lock timeout", + "meta": {"code": "55P03", "message": "canceling statement due to lock timeout"}, + } + } + ) + + outcomes: Final = iter([_lock_timeout_error(), None]) + + def first_attempt_locks_out() -> int: + outcome: Final = next(outcomes) + if outcome is not None: + raise outcome + return 1 + + prisma_client: Final = _RecordingPrisma(execute_raw=first_attempt_locks_out) + proxy_logging: Final = MagicMock() + proxy_logging.failure_handler = AsyncMock() + + async def fake_sleep(seconds: float) -> None: + return None + + monkeypatch.setattr("litellm.proxy.db.db_spend_update_writer.asyncio.sleep", fake_sleep) + daily_spend_transactions: Final = {"k1": _daily_txn()} + await DBSpendUpdateWriter._update_daily_spend( + n_retry_times=3, + prisma_client=prisma_client, + proxy_logging_obj=proxy_logging, + daily_spend_transactions=daily_spend_transactions, + entity_type="user", + entity_id_field="user_id", + ) + + assert len(prisma_client.db.statements) == 2, ( + "a 55P03 lock_timeout cancels the upsert before it applies, so the writer must " + "resend it in place instead of only requeueing it for a next tick a shutdown " + "flush never gets" + ) + assert daily_spend_transactions == {}, "the retried batch must drain the transactions dict" + proxy_logging.failure_handler.assert_not_called() + + @pytest.mark.asyncio async def test_update_daily_spend_sorting(): """ From 3571b5dc724453ebdd7d8941a20330e99224045d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 12:27:52 -0700 Subject: [PATCH 56/65] fix(vertex_ai): forward the inline-tools-2026-09-15 beta to Vertex and Anthropic (#45297) * fix(vertex_ai): forward the inline-tools-2026-09-15 beta to Vertex and Anthropic * test(vertex_ai): trim the inline-tools beta test docstrings and type their locals * test(vertex_ai): add audit cells for inline-tools beta forwarding on Vertex Anthropic * test(vertex_ai): send legal whitespace in the hostile beta header audit cell * test(vertex_ai): close the SDK clients in the inline-tools audit cells --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/anthropic_beta_headers_config.json | 6 +- ...vertex_anthropic_inline_tools_beta_wire.py | 589 ++++++++++++++++++ ...artner_models_anthropic_messages_config.py | 35 ++ ...partner_models_anthropic_transformation.py | 29 + .../test_anthropic_beta_headers_filtering.py | 16 +- 5 files changed, 672 insertions(+), 3 deletions(-) create mode 100644 tests/integration/providers/test_vertex_anthropic_inline_tools_beta_wire.py diff --git a/litellm/anthropic_beta_headers_config.json b/litellm/anthropic_beta_headers_config.json index 332d9b3ad9d..7690372a214 100644 --- a/litellm/anthropic_beta_headers_config.json +++ b/litellm/anthropic_beta_headers_config.json @@ -36,7 +36,8 @@ "web-search-2025-03-05": "web-search-2025-03-05", "mid-conversation-output-config-2026-07-01": "mid-conversation-output-config-2026-07-01", "thinking-display-updates-2026-08-18": "thinking-display-updates-2026-08-18", - "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01" + "mid-conversation-tool-changes-2026-07-01": "mid-conversation-tool-changes-2026-07-01", + "inline-tools-2026-09-15": "inline-tools-2026-09-15" }, "azure_ai": { "advisor-tool-2026-03-01": null, @@ -214,7 +215,8 @@ "token-efficient-tools-2025-02-19": null, "tool-search-tool-2025-10-19": "tool-search-tool-2025-10-19", "web-fetch-2025-09-10": null, - "web-search-2025-03-05": "web-search-2025-03-05" + "web-search-2025-03-05": "web-search-2025-03-05", + "inline-tools-2026-09-15": "inline-tools-2026-09-15" }, "databricks": { "advisor-tool-2026-03-01": null, diff --git a/tests/integration/providers/test_vertex_anthropic_inline_tools_beta_wire.py b/tests/integration/providers/test_vertex_anthropic_inline_tools_beta_wire.py new file mode 100644 index 00000000000..5061f0b34a4 --- /dev/null +++ b/tests/integration/providers/test_vertex_anthropic_inline_tools_beta_wire.py @@ -0,0 +1,589 @@ +import asyncio +import json +import re +import signal +import threading +import uuid +from collections.abc import Callable, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import ExitStack +from dataclasses import dataclass +from pathlib import Path +from queue import SimpleQueue +from typing import Final + +import anthropic +import httpx +import openai +import psutil +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually +from integration._support.process import graceful_stop_seconds, owned_proxy_process +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +_INLINE_TOOLS: Final = "inline-tools-2026-09-15" +_THINKING: Final = "interleaved-thinking-2025-05-14" +_VERTEX_UNSUPPORTED: Final = "effort-2025-11-24" +_UNKNOWN: Final = "not-a-beta-2099-01-01" +_BACKEND: Final = "claude-opus-5-5" +_PROJECT: Final = "scripted-project" +_LOCATION: Final = "us-east5" +_MODEL_PATH: Final = f"/v1/projects/{_PROJECT}/locations/{_LOCATION}/publishers/anthropic/models/{_BACKEND}" +_VERTEX_TARGET: Final = f"{_MODEL_PATH}:rawPredict" +_VERTEX_STREAM_TARGET: Final = f"{_MODEL_PATH}:streamRawPredict?alt=sse" +_ANTHROPIC_TARGET: Final = "/v1/messages" +_ANTHROPIC_KEY: Final = "synthetic-anthropic-key" +_REPLY_TEXT: Final = "inline tools beta control" +_OWNED_MODEL: Final = "partner-claude" +_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +_OWNED_PROXY_CELL_SECONDS: Final = 2 * graceful_stop_seconds() + 120 + +_REPLY: Final[dict[str, JsonValue]] = { + "id": "msg_inline_tools_beta", + "type": "message", + "role": "assistant", + "model": _BACKEND, + "content": [{"type": "text", "text": _REPLY_TEXT}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 5, "output_tokens": 3}, +} +_EVENTS: Final[tuple[tuple[str, dict[str, JsonValue]], ...]] = ( + ( + "message_start", + { + "type": "message_start", + "message": {**_REPLY, "content": [], "stop_reason": None, "usage": {"input_tokens": 5, "output_tokens": 1}}, + }, + ), + ("content_block_start", {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}), + ( + "content_block_delta", + {"type": "content_block_delta", "index": 0, "delta": {"type": "text_delta", "text": _REPLY_TEXT}}, + ), + ("content_block_stop", {"type": "content_block_stop", "index": 0}), + ( + "message_delta", + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn", "stop_sequence": None}, + "usage": {"output_tokens": 3}, + }, + ), + ("message_stop", {"type": "message_stop"}), +) +_SSE: Final = tuple(f"event: {name}\ndata: {json.dumps(data)}\n\n".encode() for name, data in _EVENTS) +_REPLY_BODY: Final = json.dumps(_REPLY).encode() + + +@dataclass(frozen=True, slots=True) +class _Delivery: + target: str + beta: str | None + body: str + + +def _streams(request: Request) -> bool: + return request.target == _VERTEX_STREAM_TARGET or _JSON_OBJECT.validate_json(request.body).get("stream") is True + + +def _peer(request: Request) -> Reply: + if request.target in (_VERTEX_TARGET, _VERTEX_STREAM_TARGET): + assert request.headers["authorization"] == "Bearer scripted-token", request.headers + elif request.target == _ANTHROPIC_TARGET: + assert request.headers["x-api-key"] == _ANTHROPIC_KEY, request.headers + else: + return Reply(status=404, body=json.dumps({"error": f"unscripted target {request.target}"}).encode()) + return Reply(content_type="text/event-stream", chunks=_SSE) if _streams(request) else Reply(body=_REPLY_BODY) + + +def _holding(held: SimpleQueue[str], release: threading.Event) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + held.put(request.target) + assert release.wait(timeout=60), "Held request was never released" + return _peer(request) + + return respond + + +def _delivered(wire: Wire) -> tuple[_Delivery, ...]: + return tuple( + _Delivery(request.target, request.headers.get("anthropic-beta"), request.body.decode()) + for request in wire.drain() + ) + + +def _one_delivery(wire: Wire, target: str, marker: str) -> str | None: + (sent,) = _delivered(wire) + assert sent.target == target, sent.target + assert marker in sent.body, sent.body + return sent.beta + + +def _vertex_deployment(gateway: Gateway, scenario: Scenario, api_base: str) -> str: + return scenario.model( + model=f"vertex_ai/{_BACKEND}", + api_base=api_base, + api_key=None, + vertex_project=_PROJECT, + vertex_location=_LOCATION, + vertex_credentials=service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")), + ) + + +def _anthropic_deployment(scenario: Scenario, api_base: str) -> str: + return scenario.model(model=f"anthropic/{_BACKEND}", api_base=api_base, api_key=_ANTHROPIC_KEY) + + +def _marker(label: str) -> str: + return f"{label} {uuid.uuid4().hex}" + + +def _messages_body(model: str, marker: str, stream: bool = False) -> dict[str, JsonValue]: + return {"model": model, "max_tokens": 16, "messages": [{"role": "user", "content": marker}], "stream": stream} + + +def _chat_body(model: str, marker: str, stream: bool = False) -> dict[str, JsonValue]: + return {"model": model, "messages": [{"role": "user", "content": marker}], "stream": stream} + + +def _responses_body(model: str, marker: str, stream: bool = False) -> dict[str, JsonValue]: + return {"model": model, "input": marker, "stream": stream} + + +def _beta_headers(value: str | None) -> dict[str, str]: + return {} if value is None else {"anthropic-beta": value} + + +def _generated(gateway: Gateway, path: str, body: Mapping[str, JsonValue], beta: str | None) -> httpx.Response: + response: Final = gateway.request("POST", path, body, headers=_beta_headers(beta)) + assert response.status_code == 200, response.text + assert _REPLY_TEXT in response.text, response.text + return response + + +def _vertex_target(stream: bool) -> str: + return _VERTEX_STREAM_TARGET if stream else _VERTEX_TARGET + + +def _clients(stack: ExitStack, base_url: str, count: int) -> tuple[httpx.Client, ...]: + return tuple( + stack.enter_context(httpx.Client(base_url=base_url, timeout=60, trust_env=False)) for _ in range(count) + ) + + +def _post(client: httpx.Client, key: str, path: str, body: Mapping[str, JsonValue], beta: str) -> int: + return client.post( + path, json=dict(body), headers={"Authorization": f"Bearer {key}", **_beta_headers(beta)} + ).status_code + + +def _post_or_dropped(client: httpx.Client, key: str, path: str, body: Mapping[str, JsonValue], beta: str) -> int | None: + try: + return _post(client, key, path, body, beta) + except httpx.TransportError: + return None + + +def _local_port(client: httpx.Client) -> int: + with client.stream("GET", "/health/liveliness") as response: + port: Final = int(response.extensions["network_stream"].get_extra_info("client_addr")[1]) + response.read() + assert response.status_code == 200, response.text + return port + + +def _accepted_client_ports(pid: int, proxy_port: int) -> frozenset[int]: + return frozenset( + connection.raddr.port + for connection in psutil.Process(pid).net_connections(kind="tcp") + if connection.raddr and connection.laddr.port == proxy_port + ) + + +def _burst_bodies(model: str, label: str, count: int) -> tuple[tuple[str, dict[str, JsonValue], str], ...]: + markers: Final = tuple(_marker(label) for _ in range(count)) + return tuple( + ( + "/v1/messages" if index % 2 == 0 else "/v1/chat/completions", + _messages_body(model, marker, stream=index % 4 == 2) + if index % 2 == 0 + else _chat_body(model, marker, stream=index % 4 == 3), + marker, + ) + for index, marker in enumerate(markers) + ) + + +def _owned_config(path: Path, gateway: Gateway, api_base: str) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + path.write_text( + yaml.safe_dump( + { + **config, + "model_list": [ + { + "model_name": _OWNED_MODEL, + "litellm_params": { + "model": f"vertex_ai/{_BACKEND}", + "api_base": api_base, + "vertex_project": _PROJECT, + "vertex_location": _LOCATION, + "vertex_credentials": service_account_json(_PROJECT, gateway.upstream_url.rstrip("/")), + }, + } + ], + } + ) + ) + return path + + +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +def test_vertex_messages_forwards_the_inline_tools_beta(gateway: Gateway, stream: bool) -> None: + marker: Final = _marker("vertex messages") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + response: Final = _generated(gateway, "/v1/messages", _messages_body(model, marker, stream), _INLINE_TOOLS) + assert not stream or "event: message_stop" in response.text, response.text + assert _one_delivery(wire, _vertex_target(stream), marker) == _INLINE_TOOLS + + +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +def test_vertex_chat_completions_forwards_the_inline_tools_beta(gateway: Gateway, stream: bool) -> None: + marker: Final = _marker("vertex chat") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + response: Final = _generated(gateway, "/v1/chat/completions", _chat_body(model, marker, stream), _INLINE_TOOLS) + assert not stream or response.text.rstrip().endswith("data: [DONE]"), response.text + assert _one_delivery(wire, _vertex_target(stream), marker) == _INLINE_TOOLS + + +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +def test_vertex_responses_forwards_the_inline_tools_beta(gateway: Gateway, stream: bool) -> None: + marker: Final = _marker("vertex responses") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + _generated(gateway, "/v1/responses", _responses_body(model, marker, stream), _INLINE_TOOLS) + assert _one_delivery(wire, _vertex_target(stream), marker) == _INLINE_TOOLS + + +def test_vertex_messages_through_the_anthropic_sdk_forwards_the_inline_tools_beta(gateway: Gateway) -> None: + marker: Final = _marker("anthropic sdk sync") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + with anthropic.Anthropic( + base_url=str(gateway.client.base_url), + api_key=gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False, timeout=60), + ) as client: + reply: Final = client.beta.messages.create( + model=model, max_tokens=16, messages=[{"role": "user", "content": marker}], betas=[_INLINE_TOOLS] + ) + assert _REPLY_TEXT in reply.model_dump_json(), reply + assert _one_delivery(wire, _VERTEX_TARGET, marker) == _INLINE_TOOLS + + +def test_vertex_messages_through_the_async_anthropic_sdk_forwards_the_inline_tools_beta(gateway: Gateway) -> None: + marker: Final = _marker("anthropic sdk async") + + async def generate(model: str) -> str: + async with anthropic.AsyncAnthropic( + base_url=str(gateway.client.base_url), + api_key=gateway.key, + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False, timeout=60), + ) as client: + reply: Final = await client.beta.messages.create( + model=model, max_tokens=16, messages=[{"role": "user", "content": marker}], betas=[_INLINE_TOOLS] + ) + return reply.model_dump_json() + + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + assert _REPLY_TEXT in asyncio.run(generate(model)) + assert _one_delivery(wire, _VERTEX_TARGET, marker) == _INLINE_TOOLS + + +def test_vertex_chat_through_the_openai_sdk_forwards_the_inline_tools_beta(gateway: Gateway) -> None: + marker: Final = _marker("openai sdk sync") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + with openai.OpenAI( + base_url=f"{gateway.client.base_url}/v1", + api_key=gateway.key, + max_retries=0, + http_client=httpx.Client(trust_env=False, timeout=60), + ) as client: + reply: Final = client.chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}], extra_headers=_beta_headers(_INLINE_TOOLS) + ) + assert reply.choices[0].message.content == _REPLY_TEXT, reply + assert _one_delivery(wire, _VERTEX_TARGET, marker) == _INLINE_TOOLS + + +def test_vertex_chat_through_the_async_openai_sdk_forwards_the_inline_tools_beta(gateway: Gateway) -> None: + marker: Final = _marker("openai sdk async") + + async def generate(model: str) -> str | None: + async with openai.AsyncOpenAI( + base_url=f"{gateway.client.base_url}/v1", + api_key=gateway.key, + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False, timeout=60), + ) as client: + reply: Final = await client.chat.completions.create( + model=model, messages=[{"role": "user", "content": marker}], extra_headers=_beta_headers(_INLINE_TOOLS) + ) + return reply.choices[0].message.content + + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + assert asyncio.run(generate(model)) == _REPLY_TEXT + assert _one_delivery(wire, _VERTEX_TARGET, marker) == _INLINE_TOOLS + + +@pytest.mark.parametrize("stream", [False, True], ids=["non_stream", "stream"]) +def test_anthropic_chat_completions_forwards_the_inline_tools_beta(gateway: Gateway, stream: bool) -> None: + marker: Final = _marker("anthropic chat") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _anthropic_deployment(scenario, wire.url) + response: Final = _generated(gateway, "/v1/chat/completions", _chat_body(model, marker, stream), _INLINE_TOOLS) + assert not stream or response.text.rstrip().endswith("data: [DONE]"), response.text + assert _one_delivery(wire, _ANTHROPIC_TARGET, marker) == _INLINE_TOOLS + + +def test_anthropic_messages_passes_the_inline_tools_beta_through_unfiltered(gateway: Gateway) -> None: + marker: Final = _marker("anthropic messages") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _anthropic_deployment(scenario, wire.url) + _generated(gateway, "/v1/messages", _messages_body(model, marker), f"{_INLINE_TOOLS},{_UNKNOWN}") + assert _one_delivery(wire, _ANTHROPIC_TARGET, marker) == f"{_INLINE_TOOLS},{_UNKNOWN}" + + +def test_vertex_messages_drops_an_unsupported_beta_sent_next_to_inline_tools(gateway: Gateway) -> None: + marker: Final = _marker("vertex unsupported sibling") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + _generated(gateway, "/v1/messages", _messages_body(model, marker), f"{_VERTEX_UNSUPPORTED},{_INLINE_TOOLS}") + assert _one_delivery(wire, _VERTEX_TARGET, marker) == _INLINE_TOOLS + + +def test_vertex_messages_forwards_inline_tools_with_another_supported_beta(gateway: Gateway) -> None: + marker: Final = _marker("vertex supported sibling") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + _generated(gateway, "/v1/messages", _messages_body(model, marker), f"{_THINKING},{_INLINE_TOOLS}") + assert _one_delivery(wire, _VERTEX_TARGET, marker) == f"{_INLINE_TOOLS},{_THINKING}" + + +@pytest.mark.parametrize( + "beta", + [None, _UNKNOWN, "", "5", f"{_UNKNOWN}," * 240, _INLINE_TOOLS.upper()], + ids=["absent", "unknown", "empty", "int", "5kb_junk", "upper_case"], +) +def test_vertex_messages_sends_no_beta_header_when_nothing_survives_the_filter( + gateway: Gateway, beta: str | None +) -> None: + marker: Final = _marker("vertex nothing survives") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + _generated(gateway, "/v1/messages", _messages_body(model, marker), beta) + assert _one_delivery(wire, _VERTEX_TARGET, marker) is None + + +@pytest.mark.parametrize( + "beta", + [f"{_INLINE_TOOLS} , {_INLINE_TOOLS}", f"{_UNKNOWN}," * 240 + _INLINE_TOOLS], + ids=["duplicated_with_spaces", "inside_5kb_junk"], +) +def test_vertex_messages_forwards_inline_tools_once_from_a_hostile_header_value(gateway: Gateway, beta: str) -> None: + marker: Final = _marker("vertex hostile value") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + _generated(gateway, "/v1/messages", _messages_body(model, marker), beta) + assert _one_delivery(wire, _VERTEX_TARGET, marker) == _INLINE_TOOLS + + +def test_vertex_messages_forwards_inline_tools_once_from_a_repeated_header_line(gateway: Gateway) -> None: + marker: Final = _marker("vertex repeated line") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + response: Final = gateway.client.post( + "/v1/messages", + json=_messages_body(model, marker), + headers=httpx.Headers( + [ + ("Authorization", f"Bearer {gateway.key}"), + ("anthropic-beta", _INLINE_TOOLS), + ("anthropic-beta", _INLINE_TOOLS), + ] + ), + ) + assert response.status_code == 200, response.text + assert _one_delivery(wire, _VERTEX_TARGET, marker) == _INLINE_TOOLS + + +def test_vertex_messages_unauthenticated_request_with_the_beta_never_reaches_the_peer(gateway: Gateway) -> None: + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + response: Final = gateway.request( + "POST", + "/v1/messages", + _messages_body(model, _marker("unauthenticated")), + key="sk-not-a-key-this-proxy-issued", + headers=_beta_headers(_INLINE_TOOLS), + ) + assert response.status_code == 401, response.text + assert wire.drain() == () + + +def test_reloading_the_allowlist_keeps_forwarding_inline_tools(gateway: Gateway) -> None: + marker: Final = _marker("after reload") + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + reloaded: Final = gateway.request("POST", "/reload/anthropic_beta_headers") + assert reloaded.status_code == 200, reloaded.text + status: Final = gateway.request("GET", "/schedule/anthropic_beta_headers_reload/status") + assert status.status_code == 200, status.text + assert _JSON_OBJECT.validate_json(status.content)["scheduled"] is False, status.text + _generated(gateway, "/v1/messages", _messages_body(model, marker), _INLINE_TOOLS) + assert _one_delivery(wire, _VERTEX_TARGET, marker) == _INLINE_TOOLS + + +def test_vertex_messages_repeated_requests_each_carry_the_inline_tools_beta(gateway: Gateway) -> None: + markers: Final = tuple(_marker("repeated") for _ in range(2)) + with wire_server(_peer) as wire, gateway.scenario() as scenario: + model: Final = _vertex_deployment(gateway, scenario, wire.url) + for marker in markers: + _generated(gateway, "/v1/messages", _messages_body(model, marker), _INLINE_TOOLS) + delivered: Final = _delivered(wire) + assert tuple((sent.target, sent.beta) for sent in delivered) == ((_VERTEX_TARGET, _INLINE_TOOLS),) * 2 + assert tuple(marker in sent.body for marker, sent in zip(markers, delivered, strict=True)) == (True, True) + + +def test_concurrent_burst_across_both_workers_forwards_inline_tools_on_every_request(gateway: Gateway) -> None: + with ExitStack() as stack: + wire: Final = stack.enter_context(wire_server(_peer)) + scenario: Final = stack.enter_context(gateway.scenario()) + model: Final = _vertex_deployment(gateway, scenario, wire.url) + burst: Final = _burst_bodies(model, "burst", 12) + clients: Final = _clients(stack, str(gateway.client.base_url), len(burst)) + pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(burst))) + statuses: Final = tuple( + pool.map( + lambda pair: _post(pair[0], gateway.key, pair[1][0], pair[1][1], _INLINE_TOOLS), + zip(clients, burst, strict=True), + ) + ) + assert statuses == (200,) * len(burst), statuses + delivered: Final = _delivered(wire) + assert tuple(sent.beta for sent in delivered) == (_INLINE_TOOLS,) * len(burst), delivered + seen: Final = tuple(sum(marker in sent.body for sent in delivered) for _, _, marker in burst) + assert seen == (1,) * len(burst), seen + + +def test_upstream_outage_mid_run_recovers_with_inline_tools_still_forwarded(gateway: Gateway) -> None: + with ExitStack() as stack: + clients: Final = _clients(stack, str(gateway.client.base_url), 8) + pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) + scenario: Final = stack.enter_context(gateway.scenario()) + + def wave(label: str) -> tuple[tuple[int | None, str], ...]: + markers: Final = tuple(_marker(label) for _ in clients) + statuses: Final = tuple( + pool.map( + lambda pair: _post_or_dropped( + pair[0], gateway.key, "/v1/messages", _messages_body(model, pair[1]), _INLINE_TOOLS + ), + zip(clients, markers, strict=True), + ) + ) + return tuple(zip(statuses, markers, strict=True)) + + def forwarded(wire: Wire, results: tuple[tuple[int | None, str], ...]) -> None: + assert tuple(status for status, _ in results) == (200,) * len(clients), results + delivered: Final = _delivered(wire) + assert tuple(sent.beta for sent in delivered) == (_INLINE_TOOLS,) * len(clients), delivered + seen: Final = tuple(sum(marker in sent.body for sent in delivered) for _, marker in results) + assert seen == (1,) * len(clients), seen + + with wire_server(_peer) as wire: + port: Final = int(wire.url.rsplit(":", 1)[1]) + model: Final = _vertex_deployment(gateway, scenario, wire.url) + forwarded(wire, wave("before outage")) + outage: Final = wave("during outage") + assert all(status is not None and status >= 500 for status, _ in outage), outage + assert gateway.request("GET", "/health/liveliness").status_code == 200 + with wire_server(_peer, port=port) as revived: + forwarded(revived, wave("after outage")) + + +@pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) +def test_worker_sigkill_mid_burst_leaves_the_replacement_forwarding_inline_tools( + gateway: Gateway, tmp_path: Path +) -> None: + held: Final[SimpleQueue[str]] = SimpleQueue() + release: Final = threading.Event() + with ExitStack() as stack: + wire: Final = stack.enter_context(wire_server(_holding(held, release))) + stack.callback(release.set) + config: Final = _owned_config(tmp_path / "inline-tools-worker-kill.yaml", gateway, wire.url) + owned: Final = stack.enter_context( + owned_proxy_process( + gateway, tmp_path, {"LITELLM_LOCAL_ANTHROPIC_BETA_HEADERS": "true"}, config=config, workers=2 + ) + ) + proxy_url: Final = owned.gateway.client.base_url + workers: Final = eventually( + lambda: tuple(int(pid) for pid in _STARTED_WORKER.findall(owned.log.read_text())), + lambda pids: len(pids) == 2, + seconds=30, + ) + clients: Final = _clients(stack, str(proxy_url), 12) + pool: Final = stack.enter_context(ThreadPoolExecutor(max_workers=len(clients))) + ports: Final = tuple(_local_port(client) for client in clients) + markers: Final = tuple(_marker("worker kill") for _ in clients) + futures: Final = tuple( + pool.submit( + _post_or_dropped, + client, + gateway.key, + "/v1/messages", + _messages_body(_OWNED_MODEL, marker), + _INLINE_TOOLS, + ) + for client, marker in zip(clients, markers, strict=True) + ) + eventually(held.qsize, lambda size: size == len(clients), seconds=30) + shares: Final = {pid: _accepted_client_ports(pid, proxy_url.port or 0) & frozenset(ports) for pid in workers} + assert sum(map(len, shares.values())) == len(clients), shares + victim: Final = min((pid for pid in workers if shares[pid]), key=lambda pid: len(shares[pid])) + psutil.Process(victim).send_signal(signal.SIGKILL) + release.set() + results: Final = tuple(future.result(timeout=60) for future in futures) + for port, result in zip(ports, results, strict=True): + assert result == (None if port in shares[victim] else 200), (port, result, shares) + eventually(lambda: len(_STARTED_WORKER.findall(owned.log.read_text())), lambda started: started >= 3, 120) + second_wave: Final = tuple(_marker("after kill") for _ in range(6)) + for marker in second_wave: + assert ( + _post( + owned.gateway.client, + gateway.key, + "/v1/messages", + _messages_body(_OWNED_MODEL, marker), + _INLINE_TOOLS, + ) + == 200 + ) + delivered: Final = _delivered(wire) + assert tuple(sent.beta for sent in delivered) == (_INLINE_TOOLS,) * (len(clients) + len(second_wave)), delivered + seen: Final = tuple(sum(marker in sent.body for sent in delivered) for marker in markers + second_wave) + assert seen == (1,) * len(seen), seen + assert owned.process.poll() is None diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py index 67a32cc82bc..1452f77604f 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_messages_config.py @@ -1,6 +1,7 @@ import copy import json import os +from typing import Final from unittest.mock import MagicMock, patch import pytest @@ -180,6 +181,40 @@ def test_no_per_message_output_config_leaves_per_turn_control_beta_out(): assert "per-turn-control-2026-07-01" not in headers.get("anthropic-beta", "") +def test_inline_tools_beta_reaches_the_vertex_messages_request(local_beta_headers_config: None) -> None: + """Vertex rejects a `tool_addition` system message unless inline-tools-2026-09-15 is on the request, so the + /v1/messages beta filter must keep the header the client sent (source and date in + tests/unit/test_anthropic_beta_headers_filtering.py).""" + from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta + + messages: Final[list[dict[str, object]]] = [ + {"role": "user", "content": "What is the weather in Paris?"}, + { + "role": "system", + "content": [ + { + "type": "tool_addition", + "tool": { + "type": "tool_definition", + "definition": { + "name": "db_query", + "description": "Run a read-only SQL query", + "input_schema": {"type": "object", "properties": {"sql": {"type": "string"}}}, + }, + }, + } + ], + }, + ] + + filtered: Final = update_headers_with_filtered_beta( + headers=_validate_vertex_headers({"anthropic-beta": "inline-tools-2026-09-15"}, messages), + provider="vertex_ai", + ) + + assert filtered["anthropic-beta"].split(",").count("inline-tools-2026-09-15") == 1 + + def test_web_search_header_not_added_without_tool(): """Test that beta header is NOT added when web search tool is not present""" config = VertexAIPartnerModelsAnthropicMessagesConfig() diff --git a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py index ca4a3dedb4a..0208d87b354 100644 --- a/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py +++ b/tests/unit/llms/vertex_ai/vertex_ai_partner_models/anthropic/test_vertex_ai_partner_models_anthropic_transformation.py @@ -1,11 +1,13 @@ import copy import json +from typing import Final import pytest from litellm.anthropic_beta_headers_manager import ( update_headers_with_filtered_beta, + update_request_with_filtered_beta, ) from litellm.llms.vertex_ai.vertex_ai_partner_models.anthropic.transformation import ( VertexAIAnthropicConfig, @@ -382,6 +384,33 @@ def test_vertex_ai_anthropic_extra_headers_beta_propagation(): assert "interleaved-thinking-2025-05-14" in headers["anthropic-beta"] +def test_vertex_ai_anthropic_inline_tools_beta_survives_the_chat_beta_filter(local_beta_headers_config: None) -> None: + """The chat path runs the Vertex beta filter over both the header and the `anthropic_beta` body field right + before the request goes out, so inline-tools-2026-09-15 must survive both (source and date in + tests/unit/test_anthropic_beta_headers_filtering.py).""" + config: Final = VertexAIAnthropicConfig() + headers: Final[dict[str, str]] = {} + optional_params: Final[dict[str, object]] = { + "max_tokens": 100, + "is_vertex_request": True, + "extra_headers": {"anthropic-beta": "inline-tools-2026-09-15"}, + } + + request_data: Final = config.transform_request( + model="claude-opus-5-5", + messages=[{"role": "user", "content": "Hello"}], + optional_params=optional_params, + litellm_params={}, + headers=headers, + ) + filtered_headers, filtered_request = update_request_with_filtered_beta( + headers=headers, request_data=request_data, provider="vertex_ai" + ) + + assert filtered_headers["anthropic-beta"].split(",").count("inline-tools-2026-09-15") == 1 + assert "inline-tools-2026-09-15" in filtered_request["anthropic_beta"] + + def test_vertex_ai_anthropic_extra_headers_beta_merged_with_auto_betas(): """Test that extra_headers betas are merged with auto-detected betas rather than replacing them.""" diff --git a/tests/unit/test_anthropic_beta_headers_filtering.py b/tests/unit/test_anthropic_beta_headers_filtering.py index 1a6899f16ba..2a85122d91c 100644 --- a/tests/unit/test_anthropic_beta_headers_filtering.py +++ b/tests/unit/test_anthropic_beta_headers_filtering.py @@ -10,7 +10,7 @@ This test validates: import json import os -from typing import Dict, List +from typing import Dict, Final, List from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -458,6 +458,20 @@ class TestAnthropicBetaHeadersFiltering: assert filtered == ["dangerous-tool-use-2026-09-03"] + @pytest.mark.parametrize("provider", ["anthropic", "vertex_ai"]) + def test_inline_tools_forwarded(self, provider: str) -> None: + """inline-tools-2026-09-15 lets a `tool_addition` system message carry a new tool mid-conversation. + Anthropic documents it for the Claude API + (https://platform.claude.com/docs/en/build-with-claude/mid-conversation-system-messages, fetched 2026-10-07), + and a live Vertex rawPredict call on claude-opus-5-5 honored it on 2026-10-07 while answering 400 on the + `tool_addition` block without it.""" + filtered: Final = filter_and_transform_beta_headers( + beta_headers=["inline-tools-2026-09-15"], + provider=provider, + ) + + assert filtered == ["inline-tools-2026-09-15"] + def test_null_value_headers_filtered(self): """Test that headers with null values are always filtered out.""" for provider in [ From 29f48e62bf414e6615074ea8f298796ae7fa99c6 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Thu, 8 Oct 2026 12:48:16 -0700 Subject: [PATCH 57/65] feat(fireworks_ai): forward the LiteLLM user id as user behind fireworks_forward_user_id (#45265) * feat(fireworks_ai): forward the LiteLLM user id as user behind fireworks_forward_user_id * fix(fireworks_ai): keep the forwarded user id when the caller sends extra_body.user --- .../litellm_core_utils/get_litellm_params.py | 1 + .../llms/fireworks_ai/chat/transformation.py | 15 ++- litellm/llms/fireworks_ai/common_utils.py | 27 ++++ .../fireworks_ai/responses/transformation.py | 26 +++- litellm/main.py | 1 + litellm/proxy/auth/auth_utils.py | 3 + litellm/types/litellm_params.py | 1 + litellm/types/router.py | 4 + .../test_fireworks_ai_chat_transformation.py | 123 ++++++++++++++++++ ...t_fireworks_ai_responses_transformation.py | 59 +++++++++ tests/unit/proxy/auth/test_auth_utils.py | 35 +++++ tests/unit/types/test_litellm_params.py | 1 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 10 ++ 13 files changed, 304 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/get_litellm_params.py b/litellm/litellm_core_utils/get_litellm_params.py index 1a48c00bdc2..bab516f54cc 100644 --- a/litellm/litellm_core_utils/get_litellm_params.py +++ b/litellm/litellm_core_utils/get_litellm_params.py @@ -67,6 +67,7 @@ OPTIONAL_KWARGS_KEYS: Final = ( "itpm", "otpm", "use_xai_oauth", + "fireworks_forward_user_id", PROVIDER_AFFINITY_HEADER_KWARG_KEY, } ) diff --git a/litellm/llms/fireworks_ai/chat/transformation.py b/litellm/llms/fireworks_ai/chat/transformation.py index a7a417db2c9..2c240544ee8 100644 --- a/litellm/llms/fireworks_ai/chat/transformation.py +++ b/litellm/llms/fireworks_ai/chat/transformation.py @@ -43,7 +43,9 @@ from ..common_utils import ( FIREROUTER, FireworksAIException, FireworksAIMixin, + get_fireworks_forwarded_user_id, resolve_fireworks_resource_name, + without_caller_user, ) if TYPE_CHECKING: @@ -674,13 +676,24 @@ class FireworksAIConfig(FireworksAIMixin, OpenAIGPTConfig): **stream_options, "include_usage": True, } - return super().transform_request( + request: Final = super().transform_request( model=resolved_model, messages=messages, optional_params=optional_params, litellm_params=litellm_params, headers=headers, ) + forwarded_user_id: Final = get_fireworks_forwarded_user_id(litellm_params) + return request if forwarded_user_id is None else {**request, "user": forwarded_user_id} + + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, object]: + return without_caller_user(extra_body, get_fireworks_forwarded_user_id(litellm_params)) def _handle_message_content_with_tool_calls( self, diff --git a/litellm/llms/fireworks_ai/common_utils.py b/litellm/llms/fireworks_ai/common_utils.py index 17fadf7ae0f..10314d3fac4 100644 --- a/litellm/llms/fireworks_ai/common_utils.py +++ b/litellm/llms/fireworks_ai/common_utils.py @@ -49,6 +49,33 @@ def with_fireworks_session_affinity( return MappingProxyType({**headers, "x-session-affinity": session_id}) +FIREWORKS_FORWARD_USER_ID_PARAM: Final = "fireworks_forward_user_id" + + +def _authenticated_user_id(metadata: object) -> str | None: + user_id: Final = metadata.get("user_api_key_user_id") if isinstance(metadata, Mapping) else None + return user_id if isinstance(user_id, str) and user_id else None + + +def get_fireworks_forwarded_user_id(litellm_params: Mapping[str, object]) -> str | None: + if litellm_params.get(FIREWORKS_FORWARD_USER_ID_PARAM) is not True: + return None + return next( + ( + user_id + for key in ("metadata", "litellm_metadata") + if (user_id := _authenticated_user_id(litellm_params.get(key))) is not None + ), + None, + ) + + +def without_caller_user(extra_body: Mapping[str, object], forwarded_user_id: str | None) -> Mapping[str, object]: + if forwarded_user_id is None: + return extra_body + return MappingProxyType({key: value for key, value in extra_body.items() if key != "user"}) + + def resolve_fireworks_api_key(api_key: str | None) -> str | None: return api_key or ( get_secret_str("FIREWORKS_API_KEY") diff --git a/litellm/llms/fireworks_ai/responses/transformation.py b/litellm/llms/fireworks_ai/responses/transformation.py index c1010102093..30eaa5a7ca2 100644 --- a/litellm/llms/fireworks_ai/responses/transformation.py +++ b/litellm/llms/fireworks_ai/responses/transformation.py @@ -7,9 +7,12 @@ import httpx from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam from litellm.llms.fireworks_ai.common_utils import ( + FIREWORKS_FORWARD_USER_ID_PARAM, + get_fireworks_forwarded_user_id, resolve_fireworks_api_key, resolve_fireworks_resource_name, with_fireworks_session_affinity, + without_caller_user, ) from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.secret_managers.main import get_secret_str @@ -31,6 +34,16 @@ def _session_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, object ) +def _forwarded_user_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, object]: + extras: Final[Mapping[str, object]] = litellm_params.model_extra or MappingProxyType({}) + return MappingProxyType( + { + FIREWORKS_FORWARD_USER_ID_PARAM: litellm_params.fireworks_forward_user_id, + "litellm_metadata": extras.get("litellm_metadata"), + } + ) + + _INSTRUCTION_ROLES: Final = frozenset({"system", "developer"}) @@ -163,13 +176,24 @@ class FireworksAIResponsesAPIConfig(OpenAIResponsesAPIConfig): *instruction_entries, ) } - return super().transform_responses_api_request( + request: Final = super().transform_responses_api_request( model=resolve_fireworks_resource_name(model), input=folded_input, response_api_optional_request_params=folded_params, litellm_params=litellm_params, headers=headers, ) + forwarded_user_id: Final = get_fireworks_forwarded_user_id(_forwarded_user_params(litellm_params)) + return request if forwarded_user_id is None else {**request, "user": forwarded_user_id} + + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: GenericLiteLLMParams, + ) -> Mapping[str, object]: + return without_caller_user(extra_body, get_fireworks_forwarded_user_id(_forwarded_user_params(litellm_params))) def transform_delete_response_api_response( self, diff --git a/litellm/main.py b/litellm/main.py index 37b290f51ba..25c73ea281e 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -5737,6 +5737,7 @@ def completion( *ANTHROPIC_WIF_KWARGS_KEYS, *OPENAI_WIF_KWARGS_KEYS, PROVIDER_AFFINITY_HEADER_KWARG_KEY, + "fireworks_forward_user_id", ) if key in kwargs }, diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 2bd10b90101..d31ce4dae03 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -431,6 +431,9 @@ _BANNED_REQUEST_BODY_PARAMS: Final[tuple[str, ...]] = ( # so a caller-supplied value picks a transport and a callback surface the # admin did not choose. "rust", + # Deployment opt-in: a caller-supplied false would switch off identity + # forwarding and let the caller choose the `user` Fireworks sees. + "fireworks_forward_user_id", # SDK-only field; also rejected outright in is_request_body_safe. "model_list", "vertex_ai_credentials", diff --git a/litellm/types/litellm_params.py b/litellm/types/litellm_params.py index 9119fa16b5d..8b254cb4b67 100644 --- a/litellm/types/litellm_params.py +++ b/litellm/types/litellm_params.py @@ -87,6 +87,7 @@ class ProviderConnection: litellm_credential_name: str | None = None configurable_clientside_auth_params: "Sequence[str | ConfigurableClientsideParamsCustomAuth] | None" = None use_xai_oauth: bool | None = None + fireworks_forward_user_id: bool | None = None @dataclass(frozen=True, slots=True, kw_only=True) diff --git a/litellm/types/router.py b/litellm/types/router.py index a66c4571b39..67683e400ed 100644 --- a/litellm/types/router.py +++ b/litellm/types/router.py @@ -504,6 +504,10 @@ class GenericLiteLLMParams(CredentialLiteLLMParams, CustomPricingLiteLLMParams): default=False, description="Use stored xAI OAuth credentials when no xAI API key is configured.", ) + fireworks_forward_user_id: bool | None = Field( + default=None, + description="Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests.", + ) model_config = ConfigDict(extra="allow", arbitrary_types_allowed=True) merge_reasoning_content_in_choices: bool | None = False model_info: dict | None = None diff --git a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py index 9de8d4c4135..5a013b19a95 100644 --- a/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py +++ b/tests/unit/llms/fireworks_ai/chat/test_fireworks_ai_chat_transformation.py @@ -2008,3 +2008,126 @@ VISION_MODEL = next( for key, info in litellm.model_cost.items() if key.startswith("fireworks_ai/accounts/fireworks/models/") and info.get("supports_vision") is True ) + + +def _fireworks_chat_client() -> MagicMock: + body: Final = { + "id": "chat-user-attribution", + "object": "chat.completion", + "created": 1, + "model": "accounts/fireworks/models/kimi-k3", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + raw_response: Final = MagicMock() + raw_response.status_code = 200 + raw_response.headers = {} + raw_response.text = json.dumps(body) + raw_response.json = lambda: body + client: Final = MagicMock(spec=HTTPHandler) + client.post.return_value = raw_response + return client + + +@pytest.mark.parametrize( + "call_kwargs, expected_user", + [ + pytest.param( + {"fireworks_forward_user_id": True, "metadata": {"user_api_key_user_id": "dev-alice"}}, + "dev-alice", + id="opted-in-sends-litellm-user-id", + ), + pytest.param( + {"fireworks_forward_user_id": True, "metadata": {"user_api_key_user_id": "dev-alice"}, "user": "caller"}, + "dev-alice", + id="litellm-user-id-replaces-caller-user", + ), + pytest.param( + {"fireworks_forward_user_id": True, "litellm_metadata": {"user_api_key_user_id": "dev-bob"}}, + "dev-bob", + id="reads-litellm-metadata", + ), + pytest.param( + { + "fireworks_forward_user_id": True, + "metadata": {"tags": ["caller-tag"]}, + "litellm_metadata": {"user_api_key_user_id": "dev-bob"}, + }, + "dev-bob", + id="reads-litellm-metadata-next-to-caller-metadata", + ), + pytest.param( + {"fireworks_forward_user_id": True, "metadata": {"user_api_key_user_id": ""}, "user": "caller"}, + "caller", + id="empty-litellm-user-id-keeps-caller-user", + ), + pytest.param( + {"fireworks_forward_user_id": True, "metadata": {"user_api_key_user_id": None}, "user": "caller"}, + "caller", + id="no-litellm-user-id-keeps-caller-user", + ), + pytest.param( + {"fireworks_forward_user_id": True, "metadata": {"user_api_key_user_id": None}}, + None, + id="no-litellm-user-id-sends-no-user", + ), + pytest.param( + {"metadata": {"user_api_key_user_id": "dev-alice"}, "user": "caller"}, + "caller", + id="not-opted-in-keeps-caller-user", + ), + pytest.param( + {"metadata": {"user_api_key_user_id": "dev-alice"}}, + None, + id="not-opted-in-sends-no-user", + ), + pytest.param( + {"fireworks_forward_user_id": "true", "metadata": {"user_api_key_user_id": "dev-alice"}}, + None, + id="non-bool-flag-is-off", + ), + pytest.param( + { + "fireworks_forward_user_id": True, + "metadata": {"user_api_key_user_id": "dev-alice"}, + "extra_body": {"user": "dev-bob", "prompt_cache_max_len": 1}, + }, + "dev-alice", + id="litellm-user-id-replaces-extra-body-user", + ), + pytest.param( + {"metadata": {"user_api_key_user_id": "dev-alice"}, "extra_body": {"user": "dev-bob"}}, + "dev-bob", + id="not-opted-in-keeps-extra-body-user", + ), + ], +) +def test_completion_forwards_litellm_user_id_as_user(call_kwargs: dict[str, object], expected_user: str | None) -> None: + client: Final = _fireworks_chat_client() + litellm.completion( + model="fireworks_ai/accounts/fireworks/models/kimi-k3", + messages=[{"role": "user", "content": "hi"}], + api_key="fw-test-key", + client=client, + **call_kwargs, + ) + request_body: Final = json.loads(client.post.call_args.kwargs["data"]) + assert request_body.get("user") == expected_user + assert "fireworks_forward_user_id" not in request_body + + +def test_completion_forwards_litellm_user_id_when_streaming() -> None: + client: Final = _fireworks_chat_client() + client.post.return_value.iter_lines = lambda: iter(()) + litellm.completion( + model="fireworks_ai/accounts/fireworks/models/kimi-k3", + messages=[{"role": "user", "content": "hi"}], + api_key="fw-test-key", + client=client, + stream=True, + fireworks_forward_user_id=True, + metadata={"user_api_key_user_id": "dev-alice"}, + ) + request_body: Final = json.loads(client.post.call_args.kwargs["data"]) + assert request_body["user"] == "dev-alice" + assert request_body["stream"] is True diff --git a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py index 735d0b1125f..878e0285cd1 100644 --- a/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py +++ b/tests/unit/llms/fireworks_ai/responses/test_fireworks_ai_responses_transformation.py @@ -608,3 +608,62 @@ def test_streaming_responses_call_hits_native_endpoint_and_yields_every_firework assert tuple(event.type for event in received) == tuple(event["type"] for event in FIREWORKS_SSE_EVENTS) assert "".join(event.delta for event in received if event.type == "response.output_text.delta") == "pong" assert received[-1].response.usage.output_tokens == 89 + + +@pytest.mark.parametrize( + "call_kwargs, expected_user", + [ + pytest.param( + {"fireworks_forward_user_id": True, "litellm_metadata": {"user_api_key_user_id": "dev-alice"}}, + "dev-alice", + id="opted-in-sends-litellm-user-id", + ), + pytest.param( + { + "fireworks_forward_user_id": True, + "litellm_metadata": {"user_api_key_user_id": "dev-alice"}, + "user": "caller", + }, + "dev-alice", + id="litellm-user-id-replaces-caller-user", + ), + pytest.param( + {"fireworks_forward_user_id": True, "metadata": {"user_api_key_user_id": "dev-alice"}}, + None, + id="ignores-caller-responses-metadata", + ), + pytest.param( + {"fireworks_forward_user_id": True, "user": "caller"}, + "caller", + id="no-litellm-user-id-keeps-caller-user", + ), + pytest.param( + {"litellm_metadata": {"user_api_key_user_id": "dev-alice"}}, + None, + id="not-opted-in-sends-no-user", + ), + pytest.param( + { + "fireworks_forward_user_id": True, + "litellm_metadata": {"user_api_key_user_id": "dev-alice"}, + "extra_body": {"user": "dev-bob"}, + }, + "dev-alice", + id="litellm-user-id-replaces-extra-body-user", + ), + pytest.param( + {"litellm_metadata": {"user_api_key_user_id": "dev-alice"}, "extra_body": {"user": "dev-bob"}}, + "dev-bob", + id="not-opted-in-keeps-extra-body-user", + ), + ], +) +def test_responses_call_forwards_litellm_user_id_as_user( + call_kwargs: Mapping[str, object], expected_user: str | None +) -> None: + client: Final = _mock_http_client(_fireworks_response("accounts/fireworks/models/kimi-k3")) + with patch(HTTPX_CLIENT_FACTORY, return_value=client): + litellm.responses(model="fireworks_ai/kimi-k3", input="hi", api_key="fw-test-key", **call_kwargs) + _, _, body = _sent_request(client) + assert body.get("user") == expected_user + assert "fireworks_forward_user_id" not in body diff --git a/tests/unit/proxy/auth/test_auth_utils.py b/tests/unit/proxy/auth/test_auth_utils.py index 130d29a9560..e2cec1cc5a2 100644 --- a/tests/unit/proxy/auth/test_auth_utils.py +++ b/tests/unit/proxy/auth/test_auth_utils.py @@ -4,6 +4,7 @@ Unit tests for auth_utils functions related to rate limiting and customer ID ext import base64 import logging +from collections.abc import Callable from typing import Final, Optional from unittest.mock import MagicMock, patch @@ -2877,6 +2878,40 @@ class TestIsRequestBodySafeBlocksClaudePlatformWorkspaceOverride: is True ) + +class TestIsRequestBodySafeBlocksFireworksForwardUserId: + @pytest.mark.parametrize("value", [True, False]) + @pytest.mark.parametrize( + "body_for", + [ + pytest.param(lambda value: {"fireworks_forward_user_id": value}, id="root"), + pytest.param(lambda value: {"extra_body": {"fireworks_forward_user_id": value}}, id="extra_body"), + pytest.param(lambda value: {"metadata": {"fireworks_forward_user_id": value}}, id="metadata"), + ], + ) + def test_fireworks_forward_user_id_in_request_body_is_rejected( + self, body_for: Callable[[bool], dict[str, object]], value: bool + ) -> None: + with pytest.raises(ValueError, match="fireworks_forward_user_id"): + is_request_body_safe( + request_body={"model": "fireworks-model", "user": "someone-else", **body_for(value)}, + general_settings={}, + llm_router=None, + model="fireworks-model", + ) + + def test_admin_opt_in_proxy_wide_allows_fireworks_forward_user_id(self) -> None: + assert ( + is_request_body_safe( + request_body={"model": "fireworks-model", "fireworks_forward_user_id": False}, + general_settings={"allow_client_side_credentials": True}, + llm_router=None, + model="fireworks-model", + ) + is True + ) + + class TestIsRequestBodySafeBlocksRustOptIn: """``rust`` hands the whole call to the Rust core, which signs and sends with its own HTTP client rather than the one the deployment configured, and diff --git a/tests/unit/types/test_litellm_params.py b/tests/unit/types/test_litellm_params.py index 8d163731a51..f8e2befe237 100644 --- a/tests/unit/types/test_litellm_params.py +++ b/tests/unit/types/test_litellm_params.py @@ -85,6 +85,7 @@ CONNECTION_NAMES: Final = ( "litellm_credential_name", "configurable_clientside_auth_params", "use_xai_oauth", + "fireworks_forward_user_id", "aws_batch_role_arn", "s3_bucket_name", "s3_region_name", diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 769798b5134..2a2b5acae78 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -35997,6 +35997,11 @@ export interface components { default_api_key_tpm_limit?: number | null; /** Drop Params */ drop_params?: boolean | string | null; + /** + * Fireworks Forward User Id + * @description Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests. + */ + fireworks_forward_user_id?: boolean | null; /** Gcs Bucket Name */ gcs_bucket_name?: string | null; /** Google Maps Grounding Cost Per Query */ @@ -51194,6 +51199,11 @@ export interface components { default_api_key_tpm_limit?: number | null; /** Drop Params */ drop_params?: boolean | string | null; + /** + * Fireworks Forward User Id + * @description Send the LiteLLM user id of the calling key as the `user` field on Fireworks AI chat, responses and messages requests. + */ + fireworks_forward_user_id?: boolean | null; /** Gcs Bucket Name */ gcs_bucket_name?: string | null; /** Google Maps Grounding Cost Per Query */ From 95dd90f6371599753d9b0d966fcfd5d1c0c114f6 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:12:42 -0700 Subject: [PATCH 58/65] fix(model_prices): add Bedrock flex prices, Anthropic web search flags, Gemini shutdown dates, OpenRouter alias drift, Mistral Large 4 context (#44910) * feat(model_prices): add Mistral Large 4 and its OpenRouter route Co-authored-by: moyai-devin-berriai[bot] <336287033+moyai-devin-berriai[bot]@users.noreply.github.com> Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): sync Gemini API shutdown dates for TTS, Live, image and Omni previews Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model-prices): correct Claude Haiku 5.5 flags, add Anthropic web search flags and OpenRouter Claude Haiku 5.5 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): add Bedrock flex tier prices and clear lapsed Together DeepSeek V4 dates Absorbs #45320 (Bedrock flex prices, verified against the AWS Price List API) and the DeepSeek V4 date removals from #45307 (verified against Together docs and API) Co-authored-by: Roi Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(model_prices): Mistral Large 4 context, OpenRouter Claude latest aliases and batch pricing Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(mistral): expect 1048576 context for Mistral Large 4 Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: moyai-devin-berriai[bot] <336287033+moyai-devin-berriai[bot]@users.noreply.github.com> Co-authored-by: Roi --- ...odel_prices_and_context_window_backup.json | 338 ++++++++++++++++-- model_prices_and_context_window.json | 338 ++++++++++++++++-- .../test_mistral_large_4_model_metadata.py | 2 +- 3 files changed, 613 insertions(+), 65 deletions(-) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index 5b9df8cafde..663c6d1a2a7 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -342,12 +342,14 @@ }, "writer.palmyra-vision-7b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 4096, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_vision": true }, @@ -486,13 +488,16 @@ }, "us.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_flex": 4.125e-08, "input_cost_per_token": 3.3e-07, + "input_cost_per_token_flex": 1.65e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-06, + "output_cost_per_token_flex": 1.375e-06, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -556,13 +561,16 @@ }, "amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_flex": 1e-07, "input_cost_per_token": 8e-07, + "input_cost_per_token_flex": 4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, + "output_cost_per_token_flex": 1.6e-06, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -3188,13 +3196,16 @@ }, "apac.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.1e-07, + "cache_read_input_token_cost_flex": 1.05e-07, "input_cost_per_token": 8.4e-07, + "input_cost_per_token_flex": 4.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.36e-06, + "output_cost_per_token_flex": 1.68e-06, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -13076,12 +13087,14 @@ }, "bedrock/ap-northeast-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13094,12 +13107,14 @@ "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13110,6 +13125,7 @@ }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13123,29 +13139,34 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13156,12 +13177,14 @@ }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13173,12 +13196,14 @@ "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true @@ -13221,12 +13246,14 @@ }, "bedrock/ap-south-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13239,12 +13266,14 @@ "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13255,6 +13284,7 @@ }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13268,29 +13298,34 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7.1e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.94e-06, + "output_cost_per_token_flex": 1.47e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13301,12 +13336,14 @@ }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13317,6 +13354,7 @@ }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, + "input_cost_per_token_flex": 1.55e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13330,16 +13368,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.24e-06 + "output_cost_per_token": 1.24e-06, + "output_cost_per_token_flex": 6.2e-07 }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13352,12 +13393,14 @@ "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13368,6 +13411,7 @@ }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13381,16 +13425,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13401,12 +13448,14 @@ }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13435,12 +13484,14 @@ }, "bedrock/eu-north-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13453,12 +13504,14 @@ "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13469,6 +13522,7 @@ }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13482,16 +13536,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13590,12 +13647,14 @@ "bedrock/eu-central-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13606,6 +13665,7 @@ }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13619,16 +13679,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13658,12 +13721,14 @@ "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13674,6 +13739,7 @@ }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13687,16 +13753,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13726,12 +13795,14 @@ "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4.7e-07, + "input_cost_per_token_flex": 2.35e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_flex": 9.3e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13742,6 +13813,7 @@ }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, + "input_cost_per_token_flex": 2.35e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13755,16 +13827,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.86e-06 + "output_cost_per_token": 1.86e-06, + "output_cost_per_token_flex": 9.3e-07 }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_flex": 1.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, + "output_cost_per_token_flex": 5.05e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -13776,12 +13851,14 @@ }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, + "input_cost_per_token_flex": 3.9e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_flex": 9.3e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13824,12 +13901,14 @@ "bedrock/eu-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13840,6 +13919,7 @@ }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13853,16 +13933,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13909,12 +13992,14 @@ }, "bedrock/sa-east-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13927,12 +14012,14 @@ "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13943,6 +14030,7 @@ }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13956,29 +14044,34 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13989,12 +14082,14 @@ }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14142,12 +14237,14 @@ }, "bedrock/us-east-1/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14160,12 +14257,14 @@ "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14176,7 +14275,9 @@ }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -14193,24 +14294,28 @@ "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14221,12 +14326,14 @@ }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14237,12 +14344,14 @@ }, "bedrock/us-east-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14255,12 +14364,14 @@ "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14271,6 +14382,7 @@ }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -14284,29 +14396,34 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.2e-06 + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14317,12 +14434,14 @@ }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14778,12 +14897,14 @@ }, "bedrock/us-west-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14796,12 +14917,14 @@ "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14812,7 +14935,9 @@ }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -14829,24 +14954,28 @@ "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14857,12 +14986,14 @@ }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -15065,6 +15196,7 @@ "source": "https://developers.openai.com/api/docs/pricing" }, "claude-haiku-4-5-20251001": { + "supports_web_search": true, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_creation_input_token_cost_batches": 6.25e-07, @@ -15207,6 +15339,7 @@ "source": "https://docs.anthropic.com/en/docs/about-claude/pricing" }, "claude-sonnet-5": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -15297,6 +15430,7 @@ "supports_web_search": true }, "claude-sonnet-4-6": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -15370,6 +15504,7 @@ "output_cost_per_token_batches": 7.5e-06 }, "claude-opus-4-5-20251101": { + "supports_web_search": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -15439,6 +15574,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -15524,6 +15660,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -15611,6 +15748,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -15655,6 +15793,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-fable-5-1": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -15700,6 +15839,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-5-5": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_1hr": 8e-06, @@ -15745,6 +15885,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-5": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -15792,6 +15933,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-8": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -22529,12 +22671,14 @@ }, "deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 164000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -23210,13 +23354,16 @@ }, "eu.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.625e-07, + "cache_read_input_token_cost_flex": 1.3125e-07, "input_cost_per_token": 1.05e-06, + "input_cost_per_token_flex": 5.25e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 4.2e-06, + "output_cost_per_token_flex": 2.1e-06, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true, @@ -29280,7 +29427,7 @@ }, "gemini/gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-10-02", + "deprecation_date": "2027-03-15", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, @@ -29730,6 +29877,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-2.5-flash-preview-tts": { + "deprecation_date": "2026-11-17", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -30303,7 +30451,7 @@ "supports_video_input": true, "supports_vision": true, "tpm": 800000, - "deprecation_date": "2026-09-30" + "deprecation_date": "2026-10-22" }, "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -30810,6 +30958,7 @@ }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -32158,12 +32307,14 @@ "google.gemma-3-12b-it": { "deprecation_date": "2027-03-30", "input_cost_per_token": 9e-08, + "input_cost_per_token_flex": 5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.9e-07, + "output_cost_per_token_flex": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-12b-it.html", "supports_audio_input": false, "supports_function_calling": true, @@ -32174,12 +32325,14 @@ "google.gemma-3-27b-it": { "deprecation_date": "2027-03-30", "input_cost_per_token": 2.3e-07, + "input_cost_per_token_flex": 1.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 3.8e-07, + "output_cost_per_token_flex": 1.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-27b-pt.html", "supports_audio_input": false, "supports_function_calling": true, @@ -32190,12 +32343,14 @@ "google.gemma-3-4b-it": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-08, + "input_cost_per_token_flex": 2e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-08, + "output_cost_per_token_flex": 4e-08, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-4b-it.html", "supports_audio_input": false, "supports_function_calling": true, @@ -38466,12 +38621,14 @@ "minimax.minimax-m2": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2.html", "supports_audio_input": false, "supports_function_calling": true, @@ -38483,12 +38640,14 @@ "minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -38499,7 +38658,9 @@ }, "minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, @@ -38641,12 +38802,14 @@ "mistral.devstral-2-123b": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-07, + "input_cost_per_token_flex": 2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_flex": 1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -38657,12 +38820,14 @@ }, "mistral.magistral-small-2509": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 40000, "max_tokens": 40000, "mode": "chat", "output_cost_per_token": 1.5e-06, + "output_cost_per_token_flex": 7.5e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38673,12 +38838,14 @@ }, "mistral.ministral-3-14b-instruct": { "input_cost_per_token": 2e-07, + "input_cost_per_token_flex": 1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2e-07, + "output_cost_per_token_flex": 1e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38689,12 +38856,14 @@ }, "mistral.ministral-3-3b-instruct": { "input_cost_per_token": 1e-07, + "input_cost_per_token_flex": 5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-07, + "output_cost_per_token_flex": 5e-08, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38705,12 +38874,14 @@ }, "mistral.ministral-3-8b-instruct": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-07, + "output_cost_per_token_flex": 7e-08, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38755,12 +38926,14 @@ }, "mistral.mistral-large-3-675b-instruct": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06, + "output_cost_per_token_flex": 7.5e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38796,12 +38969,14 @@ "mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-08, + "input_cost_per_token_flex": 2e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, + "output_cost_per_token_flex": 2e-08, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-voxtral-mini-3b-2507.html", "supports_audio_input": true, "supports_system_messages": true, @@ -38810,12 +38985,14 @@ "mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", "input_cost_per_token": 1e-07, + "input_cost_per_token_flex": 5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, + "output_cost_per_token_flex": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-voxtral-small-24b-2507.html", "supports_audio_input": true, "supports_system_messages": true, @@ -39297,8 +39474,8 @@ "cache_read_input_token_cost": 6.8e-08, "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", - "max_input_tokens": 524288, - "max_tokens": 524288, + "max_input_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -39318,8 +39495,8 @@ "cache_read_input_token_cost": 6.8e-08, "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", - "max_input_tokens": 524288, - "max_tokens": 524288, + "max_input_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -39644,12 +39821,14 @@ }, "moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -40746,12 +40925,14 @@ "nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", "input_cost_per_token": 2e-07, + "input_cost_per_token_flex": 1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_audio_input": false, "supports_function_calling": true, @@ -40762,12 +40943,14 @@ "nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-08, + "input_cost_per_token_flex": 3e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.3e-07, + "output_cost_per_token_flex": 1.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_audio_input": false, "supports_function_calling": true, @@ -40777,12 +40960,14 @@ }, "nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 6e-08, + "input_cost_per_token_flex": 3e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.4e-07, + "output_cost_per_token_flex": 1.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -40794,12 +40979,14 @@ }, "nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.5e-07, + "output_cost_per_token_flex": 3.25e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -42166,12 +42353,14 @@ }, "openai.gpt-oss-safeguard-120b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -42181,12 +42370,14 @@ }, "openai.gpt-oss-safeguard-20b": { "input_cost_per_token": 7e-08, + "input_cost_per_token_flex": 3e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2e-07, + "output_cost_per_token_flex": 1e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -42453,6 +42644,33 @@ "prompt_cache_min_tokens": 512, "supports_sampling_params": false }, + "openrouter/anthropic/claude-haiku-5.5:batch": { + "cache_creation_input_token_cost": 6.25e-08, + "cache_creation_input_token_cost_above_100k_tokens": 3.125e-07, + "cache_creation_input_token_cost_above_1hr": 1e-07, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 5e-07, + "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_above_100k_tokens": 2.5e-08, + "input_cost_per_token": 5e-08, + "input_cost_per_token_above_100k_tokens": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "output_cost_per_token_above_100k_tokens": 1.25e-06, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, @@ -45325,12 +45543,14 @@ }, "qwen.qwen3-235b-a22b-2507-v1:0": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_flex": 1.1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, + "output_cost_per_token_flex": 4.4e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -45356,12 +45576,14 @@ }, "qwen.qwen3-32b-v1:0": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-32b.html", "supports_function_calling": true, "supports_reasoning": true, @@ -45498,12 +45720,14 @@ }, "qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -47162,7 +47386,6 @@ }, "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-09-29", "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -47192,7 +47415,6 @@ }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { "cache_read_input_token_cost": 1.3e-07, - "deprecation_date": "2026-09-29", "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -47404,13 +47626,16 @@ }, "us.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_flex": 1e-07, "input_cost_per_token": 8e-07, + "input_cost_per_token_flex": 4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, + "output_cost_per_token_flex": 1.6e-06, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -48126,12 +48351,14 @@ }, "us.deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -48142,12 +48369,14 @@ }, "eu.deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -52944,12 +53173,14 @@ "zai.glm-4.7": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2.2e-06, + "output_cost_per_token_flex": 1.1e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -52980,12 +53211,14 @@ "zai.glm-4.7-flash": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7e-08, + "input_cost_per_token_flex": 3.5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -57565,6 +57798,7 @@ "gemini_native_audio": true }, "gemini-2.5-flash-native-audio-preview-12-2025": { + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -57595,6 +57829,7 @@ "supports_web_search": true }, "gemini-3.1-flash-live-preview": { + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -57756,6 +57991,7 @@ "gemini_native_audio": true }, "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -57789,6 +58025,7 @@ "supports_reasoning": true }, "gemini/gemini-3.1-flash-live-preview": { + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -57828,6 +58065,7 @@ "supports_reasoning": true }, "gemini/gemini-3.1-flash-tts-preview": { + "deprecation_date": "2026-11-17", "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", @@ -57899,6 +58137,7 @@ "supports_prompt_caching": true }, "gemini-2.5-flash-preview-tts": { + "deprecation_date": "2026-11-17", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -58289,7 +58528,9 @@ }, "bedrock_mantle/openai.gpt-oss-120b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, @@ -58308,7 +58549,9 @@ }, "bedrock_mantle/openai.gpt-oss-20b": { "input_cost_per_token": 7e-08, + "input_cost_per_token_flex": 3.5e-08, "output_cost_per_token": 3e-07, + "output_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, @@ -58327,7 +58570,9 @@ }, "bedrock_mantle/openai.gpt-oss-safeguard-120b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, @@ -58344,7 +58589,9 @@ }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { "input_cost_per_token": 7e-08, + "input_cost_per_token_flex": 3e-08, "output_cost_per_token": 2e-07, + "output_cost_per_token_flex": 1e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, @@ -59414,7 +59661,9 @@ }, "bedrock_mantle/google.gemma-4-31b": { "input_cost_per_token": 1.4e-07, + "input_cost_per_token_flex": 7e-08, "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, @@ -59433,7 +59682,9 @@ }, "bedrock_mantle/google.gemma-4-26b-a4b": { "input_cost_per_token": 1.3e-07, + "input_cost_per_token_flex": 6.5e-08, "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, @@ -59452,7 +59703,9 @@ }, "bedrock_mantle/google.gemma-4-e2b": { "input_cost_per_token": 4e-08, + "input_cost_per_token_flex": 2e-08, "output_cost_per_token": 8e-08, + "output_cost_per_token_flex": 4e-08, "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -66556,7 +66809,9 @@ "bedrock_mantle/deepseek.v3.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 5.8e-07, + "input_cost_per_token_flex": 2.9e-07, "output_cost_per_token": 1.68e-06, + "output_cost_per_token_flex": 8.4e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 8000, @@ -66572,7 +66827,9 @@ "bedrock_mantle/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "output_cost_per_token": 2.5e-06, + "output_cost_per_token_flex": 1.25e-06, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, @@ -66588,7 +66845,9 @@ }, "bedrock_mantle/qwen.qwen3-235b-a22b-2507": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_flex": 1.1e-07, "output_cost_per_token": 8.8e-07, + "output_cost_per_token_flex": 4.4e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, @@ -66604,7 +66863,9 @@ }, "bedrock_mantle/qwen.qwen3-32b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 32000, "max_output_tokens": 8000, @@ -66621,7 +66882,9 @@ "bedrock_mantle/qwen.qwen3-coder-30b-a3b-instruct": { "deprecation_date": "2027-03-30", "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, @@ -66637,7 +66900,9 @@ "bedrock_mantle/qwen.qwen3-coder-480b-a35b-instruct": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4.5e-07, + "input_cost_per_token_flex": 2.25e-07, "output_cost_per_token": 1.8e-06, + "output_cost_per_token_flex": 9e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 16000, @@ -66652,7 +66917,9 @@ }, "bedrock_mantle/qwen.qwen3-next-80b-a3b-instruct": { "input_cost_per_token": 1.4e-07, + "input_cost_per_token_flex": 7e-08, "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, @@ -66668,7 +66935,9 @@ }, "bedrock_mantle/qwen.qwen3-vl-235b-a22b-instruct": { "input_cost_per_token": 5.3e-07, + "input_cost_per_token_flex": 2.6e-07, "output_cost_per_token": 2.66e-06, + "output_cost_per_token_flex": 1.33e-06, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, @@ -73491,16 +73760,21 @@ "supports_web_search": true }, "openrouter/~anthropic/claude-haiku-latest": { - "cache_creation_input_token_cost": 1.25e-06, - "cache_creation_input_token_cost_above_1hr": 2e-06, - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 1e-06, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08, + "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5e-06, + "output_cost_per_token": 5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73537,7 +73811,7 @@ "openrouter/~anthropic/claude-sonnet-latest": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 2e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, @@ -73759,8 +74033,8 @@ "openrouter/~openai/gpt-sol-latest": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_above_272k_tokens": 2e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "openrouter", @@ -80210,7 +80484,7 @@ "openrouter/anthropic/claude-sonnet-5.5:batch": { "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, - "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 1e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, @@ -80824,7 +81098,7 @@ "cache_read_input_token_cost": 7e-08, "input_cost_per_token": 6.8e-07, "litellm_provider": "openrouter", - "max_input_tokens": 524288, + "max_input_tokens": 1048576, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index 5b9df8cafde..663c6d1a2a7 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -342,12 +342,14 @@ }, "writer.palmyra-vision-7b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 4096, "max_output_tokens": 4096, "max_tokens": 4096, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_vision": true }, @@ -486,13 +488,16 @@ }, "us.amazon.nova-2-lite-v1:0": { "cache_read_input_token_cost": 8.25e-08, + "cache_read_input_token_cost_flex": 4.125e-08, "input_cost_per_token": 3.3e-07, + "input_cost_per_token_flex": 1.65e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 64000, "max_tokens": 64000, "mode": "chat", "output_cost_per_token": 2.75e-06, + "output_cost_per_token_flex": 1.375e-06, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -556,13 +561,16 @@ }, "amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_flex": 1e-07, "input_cost_per_token": 8e-07, + "input_cost_per_token_flex": 4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, + "output_cost_per_token_flex": 1.6e-06, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -3188,13 +3196,16 @@ }, "apac.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.1e-07, + "cache_read_input_token_cost_flex": 1.05e-07, "input_cost_per_token": 8.4e-07, + "input_cost_per_token_flex": 4.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.36e-06, + "output_cost_per_token_flex": 1.68e-06, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -13076,12 +13087,14 @@ }, "bedrock/ap-northeast-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13094,12 +13107,14 @@ "bedrock/ap-northeast-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13110,6 +13125,7 @@ }, "bedrock/ap-northeast-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13123,29 +13139,34 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/ap-northeast-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/ap-northeast-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13156,12 +13177,14 @@ }, "bedrock/ap-northeast-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13173,12 +13196,14 @@ "bedrock/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true @@ -13221,12 +13246,14 @@ }, "bedrock/ap-south-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13239,12 +13266,14 @@ "bedrock/ap-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13255,6 +13284,7 @@ }, "bedrock/ap-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13268,29 +13298,34 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/ap-south-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7.1e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.94e-06, + "output_cost_per_token_flex": 1.47e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/ap-south-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13301,12 +13336,14 @@ }, "bedrock/ap-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13317,6 +13354,7 @@ }, "bedrock/ap-southeast-2/minimax.minimax-m2.5": { "input_cost_per_token": 3.1e-07, + "input_cost_per_token_flex": 1.55e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13330,16 +13368,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.24e-06 + "output_cost_per_token": 1.24e-06, + "output_cost_per_token_flex": 6.2e-07 }, "bedrock/ap-southeast-3/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13352,12 +13393,14 @@ "bedrock/ap-southeast-3/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13368,6 +13411,7 @@ }, "bedrock/ap-southeast-3/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13381,16 +13425,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/ap-southeast-3/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13401,12 +13448,14 @@ }, "bedrock/ap-southeast-3/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13435,12 +13484,14 @@ }, "bedrock/eu-north-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13453,12 +13504,14 @@ "bedrock/eu-north-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13469,6 +13522,7 @@ }, "bedrock/eu-north-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13482,16 +13536,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/eu-north-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13590,12 +13647,14 @@ "bedrock/eu-central-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13606,6 +13665,7 @@ }, "bedrock/eu-central-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13619,16 +13679,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/eu-central-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13658,12 +13721,14 @@ "bedrock/eu-west-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13674,6 +13739,7 @@ }, "bedrock/eu-west-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13687,16 +13753,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/eu-west-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13726,12 +13795,14 @@ "bedrock/eu-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4.7e-07, + "input_cost_per_token_flex": 2.35e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_flex": 9.3e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13742,6 +13813,7 @@ }, "bedrock/eu-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 4.7e-07, + "input_cost_per_token_flex": 2.35e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13755,16 +13827,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.86e-06 + "output_cost_per_token": 1.86e-06, + "output_cost_per_token_flex": 9.3e-07 }, "bedrock/eu-west-2/nvidia.nemotron-super-3-120b": { "input_cost_per_token": 2.3e-07, + "input_cost_per_token_flex": 1.15e-07, "litellm_provider": "bedrock", "max_input_tokens": 256000, "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.01e-06, + "output_cost_per_token_flex": 5.05e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -13776,12 +13851,14 @@ }, "bedrock/eu-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 7.8e-07, + "input_cost_per_token_flex": 3.9e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.86e-06, + "output_cost_per_token_flex": 9.3e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13824,12 +13901,14 @@ "bedrock/eu-south-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13840,6 +13919,7 @@ }, "bedrock/eu-south-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13853,16 +13933,19 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/eu-south-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13909,12 +13992,14 @@ }, "bedrock/sa-east-1/deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -13927,12 +14012,14 @@ "bedrock/sa-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13943,6 +14030,7 @@ }, "bedrock/sa-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3.6e-07, + "input_cost_per_token_flex": 1.8e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -13956,29 +14044,34 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.44e-06 + "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07 }, "bedrock/sa-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7.3e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.03e-06, + "output_cost_per_token_flex": 1.52e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/sa-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 7.2e-07, + "input_cost_per_token_flex": 3.6e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3.6e-06, + "output_cost_per_token_flex": 1.8e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -13989,12 +14082,14 @@ }, "bedrock/sa-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.44e-06, + "output_cost_per_token_flex": 7.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14142,12 +14237,14 @@ }, "bedrock/us-east-1/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14160,12 +14257,14 @@ "bedrock/us-east-1/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14176,7 +14275,9 @@ }, "bedrock/us-east-1/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -14193,24 +14294,28 @@ "bedrock/us-east-1/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/us-east-1/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14221,12 +14326,14 @@ }, "bedrock/us-east-1/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14237,12 +14344,14 @@ }, "bedrock/us-east-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14255,12 +14364,14 @@ "bedrock/us-east-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14271,6 +14382,7 @@ }, "bedrock/us-east-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -14284,29 +14396,34 @@ "supports_audio_input": false, "supports_response_schema": true, "supports_vision": false, - "output_cost_per_token": 1.2e-06 + "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07 }, "bedrock/us-east-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/us-east-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14317,12 +14434,14 @@ }, "bedrock/us-east-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14778,12 +14897,14 @@ }, "bedrock/us-west-2/deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock", "max_input_tokens": 163840, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -14796,12 +14917,14 @@ "bedrock/us-west-2/minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14812,7 +14935,9 @@ }, "bedrock/us-west-2/minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock", "max_input_tokens": 196000, "max_output_tokens": 8192, @@ -14829,24 +14954,28 @@ "bedrock/us-west-2/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2.5e-06, + "output_cost_per_token_flex": 1.25e-06, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-moonshot-ai-kimi-k2-thinking.html", "supports_function_calling": true, "supports_reasoning": true }, "bedrock/us-west-2/moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -14857,12 +14986,14 @@ }, "bedrock/us-west-2/qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock", "max_input_tokens": 262144, "max_output_tokens": 16384, "max_tokens": 16384, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -15065,6 +15196,7 @@ "source": "https://developers.openai.com/api/docs/pricing" }, "claude-haiku-4-5-20251001": { + "supports_web_search": true, "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, "cache_creation_input_token_cost_batches": 6.25e-07, @@ -15207,6 +15339,7 @@ "source": "https://docs.anthropic.com/en/docs/about-claude/pricing" }, "claude-sonnet-5": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, @@ -15297,6 +15430,7 @@ "supports_web_search": true }, "claude-sonnet-4-6": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 3.75e-06, "cache_creation_input_token_cost_above_1hr": 6e-06, @@ -15370,6 +15504,7 @@ "output_cost_per_token_batches": 7.5e-06 }, "claude-opus-4-5-20251101": { + "supports_web_search": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, "cache_creation_input_token_cost_batches": 3.125e-06, @@ -15439,6 +15574,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-6": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -15524,6 +15660,7 @@ "prompt_cache_min_tokens": 4096 }, "claude-opus-4-7": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -15611,6 +15748,7 @@ "prompt_cache_min_tokens": 2048 }, "claude-fable-5": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -15655,6 +15793,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-fable-5-1": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 1.25e-05, "cache_creation_input_token_cost_above_1hr": 2e-05, @@ -15700,6 +15839,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-5-5": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 5e-06, "cache_creation_input_token_cost_above_1hr": 8e-06, @@ -15745,6 +15885,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-5": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -15792,6 +15933,7 @@ "source": "https://platform.claude.com/docs/en/about-claude/pricing" }, "claude-opus-4-8": { + "supports_web_search": true, "supports_anthropic_compaction": true, "cache_creation_input_token_cost": 6.25e-06, "cache_creation_input_token_cost_above_1hr": 1e-05, @@ -22529,12 +22671,14 @@ }, "deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 164000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -23210,13 +23354,16 @@ }, "eu.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2.625e-07, + "cache_read_input_token_cost_flex": 1.3125e-07, "input_cost_per_token": 1.05e-06, + "input_cost_per_token_flex": 5.25e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 4.2e-06, + "output_cost_per_token_flex": 2.1e-06, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_function_calling": true, "supports_pdf_input": true, @@ -29280,7 +29427,7 @@ }, "gemini/gemini-2.5-flash-image": { "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-10-02", + "deprecation_date": "2027-03-15", "input_cost_per_audio_token": 1e-06, "input_cost_per_token": 3e-07, "input_cost_per_token_batches": 1.5e-07, @@ -29730,6 +29877,7 @@ "web_search_billing_unit": "per_query" }, "gemini/gemini-2.5-flash-preview-tts": { + "deprecation_date": "2026-11-17", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -30303,7 +30451,7 @@ "supports_video_input": true, "supports_vision": true, "tpm": 800000, - "deprecation_date": "2026-09-30" + "deprecation_date": "2026-10-22" }, "gemini/gemini-3.1-pro-preview": { "prompt_cache_min_tokens": 4096, @@ -30810,6 +30958,7 @@ }, "gemini/gemini-2.5-pro-preview-tts": { "cache_read_input_token_cost": 1.25e-07, + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 7e-07, "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, @@ -32158,12 +32307,14 @@ "google.gemma-3-12b-it": { "deprecation_date": "2027-03-30", "input_cost_per_token": 9e-08, + "input_cost_per_token_flex": 5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.9e-07, + "output_cost_per_token_flex": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-12b-it.html", "supports_audio_input": false, "supports_function_calling": true, @@ -32174,12 +32325,14 @@ "google.gemma-3-27b-it": { "deprecation_date": "2027-03-30", "input_cost_per_token": 2.3e-07, + "input_cost_per_token_flex": 1.2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 3.8e-07, + "output_cost_per_token_flex": 1.9e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-27b-pt.html", "supports_audio_input": false, "supports_function_calling": true, @@ -32190,12 +32343,14 @@ "google.gemma-3-4b-it": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-08, + "input_cost_per_token_flex": 2e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 8e-08, + "output_cost_per_token_flex": 4e-08, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-google-gemma-3-4b-it.html", "supports_audio_input": false, "supports_function_calling": true, @@ -38466,12 +38621,14 @@ "minimax.minimax-m2": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 1000000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-minimax-minimax-m2.html", "supports_audio_input": false, "supports_function_calling": true, @@ -38483,12 +38640,14 @@ "minimax.minimax-m2.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -38499,7 +38658,9 @@ }, "minimax.minimax-m2.5": { "input_cost_per_token": 3e-07, + "input_cost_per_token_flex": 1.5e-07, "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 196000, "max_output_tokens": 8000, @@ -38641,12 +38802,14 @@ "mistral.devstral-2-123b": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-07, + "input_cost_per_token_flex": 2e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 2e-06, + "output_cost_per_token_flex": 1e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -38657,12 +38820,14 @@ }, "mistral.magistral-small-2509": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 40000, "max_tokens": 40000, "mode": "chat", "output_cost_per_token": 1.5e-06, + "output_cost_per_token_flex": 7.5e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38673,12 +38838,14 @@ }, "mistral.ministral-3-14b-instruct": { "input_cost_per_token": 2e-07, + "input_cost_per_token_flex": 1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2e-07, + "output_cost_per_token_flex": 1e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38689,12 +38856,14 @@ }, "mistral.ministral-3-3b-instruct": { "input_cost_per_token": 1e-07, + "input_cost_per_token_flex": 5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1e-07, + "output_cost_per_token_flex": 5e-08, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38705,12 +38874,14 @@ }, "mistral.ministral-3-8b-instruct": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 1.5e-07, + "output_cost_per_token_flex": 7e-08, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38755,12 +38926,14 @@ }, "mistral.mistral-large-3-675b-instruct": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 1.5e-06, + "output_cost_per_token_flex": 7.5e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -38796,12 +38969,14 @@ "mistral.voxtral-mini-3b-2507": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4e-08, + "input_cost_per_token_flex": 2e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 4e-08, + "output_cost_per_token_flex": 2e-08, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-voxtral-mini-3b-2507.html", "supports_audio_input": true, "supports_system_messages": true, @@ -38810,12 +38985,14 @@ "mistral.voxtral-small-24b-2507": { "deprecation_date": "2027-03-30", "input_cost_per_token": 1e-07, + "input_cost_per_token_flex": 5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 3e-07, + "output_cost_per_token_flex": 1.5e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-mistral-ai-voxtral-small-24b-2507.html", "supports_audio_input": true, "supports_system_messages": true, @@ -39297,8 +39474,8 @@ "cache_read_input_token_cost": 6.8e-08, "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", - "max_input_tokens": 524288, - "max_tokens": 524288, + "max_input_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -39318,8 +39495,8 @@ "cache_read_input_token_cost": 6.8e-08, "input_cost_per_token": 6.8e-07, "litellm_provider": "mistral", - "max_input_tokens": 524288, - "max_tokens": 524288, + "max_input_tokens": 1048576, + "max_tokens": 1048576, "mode": "chat", "output_cost_per_token": 2.09e-06, "reasoning_effort_levels": [ @@ -39644,12 +39821,14 @@ }, "moonshotai.kimi-k2.5": { "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 3e-06, + "output_cost_per_token_flex": 1.5e-06, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -40746,12 +40925,14 @@ "nvidia.nemotron-nano-12b-v2": { "deprecation_date": "2027-03-30", "input_cost_per_token": 2e-07, + "input_cost_per_token_flex": 1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-12b-v2-vl-bf16.html", "supports_audio_input": false, "supports_function_calling": true, @@ -40762,12 +40943,14 @@ "nvidia.nemotron-nano-9b-v2": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-08, + "input_cost_per_token_flex": 3e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.3e-07, + "output_cost_per_token_flex": 1.2e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-nvidia-nvidia-nemotron-nano-9b-v2.html", "supports_audio_input": false, "supports_function_calling": true, @@ -40777,12 +40960,14 @@ }, "nvidia.nemotron-nano-3-30b": { "input_cost_per_token": 6e-08, + "input_cost_per_token_flex": 3e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 8000, "max_tokens": 8000, "mode": "chat", "output_cost_per_token": 2.4e-07, + "output_cost_per_token_flex": 1.2e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -40794,12 +40979,14 @@ }, "nvidia.nemotron-super-3-120b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 32000, "max_tokens": 32000, "mode": "chat", "output_cost_per_token": 6.5e-07, + "output_cost_per_token_flex": 3.25e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -42166,12 +42353,14 @@ }, "openai.gpt-oss-safeguard-120b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -42181,12 +42370,14 @@ }, "openai.gpt-oss-safeguard-20b": { "input_cost_per_token": 7e-08, + "input_cost_per_token_flex": 3e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 128000, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 2e-07, + "output_cost_per_token_flex": 1e-07, "source": "https://aws.amazon.com/bedrock/pricing/", "supports_audio_input": false, "supports_function_calling": true, @@ -42453,6 +42644,33 @@ "prompt_cache_min_tokens": 512, "supports_sampling_params": false }, + "openrouter/anthropic/claude-haiku-5.5:batch": { + "cache_creation_input_token_cost": 6.25e-08, + "cache_creation_input_token_cost_above_100k_tokens": 3.125e-07, + "cache_creation_input_token_cost_above_1hr": 1e-07, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 5e-07, + "cache_read_input_token_cost": 5e-09, + "cache_read_input_token_cost_above_100k_tokens": 2.5e-08, + "input_cost_per_token": 5e-08, + "input_cost_per_token_above_100k_tokens": 2.5e-07, + "litellm_provider": "openrouter", + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, + "mode": "chat", + "output_cost_per_token": 2.5e-07, + "output_cost_per_token_above_100k_tokens": 1.25e-06, + "source": "https://openrouter.ai/api/v1/models", + "supports_audio_input": false, + "supports_function_calling": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_reasoning": true, + "supports_response_schema": true, + "supports_tool_choice": true, + "supports_vision": true, + "supports_web_search": true + }, "openrouter/anthropic/claude-haiku-4.5": { "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, @@ -45325,12 +45543,14 @@ }, "qwen.qwen3-235b-a22b-2507-v1:0": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_flex": 1.1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 262144, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 8.8e-07, + "output_cost_per_token_flex": 4.4e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_tool_choice": true, @@ -45356,12 +45576,14 @@ }, "qwen.qwen3-32b-v1:0": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 32768, "max_output_tokens": 8192, "max_tokens": 8192, "mode": "chat", "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-qwen-qwen3-32b.html", "supports_function_calling": true, "supports_reasoning": true, @@ -45498,12 +45720,14 @@ }, "qwen.qwen3-coder-next": { "input_cost_per_token": 5e-07, + "input_cost_per_token_flex": 2.5e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 256000, "max_output_tokens": 16000, "max_tokens": 16000, "mode": "chat", "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "supports_function_calling": true, "supports_system_messages": true, "supports_tool_choice": true, @@ -47162,7 +47386,6 @@ }, "together_ai/deepseek-ai/DeepSeek-V4-Flash-0731": { "cache_read_input_token_cost": 3e-08, - "deprecation_date": "2026-09-29", "input_cost_per_token": 1.4e-07, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -47192,7 +47415,6 @@ }, "together_ai/deepseek-ai/DeepSeek-V4-Pro-0813": { "cache_read_input_token_cost": 1.3e-07, - "deprecation_date": "2026-09-29", "input_cost_per_token": 1.32e-06, "litellm_provider": "together_ai", "max_input_tokens": 1048576, @@ -47404,13 +47626,16 @@ }, "us.amazon.nova-pro-v1:0": { "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost_flex": 1e-07, "input_cost_per_token": 8e-07, + "input_cost_per_token_flex": 4e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 300000, "max_output_tokens": 10000, "max_tokens": 10000, "mode": "chat", "output_cost_per_token": 3.2e-06, + "output_cost_per_token_flex": 1.6e-06, "supports_function_calling": true, "supports_pdf_input": true, "supports_prompt_caching": true, @@ -48126,12 +48351,14 @@ }, "us.deepseek.v3.2": { "input_cost_per_token": 6.2e-07, + "input_cost_per_token_flex": 3.1e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 1.85e-06, + "output_cost_per_token_flex": 9.25e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -48142,12 +48369,14 @@ }, "eu.deepseek.v3.2": { "input_cost_per_token": 7.4e-07, + "input_cost_per_token_flex": 3.7e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 163840, "max_output_tokens": 163840, "max_tokens": 163840, "mode": "chat", "output_cost_per_token": 2.22e-06, + "output_cost_per_token_flex": 1.11e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_native_structured_output": true, @@ -52944,12 +53173,14 @@ "zai.glm-4.7": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 2.2e-06, + "output_cost_per_token_flex": 1.1e-06, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -52980,12 +53211,14 @@ "zai.glm-4.7-flash": { "deprecation_date": "2027-03-30", "input_cost_per_token": 7e-08, + "input_cost_per_token_flex": 3.5e-08, "litellm_provider": "bedrock_converse", "max_input_tokens": 203000, "max_output_tokens": 4000, "max_tokens": 4000, "mode": "chat", "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "supports_function_calling": true, "supports_reasoning": true, "supports_system_messages": true, @@ -57565,6 +57798,7 @@ "gemini_native_audio": true }, "gemini-2.5-flash-native-audio-preview-12-2025": { + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -57595,6 +57829,7 @@ "supports_web_search": true }, "gemini-3.1-flash-live-preview": { + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -57756,6 +57991,7 @@ "gemini_native_audio": true }, "gemini/gemini-2.5-flash-native-audio-preview-12-2025": { + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 3e-06, "input_cost_per_token": 5e-07, "litellm_provider": "gemini", @@ -57789,6 +58025,7 @@ "supports_reasoning": true }, "gemini/gemini-3.1-flash-live-preview": { + "deprecation_date": "2026-11-17", "input_cost_per_audio_token": 3e-06, "input_cost_per_image_token": 1e-06, "input_cost_per_token": 7.5e-07, @@ -57828,6 +58065,7 @@ "supports_reasoning": true }, "gemini/gemini-3.1-flash-tts-preview": { + "deprecation_date": "2026-11-17", "input_cost_per_token": 1e-06, "input_cost_per_token_batches": 5e-07, "litellm_provider": "gemini", @@ -57899,6 +58137,7 @@ "supports_prompt_caching": true }, "gemini-2.5-flash-preview-tts": { + "deprecation_date": "2026-11-17", "input_cost_per_token": 5e-07, "input_cost_per_token_batches": 2.5e-07, "litellm_provider": "gemini", @@ -58289,7 +58528,9 @@ }, "bedrock_mantle/openai.gpt-oss-120b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, @@ -58308,7 +58549,9 @@ }, "bedrock_mantle/openai.gpt-oss-20b": { "input_cost_per_token": 7e-08, + "input_cost_per_token_flex": 3.5e-08, "output_cost_per_token": 3e-07, + "output_cost_per_token_flex": 1.5e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16384, @@ -58327,7 +58570,9 @@ }, "bedrock_mantle/openai.gpt-oss-safeguard-120b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7e-08, "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, @@ -58344,7 +58589,9 @@ }, "bedrock_mantle/openai.gpt-oss-safeguard-20b": { "input_cost_per_token": 7e-08, + "input_cost_per_token_flex": 3e-08, "output_cost_per_token": 2e-07, + "output_cost_per_token_flex": 1e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 131072, "max_output_tokens": 16000, @@ -59414,7 +59661,9 @@ }, "bedrock_mantle/google.gemma-4-31b": { "input_cost_per_token": 1.4e-07, + "input_cost_per_token_flex": 7e-08, "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, @@ -59433,7 +59682,9 @@ }, "bedrock_mantle/google.gemma-4-26b-a4b": { "input_cost_per_token": 1.3e-07, + "input_cost_per_token_flex": 6.5e-08, "output_cost_per_token": 4e-07, + "output_cost_per_token_flex": 2e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 256000, @@ -59452,7 +59703,9 @@ }, "bedrock_mantle/google.gemma-4-e2b": { "input_cost_per_token": 4e-08, + "input_cost_per_token_flex": 2e-08, "output_cost_per_token": 8e-08, + "output_cost_per_token_flex": 4e-08, "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 128000, @@ -66556,7 +66809,9 @@ "bedrock_mantle/deepseek.v3.1": { "deprecation_date": "2027-03-30", "input_cost_per_token": 5.8e-07, + "input_cost_per_token_flex": 2.9e-07, "output_cost_per_token": 1.68e-06, + "output_cost_per_token_flex": 8.4e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 8000, @@ -66572,7 +66827,9 @@ "bedrock_mantle/moonshotai.kimi-k2-thinking": { "deprecation_date": "2027-03-30", "input_cost_per_token": 6e-07, + "input_cost_per_token_flex": 3e-07, "output_cost_per_token": 2.5e-06, + "output_cost_per_token_flex": 1.25e-06, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, @@ -66588,7 +66845,9 @@ }, "bedrock_mantle/qwen.qwen3-235b-a22b-2507": { "input_cost_per_token": 2.2e-07, + "input_cost_per_token_flex": 1.1e-07, "output_cost_per_token": 8.8e-07, + "output_cost_per_token_flex": 4.4e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, @@ -66604,7 +66863,9 @@ }, "bedrock_mantle/qwen.qwen3-32b": { "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 32000, "max_output_tokens": 8000, @@ -66621,7 +66882,9 @@ "bedrock_mantle/qwen.qwen3-coder-30b-a3b-instruct": { "deprecation_date": "2027-03-30", "input_cost_per_token": 1.5e-07, + "input_cost_per_token_flex": 7.5e-08, "output_cost_per_token": 6e-07, + "output_cost_per_token_flex": 3e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 16000, @@ -66637,7 +66900,9 @@ "bedrock_mantle/qwen.qwen3-coder-480b-a35b-instruct": { "deprecation_date": "2027-03-30", "input_cost_per_token": 4.5e-07, + "input_cost_per_token_flex": 2.25e-07, "output_cost_per_token": 1.8e-06, + "output_cost_per_token_flex": 9e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 128000, "max_output_tokens": 16000, @@ -66652,7 +66917,9 @@ }, "bedrock_mantle/qwen.qwen3-next-80b-a3b-instruct": { "input_cost_per_token": 1.4e-07, + "input_cost_per_token_flex": 7e-08, "output_cost_per_token": 1.2e-06, + "output_cost_per_token_flex": 6e-07, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, @@ -66668,7 +66935,9 @@ }, "bedrock_mantle/qwen.qwen3-vl-235b-a22b-instruct": { "input_cost_per_token": 5.3e-07, + "input_cost_per_token_flex": 2.6e-07, "output_cost_per_token": 2.66e-06, + "output_cost_per_token_flex": 1.33e-06, "litellm_provider": "bedrock_mantle", "max_input_tokens": 256000, "max_output_tokens": 8000, @@ -73491,16 +73760,21 @@ "supports_web_search": true }, "openrouter/~anthropic/claude-haiku-latest": { - "cache_creation_input_token_cost": 1.25e-06, - "cache_creation_input_token_cost_above_1hr": 2e-06, - "cache_read_input_token_cost": 1e-07, - "input_cost_per_token": 1e-06, + "input_cost_per_token_above_100k_tokens": 5e-07, + "output_cost_per_token_above_100k_tokens": 2.5e-06, + "cache_read_input_token_cost_above_100k_tokens": 5e-08, + "cache_creation_input_token_cost_above_100k_tokens": 6.25e-07, + "cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06, + "cache_creation_input_token_cost": 1.25e-07, + "cache_creation_input_token_cost_above_1hr": 2e-07, + "cache_read_input_token_cost": 1e-08, + "input_cost_per_token": 1e-07, "litellm_provider": "openrouter", - "max_input_tokens": 200000, - "max_output_tokens": 64000, - "max_tokens": 64000, + "max_input_tokens": 1000000, + "max_output_tokens": 128000, + "max_tokens": 128000, "mode": "chat", - "output_cost_per_token": 5e-06, + "output_cost_per_token": 5e-07, "source": "https://openrouter.ai/api/v1/models", "supports_audio_input": false, "supports_function_calling": true, @@ -73537,7 +73811,7 @@ "openrouter/~anthropic/claude-sonnet-latest": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_1hr": 4e-06, - "cache_read_input_token_cost": 2e-07, + "cache_read_input_token_cost": 1e-07, "input_cost_per_token": 2e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, @@ -73759,8 +74033,8 @@ "openrouter/~openai/gpt-sol-latest": { "cache_creation_input_token_cost": 2.5e-06, "cache_creation_input_token_cost_above_272k_tokens": 5e-06, - "cache_read_input_token_cost": 2e-07, - "cache_read_input_token_cost_above_272k_tokens": 4e-07, + "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost_above_272k_tokens": 2e-07, "input_cost_per_token": 2e-06, "input_cost_per_token_above_272k_tokens": 4e-06, "litellm_provider": "openrouter", @@ -80210,7 +80484,7 @@ "openrouter/anthropic/claude-sonnet-5.5:batch": { "cache_creation_input_token_cost": 1.25e-06, "cache_creation_input_token_cost_above_1hr": 2e-06, - "cache_read_input_token_cost": 1e-07, + "cache_read_input_token_cost": 5e-08, "input_cost_per_token": 1e-06, "litellm_provider": "openrouter", "max_input_tokens": 1000000, @@ -80824,7 +81098,7 @@ "cache_read_input_token_cost": 7e-08, "input_cost_per_token": 6.8e-07, "litellm_provider": "openrouter", - "max_input_tokens": 524288, + "max_input_tokens": 1048576, "max_output_tokens": 262144, "max_tokens": 262144, "mode": "chat", diff --git a/tests/unit/test_mistral_large_4_model_metadata.py b/tests/unit/test_mistral_large_4_model_metadata.py index 924265d2b41..f45c53920dd 100644 --- a/tests/unit/test_mistral_large_4_model_metadata.py +++ b/tests/unit/test_mistral_large_4_model_metadata.py @@ -31,7 +31,7 @@ def test_metadata_and_backup(model): info = litellm.get_model_info(model) assert info["litellm_provider"] == "mistral" assert info["mode"] == "chat" - assert info["max_input_tokens"] == 524288 + assert info["max_input_tokens"] == 1048576 assert info["input_cost_per_token"] == 6.8e-07 assert info["output_cost_per_token"] == 2.09e-06 assert info["cache_read_input_token_cost"] == 6.8e-08 From f663794fcd7ad4df35c8b322c507f59452418d47 Mon Sep 17 00:00:00 2001 From: fzowl <160063452+fzowl@users.noreply.github.com> Date: Thu, 8 Oct 2026 22:12:47 +0200 Subject: [PATCH 59/65] feat(voyage): rebrand to VoyageAI by MongoDB and route MongoDB keys to ai.mongodb.com (#41812) * feat(voyage): rebrand to VoyageAI by MongoDB, refresh model list, support both contextual input shapes * fix(voyage): send auto-chunking params for flat contextual inputs A flat list[str] or bare str for the contextualized embeddings endpoint is only valid as documents with enable_auto_chunking=True and input_type=document, or as queries with input_type=query. Default those params for non-query flat inputs so the request matches the live API contract, while letting caller-set values win. Nested list[list[str]] still passes through unchanged. Tests now assert the auto-chunking params instead of only echoing inputs. * feat(voyage): route MongoDB-issued keys to ai.mongodb.com Mirror voyageai.util.get_default_base_url from the official SDK: a key with the `al-` prefix is issued by MongoDB and is only valid on ai.mongodb.com, every other key on api.voyageai.com. The choice now lives in one shared helper that the embedding, contextual, multimodal, and rerank configs all use for both the default base URL and the Authorization header, so the host and the key always agree. Also drops the voyage-4-nano and voyage-multilingual-2 model map entries: voyage-4-nano is not served on the Voyage API and voyage-multilingual-2 is an older model, so neither belongs in this change. * fix(voyage): restore MongoDB routing on contextual endpoint and re-add voyage-4-nano after upstream merge The upstream merge replaced the contextual config's get_complete_url with a Voyage-only host and dropped voyage-4-nano from the model map. Reapply the al- key routing via get_default_base_url and add voyage-4-nano (open-weight, per docs.voyageai.com) so the branch keeps its task changes on top of upstream. * fix(voyage): drop voyage-4-nano and the contextual tests upstream already covers voyage-4-nano is not served on the Voyage API, so it does not belong in the model map. The contextual input tests duplicate tests/test_litellm/llms/voyage/test_voyage_contextual_embedding.py, which landed upstream with the auto-chunking fix this branch was carrying. * test(voyage): drop sys.path.insert from the voyage common utils test * fix(voyage): route rerank on the key it authenticates with get_complete_url picked the rerank host from the environment while the auth header carried the request key, so an explicit MongoDB-issued key was posted to api.voyageai.com. validate_environment now hands the resolved key to the config instance and get_complete_url reads it back, so the host and the credential always come from one key. The config is built per request, so nothing carries over between them. * test: allow ultrafast pricing keys in model prices schema * test: drop ultrafast schema keys now added upstream * test(integration): prove voyage key-based host routing on the wire --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- README.md | 2 +- litellm/llms/voyage/common_utils.py | 35 + .../llms/voyage/embedding/transformation.py | 12 +- .../embedding/transformation_contextual.py | 12 +- .../embedding/transformation_multimodal.py | 15 +- litellm/llms/voyage/rerank/transformation.py | 22 +- .../provider_endpoints_support_backup.json | 2 +- .../provider_create_fields.json | 2 +- provider_endpoints_support.json | 2 +- tests/integration/_support/forward_proxy.py | 71 ++ .../test_voyage_mongodb_host_wire.py | 688 ++++++++++++++++++ .../test_voyage_rerank_transformation.py | 7 +- tests/unit/llms/voyage/test_common_utils.py | 164 +++++ .../test_voyage_multimodal_embedding.py | 11 +- .../src/components/provider_info_helpers.tsx | 2 +- 15 files changed, 997 insertions(+), 50 deletions(-) create mode 100644 litellm/llms/voyage/common_utils.py create mode 100644 tests/integration/_support/forward_proxy.py create mode 100644 tests/integration/providers/test_voyage_mongodb_host_wire.py create mode 100644 tests/unit/llms/voyage/test_common_utils.py diff --git a/README.md b/README.md index 7ffc44854bb..ebf75e31729 100644 --- a/README.md +++ b/README.md @@ -401,7 +401,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Vercel AI Gateway (`vercel_ai_gateway`)](https://docs.litellm.ai/docs/providers/vercel_ai_gateway) | ✅ | ✅ | ✅ | | | | | | | | | [VLLM (`vllm`)](https://docs.litellm.ai/docs/providers/vllm) | ✅ | ✅ | ✅ | | | | | | | | | [Volcengine (`volcengine`)](https://docs.litellm.ai/docs/providers/volcano) | ✅ | ✅ | ✅ | | | | | | | | -| [Voyage AI (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | | +| [VoyageAI by MongoDB (`voyage`)](https://docs.litellm.ai/docs/providers/voyage) | | | | ✅ | | | | | | | | [WandB Inference (`wandb`)](https://docs.litellm.ai/docs/providers/wandb_inference) | ✅ | ✅ | ✅ | | | | | | | | | [Watsonx Text (`watsonx_text`)](https://docs.litellm.ai/docs/providers/watsonx) | ✅ | ✅ | ✅ | | | | | | | | | [xAI (`xai`)](https://docs.litellm.ai/docs/providers/xai) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/llms/voyage/common_utils.py b/litellm/llms/voyage/common_utils.py new file mode 100644 index 00000000000..2f4f63a8a43 --- /dev/null +++ b/litellm/llms/voyage/common_utils.py @@ -0,0 +1,35 @@ +""" +Shared helpers for the Voyage (VoyageAI by MongoDB) provider. +""" + +from typing import Final + +from litellm.secret_managers.main import get_secret_str + +VOYAGE_API_BASE: Final = "https://api.voyageai.com/v1" +MONGODB_API_BASE: Final = "https://ai.mongodb.com/v1" +MONGODB_API_KEY_PREFIX: Final = "al-" + + +def get_voyage_api_key(api_key: str | None = None) -> str | None: + """Resolve the key a Voyage request will authenticate with, explicit value first.""" + return ( + api_key + or get_secret_str("VOYAGE_API_KEY") + or get_secret_str("VOYAGE_AI_API_KEY") + or get_secret_str("VOYAGE_AI_TOKEN") + ) + + +def get_default_base_url(api_key: str | None = None) -> str: + """ + Pick the host that issued the key: MongoDB-issued keys (``al-`` prefix) are only + valid on ai.mongodb.com, every other key on api.voyageai.com. + + Mirrors ``voyageai.util.get_default_base_url`` in the official SDK: + https://github.com/voyage-ai/voyageai-python/blob/main/voyageai/util.py + """ + resolved: Final = get_voyage_api_key(api_key) + if resolved is not None and resolved.startswith(MONGODB_API_KEY_PREFIX): + return MONGODB_API_BASE + return VOYAGE_API_BASE diff --git a/litellm/llms/voyage/embedding/transformation.py b/litellm/llms/voyage/embedding/transformation.py index 7d74b1e00c4..a9232ad27e2 100644 --- a/litellm/llms/voyage/embedding/transformation.py +++ b/litellm/llms/voyage/embedding/transformation.py @@ -5,7 +5,7 @@ import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig -from litellm.secret_managers.main import get_secret_str +from litellm.llms.voyage.common_utils import get_default_base_url, get_voyage_api_key from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage @@ -49,7 +49,7 @@ class VoyageEmbeddingConfig(BaseEmbeddingConfig): if not api_base.endswith("/embeddings"): api_base = f"{api_base}/embeddings" return api_base - return "https://api.voyageai.com/v1/embeddings" + return f"{get_default_base_url(api_key)}/embeddings" def get_supported_openai_params(self, model: str) -> list: return [ @@ -85,14 +85,8 @@ class VoyageEmbeddingConfig(BaseEmbeddingConfig): api_key: str | None = None, api_base: str | None = None, ) -> dict: - if api_key is None: - api_key = ( - get_secret_str("VOYAGE_API_KEY") - or get_secret_str("VOYAGE_AI_API_KEY") - or get_secret_str("VOYAGE_AI_TOKEN") - ) return { - "Authorization": f"Bearer {api_key}", + "Authorization": f"Bearer {get_voyage_api_key(api_key)}", } def transform_embedding_request( diff --git a/litellm/llms/voyage/embedding/transformation_contextual.py b/litellm/llms/voyage/embedding/transformation_contextual.py index 870de8756bb..f5e7fdb48f5 100644 --- a/litellm/llms/voyage/embedding/transformation_contextual.py +++ b/litellm/llms/voyage/embedding/transformation_contextual.py @@ -11,7 +11,7 @@ import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig -from litellm.secret_managers.main import get_secret_str +from litellm.llms.voyage.common_utils import get_default_base_url, get_voyage_api_key from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage @@ -58,7 +58,7 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): if not api_base.endswith("/contextualizedembeddings"): api_base = f"{api_base}/contextualizedembeddings" return api_base - return "https://api.voyageai.com/v1/contextualizedembeddings" + return f"{get_default_base_url(api_key)}/contextualizedembeddings" def get_supported_openai_params(self, model: str) -> list: # mutable-ok: base class signature return ["encoding_format", "dimensions"] @@ -91,14 +91,8 @@ class VoyageContextualEmbeddingConfig(BaseEmbeddingConfig): api_key: str | None = None, api_base: str | None = None, ) -> dict: - if api_key is None: - api_key = ( - get_secret_str("VOYAGE_API_KEY") - or get_secret_str("VOYAGE_AI_API_KEY") - or get_secret_str("VOYAGE_AI_TOKEN") - ) return { - "Authorization": f"Bearer {api_key}", + "Authorization": f"Bearer {get_voyage_api_key(api_key)}", } AUTO_CHUNK_SIZE: Final = 32000 diff --git a/litellm/llms/voyage/embedding/transformation_multimodal.py b/litellm/llms/voyage/embedding/transformation_multimodal.py index 814d5ab7eb0..035765b6691 100644 --- a/litellm/llms/voyage/embedding/transformation_multimodal.py +++ b/litellm/llms/voyage/embedding/transformation_multimodal.py @@ -13,7 +13,7 @@ import httpx from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.base_llm.chat.transformation import BaseLLMException from litellm.llms.base_llm.embedding.transformation import BaseEmbeddingConfig -from litellm.secret_managers.main import get_secret_str +from litellm.llms.voyage.common_utils import get_default_base_url, get_voyage_api_key from litellm.types.llms.openai import AllEmbeddingInputValues, AllMessageValues from litellm.types.utils import EmbeddingResponse, Usage @@ -58,7 +58,7 @@ class VoyageMultimodalEmbeddingConfig(BaseEmbeddingConfig): if not api_base.endswith("/multimodalembeddings"): api_base = f"{api_base}/multimodalembeddings" return api_base - return "https://api.voyageai.com/v1/multimodalembeddings" + return f"{get_default_base_url(api_key)}/multimodalembeddings" def get_supported_openai_params(self, model: str) -> list: return ["dimensions"] @@ -84,19 +84,14 @@ class VoyageMultimodalEmbeddingConfig(BaseEmbeddingConfig): api_key: str | None = None, api_base: str | None = None, ) -> dict: - if api_key is None: - api_key = ( - get_secret_str("VOYAGE_API_KEY") - or get_secret_str("VOYAGE_AI_API_KEY") - or get_secret_str("VOYAGE_AI_TOKEN") - ) - if not api_key: + resolved_api_key: Final = get_voyage_api_key(api_key) + if not resolved_api_key: raise ValueError( "Voyage API key is required for multimodal embeddings. " "Set VOYAGE_API_KEY / VOYAGE_AI_API_KEY / VOYAGE_AI_TOKEN " "or pass `api_key` explicitly." ) - return {"Authorization": f"Bearer {api_key}"} + return {"Authorization": f"Bearer {resolved_api_key}"} def _normalize_content_item(self, item: dict[str, object]) -> dict[str, object]: item_type: Final = item.get("type") diff --git a/litellm/llms/voyage/rerank/transformation.py b/litellm/llms/voyage/rerank/transformation.py index 3acb2f2ed58..121c18e82ae 100644 --- a/litellm/llms/voyage/rerank/transformation.py +++ b/litellm/llms/voyage/rerank/transformation.py @@ -13,7 +13,7 @@ from pydantic import ConfigDict, TypeAdapter from litellm._uuid import uuid from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj from litellm.llms.base_llm.rerank.transformation import BaseRerankConfig -from litellm.secret_managers.main import get_secret_str +from litellm.llms.voyage.common_utils import get_default_base_url, get_voyage_api_key from litellm.types.rerank import ( RerankBilledUnits, RerankResponse, @@ -31,6 +31,16 @@ _STR: Final = TypeAdapter(str) class VoyageRerankConfig(BaseRerankConfig): + """ + ``validate_environment`` stores the credential it authenticates with so ``get_complete_url`` + can select the host that issued it. ``ProviderConfigManager.get_provider_rerank_config`` + builds this config per request, so that key never reaches another one. + """ + + def __init__(self) -> None: + super().__init__() + self._api_key: str | None = None + def get_supported_cohere_rerank_params(self, model: str) -> list: return ["query", "documents", "top_n", "return_documents"] @@ -66,7 +76,7 @@ class VoyageRerankConfig(BaseRerankConfig): optional_params: dict | None = None, ) -> str: if api_base is None: - return "https://api.voyageai.com/v1/rerank" + return f"{get_default_base_url(self._api_key)}/rerank" api_base = api_base.rstrip("/") if not api_base.endswith("/v1/rerank"): if api_base.endswith("/v1"): @@ -148,12 +158,12 @@ class VoyageRerankConfig(BaseRerankConfig): optional_params: dict | None = None, litellm_params: Mapping[str, object] | None = None, ) -> dict: - if api_key is None: - api_key = get_secret_str("VOYAGE_API_KEY") or get_secret_str("VOYAGE_AI_API_KEY") - if api_key is None: + resolved_api_key: Final = get_voyage_api_key(api_key) + if resolved_api_key is None: raise ValueError("Voyage AI API key is required. Set via `api_key` parameter or `VOYAGE_API_KEY` env var.") + self._api_key = resolved_api_key return { - "Authorization": f"Bearer {api_key}", + "Authorization": f"Bearer {resolved_api_key}", "content-type": "application/json", } diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index 0e325bb61fe..d9a6feb625d 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2354,7 +2354,7 @@ } }, "voyage": { - "display_name": "Voyage AI (`voyage`)", + "display_name": "VoyageAI by MongoDB (`voyage`)", "url": "https://docs.litellm.ai/docs/providers/voyage", "endpoints": { "chat_completions": false, diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index ae07d5d6bd9..965d8dae4d2 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -3583,7 +3583,7 @@ }, { "provider": "Voyage", - "provider_display_name": "Voyage AI", + "provider_display_name": "VoyageAI by MongoDB", "litellm_provider": "voyage", "credential_fields": [ { diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 00904219b81..62e23388581 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2711,7 +2711,7 @@ } }, "voyage": { - "display_name": "Voyage AI (`voyage`)", + "display_name": "VoyageAI by MongoDB (`voyage`)", "url": "https://docs.litellm.ai/docs/providers/voyage", "endpoints": { "chat_completions": false, diff --git a/tests/integration/_support/forward_proxy.py b/tests/integration/_support/forward_proxy.py new file mode 100644 index 00000000000..9f5cff94277 --- /dev/null +++ b/tests/integration/_support/forward_proxy.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +import threading +from collections.abc import Generator, Mapping +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from queue import SimpleQueue +from typing import Final + + +@dataclass(frozen=True, slots=True) +class Tunnel: + method: str + target: str + headers: Mapping[str, str] + + +@dataclass(frozen=True, slots=True) +class ForwardProxy: + url: str + port: int + received: SimpleQueue[Tunnel] + + def drain(self) -> tuple[Tunnel, ...]: + return tuple(self.received.get_nowait() for _ in range(self.received.qsize())) + + def targets(self) -> tuple[str, ...]: + return tuple(tunnel.target for tunnel in self.drain()) + + +@contextmanager +def refusing_forward_proxy(port: int = 0) -> Generator[ForwardProxy, None, None]: + """Owned HTTP forward proxy: records the host each CONNECT asks for and refuses the tunnel with 403. + + A litellm proxy booted with ``HTTPS_PROXY`` pointed here makes the host it dials for a provider + observable without a byte leaving the box; the caller sees litellm's own connection error. + """ + received: Final[SimpleQueue[Tunnel]] = SimpleQueue() + + class Handler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + timeout = 5 + + def refuse(self) -> None: + received.put(Tunnel(self.command, self.path, {name.lower(): value for name, value in self.headers.items()})) + self.send_response(403) + self.send_header("content-length", "0") + self.send_header("connection", "close") + self.end_headers() + self.close_connection = True + + do_CONNECT = refuse + do_GET = refuse + do_POST = refuse + do_PUT = refuse + do_DELETE = refuse + + def log_message(self, format: str, *args: object) -> None: + pass + + with ThreadingHTTPServer(("127.0.0.1", port), Handler) as server: + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + thread.start() + try: + yield ForwardProxy(f"http://127.0.0.1:{server.server_port}", server.server_port, received) + finally: + server.shutdown() + thread.join(timeout=6) + assert not thread.is_alive(), "Owned forward proxy survived cleanup" + server.server_close() diff --git a/tests/integration/providers/test_voyage_mongodb_host_wire.py b/tests/integration/providers/test_voyage_mongodb_host_wire.py new file mode 100644 index 00000000000..1a6ed907c9b --- /dev/null +++ b/tests/integration/providers/test_voyage_mongodb_host_wire.py @@ -0,0 +1,688 @@ +import asyncio +import json +import os +import re +import signal +import socket +import uuid +from collections.abc import Callable, Iterator +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import psutil +import pytest +import yaml +from integration._support.client import JSON_OBJECT, Gateway, eventually, gateway_from_environment, object_value +from integration._support.database import read_rows +from integration._support.forward_proxy import ForwardProxy, refusing_forward_proxy +from integration._support.process import owned_proxy, owned_proxy_process +from integration._support.wire import Reply, Request, wire_server +from openai import APIStatusError, AsyncOpenAI, OpenAI +from pydantic import JsonValue, TypeAdapter + +EMBEDDING_MODEL: Final = "voyage/voyage-3.5" +CONTEXTUAL_MODEL: Final = "voyage/voyage-context-3" +MULTIMODAL_MODEL: Final = "voyage/voyage-multimodal-3" +RERANK_MODEL: Final = "voyage/rerank-2.5" +MONGODB_KEY: Final = "al-synthetic-mongodb-issued-key" +VOYAGE_KEY: Final = "pa-synthetic-voyage-issued-key" +ENV_TOKEN: Final = "al-synthetic-env-token" +ENV_PRIMARY: Final = "pa-synthetic-env-primary" +ENV_SECONDARY: Final = "al-synthetic-env-secondary" +MONGODB_HOST: Final = "ai.mongodb.com:443" +VOYAGE_HOST: Final = "api.voyageai.com:443" +NO_CACHE: Final[dict[str, JsonValue]] = {"cache": {"no-cache": True}} +REFUSED_STATUS: Final = 500 +REFUSED_MARKER: Final = "403" +OUTAGE_MARKER: Final = "APIConnectionError" +MONGODB_EMBEDDINGS: Final = "voyage-audit-mongodb-embeddings" +MONGODB_CONTEXTUAL: Final = "voyage-audit-mongodb-contextual" +MONGODB_MULTIMODAL: Final = "voyage-audit-mongodb-multimodal" +MONGODB_RERANK: Final = "voyage-audit-mongodb-rerank" +VOYAGE_EMBEDDINGS: Final = "voyage-audit-voyage-embeddings" +VOYAGE_RERANK: Final = "voyage-audit-voyage-rerank" +BLANK_EMBEDDINGS: Final = "voyage-audit-blank-key-embeddings" +BLANK_RERANK: Final = "voyage-audit-blank-key-rerank" +NULL_RERANK: Final = "voyage-audit-null-key-rerank" +SCRIPTED_CHAT: Final = "voyage-audit-scripted-chat" +TOKEN_EMBEDDINGS: Final = "voyage-audit-token-embeddings" +TOKEN_RERANK: Final = "voyage-audit-token-rerank" +TOKEN_YAML_RERANK: Final = "voyage-audit-token-yaml-rerank" +TOKEN_BLANK_EMBEDDINGS: Final = "voyage-audit-token-blank-embeddings" +TOKEN_BLANK_RERANK: Final = "voyage-audit-token-blank-rerank" +TOKEN_NULL_RERANK: Final = "voyage-audit-token-null-rerank" +PRECEDENCE_EMBEDDINGS: Final = "voyage-audit-precedence-embeddings" +PRECEDENCE_RERANK: Final = "voyage-audit-precedence-rerank" +PRECEDENCE_EXPLICIT_EMBEDDINGS: Final = "voyage-audit-precedence-explicit-embeddings" +PRECEDENCE_EXPLICIT_RERANK: Final = "voyage-audit-precedence-explicit-rerank" +VECTOR: Final = [0.1, 0.2, 0.3] +_STARTED_WORKER: Final = re.compile(r"Started server process \[(\d+)\]") +_PROXY_VARIABLES: Final = frozenset({"HTTPS_PROXY", "HTTP_PROXY", "ALL_PROXY", "NO_PROXY"}) +_JSON_OBJECTS: Final = TypeAdapter(list[dict[str, JsonValue]]) +_WORKER_PIDS: Final = TypeAdapter(list[int]) +_load_yaml: Final[Callable[[str], object]] = yaml.safe_load +_OWNED_PROXY_CELL_SECONDS: Final = 2 * max(30.0, float(os.environ.get("INTEGRATION_PROXY_READY_SECONDS") or 70)) + 120 +pytestmark: Final = pytest.mark.timeout(_OWNED_PROXY_CELL_SECONDS) + + +def _deployment(name: str, model: str, mode: str, **litellm_params: JsonValue) -> dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": {"model": model, **litellm_params}, + "model_info": {"id": name, "mode": mode}, + } + + +def _isolated_deployments(upstream_url: str) -> list[dict[str, JsonValue]]: + return [ + _deployment(MONGODB_EMBEDDINGS, EMBEDDING_MODEL, "embedding", api_key=MONGODB_KEY), + _deployment(MONGODB_CONTEXTUAL, CONTEXTUAL_MODEL, "embedding", api_key=MONGODB_KEY), + _deployment(MONGODB_MULTIMODAL, MULTIMODAL_MODEL, "embedding", api_key=MONGODB_KEY), + _deployment(MONGODB_RERANK, RERANK_MODEL, "rerank", api_key=MONGODB_KEY), + _deployment(VOYAGE_EMBEDDINGS, EMBEDDING_MODEL, "embedding", api_key=VOYAGE_KEY), + _deployment(VOYAGE_RERANK, RERANK_MODEL, "rerank", api_key=VOYAGE_KEY), + _deployment(BLANK_EMBEDDINGS, EMBEDDING_MODEL, "embedding", api_key=""), + _deployment(BLANK_RERANK, RERANK_MODEL, "rerank", api_key=""), + _deployment(NULL_RERANK, RERANK_MODEL, "rerank", api_key=None), + _deployment( + SCRIPTED_CHAT, + "openai/gpt-4o-mini", + "chat", + api_key="integration-provider-key", + api_base=f"{upstream_url}/v1", + ), + ] + + +def _token_only_deployments() -> list[dict[str, JsonValue]]: + return [ + _deployment(TOKEN_EMBEDDINGS, EMBEDDING_MODEL, "embedding"), + _deployment(TOKEN_RERANK, RERANK_MODEL, "rerank"), + _deployment(TOKEN_YAML_RERANK, RERANK_MODEL, "rerank", api_key="os.environ/VOYAGE_AI_TOKEN"), + _deployment(TOKEN_BLANK_EMBEDDINGS, EMBEDDING_MODEL, "embedding", api_key=""), + _deployment(TOKEN_BLANK_RERANK, RERANK_MODEL, "rerank", api_key=""), + _deployment(TOKEN_NULL_RERANK, RERANK_MODEL, "rerank", api_key=None), + ] + + +def _precedence_deployments() -> list[dict[str, JsonValue]]: + return [ + _deployment(PRECEDENCE_EMBEDDINGS, EMBEDDING_MODEL, "embedding"), + _deployment(PRECEDENCE_RERANK, RERANK_MODEL, "rerank"), + _deployment(PRECEDENCE_EXPLICIT_EMBEDDINGS, EMBEDDING_MODEL, "embedding", api_key=MONGODB_KEY), + _deployment(PRECEDENCE_EXPLICIT_RERANK, RERANK_MODEL, "rerank", api_key=MONGODB_KEY), + ] + + +def _write_config(directory: Path, name: str, model_list: list[dict[str, JsonValue]]) -> Path: + base: Final = JSON_OBJECT.validate_python(_load_yaml(Path("tests/integration/proxy_config.yaml").read_text())) + configuration: Final = { + **base, + "model_list": model_list, + "router_settings": {"disable_cooldowns": True, "num_retries": 0}, + } + path: Final = directory / f"{name}.yaml" + path.write_text(yaml.safe_dump(configuration)) + return path + + +def _scrubbed() -> tuple[str, ...]: + return tuple(name for name in os.environ if name.upper().startswith("VOYAGE_") or name.upper() in _PROXY_VARIABLES) + + +def _forward_environment(forward_url: str, **extra: str) -> dict[str, str]: + return {"HTTPS_PROXY": forward_url, "NO_PROXY": "127.0.0.1,localhost", **extra} + + +@pytest.fixture(scope="module") +def forward() -> Iterator[ForwardProxy]: + with refusing_forward_proxy() as proxy: + yield proxy + + +@pytest.fixture(scope="module") +def module_gateway() -> Iterator[Gateway]: + with gateway_from_environment() as value: + yield value + + +@pytest.fixture(scope="module") +def isolated( + module_gateway: Gateway, forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("voyage-isolated") + config: Final = _write_config(directory, "isolated", _isolated_deployments(module_gateway.upstream_url)) + with owned_proxy( + module_gateway, + directory, + _forward_environment(forward.url), + config=config, + remove_environment=_scrubbed(), + workers=2, + ) as candidate: + yield candidate + + +@pytest.fixture(scope="module") +def token_only( + module_gateway: Gateway, forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("voyage-token-only") + config: Final = _write_config(directory, "token-only", _token_only_deployments()) + with owned_proxy( + module_gateway, + directory, + _forward_environment(forward.url, VOYAGE_AI_TOKEN=ENV_TOKEN), + config=config, + remove_environment=_scrubbed(), + ) as candidate: + yield candidate + + +@pytest.fixture(scope="module") +def precedence( + module_gateway: Gateway, forward: ForwardProxy, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Gateway]: + directory: Final = tmp_path_factory.mktemp("voyage-precedence") + config: Final = _write_config(directory, "precedence", _precedence_deployments()) + with owned_proxy( + module_gateway, + directory, + _forward_environment( + forward.url, + VOYAGE_API_KEY=ENV_PRIMARY, + VOYAGE_AI_API_KEY=ENV_SECONDARY, + VOYAGE_AI_TOKEN=ENV_TOKEN, + ), + config=config, + remove_environment=_scrubbed(), + ) as candidate: + yield candidate + + +def _embedding_body(model: str, **extra: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "input": f"host selection {uuid.uuid4().hex}", **NO_CACHE, **extra} + + +def _rerank_body(model: str, **extra: JsonValue) -> dict[str, JsonValue]: + return { + "model": model, + "query": f"host selection {uuid.uuid4().hex}", + "documents": ["first document", "second document"], + **extra, + } + + +def _embed(candidate: Gateway, model: str, **extra: JsonValue) -> httpx.Response: + return candidate.request("POST", "/v1/embeddings", _embedding_body(model, **extra)) + + +def _rerank(candidate: Gateway, model: str, path: str = "/v1/rerank", **extra: JsonValue) -> httpx.Response: + return candidate.request("POST", path, _rerank_body(model, **extra)) + + +def _error_message(call_id: str) -> str: + rows: Final = eventually( + lambda: read_rows('SELECT metadata FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)), + lambda values: len(values) == 1, + seconds=70, + ) + metadata: Final = rows[0]["metadata"] + parsed: Final = json.loads(metadata) if isinstance(metadata, str) else object_value(metadata) + return str(object_value(parsed["error_information"])["error_message"]) + + +def _assert_refused(response: httpx.Response) -> None: + assert response.status_code == REFUSED_STATUS, response.text + assert REFUSED_MARKER in response.text, response.text + assert REFUSED_MARKER in _error_message(response.headers["x-litellm-call-id"]), response.text + + +def _assert_dialed(forward: ForwardProxy, response: httpx.Response, host: str) -> None: + _assert_refused(response) + assert forward.targets() == (host,), response.text + + +def test_scripted_chat_keeps_serving_behind_the_forward_proxy(isolated: Gateway, forward: ForwardProxy) -> None: + forward.drain() + reply: Final = isolated.chat(SCRIPTED_CHAT, text=f"forward proxy control {uuid.uuid4().hex}") + rows: Final = eventually( + lambda: read_rows('SELECT model FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (str(reply["id"]),)), + lambda values: len(values) == 1, + seconds=70, + ) + assert rows[0]["model"] == "openai/gpt-4o-mini", rows + assert forward.targets() == () + + +def test_mongodb_key_embeddings_dial_ai_mongodb_openai_sdk_sync(isolated: Gateway, forward: ForwardProxy) -> None: + forward.drain() + with OpenAI( + api_key=isolated.key, + base_url=str(isolated.client.base_url).rstrip("/") + "/v1", + max_retries=0, + http_client=httpx.Client(timeout=15, trust_env=False), + ) as client: + with pytest.raises(APIStatusError) as caught: + client.embeddings.create( + model=MONGODB_EMBEDDINGS, input=f"sdk sync {uuid.uuid4().hex}", extra_body=NO_CACHE + ) + assert caught.value.status_code == REFUSED_STATUS, caught.value.message + assert REFUSED_MARKER in caught.value.message, caught.value.message + assert forward.targets() == (MONGODB_HOST,) + + +async def test_mongodb_key_embeddings_dial_ai_mongodb_openai_sdk_async( + isolated: Gateway, forward: ForwardProxy +) -> None: + forward.drain() + async with AsyncOpenAI( + api_key=isolated.key, + base_url=str(isolated.client.base_url).rstrip("/") + "/v1", + max_retries=0, + http_client=httpx.AsyncClient(timeout=15, trust_env=False), + ) as client: + with pytest.raises(APIStatusError) as caught: + await client.embeddings.create( + model=MONGODB_EMBEDDINGS, input=f"sdk async {uuid.uuid4().hex}", extra_body=NO_CACHE + ) + assert caught.value.status_code == REFUSED_STATUS, caught.value.message + assert REFUSED_MARKER in caught.value.message, caught.value.message + assert forward.targets() == (MONGODB_HOST,) + + +@pytest.mark.parametrize("deployment", (MONGODB_CONTEXTUAL, MONGODB_MULTIMODAL)) +def test_mongodb_key_other_embedding_shapes_dial_ai_mongodb( + isolated: Gateway, forward: ForwardProxy, deployment: str +) -> None: + forward.drain() + _assert_dialed(forward, _embed(isolated, deployment), MONGODB_HOST) + + +@pytest.mark.parametrize("path", ("/v1/rerank", "/rerank", "/v2/rerank")) +def test_mongodb_key_rerank_dials_ai_mongodb(isolated: Gateway, forward: ForwardProxy, path: str) -> None: + forward.drain() + _assert_dialed(forward, _rerank(isolated, MONGODB_RERANK, path), MONGODB_HOST) + + +def test_voyage_key_embeddings_still_dial_api_voyageai(isolated: Gateway, forward: ForwardProxy) -> None: + forward.drain() + _assert_dialed(forward, _embed(isolated, VOYAGE_EMBEDDINGS), VOYAGE_HOST) + + +def test_voyage_key_rerank_still_dials_api_voyageai(isolated: Gateway, forward: ForwardProxy) -> None: + forward.drain() + _assert_dialed(forward, _rerank(isolated, VOYAGE_RERANK), VOYAGE_HOST) + + +def test_request_body_mongodb_key_overrides_voyage_deployment_host(isolated: Gateway, forward: ForwardProxy) -> None: + forward.drain() + _assert_dialed(forward, _embed(isolated, VOYAGE_EMBEDDINGS, api_key=MONGODB_KEY), MONGODB_HOST) + + +@pytest.mark.parametrize("deployment", (MONGODB_EMBEDDINGS, MONGODB_RERANK)) +def test_health_check_dials_ai_mongodb_for_mongodb_key( + isolated: Gateway, forward: ForwardProxy, deployment: str +) -> None: + forward.drain() + response: Final = isolated.request("GET", "/health", params={"model": deployment}) + assert response.status_code == 503, response.text + report: Final = JSON_OBJECT.validate_json(response.content) + assert report["unhealthy_count"] == 1 and report["healthy_count"] == 0, report + unhealthy: Final = _JSON_OBJECTS.validate_python(report["unhealthy_endpoints"]) + assert len(unhealthy) == 1 and REFUSED_MARKER in str(unhealthy[0]["error"]), report + assert forward.targets() == (MONGODB_HOST,), report + + +@pytest.mark.parametrize( + ("mode", "model"), (("embedding", EMBEDDING_MODEL), ("rerank", RERANK_MODEL)), ids=("embedding", "rerank") +) +def test_test_connection_dials_ai_mongodb_for_mongodb_key( + isolated: Gateway, forward: ForwardProxy, mode: str, model: str +) -> None: + forward.drain() + report: Final = isolated.post( + "/health/test_connection", {"litellm_params": {"model": model, "api_key": MONGODB_KEY}, "mode": mode} + ) + assert report["status"] == "error", report + assert REFUSED_MARKER in json.dumps(report), report + assert forward.targets() == (MONGODB_HOST,), report + + +@pytest.mark.parametrize("api_key", (5, ["al-list-member"]), ids=("int", "list")) +def test_non_string_request_body_key_is_rejected_by_validation_before_any_dial( + isolated: Gateway, forward: ForwardProxy, api_key: JsonValue +) -> None: + forward.drain() + response: Final = _embed(isolated, MONGODB_EMBEDDINGS, api_key=api_key) + assert response.status_code == 500, response.text + assert "LiteLLM_Params" in response.text and "api_key" in response.text, response.text + assert forward.targets() == () + + +def test_empty_request_body_key_falls_back_to_api_voyageai(isolated: Gateway, forward: ForwardProxy) -> None: + forward.drain() + _assert_dialed(forward, _embed(isolated, MONGODB_EMBEDDINGS, api_key=""), VOYAGE_HOST) + + +def test_five_kilobyte_mongodb_request_body_key_dials_ai_mongodb(isolated: Gateway, forward: ForwardProxy) -> None: + forward.drain() + _assert_dialed(forward, _embed(isolated, VOYAGE_EMBEDDINGS, api_key=MONGODB_KEY + "x" * 5000), MONGODB_HOST) + + +def test_duplicate_request_body_key_counts_once(isolated: Gateway, forward: ForwardProxy) -> None: + forward.drain() + raw: Final = ( + '{"model": "%s", "input": "duplicate key %s", "cache": {"no-cache": true}, "api_key": "%s", "api_key": "%s"}' + % (VOYAGE_EMBEDDINGS, uuid.uuid4().hex, MONGODB_KEY, MONGODB_KEY) + ) + response: Final = isolated.client.post( + "/v1/embeddings", + content=raw.encode(), + headers={"Authorization": f"Bearer {isolated.key}", "content-type": "application/json"}, + ) + _assert_dialed(forward, response, MONGODB_HOST) + + +def test_blank_deployment_key_without_env_dials_api_voyageai_for_embeddings( + isolated: Gateway, forward: ForwardProxy +) -> None: + forward.drain() + _assert_dialed(forward, _embed(isolated, BLANK_EMBEDDINGS), VOYAGE_HOST) + + +@pytest.mark.parametrize("deployment", (BLANK_RERANK, NULL_RERANK), ids=("blank", "null")) +def test_rerank_without_any_key_fails_before_dialing(isolated: Gateway, forward: ForwardProxy, deployment: str) -> None: + forward.drain() + response: Final = _rerank(isolated, deployment) + assert response.status_code == 500, response.text + assert "Voyage AI API key is required" in response.text, response.text + assert forward.targets() == () + + +@dataclass(frozen=True, slots=True) +class _Probe: + host: str + response: httpx.Response + + +def _burst_plan(count: int) -> tuple[tuple[str, str, str], ...]: + shapes: Final = ( + ("/v1/embeddings", MONGODB_EMBEDDINGS, MONGODB_HOST), + ("/v1/rerank", MONGODB_RERANK, MONGODB_HOST), + ("/v1/embeddings", VOYAGE_EMBEDDINGS, VOYAGE_HOST), + ("/v1/rerank", VOYAGE_RERANK, VOYAGE_HOST), + ) + return tuple(shapes[index % len(shapes)] for index in range(count)) + + +async def _burst(base_url: str, key: str, count: int, *, tolerate_transport_errors: bool = False) -> tuple[_Probe, ...]: + async def one(client: httpx.AsyncClient, path: str, deployment: str, host: str) -> _Probe: + body: Final = _embedding_body(deployment) if path == "/v1/embeddings" else _rerank_body(deployment) + response: Final = await client.post(path, json=body, headers={"Authorization": f"Bearer {key}"}) + return _Probe(host, response) + + async with httpx.AsyncClient(base_url=base_url, timeout=30, trust_env=False) as client: + results: Final = await asyncio.gather( + *(one(client, path, deployment, host) for path, deployment, host in _burst_plan(count)), + return_exceptions=tolerate_transport_errors, + ) + for result in results: + assert not isinstance(result, BaseException) or isinstance(result, httpx.TransportError), repr(result) + return tuple(result for result in results if isinstance(result, _Probe)) + + +def _host_counts(targets: tuple[str, ...]) -> dict[str, int]: + return {host: targets.count(host) for host in sorted(set(targets))} + + +async def test_concurrent_mixed_keys_each_dial_their_own_host(isolated: Gateway, forward: ForwardProxy) -> None: + forward.drain() + served: Final = await _burst(str(isolated.client.base_url), isolated.key, 24) + assert len(served) == 24 + for probe in served: + _assert_refused(probe.response) + assert _host_counts(forward.targets()) == {MONGODB_HOST: 12, VOYAGE_HOST: 12} + + +def test_voyage_ai_token_alone_routes_embeddings_to_ai_mongodb(token_only: Gateway, forward: ForwardProxy) -> None: + forward.drain() + _assert_dialed(forward, _embed(token_only, TOKEN_EMBEDDINGS), MONGODB_HOST) + + +@pytest.mark.parametrize( + "deployment", (TOKEN_RERANK, TOKEN_YAML_RERANK, TOKEN_NULL_RERANK), ids=("missing", "yaml-env", "null") +) +def test_voyage_ai_token_alone_routes_rerank_to_ai_mongodb( + token_only: Gateway, forward: ForwardProxy, deployment: str +) -> None: + forward.drain() + _assert_dialed(forward, _rerank(token_only, deployment), MONGODB_HOST) + + +@pytest.mark.parametrize( + ("path", "deployment"), + (("/v1/embeddings", TOKEN_BLANK_EMBEDDINGS), ("/v1/rerank", TOKEN_BLANK_RERANK)), + ids=("embeddings", "rerank"), +) +def test_blank_deployment_key_falls_through_to_voyage_ai_token( + token_only: Gateway, forward: ForwardProxy, path: str, deployment: str +) -> None: + forward.drain() + response: Final = ( + _embed(token_only, deployment) if path == "/v1/embeddings" else _rerank(token_only, deployment, path) + ) + _assert_dialed(forward, response, MONGODB_HOST) + + +@pytest.mark.parametrize( + ("path", "deployment"), + (("/v1/embeddings", PRECEDENCE_EMBEDDINGS), ("/v1/rerank", PRECEDENCE_RERANK)), + ids=("embeddings", "rerank"), +) +def test_voyage_api_key_wins_over_mongodb_fallback_env( + precedence: Gateway, forward: ForwardProxy, path: str, deployment: str +) -> None: + forward.drain() + response: Final = ( + _embed(precedence, deployment) if path == "/v1/embeddings" else _rerank(precedence, deployment, path) + ) + _assert_dialed(forward, response, VOYAGE_HOST) + + +@pytest.mark.parametrize( + ("path", "deployment"), + (("/v1/embeddings", PRECEDENCE_EXPLICIT_EMBEDDINGS), ("/v1/rerank", PRECEDENCE_EXPLICIT_RERANK)), + ids=("embeddings", "rerank"), +) +def test_explicit_mongodb_deployment_key_wins_over_voyage_env( + precedence: Gateway, forward: ForwardProxy, path: str, deployment: str +) -> None: + forward.drain() + response: Final = ( + _embed(precedence, deployment) if path == "/v1/embeddings" else _rerank(precedence, deployment, path) + ) + _assert_dialed(forward, response, MONGODB_HOST) + + +def _embedding_peer(request: Request) -> Reply: + assert request.method == "POST", request.method + assert request.headers["authorization"] == f"Bearer {MONGODB_KEY}", request.headers + body: Final = JSON_OBJECT.validate_json(request.body) + return Reply( + body=json.dumps( + { + "object": "list", + "data": [{"object": "embedding", "embedding": VECTOR, "index": 0}], + "model": str(body["model"]), + "usage": {"total_tokens": 7}, + } + ).encode() + ) + + +def _rerank_peer(request: Request) -> Reply: + assert request.method == "POST", request.method + assert request.headers["authorization"] == f"Bearer {MONGODB_KEY}", request.headers + return Reply( + body=json.dumps( + { + "object": "list", + "data": [{"index": 1, "relevance_score": 0.9}, {"index": 0, "relevance_score": 0.1}], + "model": "rerank-2.5", + "usage": {"total_tokens": 11}, + } + ).encode() + ) + + +def _spend_api_base(call_id: str) -> str: + rows: Final = eventually( + lambda: read_rows('SELECT api_base FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)), + lambda values: len(values) == 1, + seconds=70, + ) + return str(rows[0]["api_base"]) + + +@pytest.mark.parametrize( + ("model", "target"), + ( + (EMBEDDING_MODEL, "/embeddings"), + (CONTEXTUAL_MODEL, "/contextualizedembeddings"), + (MULTIMODAL_MODEL, "/multimodalembeddings"), + ), + ids=("embeddings", "contextual", "multimodal"), +) +def test_explicit_api_base_keeps_mongodb_key_embeddings_on_that_base(gateway: Gateway, model: str, target: str) -> None: + with wire_server(_embedding_peer) as wire, gateway.scenario() as scenario: + deployment: Final = scenario.model(model=model, api_key=MONGODB_KEY, api_base=wire.url) + response: Final = _embed(gateway, deployment) + assert response.status_code == 200, response.text + data: Final = _JSON_OBJECTS.validate_python(JSON_OBJECT.validate_json(response.content)["data"]) + assert data[0]["embedding"] == VECTOR, response.text + received: Final = wire.drain() + assert tuple(request.target for request in received) == (target,), received + assert _spend_api_base(response.headers["x-litellm-call-id"]).startswith(wire.url), response.text + + +@pytest.mark.parametrize("suffix", ("", "/v1"), ids=("bare", "v1")) +def test_explicit_api_base_keeps_mongodb_key_rerank_on_that_base(gateway: Gateway, suffix: str) -> None: + with wire_server(_rerank_peer) as wire, gateway.scenario() as scenario: + deployment: Final = scenario.model(model=RERANK_MODEL, api_key=MONGODB_KEY, api_base=wire.url + suffix) + response: Final = _rerank(gateway, deployment) + assert response.status_code == 200, response.text + payload: Final = JSON_OBJECT.validate_json(response.content) + results: Final = _JSON_OBJECTS.validate_python(payload["results"]) + assert [result["index"] for result in results] == [1, 0], response.text + received: Final = wire.drain() + assert tuple(request.target for request in received) == ("/v1/rerank",), received + assert _spend_api_base(str(payload["id"])).startswith(wire.url), response.text + + +def test_public_provider_fields_name_voyage_as_mongodb(gateway: Gateway) -> None: + response: Final = gateway.request("GET", "/public/providers/fields") + assert response.status_code == 200, response.text + voyage: Final = [ + entry for entry in _JSON_OBJECTS.validate_json(response.content) if entry["litellm_provider"] == "voyage" + ] + assert [entry["provider_display_name"] for entry in voyage] == ["VoyageAI by MongoDB"], response.text + + +def test_public_endpoints_name_voyage_as_mongodb(gateway: Gateway) -> None: + response: Final = gateway.request("GET", "/public/endpoints") + assert response.status_code == 200, response.text + endpoints: Final = _JSON_OBJECTS.validate_python(JSON_OBJECT.validate_json(response.content)["endpoints"]) + names: Final = tuple(_voyage_display_names(endpoints)) + assert names and set(names) == {"VoyageAI by MongoDB"}, response.text + + +def _voyage_display_names(endpoints: list[dict[str, JsonValue]]) -> Iterator[str]: + for endpoint in endpoints: + for provider in _JSON_OBJECTS.validate_python(endpoint["providers"]): + if provider["slug"] == "voyage": + yield str(provider["display_name"]) + + +def _served_call_ids(served: tuple[_Probe, ...]) -> tuple[str, ...]: + return tuple(probe.response.headers["x-litellm-call-id"] for probe in served) + + +def _spend_rows(call_id: str) -> list[dict[str, JsonValue]]: + return read_rows('SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id=%s', (call_id,)) + + +def _reserve_port() -> int: + with socket.socket() as reserve: + reserve.bind(("127.0.0.1", 0)) + address: Final[Callable[[], tuple[str, int]]] = reserve.getsockname + return address()[1] + + +async def test_forward_proxy_outage_mid_traffic_recovers_with_the_right_hosts(gateway: Gateway, tmp_path: Path) -> None: + port: Final = _reserve_port() + config: Final = _write_config(tmp_path, "outage", _isolated_deployments(gateway.upstream_url)) + with owned_proxy( + gateway, + tmp_path, + _forward_environment(f"http://127.0.0.1:{port}"), + config=config, + remove_environment=_scrubbed(), + workers=2, + ) as candidate: + with refusing_forward_proxy(port=port) as before: + first: Final = await _burst(str(candidate.client.base_url), candidate.key, 12) + before_counts: Final = _host_counts(before.targets()) + during: Final = await _burst(str(candidate.client.base_url), candidate.key, 12) + liveliness: Final = candidate.request("GET", "/health/liveliness") + assert liveliness.status_code == 200, liveliness.text + with refusing_forward_proxy(port=port) as after: + third: Final = await _burst(str(candidate.client.base_url), candidate.key, 12) + after_counts: Final = _host_counts(after.targets()) + assert len(first) == len(during) == len(third) == 12 + for probe in (*first, *third): + _assert_refused(probe.response) + for probe in during: + assert probe.response.status_code == REFUSED_STATUS, probe.response.text + assert OUTAGE_MARKER in probe.response.text, probe.response.text + assert OUTAGE_MARKER in _error_message(probe.response.headers["x-litellm-call-id"]) + assert before_counts == {MONGODB_HOST: 6, VOYAGE_HOST: 6} + assert after_counts == {MONGODB_HOST: 6, VOYAGE_HOST: 6} + + +async def test_worker_sigkill_mid_traffic_leaves_the_sibling_routing_by_key(gateway: Gateway, tmp_path: Path) -> None: + config: Final = _write_config(tmp_path, "sigkill", _isolated_deployments(gateway.upstream_url)) + with ( + refusing_forward_proxy() as forward, + owned_proxy_process( + gateway, + tmp_path, + _forward_environment(forward.url), + config=config, + remove_environment=_scrubbed(), + workers=2, + ) as owned, + ): + candidate: Final = owned.gateway + workers: Final = eventually( + lambda: _WORKER_PIDS.validate_python(_STARTED_WORKER.findall(owned.log.read_text())), + lambda pids: len(pids) == 2, + seconds=60, + ) + burst: Final = asyncio.create_task( + _burst(str(candidate.client.base_url), candidate.key, 20, tolerate_transport_errors=True) + ) + await asyncio.to_thread(eventually, lambda: forward.received.qsize(), lambda size: size >= 4, 60) + victim: Final = psutil.Process(workers[0]) + victim.suspend() + victim.send_signal(signal.SIGKILL) + served: Final = await burst + for probe in served: + assert probe.response.status_code == REFUSED_STATUS, probe.response.text + assert set(forward.targets()) <= {MONGODB_HOST, VOYAGE_HOST} + follow_up: Final = _rerank(candidate, MONGODB_RERANK) + _assert_dialed(forward, follow_up, MONGODB_HOST) + duplicates: Final = tuple(call_id for call_id in _served_call_ids(served) if len(_spend_rows(call_id)) > 1) + assert duplicates == (), duplicates diff --git a/tests/unit/llms/voyage/rerank/test_voyage_rerank_transformation.py b/tests/unit/llms/voyage/rerank/test_voyage_rerank_transformation.py index 777e52f4e2a..15689f43bd0 100644 --- a/tests/unit/llms/voyage/rerank/test_voyage_rerank_transformation.py +++ b/tests/unit/llms/voyage/rerank/test_voyage_rerank_transformation.py @@ -295,11 +295,10 @@ class TestVoyageRerankTransform: assert "top_n" in supported_params assert "return_documents" in supported_params - @patch("litellm.llms.voyage.rerank.transformation.get_secret_str") - def test_validate_environment_missing_api_key(self, mock_get_secret_str): + def test_validate_environment_missing_api_key(self, monkeypatch): """Test that validate_environment raises error when API key is missing.""" - # Mock get_secret_str to return None for both environment variables - mock_get_secret_str.return_value = None + for env_var in ("VOYAGE_API_KEY", "VOYAGE_AI_API_KEY", "VOYAGE_AI_TOKEN"): + monkeypatch.delenv(env_var, raising=False) with pytest.raises(ValueError, match="Voyage AI API key is required"): self.config.validate_environment( headers={}, diff --git a/tests/unit/llms/voyage/test_common_utils.py b/tests/unit/llms/voyage/test_common_utils.py new file mode 100644 index 00000000000..511391093f3 --- /dev/null +++ b/tests/unit/llms/voyage/test_common_utils.py @@ -0,0 +1,164 @@ +import pytest + +from litellm.llms.voyage.common_utils import ( + MONGODB_API_BASE, + VOYAGE_API_BASE, + get_default_base_url, + get_voyage_api_key, +) +from litellm.llms.voyage.embedding.transformation import VoyageEmbeddingConfig +from litellm.llms.voyage.embedding.transformation_contextual import ( + VoyageContextualEmbeddingConfig, +) +from litellm.llms.voyage.embedding.transformation_multimodal import ( + VoyageMultimodalEmbeddingConfig, +) +from litellm.llms.voyage.rerank.transformation import VoyageRerankConfig + +VOYAGE_KEY_ENV_VARS = ("VOYAGE_API_KEY", "VOYAGE_AI_API_KEY", "VOYAGE_AI_TOKEN") + + +@pytest.fixture(autouse=True) +def clear_voyage_env(monkeypatch): + for env_var in VOYAGE_KEY_ENV_VARS: + monkeypatch.delenv(env_var, raising=False) + + +def test_mongodb_key_routes_to_mongodb_host(): + """MongoDB-issued keys carry the `al-` prefix and are only valid on ai.mongodb.com""" + assert get_default_base_url("al-1234567890") == MONGODB_API_BASE + + +@pytest.mark.parametrize("api_key", ["pa-1234567890", "sk-1234567890", "al", "", None]) +def test_non_mongodb_key_routes_to_voyage_host(api_key): + assert get_default_base_url(api_key) == VOYAGE_API_BASE + + +@pytest.mark.parametrize("env_var", VOYAGE_KEY_ENV_VARS) +def test_mongodb_key_from_any_supported_env_var_routes_to_mongodb_host(monkeypatch, env_var): + monkeypatch.setenv(env_var, "al-from-env") + assert get_default_base_url() == MONGODB_API_BASE + + +def test_explicit_key_wins_over_env_for_routing(monkeypatch): + monkeypatch.setenv("VOYAGE_API_KEY", "al-from-env") + assert get_default_base_url("pa-explicit") == VOYAGE_API_BASE + + +@pytest.mark.parametrize( + "config, endpoint", + [ + (VoyageEmbeddingConfig(), "embeddings"), + (VoyageContextualEmbeddingConfig(), "contextualizedembeddings"), + (VoyageMultimodalEmbeddingConfig(), "multimodalembeddings"), + ], +) +@pytest.mark.parametrize("api_key, expected_host", [("al-key", MONGODB_API_BASE), ("pa-key", VOYAGE_API_BASE)]) +def test_embedding_configs_route_by_key_prefix(config, endpoint, api_key, expected_host): + url = config.get_complete_url(None, api_key, "voyage-3", {}, {}) + assert url == f"{expected_host}/{endpoint}" + + +@pytest.mark.parametrize( + "config, endpoint", + [ + (VoyageEmbeddingConfig(), "embeddings"), + (VoyageContextualEmbeddingConfig(), "contextualizedembeddings"), + (VoyageMultimodalEmbeddingConfig(), "multimodalembeddings"), + ], +) +def test_explicit_api_base_overrides_key_routing(config, endpoint): + url = config.get_complete_url("https://gateway.internal/v1", "al-key", "voyage-3", {}, {}) + assert url == f"https://gateway.internal/v1/{endpoint}" + + +@pytest.mark.parametrize("api_key, expected_host", [("al-key", MONGODB_API_BASE), ("pa-key", VOYAGE_API_BASE)]) +def test_rerank_routes_by_request_key_prefix(api_key, expected_host): + config = VoyageRerankConfig() + config.validate_environment({}, "rerank-2.5", api_key=api_key) + + assert config.get_complete_url(None, "rerank-2.5") == f"{expected_host}/rerank" + + +@pytest.mark.parametrize("api_key, expected_host", [("al-key", MONGODB_API_BASE), ("pa-key", VOYAGE_API_BASE)]) +def test_rerank_routes_by_env_key_prefix(monkeypatch, api_key, expected_host): + monkeypatch.setenv("VOYAGE_API_KEY", api_key) + config = VoyageRerankConfig() + config.validate_environment({}, "rerank-2.5") + + assert config.get_complete_url(None, "rerank-2.5") == f"{expected_host}/rerank" + + +def test_rerank_request_key_beats_env_key_for_routing(monkeypatch): + """A MongoDB key on the request must not be posted to the Voyage host the env key names""" + monkeypatch.setenv("VOYAGE_API_KEY", "pa-from-env") + config = VoyageRerankConfig() + + headers = config.validate_environment({}, "rerank-2.5", api_key="al-on-request") + + assert headers["Authorization"] == "Bearer al-on-request" + assert config.get_complete_url(None, "rerank-2.5") == f"{MONGODB_API_BASE}/rerank" + + +def test_rerank_config_is_built_per_request_so_keys_cannot_leak(monkeypatch): + """get_complete_url reads a key off the instance, so each request must get its own instance""" + import litellm + from litellm.types.utils import LlmProviders + from litellm.utils import ProviderConfigManager + + monkeypatch.delenv("VOYAGE_API_KEY", raising=False) + first = ProviderConfigManager.get_provider_rerank_config( + model="rerank-2.5", provider=LlmProviders.VOYAGE, api_base=None, present_version_params=[] + ) + second = ProviderConfigManager.get_provider_rerank_config( + model="rerank-2.5", provider=LlmProviders.VOYAGE, api_base=None, present_version_params=[] + ) + assert isinstance(first, litellm.VoyageRerankConfig) and first is not second + + first.validate_environment({}, "rerank-2.5", api_key="al-first-request") + second.validate_environment({}, "rerank-2.5", api_key="pa-second-request") + + assert first.get_complete_url(None, "rerank-2.5") == f"{MONGODB_API_BASE}/rerank" + assert second.get_complete_url(None, "rerank-2.5") == f"{VOYAGE_API_BASE}/rerank" + + +def test_rerank_falls_back_to_env_when_validate_environment_did_not_run(monkeypatch): + """A caller that skips validate_environment keeps the pre-existing env-only behaviour""" + monkeypatch.setenv("VOYAGE_API_KEY", "al-from-env") + + assert VoyageRerankConfig().get_complete_url(None, "rerank-2.5") == f"{MONGODB_API_BASE}/rerank" + + +@pytest.mark.parametrize( + "config", + [VoyageEmbeddingConfig(), VoyageContextualEmbeddingConfig(), VoyageMultimodalEmbeddingConfig()], +) +def test_auth_header_uses_the_key_the_url_was_routed_on(monkeypatch, config): + """The host is picked from a key, so the Authorization header has to carry that same key""" + monkeypatch.setenv("VOYAGE_AI_TOKEN", "al-from-env") + + headers = config.validate_environment({}, "voyage-3", [], {}, {}) + url = config.get_complete_url(None, None, "voyage-3", {}, {}) + + assert headers["Authorization"] == "Bearer al-from-env" + assert url.startswith(MONGODB_API_BASE) + + +def test_rerank_auth_header_uses_the_key_the_url_was_routed_on(monkeypatch): + monkeypatch.setenv("VOYAGE_AI_TOKEN", "al-from-env") + config = VoyageRerankConfig() + + headers = config.validate_environment({}, "rerank-2.5") + + assert headers["Authorization"] == "Bearer al-from-env" + assert config.get_complete_url(None, "rerank-2.5").startswith(MONGODB_API_BASE) + + +def test_get_voyage_api_key_prefers_env_vars_in_documented_order(monkeypatch): + monkeypatch.setenv("VOYAGE_AI_API_KEY", "second") + monkeypatch.setenv("VOYAGE_AI_TOKEN", "third") + assert get_voyage_api_key() == "second" + + monkeypatch.setenv("VOYAGE_API_KEY", "first") + assert get_voyage_api_key() == "first" + assert get_voyage_api_key("explicit") == "explicit" diff --git a/tests/unit/llms/voyage/test_voyage_multimodal_embedding.py b/tests/unit/llms/voyage/test_voyage_multimodal_embedding.py index f3e6885cbe6..d13610ada17 100644 --- a/tests/unit/llms/voyage/test_voyage_multimodal_embedding.py +++ b/tests/unit/llms/voyage/test_voyage_multimodal_embedding.py @@ -172,15 +172,12 @@ class TestVoyageMultimodalEmbeddings: assert headers == {"Authorization": "Bearer test-key"} def test_validate_environment_uses_secret_fallback(self, monkeypatch): - import litellm.llms.voyage.embedding.transformation_multimodal as module from litellm.llms.voyage.embedding.transformation_multimodal import ( VoyageMultimodalEmbeddingConfig, ) - def fake_get_secret(name): - return "secret-key" if name == "VOYAGE_AI_API_KEY" else None - - monkeypatch.setattr(module, "get_secret_str", fake_get_secret) + monkeypatch.delenv("VOYAGE_API_KEY", raising=False) + monkeypatch.setenv("VOYAGE_AI_API_KEY", "secret-key") config = VoyageMultimodalEmbeddingConfig() headers = config.validate_environment( {}, "voyage-multimodal-3.5", [], {}, {}, api_key=None @@ -188,12 +185,12 @@ class TestVoyageMultimodalEmbeddings: assert headers == {"Authorization": "Bearer secret-key"} def test_validate_environment_raises_without_api_key(self, monkeypatch): - import litellm.llms.voyage.embedding.transformation_multimodal as module from litellm.llms.voyage.embedding.transformation_multimodal import ( VoyageMultimodalEmbeddingConfig, ) - monkeypatch.setattr(module, "get_secret_str", lambda name: None) + for env_var in ("VOYAGE_API_KEY", "VOYAGE_AI_API_KEY", "VOYAGE_AI_TOKEN"): + monkeypatch.delenv(env_var, raising=False) config = VoyageMultimodalEmbeddingConfig() with pytest.raises(ValueError, match='Voyage API key is required for multimodal embeddings\\. Set') as exc_info: config.validate_environment( diff --git a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx index 5a57fc23dde..177d4be3247 100644 --- a/ui/litellm-dashboard/src/components/provider_info_helpers.tsx +++ b/ui/litellm-dashboard/src/components/provider_info_helpers.tsx @@ -182,7 +182,7 @@ export enum Providers { VERTEX_AI_BETA = "Vertex Ai Beta", VLLM = "Local vLLM", VolcEngine = "VolcEngine", - Voyage = "Voyage AI", + Voyage = "VoyageAI by MongoDB", WANDB = "Wandb", WATSONX = "Watsonx", WATSONX_TEXT = "Watsonx Text", From 56bafc29cf7601304c5f2e7dbb8e18d614d91fc2 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:20:03 -0700 Subject: [PATCH 60/65] feat(gemini): accept file content blocks with video_metadata on multimodal embeddings (#45305) * feat(gemini): accept file content blocks with video_metadata on multimodal embeddings * test(gemini): annotate the embedding mock helper and keep the type comment short * fix(gemini-embeddings): keep file blocks through a partial embedding cache hit and validate video_metadata strictly * test(caching): type the partial-hit cache test and pin its clock * fix(gemini-embeddings): accept the Files API URI /v1/files returns as a file block's file_id * fix(gemini-embeddings): 400 on bad input shapes, drop unknown block keys under drop_params, cache multi-block inputs A bare object `input` answered 500 from both the caching handler and the transformation; both now answer 400. An unresolved `files/` reference on vertex_ai/ answered a ValueError 500; it now answers 400 naming the gemini/ provider. An empty `format` passed through to the provider; it now answers 400 naming file.format. Unknown block keys (`detail`, an unknown video_metadata key, a top-level block key) are dropped under drop_params, global or per-request, and still answer 400 without it. The embedding cache counted file blocks against `max_messages`, so a request with 5 or more blocks was never cached; file blocks no longer count. * fix(caching): answer 400 for an object embedding input on the cache lookup * test(gemini-embeddings): annotate the new embedding tests with return types and Final * test(gemini-embeddings): annotate the transformation and files batch tests with return types and Final * test(gemini-embeddings): audit file content blocks on the wire, in the cache, in batch uploads and under chaos Integration cells for gemini/ and vertex_ai/ embedding file blocks with video_metadata: the batchEmbedContents and embedContent wire shapes, format overrides, nested and repeated inputs, every malformed block answering 400 before any provider call, drop_params at the request, deployment and YAML levels, the OpenAI SDK sync and async clients, Redis cache fills, hits, partial hits and the metadata in the key with zero-spend hit rows, chat, responses and messages controls, the vertex_ai/ batch JSONL upload, a mixed fast/slow/malformed/dropped burst and a worker kill. The wire peer now reads chunked request bodies, which the streamed GCS media upload sends * test(integration): treat an upload the client abandons before its terminating chunk as a disconnect, never a stored request --------- Co-authored-by: mateo-berri <277851410+mateo-berri@users.noreply.github.com> --- litellm/caching/caching.py | 14 +- litellm/caching/caching_handler.py | 21 +- litellm/llms/vertex_ai/common_utils.py | 16 + .../llms/vertex_ai/files/transformation.py | 21 +- .../llms/vertex_ai/gemini/transformation.py | 11 +- .../batch_embed_content_handler.py | 30 +- .../batch_embed_content_transformation.py | 342 +++++++++++++----- litellm/main.py | 6 + litellm/types/llms/vertex_ai.py | 15 +- tests/integration/_support/wire.py | 49 ++- .../test_embedding_file_block_cache.py | 216 +++++++++++ .../test_gemini_chat_video_metadata_wire.py | 144 ++++++++ .../test_gemini_embedding_file_block_chaos.py | 223 ++++++++++++ .../test_gemini_embedding_file_block_wire.py | 294 +++++++++++++++ ..._vertex_embedding_batch_file_block_wire.py | 109 ++++++ .../test_vertex_embedding_file_block_wire.py | 118 ++++++ tests/unit/caching/test_caching.py | 2 + tests/unit/caching/test_caching_handler.py | 86 +++++ .../test_vertex_ai_files_transformation.py | 27 ++ .../test_batch_embed_content_handler.py | 39 ++ ...test_batch_embed_content_transformation.py | 251 ++++++++++++- .../vertex_ai/test_gemini_batch_embeddings.py | 164 +++++++++ 22 files changed, 2048 insertions(+), 150 deletions(-) create mode 100644 tests/integration/caching/test_embedding_file_block_cache.py create mode 100644 tests/integration/providers/test_gemini_chat_video_metadata_wire.py create mode 100644 tests/integration/providers/test_gemini_embedding_file_block_chaos.py create mode 100644 tests/integration/providers/test_gemini_embedding_file_block_wire.py create mode 100644 tests/integration/providers/test_vertex_embedding_batch_file_block_wire.py create mode 100644 tests/integration/providers/test_vertex_embedding_file_block_wire.py create mode 100644 tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py diff --git a/litellm/caching/caching.py b/litellm/caching/caching.py index 85ef5a93937..b928f33e961 100644 --- a/litellm/caching/caching.py +++ b/litellm/caching/caching.py @@ -71,15 +71,25 @@ class CacheMode(str, Enum): #### LiteLLM.Completion / Embedding Cache #### +def _is_conversation_item(item: object) -> bool: + if isinstance(item, BaseModel): + return True + if not isinstance(item, Mapping): + return False + block: Final = cast(Mapping[str, object], item) # cast-ok: isinstance leaves the key and value types unknown + return block.get("type") != "file" + + def _request_message_count(kwargs: Mapping[str, object]) -> int: - """Chat and Messages API `messages`, else Responses API `input` items; embedding `input` strings count as none""" + """Chat and Messages API `messages`, else Responses API `input` items; embedding strings and file blocks count as none""" messages: Final = kwargs.get("messages") if isinstance(messages, list): return len(messages) input_items: Final = kwargs.get("input") if not isinstance(input_items, list): return 0 - return sum(1 for item in input_items if isinstance(item, (Mapping, BaseModel))) + items: Final = cast(list[object], input_items) # cast-ok: isinstance leaves the element type unknown + return sum(1 for item in items if _is_conversation_item(item)) class Cache: diff --git a/litellm/caching/caching_handler.py b/litellm/caching/caching_handler.py index f9215864d1e..9ffc95c6260 100644 --- a/litellm/caching/caching_handler.py +++ b/litellm/caching/caching_handler.py @@ -21,7 +21,7 @@ import time from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable, Generator, Mapping from typing import TYPE_CHECKING, Any, Final, Optional, TypeVar -from pydantic import ConfigDict, ValidationError +from pydantic import ConfigDict, SkipValidation, ValidationError import litellm from litellm._internal_context import post_response_phase @@ -39,7 +39,7 @@ from litellm.litellm_core_utils.logging_utils import ( from litellm.types.caching import CACHED_STREAM_EVENTS_KEY, EMBEDDING_CACHE_FORMAT_VERSION, CachedEmbedding from litellm.types.integrations.custom_logger import converted_stream_requested from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.llms.openai import ResponsesAPIResponse +from litellm.types.llms.openai import ChatCompletionFileObject, ResponsesAPIResponse from litellm.types.rerank import RerankResponse from litellm.types.utils import ( CachingDetails, @@ -71,6 +71,8 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +EmbeddingCacheInputElement = str | list[int] | ChatCompletionFileObject + class CachingHandlerResponse(LiteLLMBaseModel): """ @@ -82,7 +84,7 @@ class CachingHandlerResponse(LiteLLMBaseModel): cached_result: object | None = None final_embedding_cached_response: EmbeddingResponse | None = None embedding_all_elements_cache_hit: bool = False # this is set to True when all elements in the list have a cache hit in the embedding cache, if true return the final_embedding_cached_response no need to make an API call - embedding_uncached_input: list[str | list[int]] | None = None + embedding_uncached_input: SkipValidation[list[EmbeddingCacheInputElement]] | None = None in_memory_cache_obj: Final = InMemoryCache() @@ -508,7 +510,7 @@ class LLMCachingHandler: _sync_get_cache = sync_get_cache - def handle_kwargs_input_list_or_str(self, kwargs: dict[str, object]) -> list[str]: + def handle_kwargs_input_list_or_str(self, kwargs: dict[str, object]) -> list[EmbeddingCacheInputElement]: """ Handles the input of kwargs['input'] being a list or a string """ @@ -517,7 +519,11 @@ class LLMCachingHandler: elif isinstance(kwargs["input"], list): return kwargs["input"] else: - raise ValueError("input must be a string or a list") + raise litellm.BadRequestError( + message="input must be a string or a list of strings and content blocks", + model=str(kwargs.get("model")), + llm_provider=str(kwargs.get("custom_llm_provider")), + ) def _extract_model_from_cached_results(self, non_null_list: list[tuple[int, CachedEmbedding]]) -> str | None: """ @@ -851,10 +857,7 @@ class LLMCachingHandler: self.request_kwargs = _drop_logging_obj_from_kwargs(new_kwargs) cached_result: object | None = None if call_type == CallTypes.aembedding.value: - if isinstance(new_kwargs["input"], str): - new_kwargs["input"] = [new_kwargs["input"]] - elif not isinstance(new_kwargs["input"], list): - raise ValueError("input must be a string or a list") + new_kwargs["input"] = self.handle_kwargs_input_list_or_str(new_kwargs) tasks: Final[list[Awaitable[object]]] = [] for idx, i in enumerate(new_kwargs["input"]): preset_cache_key = litellm.cache.get_cache_key(**{**new_kwargs, "input": i}) diff --git a/litellm/llms/vertex_ai/common_utils.py b/litellm/llms/vertex_ai/common_utils.py index 8c77413bace..6b759065706 100644 --- a/litellm/llms/vertex_ai/common_utils.py +++ b/litellm/llms/vertex_ai/common_utils.py @@ -25,6 +25,22 @@ from litellm.types.llms.vertex_ai import ( from litellm.types.utils import TokenCountResponse from litellm.utils import supports_response_schema, supports_system_messages +GEMINI_VIDEO_METADATA_KEYS: Final = MappingProxyType( + {"fps": "fps", "start_offset": "startOffset", "end_offset": "endOffset"} +) + + +GEMINI_FILES_API_URI_PREFIX: Final = "https://generativelanguage.googleapis.com/v1beta/files/" + + +def gemini_video_metadata_from_openai(video_metadata: Mapping[str, object]) -> dict[str, object]: + return { + gemini_key: video_metadata[openai_key] + for openai_key, gemini_key in GEMINI_VIDEO_METADATA_KEYS.items() + if openai_key in video_metadata + } + + VERTEX_SELF_DEPLOYED_ENDPOINT_UNSUPPORTED_PARAMS: Final = frozenset( { "audio", diff --git a/litellm/llms/vertex_ai/files/transformation.py b/litellm/llms/vertex_ai/files/transformation.py index da5b974f186..09ec2ba6ee4 100644 --- a/litellm/llms/vertex_ai/files/transformation.py +++ b/litellm/llms/vertex_ai/files/transformation.py @@ -67,7 +67,7 @@ from litellm.types.llms.openai import ( OpenAIFilesPurpose, PathLike, ) -from litellm.types.llms.vertex_ai import GcsBucketResponse, GeminiEmbeddingInput +from litellm.types.llms.vertex_ai import GcsBucketResponse, GeminiEmbeddingElement, GeminiEmbeddingInput from litellm.types.utils import ( Embedding, EmbeddingResponse, @@ -555,18 +555,27 @@ def _is_responses_batch_entry(openai_entry: Mapping[str, object]) -> bool: return path == "responses" or path.endswith("/responses") +def _own_embedding_input( + element: GeminiEmbeddingElement | list[str] | list[GeminiEmbeddingElement], +) -> GeminiEmbeddingInput: + if isinstance(element, (str, list)): + return element + file_block_alone: Final[list[GeminiEmbeddingElement]] = [element] + return file_block_alone + + def _openai_embedding_input_elements( embedding_input: GeminiEmbeddingInput, -) -> tuple[str | list[str], ...]: +) -> tuple[GeminiEmbeddingInput, ...]: """ Split an OpenAI `input` into the elements that each get their own embedding. - A string is one embedding, a flat array is one embedding per element, and a nested - array is one combined embedding per inner array, matching the online - `batchEmbedContents` path. + A string or a file content block is one embedding, a flat array is one embedding + per element, and a nested array is one combined embedding per inner array, + matching the online `batchEmbedContents` path. """ if isinstance(embedding_input, list): - return tuple(embedding_input) + return tuple(_own_embedding_input(element) for element in embedding_input) return (embedding_input,) diff --git a/litellm/llms/vertex_ai/gemini/transformation.py b/litellm/llms/vertex_ai/gemini/transformation.py index 8749b81d7f1..15e5e440b91 100644 --- a/litellm/llms/vertex_ai/gemini/transformation.py +++ b/litellm/llms/vertex_ai/gemini/transformation.py @@ -57,7 +57,9 @@ from litellm.types.llms.vertex_ai import ( from litellm.types.utils import GenericImageParsingChunk, LlmProviders from ..common_utils import ( + GEMINI_FILES_API_URI_PREFIX, check_text_in_content, + gemini_video_metadata_from_openai, get_supports_response_schema, get_supports_system_message, ) @@ -69,7 +71,6 @@ _GCS_METADATA_VERTEX_BASE: object | None = None # Shared sync client for GCS JSON API metadata reads so proxy/SSL settings # from litellm's HTTP stack apply (see Greptile review on PR #27278). _GCS_METADATA_HTTP_HANDLER: HTTPHandler | None = None -GEMINI_FILES_API_URI_PREFIX: Final = "https://generativelanguage.googleapis.com/v1beta/files/" _GEMINI_MIME_TYPE_ALIASES: Final[dict[str, str]] = { "image/jpg": "image/jpeg", } @@ -197,13 +198,7 @@ def _apply_gemini_metadata( part_dict["media_resolution"] = media_resolution_enum if video_metadata is not None: - gemini_video_metadata: Final = {} - if "fps" in video_metadata: - gemini_video_metadata["fps"] = video_metadata["fps"] - if "start_offset" in video_metadata: - gemini_video_metadata["startOffset"] = video_metadata["start_offset"] - if "end_offset" in video_metadata: - gemini_video_metadata["endOffset"] = video_metadata["end_offset"] + gemini_video_metadata: Final = gemini_video_metadata_from_openai(video_metadata) if gemini_video_metadata: part_dict["video_metadata"] = gemini_video_metadata diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py index d15cf89cf47..0a800497de1 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_handler.py @@ -22,6 +22,8 @@ from litellm.types.utils import EmbeddingResponse from ..gemini.vertex_and_google_ai_studio_gemini import VertexLLM from .batch_embed_content_transformation import ( + file_reference_name, + flatten_media_sources, is_file_reference, process_embed_content_response, process_response, @@ -38,11 +40,8 @@ class GoogleBatchEmbeddings(VertexLLM): def _flatten_and_detect_file_refs( input: GeminiEmbeddingInput, ) -> tuple[list[str], bool]: - """Flatten nested input lists and detect file references.""" - input_list: Final = [input] if isinstance(input, str) else input - flat_elements: Final = [ - e for item in input_list for e in (item if isinstance(item, list) else [item]) if isinstance(e, str) - ] + """Flatten nested input lists and file content blocks into their sources and detect file references.""" + flat_elements: Final = list(flatten_media_sources(input)) has_file_refs: Final = any(is_file_reference(e) for e in flat_elements) return flat_elements, has_file_refs @@ -68,7 +67,7 @@ class GoogleBatchEmbeddings(VertexLLM): for element in input_list: if isinstance(element, str) and is_file_reference(element): - url = f"https://generativelanguage.googleapis.com/v1beta/{element}" + url = f"https://generativelanguage.googleapis.com/v1beta/{file_reference_name(element)}" headers = {"x-goog-api-key": api_key} response = sync_handler.get(url=url, headers=headers) @@ -105,7 +104,7 @@ class GoogleBatchEmbeddings(VertexLLM): for element in input_list: if isinstance(element, str) and is_file_reference(element): - url = f"https://generativelanguage.googleapis.com/v1beta/{element}" + url = f"https://generativelanguage.googleapis.com/v1beta/{file_reference_name(element)}" headers = {"x-goog-api-key": api_key} response = await async_handler.get(url=url, headers=headers) @@ -139,6 +138,7 @@ class GoogleBatchEmbeddings(VertexLLM): timeout=300, client=None, extra_headers: dict | None = None, + drop_params: bool = False, ) -> EmbeddingResponse: _auth_header, vertex_project = self._ensure_access_token( credentials=vertex_credentials, @@ -207,22 +207,26 @@ class GoogleBatchEmbeddings(VertexLLM): api_key=api_key, optional_params=optional_params, logging_obj=logging_obj, + drop_params=drop_params, ) ### TRANSFORMATION (sync path) ### request_data: VertexAIBatchEmbeddingsRequestBody | dict[str, object] + flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input) if use_embed_content: resolved_files = {} if api_key: - resolved_files = self._resolve_file_references(input=input, api_key=api_key, sync_handler=sync_handler) + resolved_files = self._resolve_file_references( + input=flat_elements, api_key=api_key, sync_handler=sync_handler + ) request_data = transform_openai_input_gemini_embed_content( input=input, model=model, optional_params=optional_params, resolved_files=resolved_files, + drop_params=drop_params, ) else: - flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input) if has_file_refs and not api_key: raise ValueError( "An API key is required to resolve Gemini file references (files/...). " @@ -238,6 +242,7 @@ class GoogleBatchEmbeddings(VertexLLM): model=model, optional_params=optional_params, resolved_files=resolved_files, + drop_params=drop_params, ) ## LOGGING @@ -294,6 +299,7 @@ class GoogleBatchEmbeddings(VertexLLM): api_key: str | None = None, optional_params: dict | None = None, logging_obj: "LiteLLMLoggingObj | None" = None, + drop_params: bool = False, ) -> EmbeddingResponse: if client is None: _params: Final = {} @@ -312,20 +318,21 @@ class GoogleBatchEmbeddings(VertexLLM): async_handler = client ### TRANSFORMATION (async path) ### + flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input) if use_embed_content: resolved_files = {} if api_key: resolved_files = await self._async_resolve_file_references( - input=input, api_key=api_key, async_handler=async_handler + input=flat_elements, api_key=api_key, async_handler=async_handler ) data = transform_openai_input_gemini_embed_content( input=input, model=model, optional_params=optional_params or {}, resolved_files=resolved_files, + drop_params=drop_params, ) else: - flat_elements, has_file_refs = self._flatten_and_detect_file_refs(input) if has_file_refs and not api_key: raise ValueError( "An API key is required to resolve Gemini file references (files/...). " @@ -341,6 +348,7 @@ class GoogleBatchEmbeddings(VertexLLM): model=model, optional_params=optional_params or {}, resolved_files=resolved_files, + drop_params=drop_params, ) ## LOGGING diff --git a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py index 9ee2b71d30c..96a785c06c9 100644 --- a/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py +++ b/litellm/llms/vertex_ai/gemini_embeddings/batch_embed_content_transformation.py @@ -4,22 +4,27 @@ Transformation logic from OpenAI /v1/embeddings format to Google AI Studio /batc Why separate file? Make it easy to see how transformation works """ -from collections.abc import Mapping, Sequence -from typing import Final +from collections.abc import Iterator, Mapping, Sequence +from typing import Annotated, Final, Literal, cast -from pydantic import TypeAdapter, ValidationError +from pydantic import ConfigDict, Field, TypeAdapter, ValidationError +from litellm.exceptions import BadRequestError +from litellm.llms.vertex_ai.common_utils import GEMINI_FILES_API_URI_PREFIX, gemini_video_metadata_from_openai +from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.llms.vertex_ai import ( BlobType, ContentType, EmbedContentRequest, FileDataType, + GeminiEmbeddingElement, GeminiEmbeddingInput, PartType, PromptTokensDetails, UsageMetadata, VertexAIBatchEmbeddingsRequestBody, VertexAIBatchEmbeddingsResponseObject, + VideoMetadataType, ) from litellm.types.utils import ( Embedding, @@ -40,9 +45,16 @@ SUPPORTED_EMBEDDING_MIME_TYPES: Final = { } +_GEMINI_API_V1BETA: Final = GEMINI_FILES_API_URI_PREFIX.removesuffix("files/") + + def is_file_reference(s: str) -> bool: - """Check if string is a Gemini file reference (files/...).""" - return isinstance(s, str) and s.startswith("files/") + """A `files/...` name or the Files API URI that `/v1/files` returns as the file id.""" + return isinstance(s, str) and (s.startswith("files/") or s.startswith(GEMINI_FILES_API_URI_PREFIX)) + + +def file_reference_name(reference: str) -> str: + return reference.removeprefix(_GEMINI_API_V1BETA) _is_file_reference = is_file_reference @@ -88,12 +100,13 @@ def _infer_mime_type_from_gcs_url(gcs_url: str) -> str: ) -def _parse_data_url(data_url: str) -> tuple[str, str]: +def _parse_data_url(data_url: str, mime_type_override: str | None = None) -> tuple[str, str]: """ Parse a data URL to extract the media type and base64 data. Args: data_url: Data URL in format: data:image/jpeg;base64,/9j/4AAQ... + mime_type_override: An explicit media type that replaces the declared one, skipping the allowlist Returns: tuple: (media_type, base64_data) @@ -110,50 +123,225 @@ def _parse_data_url(data_url: str) -> tuple[str, str]: raise ValueError(f"Invalid data URL format (missing comma): {data_url[:50]}...") metadata, base64_data = data_url.split(",", 1) + declared_media_type: Final = metadata[5:].split(";")[0] - metadata = metadata[5:] + if mime_type_override is not None: + return mime_type_override, base64_data - if ";" in metadata: - media_type = metadata.split(";")[0] - else: - media_type = metadata - - if media_type not in SUPPORTED_EMBEDDING_MIME_TYPES: + if declared_media_type not in SUPPORTED_EMBEDDING_MIME_TYPES: raise ValueError( - f"Unsupported MIME type for embedding: {media_type}. " + f"Unsupported MIME type for embedding: {declared_media_type}. " f"Supported types: {', '.join(sorted(SUPPORTED_EMBEDDING_MIME_TYPES))}" ) - return media_type, base64_data + return declared_media_type, base64_data + + +def _is_data_url(s: str) -> bool: + return s.startswith("data:") and ";base64," in s + + +class _EmbeddingVideoMetadata(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid", strict=True) + + fps: float | None = None + start_offset: str | None = None + end_offset: str | None = None + + +class _EmbeddingFile(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid") + + file_id: str | None = None + file_data: str | None = None + filename: str | None = None + format: Annotated[str, Field(min_length=1)] | None = None + video_metadata: _EmbeddingVideoMetadata | None = None + + +class _EmbeddingFileBlock(LiteLLMBaseModel): + model_config = ConfigDict(extra="forbid") + + type: Literal["file"] + file: _EmbeddingFile + + +_file_block_adapter: Final = TypeAdapter(_EmbeddingFileBlock) +_video_metadata_adapter: Final = TypeAdapter(VideoMetadataType) +_input_shape_adapter: Final[TypeAdapter[str | list[object]]] = TypeAdapter(str | list[object]) +_mapping_adapter: Final = TypeAdapter(dict[str, object]) +_BLOCK_FIELDS: Final = frozenset(_EmbeddingFileBlock.model_fields) +_FILE_FIELDS: Final = frozenset(_EmbeddingFile.model_fields) +_VIDEO_METADATA_FIELDS: Final = frozenset(_EmbeddingVideoMetadata.model_fields) +_FILE_SOURCE_FORMS: Final = "a data: URI, a gs:// URL, a files/ reference, or a Gemini Files API URI" + + +def _invalid_input(message: str) -> BadRequestError: + return BadRequestError(message=message, model=None, llm_provider="gemini") + + +def _validation_error_summary(error: ValidationError) -> str: + return "; ".join(f"{'.'.join(str(loc) for loc in detail['loc'])}: {detail['msg']}" for detail in error.errors()) + + +def _as_mapping(value: object) -> dict[str, object] | None: + try: + return _mapping_adapter.validate_python(value) + except ValidationError: + return None + + +def _only_fields(mapping: Mapping[str, object], fields: frozenset[str]) -> dict[str, object]: + return {key: value for key, value in mapping.items() if key in fields} + + +def _dropping_unsupported_keys(block: Mapping[str, object]) -> dict[str, object]: + """What `drop_params` keeps of a file content block: the keys this surface understands, at every level.""" + kept_block: Final = _only_fields(block, _BLOCK_FIELDS) + file: Final = _as_mapping(block.get("file")) + if file is None: + return kept_block + kept_file: Final = _only_fields(file, _FILE_FIELDS) + video_metadata: Final = _as_mapping(file.get("video_metadata")) + if video_metadata is None: + return {**kept_block, "file": kept_file} + return { + **kept_block, + "file": {**kept_file, "video_metadata": _only_fields(video_metadata, _VIDEO_METADATA_FIELDS)}, + } + + +def _parse_file_block(element: object, drop_params: bool) -> _EmbeddingFileBlock: + if not isinstance(element, Mapping): + raise _invalid_input( + f"Embedding input elements must be strings or file content blocks, got {type(element).__name__}" + ) + block: Final = cast(Mapping[str, object], element) # cast-ok: isinstance leaves the key and value types unknown + try: + return _file_block_adapter.validate_python(_dropping_unsupported_keys(block) if drop_params else block) + except ValidationError as error: + raise _invalid_input( + f"Invalid file content block in embedding input: {_validation_error_summary(error)}" + ) from error + + +def _file_block_source(block: _EmbeddingFileBlock) -> str: + match (block.file.file_id, block.file.file_data): + case (str() as file_id, None): + return file_id + case (None, str() as file_data): + return file_data + case (None, None): + raise _invalid_input("A file content block in embedding input needs file.file_id or file.file_data") + case _: + raise _invalid_input( + "A file content block in embedding input takes file.file_id or file.file_data, not both" + ) + + +def _gemini_video_metadata(video_metadata: _EmbeddingVideoMetadata) -> VideoMetadataType: + return _video_metadata_adapter.validate_python( + gemini_video_metadata_from_openai(video_metadata.model_dump(exclude_none=True)) + ) + + +def _source_mime_type( + source: str, + mime_type_override: str | None, + resolved_files: Mapping[str, Mapping[str, str]], +) -> str | None: + if mime_type_override is not None: + return mime_type_override + if _is_data_url(source): + try: + return _parse_data_url(source)[0] + except ValueError: + return None + if _is_gcs_url(source): + try: + return _infer_mime_type_from_gcs_url(source) + except ValueError: + return None + if is_file_reference(source): + file_info: Final = resolved_files.get(source) + return None if file_info is None else file_info.get("mime_type") + return None + + +def _media_part( + source: str, + mime_type_override: str | None, + resolved_files: Mapping[str, Mapping[str, str]], +) -> PartType: + if _is_data_url(source): + mime_type, base64_data = _parse_data_url(source, mime_type_override) + return PartType(inline_data=BlobType(mime_type=mime_type, data=base64_data)) + if _is_gcs_url(source): + gcs_mime_type: Final = mime_type_override or _infer_mime_type_from_gcs_url(source) + return PartType(file_data=FileDataType(mime_type=gcs_mime_type, file_uri=source)) + if is_file_reference(source): + file_info: Final = resolved_files.get(source) + if file_info is None: + raise _invalid_input( + f"File reference {source!r} could not be resolved: " + "Gemini Files API references are only supported through the gemini/ provider" + ) + return PartType( + file_data=FileDataType(mime_type=mime_type_override or file_info["mime_type"], file_uri=file_info["uri"]) + ) + raise _invalid_input(f"A file content block source must be {_FILE_SOURCE_FORMS}, got {source[:50]!r}") + + +def _top_level_elements( + input: GeminiEmbeddingInput, +) -> Sequence[GeminiEmbeddingElement | list[str] | list[GeminiEmbeddingElement]]: + try: + _input_shape_adapter.validate_python(input) + except ValidationError as error: + raise _invalid_input( + f"Embedding input must be a string or a list of strings and file content blocks, got {type(input).__name__}" + ) from error + return [input] if isinstance(input, str) else input + + +def _elements(input: GeminiEmbeddingInput) -> Iterator[GeminiEmbeddingElement]: + for element in _top_level_elements(input): + if isinstance(element, list): + yield from element + else: + yield element + + +def _element_source(element: GeminiEmbeddingElement) -> str: + if isinstance(element, str): + return element + return _file_block_source(_parse_file_block(element, drop_params=True)) + + +def flatten_media_sources(input: GeminiEmbeddingInput) -> tuple[str, ...]: + """Every string element plus every file content block's source, in input order.""" + return tuple(_element_source(element) for element in _elements(input)) def _is_multimodal_input(input: GeminiEmbeddingInput) -> bool: """ Check if the input contains multimodal data (data URIs, file references, - GCS URLs, or nested lists for combined embeddings). + GCS URLs, file content blocks, or nested lists for combined embeddings). Args: - input: GeminiEmbeddingInput — str, List[str], or List[List[str]] for combined embeddings + input: GeminiEmbeddingInput — str, List[element], or List[List[element]] for combined embeddings Returns: - bool: True if any element is multimodal or a nested list + bool: True if any element is multimodal """ - if isinstance(input, str): - return _is_multimodal_element(input) - - for element in input: - if isinstance(element, list): - if any(_is_multimodal_element(sub) for sub in element if isinstance(sub, str)): - return True - elif isinstance(element, str) and _is_multimodal_element(element): - return True - - return False + return any(_is_multimodal_element(element) for element in _elements(input)) -def _is_multimodal_element(element: str) -> bool: - """Check if a single string element is multimodal.""" - if element.startswith("data:") and ";base64," in element: +def _is_multimodal_element(element: GeminiEmbeddingElement) -> bool: + """Check if a single element is multimodal.""" + if not isinstance(element, str): + return True + if _is_data_url(element): return True if is_file_reference(element): return True @@ -163,37 +351,29 @@ def _is_multimodal_element(element: str) -> bool: def _build_part_for_input( - element: str, - resolved_files: dict[str, dict[str, str]] | None = None, + element: GeminiEmbeddingElement, + resolved_files: Mapping[str, Mapping[str, str]] | None = None, + drop_params: bool = False, ) -> PartType: """ Build a single PartType for an input element, handling text, data URIs, - file references, and GCS URLs. + file references, GCS URLs, and file content blocks carrying a mime type + and video_metadata. """ - resolved_files = resolved_files or {} + files: Final = resolved_files or {} - if element.startswith("data:") and ";base64," in element: - mime_type, base64_data = _parse_data_url(element) - blob: Final[BlobType] = {"mime_type": mime_type, "data": base64_data} - return PartType(inline_data=blob) - elif _is_gcs_url(element): - mime_type = _infer_mime_type_from_gcs_url(element) - file_data: Final[FileDataType] = { - "mime_type": mime_type, - "file_uri": element, - } - return PartType(file_data=file_data) - elif is_file_reference(element): - if element not in resolved_files: - raise ValueError(f"File reference {element} not resolved") - file_info: Final = resolved_files[element] - file_data_ref: Final[FileDataType] = { - "mime_type": file_info["mime_type"], - "file_uri": file_info["uri"], - } - return PartType(file_data=file_data_ref) - else: - return PartType(text=element) + if isinstance(element, str): + return _media_part(element, None, files) if _is_multimodal_element(element) else PartType(text=element) + + block: Final = _parse_file_block(element, drop_params) + part: Final = _media_part(_file_block_source(block), block.file.format, files) + if block.file.video_metadata is None: + return part + video_metadata: Final = _gemini_video_metadata(block.file.video_metadata) + if not video_metadata: + return part + part_with_metadata: Final[PartType] = {**part, "video_metadata": video_metadata} + return part_with_metadata _SUPPORTED_EMBED_PARAMS: Final = {"outputDimensionality", "taskType", "title"} @@ -214,6 +394,7 @@ def transform_openai_input_gemini_content( model: str, optional_params: dict, resolved_files: dict[str, dict[str, str]] | None = None, + drop_params: bool = False, ) -> VertexAIBatchEmbeddingsRequestBody: """ Transform OpenAI embedding input to Gemini batchEmbedContents format. @@ -234,19 +415,18 @@ def transform_openai_input_gemini_content( gemini_params: Final = _filter_embed_params(optional_params) - input_list: Final = [input] if isinstance(input, str) else input + input_list: Final = _top_level_elements(input) requests: Final[list[EmbedContentRequest]] = [] for element in input_list: if isinstance(element, list): if not element: raise ValueError("Nested input list must not be empty") - for sub in element: - if not isinstance(sub, str): - raise ValueError(f"Elements inside a nested input list must be strings, got {type(sub)}") - parts = [_build_part_for_input(sub, resolved_files=resolved_files) for sub in element] + parts = [ + _build_part_for_input(sub, resolved_files=resolved_files, drop_params=drop_params) for sub in element + ] else: - parts = [_build_part_for_input(element, resolved_files=resolved_files)] + parts = [_build_part_for_input(element, resolved_files=resolved_files, drop_params=drop_params)] request = EmbedContentRequest( model=gemini_model_name, content=ContentType(parts=parts), @@ -262,6 +442,7 @@ def transform_openai_input_gemini_embed_content( model: str, optional_params: dict, resolved_files: dict[str, dict[str, str]] | None = None, + drop_params: bool = False, ) -> dict: """ Transform OpenAI embedding input to Gemini embedContent format (multimodal). @@ -279,7 +460,7 @@ def transform_openai_input_gemini_embed_content( gemini_params: Final = _filter_embed_params(optional_params) - input_list: Final = [input] if isinstance(input, str) else input + input_list: Final = _top_level_elements(input) parts: Final[list[PartType]] = [] for element in input_list: @@ -288,9 +469,7 @@ def transform_openai_input_gemini_embed_content( "Nested (combined) embeddings are not supported on the embedContent path. " "Use the batchEmbedContents path or pass a flat list instead." ) - if not isinstance(element, str): - raise ValueError(f"Unsupported input type: {type(element)}") - parts.append(_build_part_for_input(element, resolved_files=resolved_files)) + parts.append(_build_part_for_input(element, resolved_files=resolved_files, drop_params=drop_params)) request_body: Final[dict] = { "content": ContentType(parts=parts), @@ -313,31 +492,18 @@ def _parse_usage_metadata(raw_usage_metadata: object) -> UsageMetadata | None: return None -def _flatten_input(input: GeminiEmbeddingInput) -> tuple[str, ...]: - if isinstance(input, str): - return (input,) - return tuple(sub for element in input for sub in (element if isinstance(element, list) else [element])) +def _flatten_input(input: GeminiEmbeddingInput) -> tuple[GeminiEmbeddingElement, ...]: + return tuple(_elements(input)) def _is_image_element( - element: str, + element: GeminiEmbeddingElement, resolved_files: Mapping[str, Mapping[str, str]], ) -> bool: - if element.startswith("data:") and ";base64," in element: - try: - mime_type, _ = _parse_data_url(element) - except ValueError: - return False - return mime_type in _IMAGE_MIME_TYPES - if _is_gcs_url(element): - try: - return _infer_mime_type_from_gcs_url(element) in _IMAGE_MIME_TYPES - except ValueError: - return False - if is_file_reference(element): - file_info: Final = resolved_files.get(element) - return file_info is not None and file_info.get("mime_type") in _IMAGE_MIME_TYPES - return False + if isinstance(element, str): + return _source_mime_type(element, None, resolved_files) in _IMAGE_MIME_TYPES + block: Final = _parse_file_block(element, drop_params=True) + return _source_mime_type(_file_block_source(block), block.file.format, resolved_files) in _IMAGE_MIME_TYPES def _is_image_only_input( diff --git a/litellm/main.py b/litellm/main.py index 25c73ea281e..b02ccca9ecb 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -6379,6 +6379,10 @@ def embedding( """ azure: Final = kwargs.get("azure", None) client: Final = kwargs.pop("client", None) + drop_params_kwarg: Final = ( + cast(object, kwargs["drop_params"]) if "drop_params" in kwargs else None # cast-ok: untyped request kwargs + ) + drop_unsupported_params: Final = litellm.drop_params is True or normalize_drop_params(drop_params_kwarg) is True shared_session: Final = kwargs.get("shared_session", None) max_retries: Final = kwargs.get("max_retries", None) litellm_logging_obj: Final[LiteLLMLoggingObj] = kwargs.get("litellm_logging_obj") @@ -6859,6 +6863,7 @@ def embedding( api_base=api_base, client=client, extra_headers=headers, + drop_params=drop_unsupported_params, ) elif custom_llm_provider == "vertex_ai": @@ -6911,6 +6916,7 @@ def embedding( api_base=api_base, client=client, extra_headers=headers, + drop_params=drop_unsupported_params, ) elif ( "image" in optional_params diff --git a/litellm/types/llms/vertex_ai.py b/litellm/types/llms/vertex_ai.py index ce51e46ef15..116fdda0b5d 100644 --- a/litellm/types/llms/vertex_ai.py +++ b/litellm/types/llms/vertex_ai.py @@ -2,16 +2,20 @@ from enum import Enum from typing import Any, Final, Literal, Protocol from typing_extensions import ( + ReadOnly, Required, TypedDict, ) -from litellm.types.llms.openai import EmbeddingInput +from litellm.types.llms.openai import ChatCompletionFileObject, EmbeddingInput # Gemini supports nested-list inputs (e.g. [["text", "image"]]) as an explicit # opt-in for combined embeddings — a provider-specific extension of the # OpenAI-faithful EmbeddingInput shape. -GeminiEmbeddingInput = EmbeddingInput | list[list[str]] +GeminiEmbeddingElement = str | ChatCompletionFileObject +GeminiEmbeddingInput = ( + EmbeddingInput | list[GeminiEmbeddingElement] | list[list[str]] | list[list[GeminiEmbeddingElement]] +) class FunctionResponse(TypedDict, total=False): @@ -46,6 +50,12 @@ class FunctionResponsePartType(TypedDict, total=False): file_data: FileDataType +class VideoMetadataType(TypedDict, total=False): + fps: ReadOnly[float] + startOffset: ReadOnly[str] + endOffset: ReadOnly[str] + + class PartType(TypedDict, total=False): text: str inline_data: BlobType @@ -55,6 +65,7 @@ class PartType(TypedDict, total=False): thought: bool thoughtSignature: str media_resolution: Literal["low", "medium", "high"] + video_metadata: ReadOnly[VideoMetadataType] class HttpxFunctionCall(TypedDict, total=False): diff --git a/tests/integration/_support/wire.py b/tests/integration/_support/wire.py index d118da03daf..e6a166e8457 100644 --- a/tests/integration/_support/wire.py +++ b/tests/integration/_support/wire.py @@ -3,13 +3,13 @@ from __future__ import annotations import ssl import threading import time -from collections.abc import Callable, Generator, Mapping +from collections.abc import Callable, Generator, Iterator, Mapping from contextlib import contextmanager from dataclasses import dataclass from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from queue import SimpleQueue from types import MappingProxyType -from typing import Final +from typing import BinaryIO, Final @dataclass(frozen=True, slots=True) @@ -20,6 +20,37 @@ class Request: body: bytes +class AbortedBody(Exception): + """The client closed the connection before the body it announced was complete.""" + + +def exactly(stream: BinaryIO, size: int) -> bytes: + data: Final = stream.read(size) + if len(data) < size: + raise AbortedBody + return data + + +def chunked_body(stream: BinaryIO) -> Iterator[bytes]: + while True: + size_line: Final = stream.readline() + if not size_line: + raise AbortedBody + size: Final = int(size_line.split(b";")[0].strip(), 16) + if size == 0: + while stream.readline().strip(): + pass + return + yield exactly(stream, size) + stream.readline() + + +def read_body(headers: Mapping[str, str], stream: BinaryIO) -> bytes: + if headers.get("transfer-encoding", "").lower() == "chunked": + return b"".join(chunked_body(stream)) + return exactly(stream, int(headers.get("content-length", "0"))) + + @dataclass(frozen=True, slots=True) class Reply: status: int = 200 @@ -78,12 +109,14 @@ def wire_server( connected.put(f"{self.client_address[0]}:{self.client_address[1]}") def respond(self) -> None: - request: Final = Request( - self.command, - self.path, - {name.lower(): value for name, value in self.headers.items()}, - self.rfile.read(int(self.headers.get("content-length", "0"))), - ) + headers: Final = {name.lower(): value for name, value in self.headers.items()} + try: + body: Final = read_body(headers, self.rfile) + except AbortedBody: + self.close_connection = True + disconnected.put(self.path) + return + request: Final = Request(self.command, self.path, headers, body) received.put(request) try: reply = respond(request) diff --git a/tests/integration/caching/test_embedding_file_block_cache.py b/tests/integration/caching/test_embedding_file_block_cache.py new file mode 100644 index 00000000000..9f753e996e8 --- /dev/null +++ b/tests/integration/caching/test_embedding_file_block_cache.py @@ -0,0 +1,216 @@ +from __future__ import annotations + +import asyncio +import json +import threading +import zlib +from collections.abc import Mapping +from hashlib import sha256 +from typing import Final + +import pytest +from openai import AsyncOpenAI, OpenAI +from openai.types import CreateEmbeddingResponse +from pydantic import JsonValue + +from tests.integration._support.client import Gateway, Scenario, eventually, object_value, string_value +from tests.integration._support.database import read_rows +from tests.integration._support.wire import Reply, Request, Wire, wire_server + +BACKEND: Final = "gemini-embedding-001" +TARGET: Final = f"/models/{BACKEND}:batchEmbedContents" +METADATA: Final[dict[str, JsonValue]] = {"fps": 1.0, "start_offset": "0s", "end_offset": "3s"} +WIRE_METADATA: Final[dict[str, JsonValue]] = {"fps": 1.0, "startOffset": "0s", "endOffset": "3s"} +TEXT_PART: Final[dict[str, JsonValue]] = {"text": "a red bus"} +SETTLE_SECONDS: Final = 15 +SPEND_SQL: Final = ( + 'SELECT request_id, cache_hit, spend FROM "LiteLLM_SpendLogs" WHERE api_key = %s ORDER BY "startTime", request_id' +) + + +def clip(name: str) -> str: + return f"gs://scripted-bucket/cache/{name}.mp4" + + +def block(name: str, metadata: Mapping[str, JsonValue] | None = METADATA) -> dict[str, JsonValue]: + file: Final[dict[str, JsonValue]] = {"file_id": clip(name)} + return {"type": "file", "file": file if metadata is None else {**file, "video_metadata": dict(metadata)}} + + +def gcs_part(name: str, metadata: Mapping[str, JsonValue] | None = WIRE_METADATA) -> dict[str, JsonValue]: + part: Final[dict[str, JsonValue]] = {"file_data": {"mime_type": "video/mp4", "file_uri": clip(name)}} + return part if metadata is None else {**part, "video_metadata": dict(metadata)} + + +def list_value(value: JsonValue) -> list[JsonValue]: + assert isinstance(value, list), value + return value + + +def source_of(part: JsonValue) -> str: + item: Final = object_value(part) + if "text" in item: + return string_value(item["text"]) + return string_value(object_value(item["file_data"])["file_uri"]) + + +def vector(parts: list[JsonValue]) -> list[float]: + return [zlib.crc32("|".join(source_of(part) for part in parts).encode()) / 2**32, 0.5] + + +def request_parts(request: Request) -> list[list[JsonValue]]: + body: Final = object_value(json.loads(request.body)) + return [list_value(object_value(object_value(item)["content"])["parts"]) for item in list_value(body["requests"])] + + +def embed_peer(request: Request) -> Reply: + assert request.method == "POST", request.method + assert request.target.split("?")[0] == TARGET, request.target + embeddings: Final = [{"values": vector(parts)} for parts in request_parts(request)] + return Reply(body=json.dumps({"embeddings": embeddings}).encode()) + + +def wire_parts(wire: Wire) -> list[list[JsonValue]]: + received: Final = wire.drain() + assert len(received) == 1, [item.target for item in received] + return request_parts(received[0]) + + +def embeddings_of(answer: Mapping[str, JsonValue]) -> list[JsonValue]: + data: Final = [object_value(item) for item in list_value(answer["data"])] + assert [item["index"] for item in data] == list(range(len(data))), answer + return [item["embedding"] for item in data] + + +def gemini_model(scenario: Scenario, url: str) -> str: + return scenario.model(model=f"gemini/{BACKEND}", api_key="scripted-gemini-key", api_base=url) + + +def embed_body(model: str, elements: JsonValue) -> dict[str, JsonValue]: + return {"model": model, "input": elements} + + +def spend_rows(key: str) -> list[dict[str, JsonValue]]: + return read_rows(SPEND_SQL, (sha256(key.encode()).hexdigest(),)) + + +def served_from_cache( + gateway: Gateway, wire: Wire, body: Mapping[str, JsonValue], key: str +) -> dict[str, JsonValue] | None: + answer: Final = gateway.post("/v1/embeddings", body, key=key) + return None if wire.drain() else answer + + +def await_hit(gateway: Gateway, wire: Wire, body: Mapping[str, JsonValue], key: str) -> dict[str, JsonValue]: + served: Final = eventually( + lambda: served_from_cache(gateway, wire, body, key), lambda answer: answer is not None, SETTLE_SECONDS + ) + assert served is not None + return served + + +def proxy_root(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def test_block_embedding_is_served_from_the_cache_with_a_zero_spend_row(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + key: Final = scenario.key(models=[model]) + body: Final = embed_body(model, [block("one")]) + first: Final = gateway.request("POST", "/v1/embeddings", body, key=key) + assert first.status_code == 200, first.text + first_id: Final = first.headers["x-litellm-call-id"] + assert wire_parts(wire) == [[gcs_part("one")]] + served: Final = await_hit(gateway, wire, body, key) + assert embeddings_of(served) == embeddings_of(object_value(first.json())) == [vector([gcs_part("one")])] + rows: Final = eventually( + lambda: spend_rows(key), lambda found: any(row["cache_hit"] == "True" for row in found), seconds=70 + ) + by_id: Final = {string_value(row["request_id"]): row for row in rows} + assert by_id[first_id]["cache_hit"] != "True", rows + hits: Final = [row for row in rows if row["cache_hit"] == "True"] + assert hits and all(float(str(row["spend"])) == 0 for row in hits), rows + + +def test_cached_block_is_not_resent_when_a_new_text_joins_it(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + key: Final = scenario.key(models=[model]) + cached: Final = embed_body(model, [block("one")]) + gateway.post("/v1/embeddings", cached, key=key) + assert wire_parts(wire) == [[gcs_part("one")]] + await_hit(gateway, wire, cached, key) + mixed: Final = gateway.post("/v1/embeddings", embed_body(model, [block("one"), "a red bus"]), key=key) + assert wire_parts(wire) == [[TEXT_PART]] + assert embeddings_of(mixed) == [vector([gcs_part("one")]), vector([TEXT_PART])] + + +@pytest.mark.parametrize("names", (("a", "b", "c", "d", "e"), ("same", "same")), ids=("five-distinct", "two-identical")) +def test_block_lists_are_cached_whole(gateway: Gateway, names: tuple[str, ...]) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + key: Final = scenario.key(models=[model]) + body: Final = embed_body(model, [block(name) for name in names]) + first: Final = gateway.post("/v1/embeddings", body, key=key) + assert wire_parts(wire) == [[gcs_part(name)] for name in names] + assert embeddings_of(first) == [vector([gcs_part(name)]) for name in names] + assert embeddings_of(await_hit(gateway, wire, body, key)) == embeddings_of(first) + + +def test_provider_failure_is_not_cached(gateway: Gateway) -> None: + healed: Final = threading.Event() + + def respond(request: Request) -> Reply: + if healed.is_set(): + return embed_peer(request) + return Reply(status=500, body=b'{"error": {"message": "scripted outage"}}') + + with wire_server(respond) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + key: Final = scenario.key(models=[model]) + body: Final = embed_body(model, [block("flaky")]) + failed: Final = gateway.request("POST", "/v1/embeddings", body, key=key) + assert failed.status_code >= 500, failed.text + assert wire.drain(), "the failing call never reached the provider" + healed.set() + recovered: Final = gateway.post("/v1/embeddings", body, key=key) + assert wire_parts(wire) == [[gcs_part("flaky")]] + assert embeddings_of(await_hit(gateway, wire, body, key)) == embeddings_of(recovered) + + +def test_video_metadata_is_part_of_the_cache_key(gateway: Gateway) -> None: + two_seconds: Final[dict[str, JsonValue]] = {**METADATA, "end_offset": "2s"} + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + key: Final = scenario.key(models=[model]) + settled: Final = embed_body(model, [block("keyed")]) + gateway.post("/v1/embeddings", settled, key=key) + assert wire_parts(wire) == [[gcs_part("keyed")]] + await_hit(gateway, wire, settled, key) + variants: Final = ( + (block("keyed", two_seconds), gcs_part("keyed", {**WIRE_METADATA, "endOffset": "2s"})), + (block("keyed", None), gcs_part("keyed", None)), + ) + for variant, part in variants: + gateway.post("/v1/embeddings", embed_body(model, [variant]), key=key) + assert wire_parts(wire) == [[part]] + + +def test_openai_sdk_clients_fill_and_hit_the_cache(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + key: Final = scenario.key(models=[model]) + body: Final = embed_body(model, [block("sdk")]) + with OpenAI(base_url=f"{proxy_root(gateway)}/v1", api_key=key, max_retries=0) as client: + filled: Final = client.post("/embeddings", body=body, cast_to=CreateEmbeddingResponse) + assert wire_parts(wire) == [[gcs_part("sdk")]] + await_hit(gateway, wire, body, key) + + async def drive() -> CreateEmbeddingResponse: + async with AsyncOpenAI(base_url=f"{proxy_root(gateway)}/v1", api_key=key, max_retries=0) as client: + return await client.post("/embeddings", body=body, cast_to=CreateEmbeddingResponse) + + served: Final = asyncio.run(drive()) + assert served.data[0].embedding == filled.data[0].embedding == vector([gcs_part("sdk")]) + assert wire.drain() == (), "the cached answer reached the provider again" diff --git a/tests/integration/providers/test_gemini_chat_video_metadata_wire.py b/tests/integration/providers/test_gemini_chat_video_metadata_wire.py new file mode 100644 index 00000000000..32914341483 --- /dev/null +++ b/tests/integration/providers/test_gemini_chat_video_metadata_wire.py @@ -0,0 +1,144 @@ +from __future__ import annotations + +import base64 +import json +from collections.abc import Callable +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, Scenario, object_value +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +BACKEND: Final = "gemini-3.7-flash" +PROJECT: Final = "scripted-project" +LOCATION: Final = "us-central1" +MODEL_PATH: Final = f"/v1/projects/{PROJECT}/locations/{LOCATION}/publishers/google/models/{BACKEND}" +CLIP: Final = base64.b64encode(b"\x00\x00\x00\x18ftypmp42" + bytes(24)).decode() +DATA_URI: Final = f"data:video/mp4;base64,{CLIP}" +PROMPT: Final = "describe the clip" +FILE_BLOCK: Final[dict[str, JsonValue]] = { + "type": "file", + "file": {"file_data": DATA_URI, "video_metadata": {"fps": 1.0, "start_offset": "0s", "end_offset": "3s"}}, +} +TEXT_PART: Final[dict[str, JsonValue]] = {"text": PROMPT} +BARE_VIDEO_PART: Final[dict[str, JsonValue]] = {"inline_data": {"mime_type": "video/mp4", "data": CLIP}} +VIDEO_PART: Final[dict[str, JsonValue]] = { + **BARE_VIDEO_PART, + "video_metadata": {"fps": 1.0, "startOffset": "0s", "endOffset": "3s"}, +} +REPLY: Final[dict[str, JsonValue]] = { + "candidates": [{"content": {"role": "model", "parts": [{"text": "a cat"}]}, "finishReason": "STOP"}], + "usageMetadata": {"promptTokenCount": 5, "candidatesTokenCount": 2, "totalTokenCount": 7}, +} + + +def chat_peer(path: str, authorization: str | None) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + assert request.method == "POST", request.method + assert authorization is None or request.headers["authorization"] == authorization + target, _, query = request.target.partition("?") + if target == f"{path}:streamGenerateContent": + assert "alt=sse" in query, request.target + return Reply(content_type="text/event-stream", chunks=(f"data: {json.dumps(REPLY)}\n\n".encode(),)) + assert target == f"{path}:generateContent", request.target + return Reply(body=json.dumps(REPLY).encode()) + + return respond + + +def wire_parts(wire: Wire) -> list[JsonValue]: + received: Final = wire.drain() + assert len(received) == 1, [item.target for item in received] + body: Final = object_value(json.loads(received[0].body)) + contents: Final = body["contents"] + assert isinstance(contents, list) and len(contents) == 1, body + parts: Final = object_value(contents[0])["parts"] + assert isinstance(parts, list), body + return parts + + +def gemini_model(scenario: Scenario, url: str) -> str: + return scenario.model(model=f"gemini/{BACKEND}", api_key="scripted-gemini-key", api_base=url) + + +def vertex_model(gateway: Gateway, scenario: Scenario, url: str) -> str: + return scenario.model( + model=f"vertex_ai/{BACKEND}", + api_key=None, + api_base=url, + vertex_project=PROJECT, + vertex_location=LOCATION, + vertex_credentials=service_account_json(PROJECT, gateway.upstream_url), + ) + + +def chat(gateway: Gateway, model: str, stream: bool) -> httpx.Response: + body: Final[dict[str, JsonValue]] = { + "model": model, + "messages": [{"role": "user", "content": [{"type": "text", "text": PROMPT}, FILE_BLOCK]}], + "stream": stream, + } + return gateway.request("POST", "/v1/chat/completions", body) + + +def answered(response: httpx.Response) -> None: + assert response.status_code == 200, response.text + assert "a cat" in response.text, response.text + + +@pytest.mark.parametrize("stream", (False, True), ids=("non-stream", "stream")) +def test_gemini_chat_file_block_video_metadata_reaches_the_wire(gateway: Gateway, stream: bool) -> None: + with wire_server(chat_peer(f"/models/{BACKEND}", None)) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + answered(chat(gateway, model, stream)) + assert wire_parts(wire) == [TEXT_PART, VIDEO_PART] + + +@pytest.mark.parametrize("stream", (False, True), ids=("non-stream", "stream")) +def test_vertex_chat_file_block_video_metadata_reaches_the_wire(gateway: Gateway, stream: bool) -> None: + with wire_server(chat_peer(MODEL_PATH, "Bearer scripted-token")) as wire, gateway.scenario() as scenario: + model: Final = vertex_model(gateway, scenario, wire.url) + answered(chat(gateway, model, stream)) + assert wire_parts(wire) == [TEXT_PART, VIDEO_PART] + + +def test_responses_input_file_video_reaches_the_wire_as_inline_data(gateway: Gateway) -> None: + with wire_server(chat_peer(f"/models/{BACKEND}", None)) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + body: Final[dict[str, JsonValue]] = { + "model": model, + "input": [ + { + "role": "user", + "content": [ + {"type": "input_text", "text": PROMPT}, + {"type": "input_file", "file_data": DATA_URI, "filename": "clip.mp4"}, + ], + } + ], + } + answered(gateway.request("POST", "/v1/responses", body)) + assert wire_parts(wire) == [TEXT_PART, BARE_VIDEO_PART] + + +def test_messages_document_video_reaches_the_wire_as_inline_data(gateway: Gateway) -> None: + with wire_server(chat_peer(f"/models/{BACKEND}", None)) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + body: Final[dict[str, JsonValue]] = { + "model": model, + "max_tokens": 32, + "messages": [ + { + "role": "user", + "content": [ + {"type": "text", "text": PROMPT}, + {"type": "document", "source": {"type": "base64", "media_type": "video/mp4", "data": CLIP}}, + ], + } + ], + } + answered(gateway.request("POST", "/v1/messages", body)) + assert wire_parts(wire) == [TEXT_PART, BARE_VIDEO_PART] diff --git a/tests/integration/providers/test_gemini_embedding_file_block_chaos.py b/tests/integration/providers/test_gemini_embedding_file_block_chaos.py new file mode 100644 index 00000000000..cee0758b433 --- /dev/null +++ b/tests/integration/providers/test_gemini_embedding_file_block_chaos.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import json +import os +import signal +import threading +import uuid +import zlib +from collections.abc import Callable +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from pathlib import Path +from typing import Final + +import httpx +import psutil +import pytest +from integration._support.client import Gateway, eventually, object_value, string_value +from integration._support.database import read_rows +from integration._support.process import group_members, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +BACKEND: Final = "gemini-embedding-001" +TARGET: Final = f"/models/{BACKEND}:batchEmbedContents" +END_OFFSETS: Final = {"fast": "3s", "slow": "5s", "malformed": "3s", "dropped": "9s"} +PLAN: Final = tuple(enumerate(("fast",) * 16 + ("slow",) * 8 + ("malformed",) * 8 + ("dropped",) * 8)) +BURST: Final = 24 +SPEND_SQL: Final = 'SELECT request_id FROM "LiteLLM_SpendLogs" WHERE request_id = %s' + + +@dataclass(frozen=True, slots=True) +class Attempt: + kind: str + index: int + status: int + call_id: str + + +def source(kind: str, index: int) -> str: + return f"gs://scripted-bucket/chaos/{kind}-{index}.mp4" + + +def body(model: str, kind: str, index: int) -> dict[str, JsonValue]: + fps: Final[JsonValue] = "x" if kind == "malformed" else 1.0 + metadata: Final[dict[str, JsonValue]] = {"fps": fps, "start_offset": "0s", "end_offset": END_OFFSETS[kind]} + return { + "model": model, + "input": [{"type": "file", "file": {"file_id": source(kind, index), "video_metadata": metadata}}], + } + + +def first_part(request: Request) -> dict[str, JsonValue]: + requests: Final = object_value(json.loads(request.body))["requests"] + assert isinstance(requests, list) and len(requests) == 1, requests + parts: Final = object_value(object_value(requests[0])["content"])["parts"] + assert isinstance(parts, list) and len(parts) == 1, parts + return object_value(parts[0]) + + +def source_on_the_wire(request: Request) -> str: + return string_value(object_value(first_part(request)["file_data"])["file_uri"]) + + +def embed_reply(request: Request) -> Reply: + assert request.method == "POST", request.method + assert request.target.split("?")[0] == TARGET, request.target + values: Final = [zlib.crc32(source_on_the_wire(request).encode()) / 2**32, 0.5] + return Reply(body=json.dumps({"embeddings": [{"values": values}]}).encode()) + + +def chaos_peer(healed: threading.Event) -> Callable[[Request], Reply]: + def respond(request: Request) -> Reply: + end_offset: Final = object_value(first_part(request)["video_metadata"])["endOffset"] + if end_offset == END_OFFSETS["dropped"] and not healed.is_set(): + return Reply(drop_connection=True) + reply: Final = embed_reply(request) + if end_offset == END_OFFSETS["slow"]: + return Reply(chunks=(reply.body[:8], reply.body[8:]), pause_between_chunks=1.5) + return reply + + return respond + + +def wire_sources(wire: Wire) -> tuple[str, ...]: + return tuple(source_on_the_wire(request) for request in wire.drain()) + + +def spend_row_count(call_id: str) -> int: + return len(read_rows(SPEND_SQL, (call_id,))) + + +def spend_row_counts(call_ids: tuple[str, ...]) -> tuple[int, ...]: + return tuple(spend_row_count(call_id) for call_id in call_ids) + + +def attempt(gateway: Gateway, model: str, index: int, kind: str) -> Attempt: + response: Final = gateway.request("POST", "/v1/embeddings", body(model, kind, index)) + return Attempt(kind, index, response.status_code, response.headers.get("x-litellm-call-id", "")) + + +def of_kind(attempts: tuple[Attempt, ...], *kinds: str) -> tuple[Attempt, ...]: + return tuple(item for item in attempts if item.kind in kinds) + + +@pytest.mark.timeout(240) +def test_mixed_burst_keeps_every_answer_and_spend_row_honest(gateway: Gateway) -> None: + healed: Final = threading.Event() + with wire_server(chaos_peer(healed)) as wire, gateway.scenario() as scenario: + model: Final = scenario.model(model=f"gemini/{BACKEND}", api_key="scripted-gemini-key", api_base=wire.url) + with ThreadPoolExecutor(max_workers=len(PLAN)) as pool: + attempts: Final = tuple(pool.map(lambda plan: attempt(gateway, model, plan[0], plan[1]), PLAN)) + served: Final = of_kind(attempts, "fast", "slow") + assert all(item.status == 200 for item in served), attempts + assert all(item.status == 400 for item in of_kind(attempts, "malformed")), attempts + assert all(item.status >= 500 for item in of_kind(attempts, "dropped")), attempts + reached: Final = wire_sources(wire) + assert sorted(item for item in reached if "/malformed-" not in item and "/dropped-" not in item) == sorted( + source(item.kind, item.index) for item in served + ), reached + assert not any("/malformed-" in item for item in reached), reached + assert {item for item in reached if "/dropped-" in item} == { + source(item.kind, item.index) for item in of_kind(attempts, "dropped") + }, reached + call_ids: Final = tuple(item.call_id for item in served) + assert all(call_ids), attempts + counts: Final = eventually( + lambda: spend_row_counts(call_ids), lambda found: all(count >= 1 for count in found), seconds=90 + ) + assert counts == (1,) * len(call_ids), counts + healed.set() + with ThreadPoolExecutor(max_workers=len(PLAN)) as pool: + resent: Final = tuple( + pool.map(lambda item: attempt(gateway, model, item.index, item.kind), of_kind(attempts, "dropped")) + ) + assert all(item.status == 200 for item in resent), resent + assert sorted(wire_sources(wire)) == sorted(source(item.kind, item.index) for item in resent) + + +def write_config(directory: Path, url: str, name: str) -> Path: + config: Final = directory / f"gemini_embeddings_{uuid.uuid4().hex}.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": name, + "litellm_params": { + "model": f"gemini/{BACKEND}", + "api_key": "scripted-gemini-key", + "api_base": url, + }, + } + ], + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + }, + } + ) + ) + return config + + +def worker_pids(root_pid: int) -> frozenset[int]: + def is_worker(process: psutil.Process) -> bool: + try: + return "spawn_main" in " ".join(process.cmdline()) + except (psutil.NoSuchProcess, psutil.AccessDenied, psutil.ZombieProcess): + return False + + return frozenset(process.pid for process in group_members(root_pid) if is_worker(process)) + + +def outcome(candidate: Gateway, model: str, index: int) -> str: + try: + response: Final = candidate.request("POST", "/v1/embeddings", body(model, "fast", index)) + except httpx.TransportError as error: + return f"transport:{type(error).__name__}" + assert response.status_code == 200, response.text + return f"ok:{response.headers['x-litellm-call-id']}" + + +@pytest.mark.timeout(300) +def test_block_embeddings_survive_a_worker_kill(gateway: Gateway, tmp_path: Path) -> None: + name: Final = f"gemini-embeddings-{uuid.uuid4().hex}" + with wire_server(embed_reply) as wire: + config: Final = write_config(tmp_path, wire.url, name) + with owned_proxy_process(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config, workers=2) as owned: + candidate: Final = owned.gateway + workers: Final = eventually(lambda: worker_pids(owned.process.pid), lambda pids: len(pids) == 2, seconds=30) + victim: Final = min(workers) + assert outcome(candidate, name, 0).startswith("ok:") + + def attempt_around_the_kill(index: int) -> str: + if index == 2: + os.kill(victim, signal.SIGKILL) + return outcome(candidate, name, index) + + with ThreadPoolExecutor(max_workers=BURST) as pool: + outcomes: Final = tuple(pool.map(attempt_around_the_kill, range(1, BURST + 1))) + assert outcomes.count("ok") == 0 and any(item.startswith("ok:") for item in outcomes), outcomes + assert all(item.startswith("ok:") or item.startswith("transport:") for item in outcomes), outcomes + respawned: Final = eventually( + lambda: worker_pids(owned.process.pid), + lambda pids: len(pids) == 2 and victim not in pids, + seconds=60, + ) + assert victim not in respawned, respawned + + def settled_burst() -> tuple[str, ...]: + with ThreadPoolExecutor(max_workers=BURST) as pool: + return tuple(pool.map(lambda index: outcome(candidate, name, BURST + 1 + index), range(BURST))) + + final: Final = eventually( + settled_burst, lambda values: all(v.startswith("ok:") for v in values), seconds=40 + ) + call_ids: Final = tuple(item.removeprefix("ok:") for item in (*outcomes, *final) if item.startswith("ok:")) + counts: Final = eventually( + lambda: spend_row_counts(call_ids), lambda found: all(count >= 1 for count in found), seconds=90 + ) + assert counts == (1,) * len(call_ids), counts + assert len(wire.drain()) >= len(call_ids) diff --git a/tests/integration/providers/test_gemini_embedding_file_block_wire.py b/tests/integration/providers/test_gemini_embedding_file_block_wire.py new file mode 100644 index 00000000000..d6439d27cd5 --- /dev/null +++ b/tests/integration/providers/test_gemini_embedding_file_block_wire.py @@ -0,0 +1,294 @@ +from __future__ import annotations + +import asyncio +import base64 +import json +import uuid +import zlib +from pathlib import Path +from typing import Final + +import httpx +import pytest +from integration._support.client import Gateway, Scenario, object_value, string_value +from integration._support.process import owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from openai import AsyncOpenAI, OpenAI +from openai.types import CreateEmbeddingResponse +from pydantic import JsonValue + +BACKEND: Final = "gemini-embedding-001" +TARGET: Final = f"/models/{BACKEND}:batchEmbedContents" +CLIP: Final = base64.b64encode(b"\x00\x00\x00\x18ftypmp42" + bytes(24)).decode() +DATA_URI: Final = f"data:video/mp4;base64,{CLIP}" +GCS_URI: Final = "gs://scripted-bucket/clips/animals.mp4" +METADATA: Final[dict[str, JsonValue]] = {"fps": 1.0, "start_offset": "0s", "end_offset": "3s"} +WIRE_METADATA: Final[dict[str, JsonValue]] = {"fps": 1.0, "startOffset": "0s", "endOffset": "3s"} +INLINE_DATA: Final[dict[str, JsonValue]] = {"mime_type": "video/mp4", "data": CLIP} +INLINE_PART: Final[dict[str, JsonValue]] = {"inline_data": INLINE_DATA, "video_metadata": WIRE_METADATA} +TEXT_PART: Final[dict[str, JsonValue]] = {"text": "a red bus"} +ACCEPTED_FORMS: Final = "must be a data: URI, a gs:// URL, a files/ reference, or a Gemini Files API URI" +LONG_OFFSET: Final = "9" * 5000 + "s" + + +def block(**file: JsonValue) -> dict[str, JsonValue]: + return {"type": "file", "file": file} + + +def gcs_part(uri: str = GCS_URI, metadata: dict[str, JsonValue] | None = WIRE_METADATA) -> dict[str, JsonValue]: + file_data: Final[dict[str, JsonValue]] = {"file_data": {"mime_type": "video/mp4", "file_uri": uri}} + return file_data if metadata is None else {**file_data, "video_metadata": metadata} + + +GCS_PART: Final = gcs_part() +NOISY_BLOCK: Final[dict[str, JsonValue]] = { + **block(file_id=GCS_URI, mime_type="video/mp4", video_metadata={**METADATA, "frame_rate": 2}), + "caption": "unknown block key", +} + + +def list_value(value: JsonValue) -> list[JsonValue]: + assert isinstance(value, list), value + return value + + +def source_of(part: JsonValue) -> str: + item: Final = object_value(part) + if "text" in item: + return string_value(item["text"]) + if "file_data" in item: + return string_value(object_value(item["file_data"])["file_uri"]) + return string_value(object_value(item["inline_data"])["data"]) + + +def vector(parts: list[JsonValue]) -> list[float]: + return [zlib.crc32("|".join(source_of(part) for part in parts).encode()) / 2**32, 0.5] + + +def request_parts(request: Request) -> list[list[JsonValue]]: + body: Final = object_value(json.loads(request.body)) + return [list_value(object_value(object_value(item)["content"])["parts"]) for item in list_value(body["requests"])] + + +def embed_peer(request: Request) -> Reply: + assert request.method == "POST", request.method + assert request.target.split("?")[0] == TARGET, request.target + embeddings: Final = [{"values": vector(parts)} for parts in request_parts(request)] + return Reply(body=json.dumps({"embeddings": embeddings}).encode()) + + +def wire_parts(wire: Wire) -> list[list[JsonValue]]: + received: Final = wire.drain() + assert len(received) == 1, [item.target for item in received] + return request_parts(received[0]) + + +def embeddings(response: httpx.Response) -> list[JsonValue]: + assert response.status_code == 200, response.text + data: Final = [object_value(item) for item in list_value(object_value(response.json())["data"])] + assert [item["index"] for item in data] == list(range(len(data))), response.text + assert all(item["object"] == "embedding" for item in data), response.text + return [item["embedding"] for item in data] + + +def gemini_model(scenario: Scenario, url: str, **litellm_params: JsonValue) -> str: + return scenario.model(model=f"gemini/{BACKEND}", api_key="scripted-gemini-key", api_base=url, **litellm_params) + + +def embed(gateway: Gateway, model: str, elements: JsonValue, **extra: JsonValue) -> httpx.Response: + return gateway.request("POST", "/v1/embeddings", {"model": model, "input": elements, **extra}) + + +def proxy_root(gateway: Gateway) -> str: + return str(gateway.client.base_url).rstrip("/") + + +def write_config(directory: Path, url: str, name: str) -> Path: + config: Final = directory / f"gemini_embeddings_{uuid.uuid4().hex}.yaml" + config.write_text( + json.dumps( + { + "model_list": [ + { + "model_name": name, + "litellm_params": { + "model": f"gemini/{BACKEND}", + "api_key": "scripted-gemini-key", + "api_base": url, + }, + } + ], + "litellm_settings": {"drop_params": True}, + "general_settings": { + "master_key": "os.environ/LITELLM_MASTER_KEY", + "database_url": "os.environ/DATABASE_URL", + }, + } + ) + ) + return config + + +def test_string_input_reaches_the_wire_as_a_text_part(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + assert embeddings(embed(gateway, model, ["a red bus"])) == [vector([TEXT_PART])] + assert wire_parts(wire) == [[TEXT_PART]] + + +@pytest.mark.parametrize( + ("file", "part"), + ( + pytest.param({"file_data": DATA_URI, "video_metadata": METADATA}, INLINE_PART, id="data-uri"), + pytest.param({"file_id": GCS_URI, "video_metadata": METADATA}, GCS_PART, id="gcs"), + pytest.param( + {"file_id": GCS_URI, "video_metadata": {"fps": 1, "start_offset": "", "end_offset": LONG_OFFSET}}, + gcs_part(metadata={"fps": 1, "startOffset": "", "endOffset": LONG_OFFSET}), + id="verbatim-strings-and-int-fps", + ), + pytest.param( + {"file_data": DATA_URI, "format": "video/quicktime"}, + {"inline_data": {"mime_type": "video/quicktime", "data": CLIP}}, + id="format-overrides-the-data-uri-mime", + ), + pytest.param( + {"file_id": "gs://scripted-bucket/clips/clip.bin", "format": "video/mp4"}, + gcs_part("gs://scripted-bucket/clips/clip.bin", None), + id="format-names-an-unlisted-extension", + ), + pytest.param({"file_id": GCS_URI}, gcs_part(metadata=None), id="no-metadata"), + pytest.param({"file_id": GCS_URI, "video_metadata": {}}, gcs_part(metadata=None), id="empty-metadata"), + ), +) +def test_file_block_reaches_the_wire_as_one_part( + gateway: Gateway, file: dict[str, JsonValue], part: dict[str, JsonValue] +) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + assert embeddings(embed(gateway, model, [block(**file)])) == [vector([part])] + assert wire_parts(wire) == [[part]] + + +def test_nested_text_and_block_become_one_request_with_two_parts(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + response: Final = embed(gateway, model, [["a red bus", block(file_id=GCS_URI, video_metadata=METADATA)]]) + assert embeddings(response) == [vector([TEXT_PART, GCS_PART])] + assert wire_parts(wire) == [[TEXT_PART, GCS_PART]] + + +def test_repeated_blocks_become_one_request_each(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + response: Final = embed(gateway, model, [block(file_id=GCS_URI, video_metadata=METADATA)] * 2) + assert embeddings(response) == [vector([GCS_PART])] * 2 + assert wire_parts(wire) == [[GCS_PART], [GCS_PART]] + + +@pytest.mark.parametrize( + ("element", "fragment"), + ( + pytest.param(block(file_id=GCS_URI, video_metadata={"fps": "1"}), "file.video_metadata.fps", id="fps-string"), + pytest.param( + block(file_id=GCS_URI, video_metadata={"start_offset": 0}), + "file.video_metadata.start_offset", + id="offset-int", + ), + pytest.param( + block(file_id=GCS_URI, video_metadata={"end_offset": ["3s"]}), + "file.video_metadata.end_offset", + id="offset-list", + ), + pytest.param(block(file_id=GCS_URI, video_metadata="1fps"), "file.video_metadata", id="metadata-string"), + pytest.param( + block(file_id=GCS_URI, video_metadata={**METADATA, "frame_rate": 2}), + "frame_rate", + id="unknown-metadata-key", + ), + pytest.param(block(file_id=GCS_URI, mime_type="video/mp4"), "mime_type", id="unknown-file-key"), + pytest.param({**block(file_id=GCS_URI), "caption": "x"}, "caption", id="unknown-block-key"), + pytest.param({"type": "video", "file": {"file_id": GCS_URI}}, "type", id="wrong-type"), + pytest.param( + block(file_id=GCS_URI, file_data=DATA_URI), + "takes file.file_id or file.file_data, not both", + id="both-sources", + ), + pytest.param(block(video_metadata=METADATA), "needs file.file_id or file.file_data", id="no-source"), + pytest.param(block(file_data=CLIP), ACCEPTED_FORMS, id="bare-base64"), + pytest.param(block(file_id="https://example.com/clip.mp4"), ACCEPTED_FORMS, id="http-url"), + pytest.param(block(file_id=""), ACCEPTED_FORMS, id="empty-file-id"), + pytest.param(block(file_id=GCS_URI, format=""), "file.format", id="empty-format"), + pytest.param(block(file_data=5), "file.file_data", id="file-data-int"), + pytest.param(block(file_data=[DATA_URI]), "file.file_data", id="file-data-list"), + pytest.param(1, "got int", id="int-element"), + ), +) +def test_malformed_blocks_answer_400_before_any_provider_call( + gateway: Gateway, element: JsonValue, fragment: str +) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + response: Final = embed(gateway, model, [element]) + assert response.status_code == 400, response.text + assert fragment in response.text, response.text + assert wire.drain() == (), "the rejected input reached the provider" + + +def test_bare_block_input_answers_400_before_any_provider_call(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + response: Final = embed(gateway, model, block(file_id=GCS_URI, video_metadata=METADATA)) + assert response.status_code == 400, response.text + assert "input must be a string or a list" in response.text, response.text + assert wire.drain() == (), "the rejected input reached the provider" + + +def test_request_drop_params_strips_unknown_keys_at_every_level(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + assert embeddings(embed(gateway, model, [NOISY_BLOCK], drop_params=True)) == [vector([GCS_PART])] + assert wire_parts(wire) == [[GCS_PART]] + + +def test_deployment_drop_params_strips_unknown_keys_at_every_level(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url, drop_params=True) + assert embeddings(embed(gateway, model, [NOISY_BLOCK])) == [vector([GCS_PART])] + assert wire_parts(wire) == [[GCS_PART]] + + +@pytest.mark.timeout(300) +def test_yaml_drop_params_strips_unknown_keys_at_every_level(gateway: Gateway, tmp_path: Path) -> None: + name: Final = f"gemini-embeddings-{uuid.uuid4().hex}" + with wire_server(embed_peer) as wire: + config: Final = write_config(tmp_path, wire.url, name) + with owned_proxy_process(gateway, tmp_path, {"STORE_MODEL_IN_DB": "False"}, config=config) as owned: + assert embeddings(embed(owned.gateway, name, [NOISY_BLOCK])) == [vector([GCS_PART])] + assert wire_parts(wire) == [[GCS_PART]] + + +def test_openai_sdk_clients_send_blocks_through_the_proxy(gateway: Gateway) -> None: + sync_uri: Final = "gs://scripted-bucket/clips/sync.mp4" + async_uri: Final = "gs://scripted-bucket/clips/async.mp4" + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = gemini_model(scenario, wire.url) + with OpenAI(base_url=f"{proxy_root(gateway)}/v1", api_key=gateway.key, max_retries=0) as client: + served: Final = client.post( + "/embeddings", + body={"model": model, "input": [block(file_id=sync_uri, video_metadata=METADATA)]}, + cast_to=CreateEmbeddingResponse, + ) + assert served.data[0].embedding == vector([gcs_part(sync_uri)]), served.model_dump_json() + assert wire_parts(wire) == [[gcs_part(sync_uri)]] + + async def drive() -> CreateEmbeddingResponse: + async with AsyncOpenAI(base_url=f"{proxy_root(gateway)}/v1", api_key=gateway.key, max_retries=0) as client: + return await client.post( + "/embeddings", + body={"model": model, "input": [block(file_id=async_uri, video_metadata=METADATA)]}, + cast_to=CreateEmbeddingResponse, + ) + + served_async: Final = asyncio.run(drive()) + assert served_async.data[0].embedding == vector([gcs_part(async_uri)]), served_async.model_dump_json() + assert wire_parts(wire) == [[gcs_part(async_uri)]] diff --git a/tests/integration/providers/test_vertex_embedding_batch_file_block_wire.py b/tests/integration/providers/test_vertex_embedding_batch_file_block_wire.py new file mode 100644 index 00000000000..042ca8b0889 --- /dev/null +++ b/tests/integration/providers/test_vertex_embedding_batch_file_block_wire.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +import json +from typing import Final +from urllib.parse import unquote + +import httpx +from integration._support.client import Gateway, Scenario +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, wire_server +from pydantic import JsonValue + +BACKEND: Final = "gemini-embedding-2-preview" +PROJECT: Final = "scripted-project" +LOCATION: Final = "us-central1" +BUCKET: Final = "scripted-bucket" +UPLOAD_PREFIX: Final = f"/upload/storage/v1/b/{BUCKET}/o?uploadType=media&name=" +GCS_URI: Final = "gs://scripted-bucket/clips/animals.mp4" +BLOCK: Final[dict[str, JsonValue]] = { + "type": "file", + "file": {"file_id": GCS_URI, "video_metadata": {"fps": 1.0, "start_offset": "0s", "end_offset": "3s"}}, +} +GCS_PART: Final[dict[str, JsonValue]] = { + "file_data": {"mime_type": "video/mp4", "file_uri": GCS_URI}, + "video_metadata": {"fps": 1.0, "startOffset": "0s", "endOffset": "3s"}, +} +TEXT_PART: Final[dict[str, JsonValue]] = {"text": "a red bus"} + + +def row(custom_id: str, model: str, elements: JsonValue) -> dict[str, JsonValue]: + return { + "custom_id": custom_id, + "method": "POST", + "url": "/v1/embeddings", + "body": {"model": model, "input": elements}, + } + + +def stored_row(key: str, parts: list[JsonValue]) -> dict[str, JsonValue]: + return {"key": key, "request": {"content": {"parts": parts}}} + + +def gcs_peer(request: Request) -> Reply: + assert request.method == "POST", request.method + assert request.target.startswith(UPLOAD_PREFIX), request.target + assert request.headers["authorization"] == "Bearer scripted-token" + name: Final = unquote(request.target.removeprefix(UPLOAD_PREFIX)) + stored: Final = { + "kind": "storage#object", + "id": f"{BUCKET}/{name}/1759950000000000", + "name": name, + "bucket": BUCKET, + "size": str(len(request.body)), + "timeCreated": "2026-10-08T19:00:00.000Z", + "contentType": "application/octet-stream", + } + return Reply(body=json.dumps(stored).encode()) + + +def vertex_batch_model(gateway: Gateway, scenario: Scenario, url: str) -> str: + return scenario.model( + model=f"vertex_ai/{BACKEND}", + api_key=None, + api_base=url, + vertex_project=PROJECT, + vertex_location=LOCATION, + vertex_credentials=service_account_json(PROJECT, gateway.upstream_url), + gcs_bucket_name=BUCKET, + ) + + +def upload(gateway: Gateway, model: str, rows: tuple[dict[str, JsonValue], ...]) -> httpx.Response: + jsonl: Final = "\n".join(json.dumps(line) for line in rows) + "\n" + return gateway.request_multipart( + "/v1/files", + {"purpose": "batch", "target_model_names": model}, + {"file": ("in.jsonl", jsonl.encode(), "application/jsonl")}, + ) + + +def test_block_rows_upload_as_embed_content_requests(gateway: Gateway) -> None: + with wire_server(gcs_peer) as wire, gateway.scenario() as scenario: + model: Final = vertex_batch_model(gateway, scenario, wire.url) + rows: Final = ( + row("mixed", model, [BLOCK, "a red bus"]), + row("bare", model, "a red bus"), + row("nested", model, [[BLOCK, "a red bus"]]), + ) + response: Final = upload(gateway, model, rows) + assert response.status_code == 200, response.text + assert response.json()["object"] == "file" and response.json()["purpose"] == "batch", response.text + uploads: Final = wire.drain() + assert len(uploads) == 1, [upload.target for upload in uploads] + stored: Final = tuple(json.loads(line) for line in uploads[0].body.decode().splitlines() if line.strip()) + assert stored == ( + stored_row("mixed#0/2", [GCS_PART]), + stored_row("mixed#1/2", [TEXT_PART]), + stored_row("bare", [TEXT_PART]), + stored_row("nested", [GCS_PART, TEXT_PART]), + ) + + +def test_bare_block_row_fails_the_upload_before_any_storage_write(gateway: Gateway) -> None: + with wire_server(gcs_peer) as wire, gateway.scenario() as scenario: + model: Final = vertex_batch_model(gateway, scenario, wire.url) + response: Final = upload(gateway, model, (row("mixed", model, [BLOCK, "a red bus"]), row("bare", model, BLOCK))) + assert response.status_code >= 400, response.text + assert "got dict" in response.text, response.text + assert wire.drain() == (), "a rejected batch file reached the bucket" diff --git a/tests/integration/providers/test_vertex_embedding_file_block_wire.py b/tests/integration/providers/test_vertex_embedding_file_block_wire.py new file mode 100644 index 00000000000..ad66412a27a --- /dev/null +++ b/tests/integration/providers/test_vertex_embedding_file_block_wire.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +import json +from typing import Final + +import httpx +from integration._support.client import Gateway, Scenario, object_value +from integration._support.vertex import service_account_json +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue + +BACKEND: Final = "gemini-embedding-2-preview" +PROJECT: Final = "scripted-project" +LOCATION: Final = "us-central1" +TARGET: Final = f"/v1/projects/{PROJECT}/locations/{LOCATION}/publishers/google/models/{BACKEND}:embedContent" +GCS_URI: Final = "gs://scripted-bucket/clips/animals.mp4" +METADATA: Final[dict[str, JsonValue]] = {"fps": 1.0, "start_offset": "0s", "end_offset": "3s"} +GCS_PART: Final[dict[str, JsonValue]] = { + "file_data": {"mime_type": "video/mp4", "file_uri": GCS_URI}, + "video_metadata": {"fps": 1.0, "startOffset": "0s", "endOffset": "3s"}, +} +TEXT_PART: Final[dict[str, JsonValue]] = {"text": "a red bus"} +VALUES: Final = [0.75, 0.25] + + +def block(**file: JsonValue) -> dict[str, JsonValue]: + return {"type": "file", "file": file} + + +def embed_peer(request: Request) -> Reply: + assert request.method == "POST", request.method + assert request.target == TARGET, request.target + assert request.headers["authorization"] == "Bearer scripted-token" + return Reply(body=json.dumps({"embedding": {"values": VALUES}}).encode()) + + +def wire_parts(wire: Wire) -> list[JsonValue]: + received: Final = wire.drain() + assert len(received) == 1, [item.target for item in received] + body: Final = object_value(json.loads(received[0].body)) + parts: Final = object_value(body["content"])["parts"] + assert isinstance(parts, list), body + return parts + + +def vertex_model(gateway: Gateway, scenario: Scenario, url: str, **litellm_params: JsonValue) -> str: + return scenario.model( + model=f"vertex_ai/{BACKEND}", + api_key=None, + api_base=url, + vertex_project=PROJECT, + vertex_location=LOCATION, + vertex_credentials=service_account_json(PROJECT, gateway.upstream_url), + **litellm_params, + ) + + +def embed(gateway: Gateway, model: str, elements: JsonValue, **extra: JsonValue) -> httpx.Response: + return gateway.request("POST", "/v1/embeddings", {"model": model, "input": elements, **extra}) + + +def single_embedding(response: httpx.Response) -> JsonValue: + assert response.status_code == 200, response.text + data: Final = object_value(response.json())["data"] + assert isinstance(data, list) and len(data) == 1, response.text + item: Final = object_value(data[0]) + assert item["index"] == 0 and item["object"] == "embedding", response.text + return item["embedding"] + + +def rejected(gateway: Gateway, wire: Wire, model: str, elements: JsonValue, fragment: str) -> None: + response: Final = embed(gateway, model, elements) + assert response.status_code == 400, response.text + assert fragment in response.text, response.text + assert wire.drain() == (), "the rejected input reached the provider" + + +def test_string_input_reaches_embed_content_as_a_text_part(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = vertex_model(gateway, scenario, wire.url) + assert single_embedding(embed(gateway, model, ["a red bus"])) == VALUES + assert wire_parts(wire) == [TEXT_PART] + + +def test_file_block_with_video_metadata_reaches_embed_content(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = vertex_model(gateway, scenario, wire.url) + assert single_embedding(embed(gateway, model, [block(file_id=GCS_URI, video_metadata=METADATA)])) == VALUES + assert wire_parts(wire) == [GCS_PART] + + +def test_request_drop_params_strips_unknown_keys_at_every_level(gateway: Gateway) -> None: + noisy: Final[dict[str, JsonValue]] = { + **block(file_id=GCS_URI, mime_type="video/mp4", video_metadata={**METADATA, "frame_rate": 2}), + "caption": "unknown block key", + } + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = vertex_model(gateway, scenario, wire.url) + assert single_embedding(embed(gateway, model, [noisy], drop_params=True)) == VALUES + assert wire_parts(wire) == [GCS_PART] + + +def test_bad_fps_answers_400_before_any_provider_call(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = vertex_model(gateway, scenario, wire.url) + rejected(gateway, wire, model, [block(file_id=GCS_URI, video_metadata={"fps": "1"})], "file.video_metadata.fps") + + +def test_files_api_reference_answers_400_naming_the_gemini_provider(gateway: Gateway) -> None: + with wire_server(embed_peer) as wire, gateway.scenario() as scenario: + model: Final = vertex_model(gateway, scenario, wire.url) + rejected( + gateway, + wire, + model, + [block(file_id="files/abc", video_metadata=METADATA)], + "Gemini Files API references are only supported through the gemini/ provider", + ) diff --git a/tests/unit/caching/test_caching.py b/tests/unit/caching/test_caching.py index a799c45e4c7..c94ee836f91 100644 --- a/tests/unit/caching/test_caching.py +++ b/tests/unit/caching/test_caching.py @@ -953,6 +953,7 @@ def test_redis_caching_multiple_namespaces(): _TOOL_TURN_ITEM: Final = {"role": "user", "content": "hi"} +_FILE_BLOCK_ITEM: Final = {"type": "file", "file": {"file_data": "data:video/mp4;base64,AAAA", "format": "video/mp4"}} @pytest.mark.parametrize( @@ -964,6 +965,7 @@ _TOOL_TURN_ITEM: Final = {"role": "user", "content": "hi"} pytest.param({"input": [_TOOL_TURN_ITEM] * 5}, False, id="five-responses-items-skip-the-cache"), pytest.param({"input": "one prompt"}, True, id="string-input-is-one-message"), pytest.param({"input": ["a", "b", "c", "d", "e"]}, True, id="embedding-strings-are-not-messages"), + pytest.param({"input": [_FILE_BLOCK_ITEM] * 5}, True, id="embedding-file-blocks-are-not-messages"), ], ) def test_should_use_cache_stops_past_the_default_max_messages(kwargs: dict[str, object], expected: bool) -> None: diff --git a/tests/unit/caching/test_caching_handler.py b/tests/unit/caching/test_caching_handler.py index 22eea9935f6..d98c02667ee 100644 --- a/tests/unit/caching/test_caching_handler.py +++ b/tests/unit/caching/test_caching_handler.py @@ -41,6 +41,7 @@ from litellm.types.utils import ( from litellm.types.llms.openai import ResponsesAPIResponse from collections.abc import Awaitable, Callable from datetime import timedelta, datetime +from typing import Final from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLogging from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper from litellm._logging import verbose_logger @@ -2714,3 +2715,88 @@ async def test_async_get_cache_forgets_the_worker_copy_of_a_stored_response_with assert lookup.cached_result is None assert await handler.dual_cache.async_get_cache(key) is None + + +@pytest.mark.asyncio +async def test_async_get_cache_partial_hit_keeps_file_block_items_uncached() -> None: + setup_cache() + fixed_start: Final = datetime(2026, 1, 1) + caching_handler: Final = LLMCachingHandler(original_function=aembedding, request_kwargs={}, start_time=fixed_start) + model: Final = "gemini/gemini-embedding-2-preview" + logging_obj: Final = LiteLLMLogging( + litellm_call_id=str(uuid.uuid4()), + call_type=CallTypes.aembedding.value, + model=model, + messages=[], + function_id=str(uuid.uuid4()), + stream=False, + start_time=fixed_start, + ) + await caching_handler.async_set_cache( + result=EmbeddingResponse(model=model, data=[Embedding(embedding=[0.1, 0.2], index=0, object="embedding")]), + original_function=aembedding, + kwargs={"model": model, "input": ["a red bus"], "caching": True}, + ) + clip_block: Final = { + "type": "file", + "file": { + "file_data": "data:video/mp4;base64,AAAA", + "format": "video/mp4", + "video_metadata": {"fps": 1, "start_offset": "0s", "end_offset": "1s"}, + }, + "detail": "left for the provider transformation to judge", + } + + cached_response: Final = await caching_handler.async_get_cache( + model=model, + original_function=aembedding, + logging_obj=logging_obj, + start_time=fixed_start, + call_type=CallTypes.aembedding.value, + kwargs={"model": model, "input": [clip_block, "a red bus"], "caching": True}, + ) + + assert cached_response.embedding_all_elements_cache_hit is False + assert cached_response.embedding_uncached_input == [clip_block] + assert cached_response.final_embedding_cached_response is not None + assert cached_response.final_embedding_cached_response.data[1].embedding == [0.1, 0.2] + assert cached_response.final_embedding_cached_response.data[0] is None + + +def test_handle_kwargs_input_answers_400_for_a_single_object_input() -> None: + caching_handler: Final = LLMCachingHandler( + original_function=aembedding, request_kwargs={}, start_time=datetime(2026, 1, 1) + ) + clip_block: Final = {"type": "file", "file": {"file_data": "data:video/mp4;base64,AAAA"}} + with pytest.raises(litellm.BadRequestError, match="string or a list"): + caching_handler.handle_kwargs_input_list_or_str( + {"model": "gemini/gemini-embedding-2-preview", "custom_llm_provider": "gemini", "input": clip_block} + ) + + +@pytest.mark.asyncio +async def test_async_get_cache_answers_400_for_a_single_object_embedding_input() -> None: + setup_cache() + fixed_start: Final = datetime(2026, 1, 1) + caching_handler: Final = LLMCachingHandler(original_function=aembedding, request_kwargs={}, start_time=fixed_start) + model: Final = "gemini/gemini-embedding-2-preview" + logging_obj: Final = LiteLLMLogging( + litellm_call_id=str(uuid.uuid4()), + call_type=CallTypes.aembedding.value, + model=model, + messages=[], + function_id=str(uuid.uuid4()), + stream=False, + start_time=fixed_start, + ) + clip_block: Final = {"type": "file", "file": {"file_data": "data:video/mp4;base64,AAAA"}} + + with pytest.raises(litellm.BadRequestError, match="string or a list"): + await caching_handler.async_get_cache( + model=model, + original_function=aembedding, + logging_obj=logging_obj, + start_time=fixed_start, + call_type=CallTypes.aembedding.value, + kwargs={"model": model, "custom_llm_provider": "gemini", "input": clip_block, "caching": True}, + ) diff --git a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py index 6f18a391f7b..dde3df6fcc0 100644 --- a/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py +++ b/tests/unit/llms/vertex_ai/files/test_vertex_ai_files_transformation.py @@ -1289,6 +1289,33 @@ class TestVertexEmbeddingsBatchInputTranslation: assert set(row["request"]) == {"content"} + def test_should_translate_a_file_block_with_video_metadata(self) -> None: + (row,) = _wrap_entries( + [ + _embeddings_entry( + body={ + "model": "gemini-embedding-2", + "input": [ + { + "type": "file", + "file": { + "file_id": "gs://my-bucket/clip.mp4", + "video_metadata": {"start_offset": "3s", "end_offset": "6s"}, + }, + } + ], + } + ) + ] + ) + + assert row["request"]["content"]["parts"] == [ + { + "file_data": {"mime_type": "video/mp4", "file_uri": "gs://my-bucket/clip.mp4"}, + "video_metadata": {"startOffset": "3s", "endOffset": "6s"}, + } + ] + def test_should_translate_multimodal_gcs_input(self): (row,) = _wrap_entries( [ diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py new file mode 100644 index 00000000000..483b350fd43 --- /dev/null +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_handler.py @@ -0,0 +1,39 @@ +from typing import Final + +import httpx +import pytest + +from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler +from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler import GoogleBatchEmbeddings + +FILES_URI: Final = "https://generativelanguage.googleapis.com/v1beta/files/clip123" +FILE_METADATA_URL: Final = "https://generativelanguage.googleapis.com/v1beta/files/clip123" + + +def _files_api_metadata(request: httpx.Request) -> httpx.Response: + if str(request.url) != FILE_METADATA_URL or request.headers.get("x-goog-api-key") != "gemini-key": + return httpx.Response(404, json={"error": {"message": "no such file"}}) + return httpx.Response(200, json={"mimeType": "video/mp4", "uri": FILES_URI}) + + +@pytest.mark.parametrize("reference", ["files/clip123", FILES_URI]) +def test_resolve_file_references_fetches_the_file_metadata_for_both_reference_forms(reference: str) -> None: + resolved: Final = GoogleBatchEmbeddings()._resolve_file_references( + input=[reference, "a red bus"], + api_key="gemini-key", + sync_handler=HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(_files_api_metadata))), + ) + assert resolved == {reference: {"mime_type": "video/mp4", "uri": FILES_URI}} + + +@pytest.mark.asyncio +@pytest.mark.parametrize("reference", ["files/clip123", FILES_URI]) +async def test_async_resolve_file_references_fetches_the_file_metadata_for_both_reference_forms( + reference: str, +) -> None: + resolved: Final = await GoogleBatchEmbeddings()._async_resolve_file_references( + input=[reference, "a red bus"], + api_key="gemini-key", + async_handler=AsyncHTTPHandler(transport=httpx.MockTransport(_files_api_metadata)), + ) + assert resolved == {reference: {"mime_type": "video/mp4", "uri": FILES_URI}} diff --git a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py index ba2b26bf0a2..6584c27b895 100644 --- a/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py +++ b/tests/unit/llms/vertex_ai/gemini_embeddings/test_batch_embed_content_transformation.py @@ -8,12 +8,17 @@ Covers: - Response processing with correct indices """ +from collections.abc import Callable +from typing import Final + import pytest import litellm +from litellm.exceptions import BadRequestError from litellm.litellm_core_utils.llm_cost_calc.utils import generic_cost_per_token from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( _build_part_for_input, + file_reference_name, _is_multimodal_input, process_embed_content_response, process_response, @@ -25,6 +30,12 @@ from litellm.types.utils import EmbeddingResponse IMAGE_DATA_URI = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII" GCS_URL = "gs://my-bucket/image.png" +VIDEO_DATA_URI = "data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAA" +FILES_URI = "https://generativelanguage.googleapis.com/v1beta/files/clip123" + + +def _file_block(**file: object) -> dict[str, object]: + return {"type": "file", "file": file} @pytest.fixture(autouse=True) @@ -51,6 +62,7 @@ class TestIsMultimodalInput: def test_file_reference(self): assert _is_multimodal_input(["files/abc123"]) is True + assert _is_multimodal_input([FILES_URI]) is True def test_mixed_text_and_image(self): assert _is_multimodal_input(["hello", IMAGE_DATA_URI]) is True @@ -59,9 +71,16 @@ class TestIsMultimodalInput: """Nested list with text is not multimodal.""" assert _is_multimodal_input([["text_a", "text_b"]]) is False + def test_file_content_block_is_multimodal(self) -> None: + assert _is_multimodal_input([_file_block(file_data=VIDEO_DATA_URI)]) is True + def test_nested_list_with_image_is_multimodal(self): assert _is_multimodal_input([["a red shoe", IMAGE_DATA_URI]]) is True + def test_single_object_input_answers_400(self) -> None: + with pytest.raises(BadRequestError, match="string or a list"): + _is_multimodal_input(_file_block(file_data=VIDEO_DATA_URI)) + class TestBuildPartForInput: def test_text_input(self): @@ -83,15 +102,13 @@ class TestBuildPartForInput: assert part["file_data"]["file_uri"] == GCS_URL def test_file_reference_resolved(self): - resolved = { - "files/abc": {"mime_type": "image/jpeg", "uri": "https://example.com/abc"} - } + resolved = {"files/abc": {"mime_type": "image/jpeg", "uri": "https://example.com/abc"}} part = _build_part_for_input("files/abc", resolved_files=resolved) assert part["file_data"] is not None assert part["file_data"]["mime_type"] == "image/jpeg" - def test_file_reference_unresolved_raises(self): - with pytest.raises(ValueError, match="not resolved"): + def test_file_reference_unresolved_answers_400_naming_the_gemini_provider(self) -> None: + with pytest.raises(BadRequestError, match="gemini/ provider"): _build_part_for_input("files/abc") @@ -124,10 +141,7 @@ class TestTransformOpenaiInputGeminiContent: ) assert len(result["requests"]) == 2 # First request is text - assert ( - result["requests"][0]["content"]["parts"][0]["text"] - == "The food was delicious" - ) + assert result["requests"][0]["content"]["parts"][0]["text"] == "The food was delicious" # Second request is image assert result["requests"][1]["content"]["parts"][0]["inline_data"] is not None @@ -226,9 +240,7 @@ class TestProcessResponse: """Test that process_response sets correct indices.""" def test_single_embedding_index(self): - predictions: VertexAIBatchEmbeddingsResponseObject = { - "embeddings": [{"values": [0.1, 0.2]}] - } + predictions: VertexAIBatchEmbeddingsResponseObject = {"embeddings": [{"values": [0.1, 0.2]}]} model_response = EmbeddingResponse() result = process_response( input="hello", @@ -279,9 +291,7 @@ class TestProcessResponse: def test_nested_input_token_counting(self): """Nested list: only plain-text sub-elements should be counted.""" - predictions: VertexAIBatchEmbeddingsResponseObject = { - "embeddings": [{"values": [0.1, 0.2]}] - } + predictions: VertexAIBatchEmbeddingsResponseObject = {"embeddings": [{"values": [0.1, 0.2]}]} result = process_response( input=[["a red shoe", IMAGE_DATA_URI]], model_response=EmbeddingResponse(), @@ -300,7 +310,7 @@ class TestProcessResponse: ) def test_nested_non_string_element_raises(self): - with pytest.raises(ValueError, match="must be strings"): + with pytest.raises(BadRequestError, match="must be strings or file content blocks, got list"): transform_openai_input_gemini_content( input=[[["doubly", "nested"]]], model="gemini-embedding-2-preview", @@ -408,3 +418,212 @@ class TestProcessEmbedContentResponseUsage: assert result.usage.prompt_tokens > 0 +class TestFileContentBlocks: + MODEL: Final = "gemini-embedding-2-preview" + CLIP_METADATA: Final = {"fps": 2, "start_offset": "3s", "end_offset": "6s"} + CLIP_PART: Final = {"fps": 2.0, "startOffset": "3s", "endOffset": "6s"} + + def test_data_uri_block_forwards_format_and_video_metadata(self) -> None: + part: Final = _build_part_for_input( + _file_block( + file_data="data:application/octet-stream;base64,QUJD", + format="video/mp4", + video_metadata=self.CLIP_METADATA, + ) + ) + assert part == { + "inline_data": {"mime_type": "video/mp4", "data": "QUJD"}, + "video_metadata": self.CLIP_PART, + } + + def test_gcs_block_infers_mime_type_and_forwards_video_metadata(self) -> None: + part: Final = _build_part_for_input( + _file_block(file_id="gs://my-bucket/clip.mp4", video_metadata={"start_offset": "0s", "end_offset": "3s"}) + ) + assert part == { + "file_data": {"mime_type": "video/mp4", "file_uri": "gs://my-bucket/clip.mp4"}, + "video_metadata": {"startOffset": "0s", "endOffset": "3s"}, + } + + def test_file_reference_block_uses_the_resolved_file(self) -> None: + resolved_files: Final = {"files/clip123": {"mime_type": "video/mp4", "uri": FILES_URI}} + part: Final = _build_part_for_input( + _file_block(file_id="files/clip123", video_metadata={"fps": 1}), + resolved_files=resolved_files, + ) + assert part == { + "file_data": {"mime_type": "video/mp4", "file_uri": FILES_URI}, + "video_metadata": {"fps": 1.0}, + } + + def test_files_api_uri_block_uses_the_resolved_file(self) -> None: + resolved_files: Final = {FILES_URI: {"mime_type": "video/mp4", "uri": FILES_URI}} + part: Final = _build_part_for_input( + _file_block(file_id=FILES_URI, video_metadata={"start_offset": "0s", "end_offset": "3s"}), + resolved_files=resolved_files, + ) + assert part == { + "file_data": {"mime_type": "video/mp4", "file_uri": FILES_URI}, + "video_metadata": {"startOffset": "0s", "endOffset": "3s"}, + } + + @pytest.mark.parametrize("reference", ["files/clip123", FILES_URI]) + def test_file_reference_name_is_the_files_name_for_both_forms(self, reference: str) -> None: + assert file_reference_name(reference) == "files/clip123" + + def test_block_without_video_metadata_sends_no_video_metadata_key(self) -> None: + part: Final = _build_part_for_input(_file_block(file_data=IMAGE_DATA_URI, filename="dot.png")) + assert part == {"inline_data": {"mime_type": "image/png", "data": IMAGE_DATA_URI.split(",", 1)[1]}} + + def test_block_with_empty_video_metadata_sends_no_video_metadata_key(self) -> None: + part: Final = _build_part_for_input(_file_block(file_data=VIDEO_DATA_URI, video_metadata={})) + assert part == {"inline_data": {"mime_type": "video/mp4", "data": VIDEO_DATA_URI.split(",", 1)[1]}} + + def test_batch_path_nested_block_and_text_share_one_request(self) -> None: + result: Final = transform_openai_input_gemini_content( + input=[[_file_block(file_data=VIDEO_DATA_URI, video_metadata=self.CLIP_METADATA), "a solid color clip"]], + model=self.MODEL, + optional_params={"dimensions": 768}, + ) + [request] = result["requests"] + assert request["outputDimensionality"] == 768 + assert request["content"]["parts"] == [ + { + "inline_data": {"mime_type": "video/mp4", "data": VIDEO_DATA_URI.split(",", 1)[1]}, + "video_metadata": self.CLIP_PART, + }, + {"text": "a solid color clip"}, + ] + + def test_batch_path_flat_block_and_text_are_separate_requests(self) -> None: + result: Final = transform_openai_input_gemini_content( + input=[_file_block(file_data=VIDEO_DATA_URI, video_metadata=self.CLIP_METADATA), "a solid color clip"], + model=self.MODEL, + optional_params={}, + ) + assert len(result["requests"]) == 2 + assert result["requests"][0]["content"]["parts"][0]["video_metadata"] == self.CLIP_PART + assert result["requests"][1]["content"]["parts"] == [{"text": "a solid color clip"}] + + def test_embed_content_path_accepts_flat_block_and_text(self) -> None: + result: Final = transform_openai_input_gemini_embed_content( + input=[_file_block(file_data=VIDEO_DATA_URI, video_metadata=self.CLIP_METADATA), "a solid color clip"], + model=self.MODEL, + optional_params={}, + ) + parts: Final = result["content"]["parts"] + assert parts[0]["video_metadata"] == self.CLIP_PART + assert parts[1] == {"text": "a solid color clip"} + + def test_embed_content_path_still_rejects_nested_lists(self) -> None: + with pytest.raises(ValueError, match="Nested"): + transform_openai_input_gemini_embed_content( + input=[[_file_block(file_data=VIDEO_DATA_URI), "a solid color clip"]], + model=self.MODEL, + optional_params={}, + ) + + @pytest.mark.parametrize( + "block, named_in_error", + [ + ( + _file_block(file_data=VIDEO_DATA_URI, video_metadata={"fps": 1, "startOffset": "1s"}), + "video_metadata.startOffset", + ), + (_file_block(file_data=VIDEO_DATA_URI, video_metadata={"fps": "fast"}), "video_metadata.fps"), + (_file_block(file_data=VIDEO_DATA_URI, video_metadata={"fps": "1"}), "video_metadata.fps"), + (_file_block(file_data=VIDEO_DATA_URI, video_metadata={"fps": True}), "video_metadata.fps"), + (_file_block(file_data=VIDEO_DATA_URI, video_metadata={"start_offset": 5}), "video_metadata.start_offset"), + (_file_block(file_data=VIDEO_DATA_URI, detail="high"), "file.detail"), + (_file_block(file_data=VIDEO_DATA_URI, format=""), "file.format"), + (_file_block(file_id="gs://my-bucket/clip.mp4", file_data=VIDEO_DATA_URI), "not both"), + (_file_block(), "needs file.file_id or file.file_data"), + ( + _file_block(file_id="https://example.com/clip.mp4"), + "a data: URI, a gs:// URL, a files/ reference, or a Gemini Files API URI", + ), + ({"type": "image_url", "image_url": {"url": IMAGE_DATA_URI}}, "Input should be 'file'"), + ], + ) + def test_malformed_block_answers_400_naming_the_field( + self, block: dict[str, object], named_in_error: str + ) -> None: + with pytest.raises(BadRequestError, match=named_in_error): + _build_part_for_input(block) + + def test_drop_params_drops_the_block_keys_this_surface_does_not_take(self) -> None: + block: Final = _file_block( + file_data=VIDEO_DATA_URI, + detail="high", + video_metadata={"fps": 1, "start_offset": "1s", "resolution": "low"}, + ) + part: Final = _build_part_for_input({**block, "cache_control": {"type": "ephemeral"}}, drop_params=True) + assert part["inline_data"]["mime_type"] == "video/mp4" + assert part["video_metadata"] == {"fps": 1.0, "startOffset": "1s"} + + def test_drop_params_still_answers_400_for_a_malformed_value(self) -> None: + block: Final = _file_block(file_data=VIDEO_DATA_URI, detail="high", video_metadata={"fps": "fast"}) + with pytest.raises(BadRequestError, match=r"video_metadata\.fps"): + _build_part_for_input(block, drop_params=True) + + @pytest.mark.parametrize( + "transform", [transform_openai_input_gemini_content, transform_openai_input_gemini_embed_content] + ) + def test_transforms_forward_drop_params_to_every_block(self, transform: Callable[..., object]) -> None: + block: Final = _file_block(file_data=VIDEO_DATA_URI, detail="high") + with pytest.raises(BadRequestError, match=r"file\.detail"): + transform(input=[block], model="gemini-embedding-2-preview", optional_params={}) + transform(input=[block], model="gemini-embedding-2-preview", optional_params={}, drop_params=True) + + def test_batch_path_forwards_drop_params_into_nested_lists(self) -> None: + block: Final = _file_block(file_data=VIDEO_DATA_URI, detail="high") + body: Final = transform_openai_input_gemini_content( + input=[[block, "a caption"]], model="gemini-embedding-2-preview", optional_params={}, drop_params=True + ) + assert len(body["requests"][0]["content"]["parts"]) == 2 + + @pytest.mark.parametrize( + "transform", [transform_openai_input_gemini_content, transform_openai_input_gemini_embed_content] + ) + def test_single_object_input_answers_400(self, transform: Callable[..., object]) -> None: + with pytest.raises(BadRequestError, match="string or a list"): + transform( + input=_file_block(file_data=VIDEO_DATA_URI), model="gemini-embedding-2-preview", optional_params={} + ) + + def test_process_response_counts_only_the_text_tokens_next_to_a_block(self) -> None: + text: Final = "a solid color clip" + with_block: Final = process_response( + input=[_file_block(file_data=VIDEO_DATA_URI, video_metadata=self.CLIP_METADATA), text], + model_response=EmbeddingResponse(), + model=self.MODEL, + _predictions={"embeddings": [{"values": [0.1]}, {"values": [0.2]}]}, + ) + text_only: Final = process_response( + input=[text], + model_response=EmbeddingResponse(), + model=self.MODEL, + _predictions={"embeddings": [{"values": [0.2]}]}, + ) + assert with_block.usage.prompt_tokens == text_only.usage.prompt_tokens > 0 + + def test_embed_content_usage_fallback_with_a_block_does_not_estimate(self) -> None: + result: Final = process_embed_content_response( + input=[_file_block(file_data=VIDEO_DATA_URI, video_metadata=self.CLIP_METADATA)], + model_response=EmbeddingResponse(), + model=self.MODEL, + response_json={"embedding": {"values": [0.1, 0.2]}}, + ) + assert result.usage.prompt_tokens == 0 + + def test_image_block_counts_as_image_only_input(self) -> None: + result: Final = process_embed_content_response( + input=[_file_block(file_data=IMAGE_DATA_URI)], + model_response=EmbeddingResponse(), + model=self.MODEL, + response_json={ + "embedding": {"values": [0.1, 0.2]}, + "usageMetadata": {"promptTokenCount": 258, "totalTokenCount": 258}, + }, + ) + assert result.usage.prompt_tokens_details.image_tokens == 258 diff --git a/tests/unit/llms/vertex_ai/test_gemini_batch_embeddings.py b/tests/unit/llms/vertex_ai/test_gemini_batch_embeddings.py index 98abf5459df..7271fb48861 100644 --- a/tests/unit/llms/vertex_ai/test_gemini_batch_embeddings.py +++ b/tests/unit/llms/vertex_ai/test_gemini_batch_embeddings.py @@ -8,11 +8,15 @@ This test ensures that: """ import json +from contextlib import AbstractContextManager +from typing import Final from unittest.mock import MagicMock, patch import pytest import litellm +import httpx + from litellm.llms.custom_httpx.http_handler import HTTPHandler from litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_transformation import ( _filter_embed_params, @@ -655,3 +659,163 @@ def test_batch_embeddings_response_has_correct_indices_and_order(): assert ( embedding.embedding == expected_values[i] ), f"embedding {i} has wrong values: {embedding.embedding}" + + +CLIP_BLOCK = { + "type": "file", + "file": { + "file_data": "data:video/mp4;base64,AAAAIGZ0eXBpc29tAAACAGlzb21pc28yYXZjMW1wNDEAAAAIZnJlZQAA", + "video_metadata": {"fps": 2, "start_offset": "3s", "end_offset": "6s"}, + }, +} +CLIP_PART_METADATA = {"fps": 2.0, "startOffset": "3s", "endOffset": "6s"} + + +def _mock_embedding_call( + response_json: dict[str, object], +) -> tuple[AbstractContextManager[MagicMock], AbstractContextManager[MagicMock], MagicMock]: + mock_get_token: Final = patch( + "litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler.GoogleBatchEmbeddings._get_token_and_url" + ) + mock_auth: Final = patch( + "litellm.llms.vertex_ai.gemini_embeddings.batch_embed_content_handler.GoogleBatchEmbeddings._ensure_access_token", + side_effect=lambda *args, **kwargs: (None, "test-project"), + ) + mock_response: Final = MagicMock() + mock_response.status_code = 200 + mock_response.json.return_value = response_json + return mock_get_token, mock_auth, mock_response + + +def test_gemini_batch_path_sends_file_block_video_metadata() -> None: + client: Final = HTTPHandler() + mock_get_token, mock_auth, mock_response = _mock_embedding_call( + {"embeddings": [{"values": [0.1, 0.2]}, {"values": [0.3, 0.4]}]} + ) + with patch.object(client, "post", return_value=mock_response) as mock_post, mock_auth, mock_get_token as token: + token.return_value = ( + {"x-goog-api-key": "test-key"}, + "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-2-preview:batchEmbedContents", + ) + response: Final = litellm.embedding( + model="gemini/gemini-embedding-2-preview", + input=[CLIP_BLOCK, "a solid color clip"], + api_key="test-key", + client=client, + ) + + request_body: Final = json.loads(mock_post.call_args.kwargs["data"]) + clip_part: Final = request_body["requests"][0]["content"]["parts"][0] + assert clip_part["inline_data"]["mime_type"] == "video/mp4" + assert clip_part["video_metadata"] == CLIP_PART_METADATA + assert request_body["requests"][1]["content"]["parts"] == [{"text": "a solid color clip"}] + assert [row["index"] for row in response.data] == [0, 1] + + +def test_vertex_embed_content_path_sends_file_block_video_metadata() -> None: + client: Final = HTTPHandler() + url: Final = "https://us-central1-aiplatform.googleapis.com/v1/projects/test/locations/us-central1/publishers/google/models/gemini-embedding-2-preview:embedContent" + mock_get_token, mock_auth, mock_response = _mock_embedding_call({"embedding": {"values": [0.1, 0.2]}}) + with patch.object(client, "post", return_value=mock_response) as mock_post, mock_auth, mock_get_token as token: + token.return_value = ({"Authorization": "Bearer test-token"}, url) + response: Final = litellm.embedding( + model="vertex_ai/gemini-embedding-2-preview", + input=[CLIP_BLOCK, "a solid color clip"], + vertex_project="test-project", + vertex_location="us-central1", + client=client, + ) + + data: Final = json.loads(mock_post.call_args.kwargs["data"]) + assert data["content"]["parts"][0]["video_metadata"] == CLIP_PART_METADATA + assert data["content"]["parts"][1] == {"text": "a solid color clip"} + assert len(response.data) == 1 + + +def test_file_block_with_files_reference_is_resolved_through_the_files_api() -> None: + client: Final = HTTPHandler() + files_uri: Final = "https://generativelanguage.googleapis.com/v1beta/files/clip123" + mock_get_token, mock_auth, mock_response = _mock_embedding_call({"embeddings": [{"values": [0.1, 0.2]}]}) + file_lookup: Final = MagicMock() + file_lookup.status_code = 200 + file_lookup.json.return_value = {"mimeType": "video/mp4", "uri": files_uri} + with ( + patch.object(client, "post", return_value=mock_response) as mock_post, + patch.object(client, "get", return_value=file_lookup) as mock_get, + mock_auth, + mock_get_token as token, + ): + token.return_value = ( + {"x-goog-api-key": "test-key"}, + "https://generativelanguage.googleapis.com/v1beta/models/gemini-embedding-2-preview:batchEmbedContents", + ) + litellm.embedding( + model="gemini/gemini-embedding-2-preview", + input=[{"type": "file", "file": {"file_id": "files/clip123", "video_metadata": {"fps": 1}}}], + api_key="test-key", + client=client, + ) + + assert mock_get.call_args.kwargs["url"] == "https://generativelanguage.googleapis.com/v1beta/files/clip123" + clip_part: Final = json.loads(mock_post.call_args.kwargs["data"])["requests"][0]["content"]["parts"][0] + assert clip_part == { + "file_data": {"mime_type": "video/mp4", "file_uri": files_uri}, + "video_metadata": {"fps": 1.0}, + } + + +CLIP_BLOCK_WITH_DETAIL: Final = {"type": "file", "file": {**CLIP_BLOCK["file"], "detail": "high"}} + + +def _recording_client(calls: list[httpx.Request], response_json: dict[str, object]) -> HTTPHandler: + def route(request: httpx.Request) -> httpx.Response: + calls.append(request) + return httpx.Response(200, json=response_json) + + return HTTPHandler(client=httpx.Client(transport=httpx.MockTransport(route))) + + +def test_gemini_drop_params_strips_the_block_keys_embeddings_do_not_take() -> None: + calls: Final[list[httpx.Request]] = [] + client: Final = _recording_client(calls, {"embeddings": [{"values": [0.1, 0.2]}]}) + response: Final = litellm.embedding( + model="gemini/gemini-embedding-2-preview", + input=[CLIP_BLOCK_WITH_DETAIL], + api_key="test-key", + client=client, + drop_params=True, + ) + sent_part: Final = json.loads(calls[0].content)["requests"][0]["content"]["parts"][0] + assert sent_part["video_metadata"] == CLIP_PART_METADATA + assert "detail" not in calls[0].content.decode() + assert response.data[0].embedding == [0.1, 0.2] + + +def test_gemini_block_detail_answers_400_without_drop_params() -> None: + calls: Final[list[httpx.Request]] = [] + client: Final = _recording_client(calls, {"embeddings": [{"values": [0.1, 0.2]}]}) + with pytest.raises(litellm.BadRequestError, match=r"file\.detail"): + litellm.embedding( + model="gemini/gemini-embedding-2-preview", input=[CLIP_BLOCK_WITH_DETAIL], api_key="test-key", client=client + ) + assert calls == [] + + +def test_vertex_drop_params_strips_the_block_keys_embeddings_do_not_take() -> None: + client: Final = HTTPHandler() + url: Final = "https://us-central1-aiplatform.googleapis.com/v1/projects/test/locations/us-central1/publishers/google/models/gemini-embedding-2-preview:embedContent" + mock_get_token, mock_auth, mock_response = _mock_embedding_call({"embedding": {"values": [0.1, 0.2]}}) + with patch.object(client, "post", return_value=mock_response) as mock_post, mock_auth, mock_get_token as token: + token.return_value = ({"Authorization": "Bearer test-token"}, url) + litellm.embedding( + model="vertex_ai/gemini-embedding-2-preview", + input=[CLIP_BLOCK_WITH_DETAIL], + vertex_project="test-project", + vertex_location="us-central1", + client=client, + drop_params=True, + ) + + data: Final = json.loads(mock_post.call_args.kwargs["data"]) + assert data["content"]["parts"][0]["video_metadata"] == CLIP_PART_METADATA + assert "detail" not in mock_post.call_args.kwargs["data"] From 9505af063911a0b935c199fb7a9f819b5224d7ae Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 8 Oct 2026 13:22:58 -0700 Subject: [PATCH 61/65] feat(rust): add the Anthropic beta header policy (#45419) * feat(rust): add the Anthropic beta header policy Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(rust): resolve known betas held as Other in AnthropicBeta::on Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rust): use named rstest cases for Other beta resolution Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * docs(rust): make the rstest named-case rule explicit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(rust): move BetaPolicy tests to the public API test crate Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm-rust/AGENTS.md | 3 + .../crates/llms-types/src/providers/AGENTS.md | 3 +- .../llms-types/src/providers/anthropic.rs | 368 +++++++++++++++++- .../crates/llms/src/anthropic/beta_headers.rs | 36 ++ litellm-rust/crates/llms/src/anthropic/mod.rs | 1 + .../llms/tests/anthropic_beta_headers.rs | 111 ++++++ 6 files changed, 519 insertions(+), 3 deletions(-) create mode 100644 litellm-rust/crates/llms/src/anthropic/beta_headers.rs create mode 100644 litellm-rust/crates/llms/tests/anthropic_beta_headers.rs diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index 442dfd8e957..2c099c6f7a2 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -17,6 +17,9 @@ For string-valued enums and their Serde conversions, follow [.agents/skills/rust Use [`#[rstest]`](https://docs.rs/rstest/latest/rstest/attr.rstest.html) for new and updated tests and [`#[fixture]`](https://docs.rs/rstest/latest/rstest/attr.fixture.html) for reusable setup, injected through typed test arguments. Express input variations as named `#[case::name(...)]` cases instead of loops or duplicated tests so each failure identifies its case. Keep behavior assertions in the test body and fixtures focused on setup. Use the workspace `rstest` dependency +- Never loop over inputs (`for`, `.iter().for_each`, `.all`) inside a test body; give each input its own `#[case::name(...)]`, or use `#[values(...)]` for a cross product +- Exception: a test pinning a Rust table against a repo-owned data file (for example `include_str!` of a JSON config) may iterate that file's entries + ## Error definitions - A crate's errors live in `src/error.rs`, defined with `thiserror`, and re-exported from `lib.rs` diff --git a/litellm-rust/crates/llms-types/src/providers/AGENTS.md b/litellm-rust/crates/llms-types/src/providers/AGENTS.md index 69712edbcbb..128d29652d1 100644 --- a/litellm-rust/crates/llms-types/src/providers/AGENTS.md +++ b/litellm-rust/crates/llms-types/src/providers/AGENTS.md @@ -4,7 +4,8 @@ These links describe the provider-specific wire types in this directory. Shared ## anthropic.rs -`AnthropicBeta` and `BetaSet` represent the `anthropic-beta` header values and their wire spelling +`AnthropicBeta`, `BetaSet` and `BetaProvider` represent the `anthropic-beta` header values, their +wire spelling and per-host support - https://platform.claude.com/docs/en/api/beta-headers.md diff --git a/litellm-rust/crates/llms-types/src/providers/anthropic.rs b/litellm-rust/crates/llms-types/src/providers/anthropic.rs index 3e0c4369b11..c2dcefce082 100644 --- a/litellm-rust/crates/llms-types/src/providers/anthropic.rs +++ b/litellm-rust/crates/llms-types/src/providers/anthropic.rs @@ -7,6 +7,34 @@ use std::{ str::FromStr, }; +/// A provider column of `litellm/anthropic_beta_headers_config.json`: which betas a host accepts +/// and under which name. +#[derive( + Clone, Copy, Debug, PartialEq, Eq, Hash, strum::AsRefStr, strum::Display, strum::EnumString, +)] +#[strum(serialize_all = "snake_case")] +pub enum BetaProvider { + Anthropic, + AzureAi, + BedrockConverse, + Bedrock, + BedrockMantle, + VertexAi, + Databricks, +} + +impl BetaProvider { + pub const ALL: &[Self] = &[ + Self::Anthropic, + Self::AzureAi, + Self::BedrockConverse, + Self::Bedrock, + Self::BedrockMantle, + Self::VertexAi, + Self::Databricks, + ]; +} + /// One value of the `anthropic-beta` header. Equality, ordering and hashing follow the wire /// string, so a value parsed from a caller's header never disagrees with the matching variant. #[derive(Clone, Debug, strum::AsRefStr, strum::Display, strum::EnumString)] @@ -35,12 +63,66 @@ pub enum AnthropicBeta { PerTurnControl20260701, #[strum(serialize = "dangerous-tool-use-2026-09-03")] DangerousToolUse20260903, + #[strum(serialize = "bash_20241022")] + Bash20241022, + #[strum(serialize = "bash_20250124")] + Bash20250124, + #[strum(serialize = "claude-code-20250219")] + ClaudeCode20250219, + #[strum(serialize = "code-execution-2025-08-25")] + CodeExecution20250825, + #[strum(serialize = "computer-use-2025-01-24")] + ComputerUse20250124, + #[strum(serialize = "computer-use-2025-11-24")] + ComputerUse20251124, + #[strum(serialize = "context-1m-2025-08-07")] + Context1m20250807, + #[strum(serialize = "effort-2025-11-24")] + Effort20251124, + #[strum(serialize = "files-api-2025-04-14")] + FilesApi20250414, + #[strum(serialize = "fine-grained-tool-streaming-2025-05-14")] + FineGrainedToolStreaming20250514, + #[strum(serialize = "interleaved-thinking-2025-05-14")] + InterleavedThinking20250514, + #[strum(serialize = "mcp-client-2025-04-04")] + McpClient20250404, + #[strum(serialize = "mcp-client-2025-11-20")] + McpClient20251120, + #[strum(serialize = "mcp-servers-2025-12-04")] + McpServers20251204, + #[strum(serialize = "mid-conversation-output-config-2026-07-01")] + MidConversationOutputConfig20260701, + #[strum(serialize = "mid-conversation-tool-changes-2026-07-01")] + MidConversationToolChanges20260701, + #[strum(serialize = "output-128k-2025-02-19")] + Output128k20250219, + #[strum(serialize = "prompt-caching-scope-2026-01-05")] + PromptCachingScope20260105, + #[strum(serialize = "skills-2025-10-02")] + Skills20251002, + #[strum(serialize = "structured-output-2024-03-01")] + StructuredOutput20240301, + #[strum(serialize = "text_editor_20241022")] + TextEditor20241022, + #[strum(serialize = "text_editor_20250124")] + TextEditor20250124, + #[strum(serialize = "thinking-binding-controls-2026-08-01")] + ThinkingBindingControls20260801, + #[strum(serialize = "thinking-display-updates-2026-08-18")] + ThinkingDisplayUpdates20260818, + #[strum(serialize = "token-efficient-tools-2025-02-19")] + TokenEfficientTools20250219, + #[strum(serialize = "tool-examples-2025-10-29")] + ToolExamples20251029, + #[strum(serialize = "tool-search-tool-2025-10-19")] + ToolSearchTool20251019, #[strum(default, transparent)] Other(String), } impl AnthropicBeta { - pub const KNOWN: [Self; 12] = [ + pub const KNOWN: [Self; 39] = [ Self::Oauth20250420, Self::WebFetch20250910, Self::WebSearch20250305, @@ -53,11 +135,145 @@ impl AnthropicBeta { Self::AdvisorTool20260301, Self::PerTurnControl20260701, Self::DangerousToolUse20260903, + Self::Bash20241022, + Self::Bash20250124, + Self::ClaudeCode20250219, + Self::CodeExecution20250825, + Self::ComputerUse20250124, + Self::ComputerUse20251124, + Self::Context1m20250807, + Self::Effort20251124, + Self::FilesApi20250414, + Self::FineGrainedToolStreaming20250514, + Self::InterleavedThinking20250514, + Self::McpClient20250404, + Self::McpClient20251120, + Self::McpServers20251204, + Self::MidConversationOutputConfig20260701, + Self::MidConversationToolChanges20260701, + Self::Output128k20250219, + Self::PromptCachingScope20260105, + Self::Skills20251002, + Self::StructuredOutput20240301, + Self::TextEditor20241022, + Self::TextEditor20250124, + Self::ThinkingBindingControls20260801, + Self::ThinkingDisplayUpdates20260818, + Self::TokenEfficientTools20250219, + Self::ToolExamples20251029, + Self::ToolSearchTool20251019, ]; pub fn as_str(&self) -> &str { self.as_ref() } + + pub fn on(&self, provider: BetaProvider) -> Option { + let resolved = match self { + Self::Other(raw) => raw.parse().unwrap(), + _ => self.clone(), + }; + if resolved.rejected_by().contains(&provider) { + return None; + } + Some(match (resolved, provider) { + ( + Self::AdvancedToolUse20251120, + BetaProvider::Bedrock | BetaProvider::BedrockMantle | BetaProvider::VertexAi, + ) => Self::ToolSearchTool20251019, + (beta, _) => beta, + }) + } + + fn rejected_by(&self) -> &'static [BetaProvider] { + match self { + Self::AdvancedToolUse20251120 | Self::ContextManagement20250627 => { + &[BetaProvider::BedrockConverse] + } + Self::AdvisorTool20260301 | Self::Compact20260904 => &[ + BetaProvider::AzureAi, + BetaProvider::BedrockConverse, + BetaProvider::Bedrock, + BetaProvider::BedrockMantle, + BetaProvider::VertexAi, + BetaProvider::Databricks, + ], + Self::Bash20241022 + | Self::Bash20250124 + | Self::McpServers20251204 + | Self::StructuredOutput20240301 + | Self::TextEditor20241022 + | Self::TextEditor20250124 => BetaProvider::ALL, + Self::ClaudeCode20250219 | Self::ToolExamples20251029 => &[ + BetaProvider::Anthropic, + BetaProvider::AzureAi, + BetaProvider::BedrockConverse, + BetaProvider::Bedrock, + BetaProvider::VertexAi, + BetaProvider::Databricks, + ], + Self::CodeExecution20250825 + | Self::FilesApi20250414 + | Self::McpClient20250404 + | Self::McpClient20251120 + | Self::PromptCachingScope20260105 + | Self::Skills20251002 + | Self::WebFetch20250910 => &[ + BetaProvider::BedrockConverse, + BetaProvider::Bedrock, + BetaProvider::BedrockMantle, + BetaProvider::VertexAi, + ], + Self::Compact20260112 => &[BetaProvider::AzureAi, BetaProvider::BedrockConverse], + Self::ComputerUse20250124 | Self::ComputerUse20251124 | Self::Context1m20250807 => &[], + Self::DangerousToolUse20260903 => { + &[BetaProvider::BedrockConverse, BetaProvider::Databricks] + } + Self::Effort20251124 => &[BetaProvider::VertexAi], + Self::FastMode20260201 | Self::Oauth20250420 => &[ + BetaProvider::AzureAi, + BetaProvider::BedrockConverse, + BetaProvider::Bedrock, + BetaProvider::BedrockMantle, + BetaProvider::VertexAi, + ], + Self::FineGrainedToolStreaming20250514 => { + &[BetaProvider::AzureAi, BetaProvider::VertexAi] + } + Self::InterleavedThinking20250514 | Self::WebSearch20250305 => { + &[BetaProvider::BedrockConverse, BetaProvider::Bedrock] + } + Self::MidConversationOutputConfig20260701 + | Self::MidConversationToolChanges20260701 => &[ + BetaProvider::AzureAi, + BetaProvider::BedrockConverse, + BetaProvider::BedrockMantle, + BetaProvider::VertexAi, + BetaProvider::Databricks, + ], + Self::Output128k20250219 | Self::TokenEfficientTools20250219 => &[ + BetaProvider::AzureAi, + BetaProvider::BedrockConverse, + BetaProvider::Bedrock, + BetaProvider::VertexAi, + ], + Self::PerTurnControl20260701 => &[ + BetaProvider::BedrockConverse, + BetaProvider::Bedrock, + BetaProvider::Databricks, + ], + Self::StructuredOutputs20251113 => &[BetaProvider::Bedrock, BetaProvider::VertexAi], + Self::ThinkingBindingControls20260801 => &[BetaProvider::AzureAi], + Self::ThinkingDisplayUpdates20260818 => &[BetaProvider::Databricks], + Self::ToolSearchTool20251019 => &[ + BetaProvider::Anthropic, + BetaProvider::AzureAi, + BetaProvider::BedrockConverse, + BetaProvider::Databricks, + ], + Self::Other(_) => BetaProvider::ALL, + } + } } impl PartialEq for AnthropicBeta { @@ -149,10 +365,20 @@ impl fmt::Display for BetaSet { #[cfg(test)] mod tests { + use std::collections::BTreeSet; + + use indexmap::IndexMap; use rstest::rstest; use super::*; + fn beta_headers_config() -> IndexMap { + serde_json::from_str(include_str!( + "../../../../../litellm/anthropic_beta_headers_config.json" + )) + .unwrap() + } + fn set(header: &str) -> BetaSet { header.parse().unwrap_or_else(|never| match never {}) } @@ -171,7 +397,34 @@ mod tests { AnthropicBeta::FastMode20260201, AnthropicBeta::AdvisorTool20260301, AnthropicBeta::PerTurnControl20260701, - AnthropicBeta::DangerousToolUse20260903 + AnthropicBeta::DangerousToolUse20260903, + AnthropicBeta::Bash20241022, + AnthropicBeta::Bash20250124, + AnthropicBeta::ClaudeCode20250219, + AnthropicBeta::CodeExecution20250825, + AnthropicBeta::ComputerUse20250124, + AnthropicBeta::ComputerUse20251124, + AnthropicBeta::Context1m20250807, + AnthropicBeta::Effort20251124, + AnthropicBeta::FilesApi20250414, + AnthropicBeta::FineGrainedToolStreaming20250514, + AnthropicBeta::InterleavedThinking20250514, + AnthropicBeta::McpClient20250404, + AnthropicBeta::McpClient20251120, + AnthropicBeta::McpServers20251204, + AnthropicBeta::MidConversationOutputConfig20260701, + AnthropicBeta::MidConversationToolChanges20260701, + AnthropicBeta::Output128k20250219, + AnthropicBeta::PromptCachingScope20260105, + AnthropicBeta::Skills20251002, + AnthropicBeta::StructuredOutput20240301, + AnthropicBeta::TextEditor20241022, + AnthropicBeta::TextEditor20250124, + AnthropicBeta::ThinkingBindingControls20260801, + AnthropicBeta::ThinkingDisplayUpdates20260818, + AnthropicBeta::TokenEfficientTools20250219, + AnthropicBeta::ToolExamples20251029, + AnthropicBeta::ToolSearchTool20251019 )] beta: AnthropicBeta, ) { @@ -181,6 +434,117 @@ mod tests { assert!(AnthropicBeta::KNOWN.contains(&beta)); } + #[rstest] + fn known_betas_are_exactly_the_config_keys() { + let config = beta_headers_config(); + let config_keys: BTreeSet = config + .iter() + .filter(|(provider, _)| provider.as_str() != "description") + .flat_map(|(_, column)| { + column + .as_object() + .into_iter() + .flat_map(|values| values.keys().cloned()) + }) + .collect(); + let known_keys: BTreeSet = AnthropicBeta::KNOWN + .iter() + .map(|beta| beta.as_str().to_string()) + .collect(); + assert_eq!(known_keys, config_keys); + } + + #[rstest] + #[case::anthropic(BetaProvider::Anthropic)] + #[case::azure_ai(BetaProvider::AzureAi)] + #[case::bedrock_converse(BetaProvider::BedrockConverse)] + #[case::bedrock(BetaProvider::Bedrock)] + #[case::bedrock_mantle(BetaProvider::BedrockMantle)] + #[case::vertex_ai(BetaProvider::VertexAi)] + #[case::databricks(BetaProvider::Databricks)] + fn provider_columns_match_beta_provider_all(#[case] provider: BetaProvider) { + let config = beta_headers_config(); + let config_columns: Vec = config + .keys() + .filter(|column| column.as_str() != "description") + .cloned() + .collect(); + let beta_provider_columns: Vec = + BetaProvider::ALL.iter().map(ToString::to_string).collect(); + assert_eq!(config_columns, beta_provider_columns); + assert_eq!( + provider.to_string().parse::().unwrap(), + provider + ); + } + + #[rstest] + fn on_matches_every_config_cell() { + let config = beta_headers_config(); + for provider in BetaProvider::ALL { + for beta in &AnthropicBeta::KNOWN { + let expected = config + .get(provider.as_ref()) + .and_then(serde_json::Value::as_object) + .and_then(|column| column.get(beta.as_str())) + .and_then(serde_json::Value::as_str) + .map(str::to_string); + let actual = beta.on(*provider).map(|name| name.to_string()); + assert_eq!(actual, expected, "provider {provider}, beta {beta}"); + if matches!(beta, AnthropicBeta::AdvancedToolUse20251120) + && matches!( + provider, + BetaProvider::Bedrock + | BetaProvider::BedrockMantle + | BetaProvider::VertexAi + ) + { + let parsed: AnthropicBeta = actual.as_deref().unwrap().parse().unwrap(); + assert!(AnthropicBeta::KNOWN.contains(&parsed)); + assert!(!matches!(parsed, AnthropicBeta::Other(_))); + } + } + } + } + + #[rstest] + #[case::renamed( + "advanced-tool-use-2025-11-20", + BetaProvider::Bedrock, + Some(AnthropicBeta::ToolSearchTool20251019) + )] + #[case::kept( + "oauth-2025-04-20", + BetaProvider::Anthropic, + Some(AnthropicBeta::Oauth20250420) + )] + #[case::rejected("effort-2025-11-24", BetaProvider::VertexAi, None)] + #[case::unknown("example-beta-2099-01-01", BetaProvider::Anthropic, None)] + fn on_resolves_known_spellings_held_as_other( + #[case] raw: &str, + #[case] provider: BetaProvider, + #[case] expected: Option, + ) { + let actual = AnthropicBeta::Other(raw.to_string()).on(provider); + assert_eq!(actual, expected); + assert!(!matches!(actual, Some(AnthropicBeta::Other(_)))); + } + + #[rstest] + #[case::anthropic(BetaProvider::Anthropic)] + #[case::azure_ai(BetaProvider::AzureAi)] + #[case::bedrock_converse(BetaProvider::BedrockConverse)] + #[case::bedrock(BetaProvider::Bedrock)] + #[case::bedrock_mantle(BetaProvider::BedrockMantle)] + #[case::vertex_ai(BetaProvider::VertexAi)] + #[case::databricks(BetaProvider::Databricks)] + fn unknown_beta_is_rejected_by_every_provider(#[case] provider: BetaProvider) { + assert_eq!( + AnthropicBeta::Other("example-beta-2099-01-01".into()).on(provider), + None + ); + } + #[test] fn unknown_values_are_kept_verbatim() { let parsed: AnthropicBeta = "claude-code-20250219".parse().unwrap(); diff --git a/litellm-rust/crates/llms/src/anthropic/beta_headers.rs b/litellm-rust/crates/llms/src/anthropic/beta_headers.rs new file mode 100644 index 00000000000..3b83b12d727 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/beta_headers.rs @@ -0,0 +1,36 @@ +use litellm_http::request::{with_header, without_headers}; +use litellm_llms_types::providers::anthropic::{BetaProvider, BetaSet}; + +use crate::{anthropic::common_utils::existing_betas, base_llm::auth::Headers}; + +const BETA_HEADER: &str = "anthropic-beta"; + +/// How the `anthropic-beta` header is treated before a request leaves for a host. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum BetaPolicy { + /// Send the header untouched and let the host decide. + Forward, + /// Keep only the betas the provider accepts, under the names it expects. + Filter(BetaProvider), + /// Remove every `anthropic-beta` header. + Drop, +} + +impl BetaPolicy { + pub fn apply(self, headers: Headers) -> Headers { + let provider = match self { + Self::Forward => return headers, + Self::Drop => return without_headers(headers, &[BETA_HEADER]), + Self::Filter(provider) => provider, + }; + let accepted: BetaSet = existing_betas(&headers) + .iter() + .filter_map(|beta| beta.on(provider)) + .collect(); + let headers = without_headers(headers, &[BETA_HEADER]); + if accepted.is_empty() { + return headers; + } + with_header(headers, BETA_HEADER, accepted.to_string()) + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/mod.rs b/litellm-rust/crates/llms/src/anthropic/mod.rs index a884c146dca..72c390cfe21 100644 --- a/litellm-rust/crates/llms/src/anthropic/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/mod.rs @@ -1,3 +1,4 @@ +pub mod beta_headers; pub mod common_utils; pub mod batches; diff --git a/litellm-rust/crates/llms/tests/anthropic_beta_headers.rs b/litellm-rust/crates/llms/tests/anthropic_beta_headers.rs new file mode 100644 index 00000000000..422c3d4b5f0 --- /dev/null +++ b/litellm-rust/crates/llms/tests/anthropic_beta_headers.rs @@ -0,0 +1,111 @@ +use litellm_llms::anthropic::beta_headers::BetaPolicy; +use litellm_llms_types::providers::anthropic::BetaProvider; +use rstest::rstest; + +fn headers(pairs: &[(&str, &str)]) -> Vec<(String, String)> { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect() +} + +#[rstest] +fn forward_keeps_headers_as_is() { + let input = headers(&[ + ( + "Anthropic-Beta", + "example-beta-2099-01-01,web-search-2025-03-05", + ), + ("x-api-key", "k"), + ]); + assert_eq!(BetaPolicy::Forward.apply(input.clone()), input); +} + +#[rstest] +fn drop_removes_every_casing() { + assert_eq!( + BetaPolicy::Drop.apply(headers(&[ + ("anthropic-beta", "a"), + ("x-api-key", "k"), + ("ANTHROPIC-BETA", "b"), + ])), + headers(&[("x-api-key", "k")]) + ); +} + +#[rstest] +#[case::bedrock(BetaProvider::Bedrock, Some("tool-search-tool-2025-10-19"))] +#[case::bedrock_mantle(BetaProvider::BedrockMantle, Some("tool-search-tool-2025-10-19"))] +#[case::vertex_ai(BetaProvider::VertexAi, Some("tool-search-tool-2025-10-19"))] +#[case::anthropic(BetaProvider::Anthropic, Some("advanced-tool-use-2025-11-20"))] +#[case::azure_ai(BetaProvider::AzureAi, Some("advanced-tool-use-2025-11-20"))] +#[case::databricks(BetaProvider::Databricks, Some("advanced-tool-use-2025-11-20"))] +#[case::bedrock_converse(BetaProvider::BedrockConverse, None)] +fn filter_renames_per_host(#[case] provider: BetaProvider, #[case] expected_beta: Option<&str>) { + let expected = match expected_beta { + Some(beta) => headers(&[("x-api-key", "k"), ("anthropic-beta", beta)]), + None => headers(&[("x-api-key", "k")]), + }; + assert_eq!( + BetaPolicy::Filter(provider).apply(headers(&[ + ("Anthropic-Beta", "advanced-tool-use-2025-11-20"), + ("x-api-key", "k"), + ])), + expected + ); +} + +#[rstest] +#[case::azure_ai(BetaProvider::AzureAi, "fast-mode-2026-02-01,example-beta-2099-01-01")] +#[case::anthropic(BetaProvider::Anthropic, "bash_20241022")] +#[case::vertex_ai(BetaProvider::VertexAi, " , ")] +fn filter_removes_header_when_nothing_survives( + #[case] provider: BetaProvider, + #[case] beta_value: &str, +) { + assert_eq!( + BetaPolicy::Filter(provider).apply(headers(&[ + ("x-api-key", "k"), + ("anthropic-beta", beta_value) + ])), + headers(&[("x-api-key", "k")]) + ); +} + +#[rstest] +#[case::anthropic( + BetaProvider::Anthropic, + "web-search-2025-03-05", + "oauth-2025-04-20,web-search-2025-03-05,example-beta-2099-01-01", + "oauth-2025-04-20,web-search-2025-03-05" +)] +#[case::vertex_ai_renames_and_deduplicates( + BetaProvider::VertexAi, + "advanced-tool-use-2025-11-20", + "tool-search-tool-2025-10-19", + "tool-search-tool-2025-10-19" +)] +fn filter_deduplicates_and_sorts( + #[case] provider: BetaProvider, + #[case] first_beta: &str, + #[case] second_beta: &str, + #[case] expected_beta: &str, +) { + assert_eq!( + BetaPolicy::Filter(provider).apply(headers(&[ + ("ANTHROPIC-BETA", first_beta), + ("x-api-key", "k"), + ("anthropic-beta", second_beta), + ])), + headers(&[("x-api-key", "k"), ("anthropic-beta", expected_beta)]) + ); +} + +#[rstest] +#[case::forward(BetaPolicy::Forward)] +#[case::drop(BetaPolicy::Drop)] +#[case::filter(BetaPolicy::Filter(BetaProvider::Bedrock))] +fn headers_without_a_beta_are_untouched(#[case] policy: BetaPolicy) { + let input = headers(&[("x-api-key", "k"), ("anthropic-version", "2023")]); + assert_eq!(policy.apply(input.clone()), input); +} From a7b04a8028cbf25ca85b740bd1c2d38dac4a3034 Mon Sep 17 00:00:00 2001 From: yuneng-jiang Date: Thu, 8 Oct 2026 13:23:43 -0700 Subject: [PATCH 62/65] test(decisions): post System One bodies to /v1/systemone in the translation bases (#45327) --- tests/integration/translation/decisions/bases/cloudflare.py | 2 +- tests/integration/translation/decisions/bases/openrouter.py | 2 +- tests/integration/translation/decisions/bases/perplexity.py | 2 +- .../integration/translation/decisions/bases/strands_decider.py | 2 +- tests/integration/translation/decisions/bases/typesafe.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/integration/translation/decisions/bases/cloudflare.py b/tests/integration/translation/decisions/bases/cloudflare.py index 1668dc215a8..ae86a2ccd87 100644 --- a/tests/integration/translation/decisions/bases/cloudflare.py +++ b/tests/integration/translation/decisions/bases/cloudflare.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ CLEF_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "cloudflare/@cf/cloudflare/clef", "state": "Ticket (billing): The export job hangs at 99% and never finishes", diff --git a/tests/integration/translation/decisions/bases/openrouter.py b/tests/integration/translation/decisions/bases/openrouter.py index f6217bcf818..b65acff6b06 100644 --- a/tests/integration/translation/decisions/bases/openrouter.py +++ b/tests/integration/translation/decisions/bases/openrouter.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ TYPESAFE_JEV_1_13_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "openrouter/typesafe/jev-1.13", "state": "Ticket (billing): The export job hangs at 99% and never finishes", diff --git a/tests/integration/translation/decisions/bases/perplexity.py b/tests/integration/translation/decisions/bases/perplexity.py index 71ef7a85220..02b1de51f24 100644 --- a/tests/integration/translation/decisions/bases/perplexity.py +++ b/tests/integration/translation/decisions/bases/perplexity.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ PPLX_DECIDER_V1_27B_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "perplexity/pplx-decider-v1-27b", "state": "Ticket (billing): The export job hangs at 99% and never finishes", diff --git a/tests/integration/translation/decisions/bases/strands_decider.py b/tests/integration/translation/decisions/bases/strands_decider.py index 83d7ae5e33a..8a1a51f8c3e 100644 --- a/tests/integration/translation/decisions/bases/strands_decider.py +++ b/tests/integration/translation/decisions/bases/strands_decider.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ STRANDS_DECIDER_2B_HOBSON_V19_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "strands_decider/strands-decider-2B-hobson-v19", "state": "Help! My payouts have been failing for 3 days!", diff --git a/tests/integration/translation/decisions/bases/typesafe.py b/tests/integration/translation/decisions/bases/typesafe.py index 2fb384a94ae..5da5b05cb09 100644 --- a/tests/integration/translation/decisions/bases/typesafe.py +++ b/tests/integration/translation/decisions/bases/typesafe.py @@ -6,7 +6,7 @@ from integration.translation.case import TranslationTestCase """ JEV_1_13_0_TEST_CASE: Final = TranslationTestCase( scenario="basic", - litellm_endpoint="/v1/decisions", + litellm_endpoint="/v1/systemone", litellm_request={ "model": "typesafe/jev-1.13.0", "state": "Ticket (billing): The export job hangs at 99% and never finishes", From 7ec2a94cd87d137bf1a4db9d288bca49932f57e6 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:41:03 -0700 Subject: [PATCH 63/65] fix(ci): drop the publicly known master key prefix from the voyage routing test (#45431) Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- tests/unit/llms/voyage/test_common_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/llms/voyage/test_common_utils.py b/tests/unit/llms/voyage/test_common_utils.py index 511391093f3..51d6c3ed0af 100644 --- a/tests/unit/llms/voyage/test_common_utils.py +++ b/tests/unit/llms/voyage/test_common_utils.py @@ -29,7 +29,7 @@ def test_mongodb_key_routes_to_mongodb_host(): assert get_default_base_url("al-1234567890") == MONGODB_API_BASE -@pytest.mark.parametrize("api_key", ["pa-1234567890", "sk-1234567890", "al", "", None]) +@pytest.mark.parametrize("api_key", ["pa-1234567890", "sk-voyage-legacy-key", "al", "", None]) def test_non_mongodb_key_routes_to_voyage_host(api_key): assert get_default_base_url(api_key) == VOYAGE_API_BASE From 85a3869dfd7c5dab6483debf540712409ad090ce Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:46:53 -0700 Subject: [PATCH 64/65] feat(guardrails): per-mode stream_scope with bedrock stream and pass-through fixes (#43801) * feat(guardrails): run each mode only on streaming, non-streaming, or both Add stream_scope so a rail can target streaming inference, non-streaming inference, or both per pre, during, and post mode Co-authored-by: Cursor * fix(guardrails): honor stream_scope in pipelines and dashboard types Pipeline steps skipped the stream_scope filter, direct construction ignored mixed-case maps, and schema.d.ts was stale. Co-authored-by: Cursor * fix(guardrails): skip unmatched stream_scope steps instead of allowing Co-authored-by: Cursor * fix(ui): keep stored stream_scope keys for modes not on screen Co-authored-by: Cursor * fix(ci): format stream_scope helpers and update fork MCP unit tests Co-authored-by: Cursor * fix(mcp): keep hang cancellation tests from timing out during setup Co-authored-by: Cursor * fix(guardrails): honor path-defined streaming for stream_scope Passthrough routes like Gemini streamGenerateContent decide streaming from the URL, so stamp that onto hook data before guardrails run. Co-authored-by: Cursor * fix(rust): copy AnthropicModelCapabilities instead of cloning Clippy treats clone-on-Copy as an error, which failed rust-lint on the messages request tests. Co-authored-by: Cursor * fix(guardrails): trust only server stream classification Co-authored-by: Cursor * fix(guardrails): keep streaming marker through deepcopy scan_raw_request snapshots copy each field, so a plain object() marker would lose identity and skip streaming-only rails. Co-authored-by: Cursor * fix(cost-map): drop duplicate perceptron-mk1.5 row Two main cost-map PRs both added the OpenRouter model, so the merge left a second key that CI rejects. Co-authored-by: Cursor * fix(rust): expect native transcription 429 as RustUpstreamError HTTP status errors from native routes map through route_error_to_pyerr, so the wheel SIGINT child was dying on an outdated RuntimeError check and never reached the hang probe. Co-authored-by: Cursor * fix(tests): follow Google Interactions OpenAPI without hardcoded names The live spec dropped CreateModelInteractionParams and renamed the item path to {interactionsId}. Misc CI failed because the compliance tests still looked those names up as literals. * test(guardrails): reproduce stream_scope bedrock and passthrough field gaps Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): cover invalid stored stream_scope reads Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): classify bedrock stream actions and keep caller is_streaming_request Co-authored-by: Shivi Jain Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(guardrails): make stream_scope_allows public and drop mutable builds Co-authored-by: Shivi Jain Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): classify pass-through and Bedrock stream scopes Co-authored-by: Shivi Jain Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): harden stream classification and validation Co-authored-by: Shivi Jain Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): add stream scope integration audit Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): avoid mutating passthrough custom body Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): run stream scope audit without enterprise license Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): keep stored scope restart cell on one worker Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): scope stream scope audit sink assertions to the rail under test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): assert logging_only scope absence behind an ordered barrier rail Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): assert one logging_only scan per phase after the barrier rail Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): set request scope on pass-through stream fixtures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): tolerate invalid YAML stream_scope in v1 guardrails list Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore: merge main into litellm_guardrail_stream_scope_fixes Update pass-through pre-call test callbacks for main's endpoint_type argument Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): add request paths to pass-through fixtures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): cover websocket pass-through stream scope and LIT-9050 outage spend rows Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * chore(lint): remove unused type discipline suppressions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(guardrails): script the vertex live upstream in the websocket stream scope test Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): keep stream marker json-serializable and restore pass-through helper names Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(lint): allow required Bedrock action re-export Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): strip the stream marker from pass-through payloads Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(guardrails): drop stream marker by value in scans and snapshots, plain-tuple stream scope state Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): render tag-scoped guardrail modes read-only in the custom code editor A guardrail whose litellm_params.mode is the tag-scoped dict {tags, default} crashed the Custom Code editor on open: normalizeMode wrapped the dict into the mode array and StreamScopeFields rendered it as a React child (error #31, whole dashboard unmounted). Treat a non-string non-array mode as no editable modes, show formatGuardrailMode(mode) in a disabled input (read-only, matching the guardrail info view), and keep mode/stream_scope out of the update payload for such guardrails. * fix(ui): resolve merge fallout in guardrails components Deduplicate toModeArray import after the merge, and move the read-only guardrail details block back into GuardrailReadOnlyDetails (now rendering the shared mode/logging-only rows plus the stream-scope detail) so guardrail_info.tsx stays under the 800-line lint budget. Guardrails UI suite: 298 passed. * fix(ui): drop duplicate toModeArray import reintroduced by merge * chore(pass-through): document the deliberate in-place marker strip as mutable-ok The clear/update on _parsed_body is load-bearing: rebinding to a fresh mapping instead breaks 75 pass-through tests because the marker-free body must propagate through the caller's request dict so downstream guardrail scans and snapshots never observe the server streaming marker. * fix(guardrails): move stream_scope after timeout in CustomGuardrail init Inserting stream_scope before the existing timeout parameter shifted the positional slot of timeout, so positional callers constructed with their timeout bound to stream_scope (ValueError) and timeout silently None. Restores the base parameter order; keyword callers are unaffected. * fix(guardrails): typing pass for the lint gates stream_scope leaves the declared constructor parameters (restoring the base positional surface; keyword construction unchanged), unknown config values crossing the new stream-scope code paths get typed locals or cast-ok boundaries, and the passthrough payload literals are annotated. All three lint gates pass against current main; the guardrail suites are unchanged (236+5019 passing; one known anyio-driver failure pre-existing on base). * style: sort cast imports for the ruff gate * chore(pass-through): drop unused BEDROCK_STREAMING_ACTIONS re-export The streaming check now uses is_bedrock_streaming_endpoint; nothing in the repo imports the name from this module. --------- Co-authored-by: Shivi Jain Co-authored-by: Cursor Co-authored-by: yucheng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: gabriele --- litellm/constants.py | 10 + litellm/integrations/custom_guardrail.py | 76 +- .../litellm_core_utils/realtime_streaming.py | 4 +- .../bedrock/passthrough/transformation.py | 7 + .../guardrail_translation/handler.py | 5 +- litellm/proxy/_lazy_openapi_snapshot.json | 56 + .../proxy/guardrails/guardrail_endpoints.py | 13 +- .../proxy/guardrails/guardrail_registry.py | 30 +- litellm/proxy/litellm_pre_call_utils.py | 5 +- .../llm_passthrough_endpoints.py | 41 +- .../pass_through_endpoints.py | 30 +- .../proxy/policy_engine/pipeline_executor.py | 39 +- litellm/types/guardrails.py | 116 +- .../proxy/policy_engine/pipeline_types.py | 2 +- .../test_guardrail_stream_scope.py | 856 +++++++ .../test_guardrail_stream_scope_chaos.py | 1079 +++++++++ .../test_guardrail_stream_scope_matrix.py | 2076 +++++++++++++++++ .../test_mcp_client.py | 5 +- .../integrations/test_custom_guardrail.py | 344 +++ ...test_bedrock_passthrough_transformation.py | 23 +- .../guardrail_translation/test_handler.py | 52 + .../mcp_server/test_mcp_server.py | 14 +- .../guardrails/test_guardrail_endpoints.py | 51 +- .../guardrails/test_guardrail_registry.py | 68 +- .../test_llm_pass_through_endpoints.py | 294 ++- .../test_pass_through_endpoints.py | 221 +- .../policy_engine/test_pipeline_executor.py | 54 + .../unit/proxy/test_litellm_pre_call_utils.py | 100 +- tests/unit/proxy/test_pricing_field_strip.py | 33 +- .../test_guardrails_case_normalization.py | 88 +- .../_components/GuardrailReadOnlyDetails.tsx | 68 + .../_components/StreamScopeFields.tsx | 95 + ...d_guardrail_form.characterization.test.tsx | 17 + .../_components/add_guardrail_form.tsx | 16 +- .../custom_code/CustomCodeModal.test.tsx | 59 +- .../custom_code/CustomCodeModal.tsx | 252 +- .../custom_code/custom_code_catalog.ts | 136 ++ .../guardrails/_components/guardrail_info.tsx | 91 +- .../guardrail_info_helpers.test.tsx | 57 + .../_components/guardrail_info_helpers.tsx | 83 + .../src/components/policies/types.ts | 2 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 14 + 42 files changed, 6315 insertions(+), 367 deletions(-) create mode 100644 tests/integration/observability/test_guardrail_stream_scope.py create mode 100644 tests/integration/observability/test_guardrail_stream_scope_chaos.py create mode 100644 tests/integration/observability/test_guardrail_stream_scope_matrix.py create mode 100644 tests/unit/llms/pass_through/guardrail_translation/test_handler.py create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailReadOnlyDetails.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/StreamScopeFields.tsx create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/custom_code_catalog.ts diff --git a/litellm/constants.py b/litellm/constants.py index c4adc0de22f..b8048ad30b4 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1,10 +1,20 @@ import os import sys +from enum import Enum from types import MappingProxyType from typing import Final, Literal from litellm.litellm_core_utils.env_utils import get_env_int, get_env_int_in_range, get_env_int_or_none +SERVER_STREAMING_CLASSIFICATION_KEY: Final = "litellm_server_streaming_classification" + + +class ServerStreamingClassification(str, Enum): + MARKER = "litellm-server-streaming" + + +SERVER_STREAMING_CLASSIFICATION_MARKER: Final = ServerStreamingClassification.MARKER + DEFER_PYDANTIC_BUILD: Final = os.getenv("DEFER_PYDANTIC_BUILD", "true") in ("true", "1", "on") DEFAULT_HEALTH_CHECK_PROMPT: Final = str(os.getenv("DEFAULT_HEALTH_CHECK_PROMPT", "test from litellm")) AZURE_DEFAULT_RESPONSES_API_VERSION: Final = str(os.getenv("AZURE_DEFAULT_RESPONSES_API_VERSION", "preview")) diff --git a/litellm/integrations/custom_guardrail.py b/litellm/integrations/custom_guardrail.py index 94f1207b9ae..383a09f0931 100644 --- a/litellm/integrations/custom_guardrail.py +++ b/litellm/integrations/custom_guardrail.py @@ -6,7 +6,7 @@ import secrets from collections.abc import Mapping, Sequence from datetime import datetime from types import MappingProxyType -from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, get_args +from typing import TYPE_CHECKING, Any, ClassVar, Final, Literal, Optional, cast, get_args import httpx @@ -21,11 +21,14 @@ from litellm.litellm_core_utils.core_helpers import ( ) from litellm.secret_managers.main import str_to_bool from litellm.types.guardrails import ( + DEFAULT_GUARDRAIL_STREAM_SCOPE, DynamicGuardrailParams, GuardrailEventHooks, + GuardrailStreamScope, LitellmParams, LoggingOnlyScope, Mode, + runtime_stream_scope, ) from litellm.types.llms.openai import AllMessageValues from litellm.types.proxy.guardrails.guardrail_hooks.base import GuardrailConfigModel @@ -49,6 +52,8 @@ from litellm.constants import ( GUARDRAIL_SCANNED_MESSAGES_CACHE_TTL_SECONDS, LOGS_GUARDRAIL_INFORMATION_MARKER, PRE_CALL_EXECUTED_GUARDRAILS_KEY, + SERVER_STREAMING_CLASSIFICATION_KEY, + SERVER_STREAMING_CLASSIFICATION_MARKER, ) from litellm.exceptions import ( BlockedPiiEntityError, @@ -173,6 +178,42 @@ def get_session_id_from_request_data(request_data: dict[str, Any]) -> str | None return None +_REALTIME_STREAMING_HOOKS: Final = frozenset({GuardrailEventHooks.realtime_input_transcription}) + + +def without_server_streaming_classification(data: Mapping[str, object]) -> dict[str, object]: + return { + key: value + for key, value in data.items() + if key != SERVER_STREAMING_CLASSIFICATION_KEY or value != SERVER_STREAMING_CLASSIFICATION_MARKER + } + + +def guardrail_request_data_with_streaming( + data: Mapping[str, object], + *, + is_streaming: bool, +) -> dict[str, object]: + data_without_server_classification: Final = without_server_streaming_classification(data) + if not is_streaming: + return data_without_server_classification + return { + **data_without_server_classification, + SERVER_STREAMING_CLASSIFICATION_KEY: SERVER_STREAMING_CLASSIFICATION_MARKER, + } + + +def _request_is_streaming(data: object, event_type: GuardrailEventHooks | None = None) -> bool: + if event_type in _REALTIME_STREAMING_HOOKS: + return True + if not isinstance(data, Mapping): + return False + return ( + data.get("stream") is True + or data.get(SERVER_STREAMING_CLASSIFICATION_KEY) is SERVER_STREAMING_CLASSIFICATION_MARKER + ) + + class CustomGuardrail(CustomLogger): # If True, during_call runs async_moderation_hook instead of the unified apply_guardrail path. use_native_during_call_hook: ClassVar[bool] = False @@ -183,6 +224,9 @@ class CustomGuardrail(CustomLogger): records_own_guardrail_information: ClassVar[bool] = False logging_only_scope: LoggingOnlyScope | None + stream_scope_default: GuardrailStreamScope = DEFAULT_GUARDRAIL_STREAM_SCOPE + stream_scope_by_hook: tuple[tuple[str, GuardrailStreamScope], ...] = () + timeout: float | httpx.Timeout | None = None def __init_subclass__(cls, **kwargs: object) -> None: # kwargs-ok: forwarded to cooperative __init_subclass__ hooks @@ -258,6 +302,8 @@ class CustomGuardrail(CustomLogger): self.run_in_parallel: bool = run_in_parallel self.scan_raw_request: bool = scan_raw_request self.only_scan_new_messages: bool = only_scan_new_messages + stream_scope_arg: Final[object] = cast(object, kwargs.pop("stream_scope", None)) # cast-ok: config + self.apply_stream_scope(stream_scope_arg) self.logging_only_scope = None if timeout is not None: self.timeout = timeout @@ -1099,6 +1145,23 @@ class CustomGuardrail(CustomLogger): return name in suppressed_compression_guardrails() + def apply_stream_scope(self, stream_scope: object) -> None: + default, by_hook = runtime_stream_scope(stream_scope) + self.stream_scope_default = default + self.stream_scope_by_hook = tuple(by_hook.items()) + + def stream_scope_allows(self, data: object, event_type: GuardrailEventHooks) -> bool: + scope: Final = next( + (scope for hook, scope in self.stream_scope_by_hook if hook == event_type.value), + self.stream_scope_default, + ) + if scope == "both": + return True + is_streaming: Final = _request_is_streaming(data, event_type) + if scope == "streaming": + return is_streaming + return not is_streaming + def should_run_guardrail( self, data, @@ -1142,8 +1205,10 @@ class CustomGuardrail(CustomLogger): data, self.event_hook, event_type ) if result is not None: - return result - return True + tagged_result: Final[bool] = bool(cast(object, result)) # cast-ok: helper return + data_obj: Final[object] = cast(object, data) # cast-ok: data param + return tagged_result and self.stream_scope_allows(data_obj, event_type) + return self.stream_scope_allows(cast(object, data), event_type) # cast-ok: data param return False if ( @@ -1167,8 +1232,9 @@ class CustomGuardrail(CustomLogger): ) result = EnterpriseCustomGuardrailHelper._should_run_if_mode_by_tag(data, self.event_hook, event_type) if result is not None: - return result - return True + mode_tag_result: Final[bool] = bool(cast(object, result)) # cast-ok: helper return + return mode_tag_result and self.stream_scope_allows(cast(object, data), event_type) # cast-ok: data + return self.stream_scope_allows(cast(object, data), event_type) # cast-ok: data param def _event_hook_is_event_type(self, event_type: GuardrailEventHooks) -> bool: """ diff --git a/litellm/litellm_core_utils/realtime_streaming.py b/litellm/litellm_core_utils/realtime_streaming.py index a781be610a6..473be107cbb 100644 --- a/litellm/litellm_core_utils/realtime_streaming.py +++ b/litellm/litellm_core_utils/realtime_streaming.py @@ -782,7 +782,7 @@ class RealTimeStreaming: isinstance(cb, CustomGuardrail) and any( cb.should_run_guardrail( - data=self.request_data, + data={**self.request_data, "stream": True}, event_type=et, ) for et in event_hooks @@ -847,7 +847,7 @@ class RealTimeStreaming: if event_hooks is None: event_hooks = [GuardrailEventHooks.realtime_input_transcription] _realtime_event_types: Final = event_hooks - _check_data: Final = {**self.request_data, "transcript": transcript} + _check_data: Final = {**self.request_data, "transcript": transcript, "stream": True} _already_run: Final[set] = set() for callback in litellm.callbacks: diff --git a/litellm/llms/bedrock/passthrough/transformation.py b/litellm/llms/bedrock/passthrough/transformation.py index c61576fa7b2..0d7f7cd3562 100644 --- a/litellm/llms/bedrock/passthrough/transformation.py +++ b/litellm/llms/bedrock/passthrough/transformation.py @@ -22,6 +22,13 @@ if TYPE_CHECKING: from litellm.types.utils import CostResponseTypes +BEDROCK_STREAMING_ACTIONS: Final = frozenset({"invoke-with-response-stream", "converse-stream"}) + + +def is_bedrock_streaming_endpoint(endpoint: str) -> bool: + return endpoint.partition("?")[0].rstrip("/").rsplit("/", 1)[-1] in BEDROCK_STREAMING_ACTIONS + + _TEXT_ONLY_DELTA_FIELDS: Final = frozenset({"content", "role"}) diff --git a/litellm/llms/pass_through/guardrail_translation/handler.py b/litellm/llms/pass_through/guardrail_translation/handler.py index 1f295a6e656..fd58ca2ddbb 100644 --- a/litellm/llms/pass_through/guardrail_translation/handler.py +++ b/litellm/llms/pass_through/guardrail_translation/handler.py @@ -10,6 +10,7 @@ from collections.abc import Mapping from typing import TYPE_CHECKING, Any, Final, Optional from litellm._logging import verbose_proxy_logger +from litellm.integrations.custom_guardrail import without_server_streaming_classification from litellm.llms.base_llm.guardrail_translation.base_translation import BaseTranslation from litellm.proxy._types import PassThroughGuardrailSettings from litellm.types.utils import GenericGuardrailAPIInputs @@ -80,7 +81,9 @@ class PassThroughEndpointHandler(BaseTranslation): from litellm.litellm_core_utils.safe_json_dumps import safe_dumps payload_to_check: Final = { - k: v for k, v in data.items() if not k.startswith("_") and k not in ("metadata", "litellm_logging_obj") + k: v + for k, v in without_server_streaming_classification(data).items() + if not k.startswith("_") and k not in ("metadata", "litellm_logging_obj") } verbose_proxy_logger.debug("PassThroughEndpointHandler: Using full payload for guardrail") return safe_dumps(payload_to_check) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 9322fb77615..7dcca8f9ec0 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -11906,6 +11906,34 @@ "description": "When True (default), after sensitive data is detected and routed, all subsequent requests in the same session will continue routing to the same model.", "title": "Sticky Session Routing" }, + "stream_scope": { + "anyOf": [ + { + "enum": [ + "streaming", + "non_streaming", + "both" + ], + "type": "string" + }, + { + "additionalProperties": { + "enum": [ + "streaming", + "non_streaming", + "both" + ], + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Whether this guardrail runs on streaming requests, non-streaming requests, or both. A string applies to every configured mode. A map overrides named modes (pre_call, during_call, post_call, ...); omitted keys default to both. Unset means both, matching historical behavior.", + "title": "Stream Scope" + }, "template_id": { "anyOf": [ { @@ -14831,6 +14859,34 @@ "description": "When True (default), after sensitive data is detected and routed, all subsequent requests in the same session will continue routing to the same model.", "title": "Sticky Session Routing" }, + "stream_scope": { + "anyOf": [ + { + "enum": [ + "streaming", + "non_streaming", + "both" + ], + "type": "string" + }, + { + "additionalProperties": { + "enum": [ + "streaming", + "non_streaming", + "both" + ], + "type": "string" + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Whether this guardrail runs on streaming requests, non-streaming requests, or both. A string applies to every configured mode. A map overrides named modes (pre_call, during_call, post_call, ...); omitted keys default to both. Unset means both, matching historical behavior.", + "title": "Stream Scope" + }, "template_id": { "anyOf": [ { diff --git a/litellm/proxy/guardrails/guardrail_endpoints.py b/litellm/proxy/guardrails/guardrail_endpoints.py index 8e1209401ab..6072d8bf22a 100644 --- a/litellm/proxy/guardrails/guardrail_endpoints.py +++ b/litellm/proxy/guardrails/guardrail_endpoints.py @@ -69,6 +69,7 @@ from litellm.types.guardrails import ( PresidioPresidioConfigModelUserInterface, SupportedGuardrailIntegrations, ToolPermissionGuardrailConfigModel, + with_tolerated_stream_scope, ) from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.proxy.guardrails.guardrail_hooks.hide_secrets import ( @@ -146,7 +147,7 @@ def _get_guardrails_list_response( GuardrailInfoResponse( guardrail_id=guardrail.get("guardrail_id"), guardrail_name=guardrail.get("guardrail_name"), - litellm_params=masked_params, + litellm_params=with_tolerated_stream_scope(masked_params), guardrail_info=guardrail.get("guardrail_info"), ) ) @@ -289,7 +290,7 @@ async def list_guardrails_v2( ) masked_litellm_params = ( parse_tolerant_litellm_params( - masked_litellm_params_dict, + with_tolerated_stream_scope(masked_litellm_params_dict), guardrail.get("guardrail_name") or "Unknown", params_model=BaseLitellmParams, ) @@ -336,7 +337,7 @@ async def list_guardrails_v2( ) masked_in_memory_litellm_params_typed = ( parse_tolerant_litellm_params( - masked_in_memory_litellm_params, + with_tolerated_stream_scope(masked_in_memory_litellm_params), guardrail.get("guardrail_name") or "Unknown", params_model=BaseLitellmParams, ) @@ -1263,7 +1264,7 @@ async def patch_guardrail( # Update litellm_params if default_on is provided or pii_entities_config is provided existing_litellm_params: Final = _as_str_object_mapping(dict(existing_guardrail.get("litellm_params", {}))) current_litellm_params: Final = parse_tolerant_litellm_params( - existing_litellm_params, + with_tolerated_stream_scope(existing_litellm_params), existing_guardrail.get("guardrail_name") or "Unknown", ) requested_litellm_params: Final[Mapping[str, object]] = ( @@ -1275,7 +1276,7 @@ async def patch_guardrail( MappingProxyType({**current_litellm_params.model_dump(exclude_unset=True), **requested_litellm_params}) ) try: - parsed_litellm_params: Final = LitellmParams(**merged_litellm_params) + parsed_litellm_params: Final = LitellmParams(**with_tolerated_stream_scope(merged_litellm_params)) except ValidationError as validation_error: raise HTTPException( status_code=422, @@ -1436,7 +1437,7 @@ async def get_guardrail_info(guardrail_id: str): ) masked_litellm_params = ( parse_tolerant_litellm_params( - masked_litellm_params_dict, + with_tolerated_stream_scope(masked_litellm_params_dict), result.get("guardrail_name") or "Unknown", params_model=BaseLitellmParams, ) diff --git a/litellm/proxy/guardrails/guardrail_registry.py b/litellm/proxy/guardrails/guardrail_registry.py index d14cc940885..439d17b8644 100644 --- a/litellm/proxy/guardrails/guardrail_registry.py +++ b/litellm/proxy/guardrails/guardrail_registry.py @@ -54,6 +54,7 @@ from litellm.types.guardrails import ( LakeraCategoryThresholds, LitellmParams, SupportedGuardrailIntegrations, + with_tolerated_stream_scope, ) from .guardrail_hooks.llm_as_a_judge import ( @@ -634,6 +635,8 @@ def _configure_callback_scoping( "skip_tool_message_in_guardrail are enabled together, which excludes every message from " "scanning, so no request content would ever be scanned. Remove one of the two." ) + if isinstance(custom_guardrail_callback, CustomGuardrail): # pyright: ignore[reportUnnecessaryIsInstance] # module-path classes may only subclass CustomLogger + custom_guardrail_callback.apply_stream_scope(litellm_params.stream_scope) _apply_configured_bool_overrides(custom_guardrail_callback, litellm_params) @@ -718,9 +721,11 @@ class InMemoryGuardrailHandler: if isinstance(litellm_params_data, dict): if reject_invalid_logging_only_scope: - litellm_params = LitellmParams(**litellm_params_data) + litellm_params = LitellmParams(**with_tolerated_stream_scope(litellm_params_data)) else: - litellm_params = parse_tolerant_litellm_params(litellm_params_data, guardrail["guardrail_name"]) + litellm_params = parse_tolerant_litellm_params( + with_tolerated_stream_scope(litellm_params_data), guardrail["guardrail_name"] + ) else: litellm_params = litellm_params_data @@ -863,14 +868,17 @@ class InMemoryGuardrailHandler: # Extract additional params from litellm_params to pass to custom guardrail # This matches the behavior of other guardrail initializers (e.g., initialize_lakera) # and aligns with the documented behavior for custom guardrails - if hasattr(litellm_params, "model_dump"): - extra_params = litellm_params.model_dump(exclude_none=True) - else: - extra_params = dict(litellm_params) if litellm_params else {} - - # Remove params that are handled explicitly or are internal - for key in ["guardrail", "mode", "default_on"]: - extra_params.pop(key, None) + excluded_extra_param_keys: Final = frozenset(("guardrail", "mode", "default_on", "stream_scope")) + extra_params_items: Final = ( + litellm_params.model_dump(exclude_none=True).items() + if hasattr(litellm_params, "model_dump") + else iter(litellm_params) + if litellm_params + else () + ) + extra_params: Final = MappingProxyType( + {key: value for key, value in extra_params_items if key not in excluded_extra_param_keys} + ) _guardrail_callback: Final = _guardrail_class( guardrail_name=guardrail["guardrail_name"], @@ -1007,7 +1015,7 @@ class InMemoryGuardrailHandler: return params.model_dump() if isinstance(params, dict): try: - return parse_tolerant_litellm_params(params, guardrail_name).model_dump() + return parse_tolerant_litellm_params(with_tolerated_stream_scope(params), guardrail_name).model_dump() except ValidationError as e: verbose_proxy_logger.warning( "Could not normalize guardrail litellm_params for comparison; treating the guardrail as changed. Error: %s", diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 46cdf328af6..66dca57e782 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -32,6 +32,7 @@ from litellm.constants import ( SESSION_ID_OMITTED_METADATA_KEY, X_LITELLM_DISABLE_CALLBACKS, ) +from litellm.integrations.custom_guardrail import without_server_streaming_classification from litellm.litellm_core_utils.core_helpers import is_codex_user_agent from litellm.litellm_core_utils.credential_accessor import CredentialAccessor from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( @@ -2094,7 +2095,9 @@ def refresh_proxy_server_request_body_snapshot( | _TRANSPORT_ONLY_CREDENTIAL_KEYS | _CALLBACK_CREDENTIAL_KEYS ) - body: Final = {k: v for k, v in data.items() if k not in _body_snapshot_exclude} + body: Final = { + k: v for k, v in without_server_streaming_classification(data).items() if k not in _body_snapshot_exclude + } proxy_server_request["body"] = body if guardrails_applied and isinstance(logging_obj, Logging): metadata: Final = data.get(get_metadata_variable_name_from_kwargs(data)) diff --git a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py index ca142088383..1eefb58d7fa 100644 --- a/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/llm_passthrough_endpoints.py @@ -44,12 +44,14 @@ from litellm.constants import ( AZURE_SPEECH_SUBSCRIPTION_KEY_HEADER, BEDROCK_AGENT_RUNTIME_PASS_THROUGH_ROUTES, ) +from litellm.integrations.custom_guardrail import guardrail_request_data_with_streaming from litellm.litellm_core_utils.aws_partition import get_aws_dns_suffix from litellm.llms.anthropic.common_utils import AnthropicModelInfo, merge_anthropic_beta_headers from litellm.llms.azure.passthrough.transformation import ( foreign_azure_deployment, is_azure_body_model_inference_endpoint, ) +from litellm.llms.bedrock.passthrough.transformation import is_bedrock_streaming_endpoint from litellm.llms.custom_httpx.http_handler import get_async_httpx_client from litellm.llms.deepgram.common_utils import ( deepgram_listen_callback_params, @@ -946,8 +948,6 @@ BEDROCK_ENDPOINT_ACTIONS: Final = { "count-tokens", } -BEDROCK_STREAMING_ACTIONS: Final = {"invoke-with-response-stream", "converse-stream"} - def is_bedrock_count_tokens_endpoint(endpoint: str) -> bool: return "count_tokens" in endpoint or "count-tokens" in endpoint @@ -1077,7 +1077,7 @@ async def handle_bedrock_passthrough_router_model( from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing # Detect streaming based on endpoint - is_streaming: Final = any(action in endpoint for action in BEDROCK_STREAMING_ACTIONS) + is_streaming: Final = is_bedrock_streaming_endpoint(endpoint) verbose_proxy_logger.debug( "Bedrock router passthrough: model='%s', endpoint='%s', streaming=%s", model, endpoint, is_streaming @@ -1085,15 +1085,19 @@ async def handle_bedrock_passthrough_router_model( # Use the common processing path (same as non-router models) # This ensures all metadata, hooks, and logging are properly initialized - data: Final[dict[str, object]] = {} + bedrock_payload: Final[dict[str, object]] = { + "model": model, + "method": request.method, + "endpoint": endpoint, + "data": request_body, + "custom_llm_provider": "bedrock", + } + data: Final[dict[str, object]] = guardrail_request_data_with_streaming( + MappingProxyType(bedrock_payload), + is_streaming=is_streaming, + ) base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data) - data["model"] = model - data["method"] = request.method - data["endpoint"] = endpoint - data["data"] = request_body - data["custom_llm_provider"] = "bedrock" - # Use the common passthrough processing to handle metadata and hooks # This also handles all response formatting (streaming/non-streaming) and exceptions try: @@ -1285,14 +1289,19 @@ async def bedrock_llm_proxy_route( "Bedrock passthrough: Using direct Bedrock model '%s' for endpoint '%s'", model, endpoint ) - data: Final[dict[str, object]] = {} + is_streaming: Final = is_bedrock_streaming_endpoint(endpoint) + passthrough_payload: Final[dict[str, object]] = { + "method": request.method, + "endpoint": endpoint, + "data": request_body, + "custom_llm_provider": "bedrock", + } + data: Final[dict[str, object]] = guardrail_request_data_with_streaming( + MappingProxyType(passthrough_payload), + is_streaming=is_streaming, + ) base_llm_response_processor: Final = ProxyBaseLLMRequestProcessing(data=data) - data["method"] = request.method - data["endpoint"] = endpoint - data["data"] = request_body - data["custom_llm_provider"] = "bedrock" - try: result: Final = await base_llm_response_processor.base_passthrough_process_llm_request( request=request, diff --git a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py index 07053ad0f32..ee958679c1f 100644 --- a/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py +++ b/litellm/proxy/pass_through_endpoints/pass_through_endpoints.py @@ -48,7 +48,11 @@ from litellm.constants import ( SESSION_ID_OMITTED_METADATA_KEY, WEBSOCKET_CLOSE_REASON_MAX_BYTES, ) -from litellm.integrations.custom_guardrail import CustomGuardrail +from litellm.integrations.custom_guardrail import ( + CustomGuardrail, + guardrail_request_data_with_streaming, + without_server_streaming_classification, +) from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.core_helpers import ( bind_budget_reservation_to_callbacks, @@ -603,6 +607,12 @@ class HttpPassThroughEndpointHelpers(BasePassthroughUtils): from litellm.proxy.proxy_server import llm_router _parsed_body = _parsed_body or {} + parsed_body_typed: Final[Mapping[str, object]] = cast(Mapping[str, object], _parsed_body) # cast-ok: json + server_marker_free_body: Final = without_server_streaming_classification(parsed_body_typed) + # The marker-free body must propagate through the caller's request dict, so + # downstream guardrail scans and snapshots never observe the server streaming marker. + _parsed_body.clear() + _parsed_body.update(server_marker_free_body) managed_model: Final = get_model_from_request( request_data=_parsed_body, route=get_request_route(request), @@ -828,7 +838,7 @@ def _build_passthrough_failure_request_payload( error response. Spend tracking only attributes a recovered cost when it comes paired with a usage object, so both keys are written together. """ - request_payload: Final[dict] = dict(parsed_body or {}) + request_payload: Final[dict] = dict(cast(Mapping[str, object], parsed_body or {})) # cast-ok: json body if kwargs: request_payload.update(kwargs) if logging_obj is not None: @@ -1163,7 +1173,7 @@ async def pass_through_request( is_multipart: Final = HttpPassThroughEndpointHelpers.is_multipart(request) and not custom_body if custom_body: - _parsed_body = custom_body + _parsed_body = dict(custom_body) elif is_multipart: # Don't parse multipart body here - it will be handled by make_multipart_http_request _parsed_body = {} @@ -1232,6 +1242,14 @@ async def pass_through_request( if _parsed_body is None: _parsed_body = {} _parsed_body["litellm_logging_obj"] = logging_obj + is_streaming_pass_through: Final = bool( + HttpPassThroughEndpointHelpers._update_stream_param_based_on_request_body( + parsed_body=_parsed_body, + stream=stream, + ) + ) + typed_body: Final[Mapping[str, object]] = cast(Mapping[str, object], _parsed_body) # cast-ok: json + _parsed_body = guardrail_request_data_with_streaming(typed_body, is_streaming=is_streaming_pass_through) ### CALL HOOKS ### - modify incoming data / reject request before calling the model _parsed_body = await proxy_logging_obj.pre_call_hook( @@ -2518,10 +2536,10 @@ async def websocket_passthrough_request( ) ### CALL HOOKS ### - modify incoming data / reject request before calling the model - websocket_data: dict[str, object] = {} - websocket_data = await proxy_logging_obj.pre_call_hook( + websocket_hook_data: Final = guardrail_request_data_with_streaming(MappingProxyType({}), is_streaming=True) + await proxy_logging_obj.pre_call_hook( user_api_key_dict=user_api_key_dict, - data=websocket_data, + data=websocket_hook_data, call_type="pass_through_endpoint", ) diff --git a/litellm/proxy/policy_engine/pipeline_executor.py b/litellm/proxy/policy_engine/pipeline_executor.py index d80eb56d8e3..5ccf2920030 100644 --- a/litellm/proxy/policy_engine/pipeline_executor.py +++ b/litellm/proxy/policy_engine/pipeline_executor.py @@ -8,7 +8,8 @@ pass/fail actions (allow, block, next, modify_response) and data forwarding. import copy import time from collections.abc import Callable, Mapping, Sequence -from typing import TYPE_CHECKING, Final, Literal, TypeVar +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, Literal, TypeVar, cast from pydantic import BaseModel @@ -29,6 +30,7 @@ from litellm.proxy.common_utils.callback_utils import add_guardrail_to_applied_g from litellm.proxy.guardrails.guardrail_hooks.unified_guardrail.unified_guardrail import ( UnifiedLLMGuardrails, ) +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.proxy.policy_engine.pipeline_types import ( PipelineExecutionResult, PipelineStep, @@ -265,6 +267,30 @@ class _LegacyHookStreamAdapter(CustomGuardrail): return recorder.inputs +_PIPELINE_EVENT_HOOKS: Final = MappingProxyType( + { + "pre_call": GuardrailEventHooks.pre_call, + "post_call": GuardrailEventHooks.post_call, + "during_call": GuardrailEventHooks.during_call, + } +) + + +def _pipeline_stream_scope_allows( + callback: CustomGuardrail, + hook_input: Mapping[str, object], + mode: str, + streaming_chunks: list[object] | None, +) -> bool: + event_type: Final = _PIPELINE_EVENT_HOOKS.get(mode) + if event_type is None: + return True + return callback.stream_scope_allows( + hook_input if streaming_chunks is None else {**hook_input, "stream": True}, + event_type, + ) + + def _prepare_hook_input( step: PipelineStep, callback: CustomGuardrail, @@ -494,7 +520,7 @@ class PipelineExecutor: streaming_chunks: list[object] | None = None, # mutable-ok: shared buffered-stream chunks, read per step endpoint_translation: "BaseTranslation | None" = None, ) -> tuple[ - Literal["pass", "fail", "error"], + Literal["pass", "fail", "error", "skip"], dict | None, str | None, Exception | None, @@ -504,7 +530,7 @@ class PipelineExecutor: Returns: Tuple of (outcome, modified_data, error_detail, original_exception): - - outcome: "pass", "fail", or "error" + - outcome: "pass", "fail", "error", or "skip" - modified_data: dict if guardrail returned modified data, else None - error_detail: error message string if fail/error, else None - original_exception: the exception the guardrail raised, so the @@ -516,6 +542,10 @@ class PipelineExecutor: verbose_proxy_logger.warning("Pipeline: guardrail '%s' not found in callbacks", step.guardrail) return ("error", None, f"Guardrail '{step.guardrail}' not found", None) + hook_data: Final[Mapping[str, object]] = cast(Mapping[str, object], data) # cast-ok: payload + if not _pipeline_stream_scope_allows(callback, hook_data, mode, streaming_chunks): + return ("skip", None, None, None) + hook_input, scans_raw_request = _prepare_hook_input(step, callback, data, raw_request_snapshot) snapshot_entries_before: Final = len(_recorded_guardrail_information(hook_input)) @@ -702,10 +732,13 @@ def _pipeline_action_for_outcome(step: PipelineStep, outcome: str) -> str: """ Map pipeline step outcome to the configured action. + - skip -> next (stream_scope mismatch; do not apply on_pass/on_fail) - pass -> on_pass - fail -> on_fail (content/policy intervention) - error -> on_error if set, else on_fail (backward compatible) """ + if outcome == "skip": + return "next" if outcome == "pass": return step.on_pass if outcome == "fail": diff --git a/litellm/types/guardrails.py b/litellm/types/guardrails.py index 201b92f8107..ecc3d0d747c 100644 --- a/litellm/types/guardrails.py +++ b/litellm/types/guardrails.py @@ -2,11 +2,12 @@ from collections.abc import Mapping from datetime import datetime from enum import Enum from types import MappingProxyType -from typing import Final, Literal +from typing import Final, Literal, cast from pydantic import ConfigDict, Field, field_validator, model_validator from typing_extensions import ReadOnly, Required, TypedDict +from litellm._logging import verbose_logger from litellm.constants import BEDROCK_APPLY_GUARDRAIL_CHUNK_BUDGET_CHARS from litellm.types.llms.base import LiteLLMBaseModel from litellm.types.proxy.guardrails.guardrail_hooks.agent_365 import ( @@ -901,6 +902,93 @@ class ContentFilterConfigModel(LiteLLMBaseModel): MCP_SECURITY_ON_VIOLATION: Final = frozenset({"block", "alert"}) +GuardrailStreamScope = Literal["streaming", "non_streaming", "both"] +DEFAULT_GUARDRAIL_STREAM_SCOPE: Final[GuardrailStreamScope] = "both" + + +class GuardrailEventHooks(str, Enum): + pre_call = "pre_call" + post_call = "post_call" + during_call = "during_call" + logging_only = "logging_only" + pre_mcp_call = "pre_mcp_call" + during_mcp_call = "during_mcp_call" + post_mcp_call = "post_mcp_call" + realtime_input_transcription = "realtime_input_transcription" + + +GUARDRAIL_EVENT_HOOK_VALUES: Final = frozenset(member.value for member in GuardrailEventHooks) + +_GUARDRAIL_STREAM_SCOPES: Final[Mapping[str, GuardrailStreamScope]] = MappingProxyType( + { + "streaming": "streaming", + "non_streaming": "non_streaming", + "both": "both", + } +) + + +def _as_guardrail_stream_scope(value: object) -> GuardrailStreamScope: + if not isinstance(value, str): + raise ValueError(f"stream_scope values must be strings, got {type(value).__name__}") + scope: Final = _GUARDRAIL_STREAM_SCOPES.get(value.lower()) + if scope is None: + raise ValueError(f"stream_scope must be one of both, streaming, non_streaming, got {value!r}") + return scope + + +def _validated_stream_scope_hook(key: object) -> str: + if not isinstance(key, str): + raise ValueError(f"stream_scope keys must be strings, got {type(key).__name__}") + hook: Final = key.lower() + if hook not in GUARDRAIL_EVENT_HOOK_VALUES: + raise ValueError( + f"stream_scope keys must be guardrail modes ({sorted(GUARDRAIL_EVENT_HOOK_VALUES)}), got {key!r}" + ) + return hook + + +def coerce_stream_scope(value: object) -> GuardrailStreamScope | dict[str, GuardrailStreamScope] | None: + if value is None: + return None + if isinstance(value, str): + return _as_guardrail_stream_scope(value) + if isinstance(value, Mapping): + scope_map: Final[Mapping[str, object]] = cast(Mapping[str, object], value) # cast-ok: keys validated below + return { + _validated_stream_scope_hook(key): _as_guardrail_stream_scope(scope) for key, scope in scope_map.items() + } + raise ValueError(f"stream_scope must be a string or mapping, got {type(value).__name__}") + + +def stored_stream_scope(value: object) -> GuardrailStreamScope | dict[str, GuardrailStreamScope] | None: + try: + return coerce_stream_scope(value) + except ValueError: + verbose_logger.warning("Ignoring invalid stored stream_scope value of type %s", type(value).__name__) + return None + + +def with_tolerated_stream_scope(params: Mapping[str, object]) -> dict[str, object]: + if "stream_scope" not in params: + return dict(params) + return { + **params, + "stream_scope": stored_stream_scope(params["stream_scope"]), + } + + +def runtime_stream_scope( + stream_scope: object, +) -> tuple[GuardrailStreamScope, MappingProxyType[str, GuardrailStreamScope]]: + coerced: Final = coerce_stream_scope(stream_scope) + if coerced is None: + return DEFAULT_GUARDRAIL_STREAM_SCOPE, MappingProxyType({}) + if isinstance(coerced, str): + return coerced, MappingProxyType({}) + return DEFAULT_GUARDRAIL_STREAM_SCOPE, MappingProxyType(coerced) + + LoggingOnlyScope = Literal["input", "output", "both"] @@ -1144,6 +1232,21 @@ class BaseLitellmParams(ContentFilterConfigModel): # works for new and patch up ), ) + stream_scope: GuardrailStreamScope | dict[str, GuardrailStreamScope] | None = Field( + default=None, + description=( + "Whether this guardrail runs on streaming requests, non-streaming requests, or both. " + "A string applies to every configured mode. A map overrides named modes " + "(pre_call, during_call, post_call, ...); omitted keys default to both. " + "Unset means both, matching historical behavior." + ), + ) + + @field_validator("stream_scope", mode="before") + @classmethod + def normalize_stream_scope(cls, v: object) -> GuardrailStreamScope | dict[str, GuardrailStreamScope] | None: + return coerce_stream_scope(v) + logging_only_scope: LoggingOnlyScope | None = Field( default=None, description=( @@ -1278,17 +1381,6 @@ class guardrailConfig(TypedDict): guardrails: list[Guardrail] -class GuardrailEventHooks(str, Enum): - pre_call = "pre_call" - post_call = "post_call" - during_call = "during_call" - logging_only = "logging_only" - pre_mcp_call = "pre_mcp_call" - during_mcp_call = "during_mcp_call" - post_mcp_call = "post_mcp_call" - realtime_input_transcription = "realtime_input_transcription" - - class DynamicGuardrailParams(TypedDict): extra_body: ReadOnly[dict[str, object]] diff --git a/litellm/types/proxy/policy_engine/pipeline_types.py b/litellm/types/proxy/policy_engine/pipeline_types.py index 5278754701d..0a089d2fe9b 100644 --- a/litellm/types/proxy/policy_engine/pipeline_types.py +++ b/litellm/types/proxy/policy_engine/pipeline_types.py @@ -87,7 +87,7 @@ class PipelineStepResult(LiteLLMBaseModel): """Result of executing a single pipeline step.""" guardrail_name: str - outcome: Literal["pass", "fail", "error"] + outcome: Literal["pass", "fail", "error", "skip"] action_taken: str modified_data: dict[str, Any] | None = None error_detail: str | None = None diff --git a/tests/integration/observability/test_guardrail_stream_scope.py b/tests/integration/observability/test_guardrail_stream_scope.py new file mode 100644 index 00000000000..8b832f07aa4 --- /dev/null +++ b/tests/integration/observability/test_guardrail_stream_scope.py @@ -0,0 +1,856 @@ +from __future__ import annotations + +import base64 +import json +import os +import uuid +from collections.abc import Iterator, Mapping, Sequence +from dataclasses import dataclass +from itertools import chain +from pathlib import Path +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, JsonValue, Scenario, eventually +from integration._support.database import read_rows, write_rows +from integration._support.process import owned_proxy_process +from integration._support.upstream import ( + _aws_event_frame, # pyright: ignore[reportPrivateUsage] # project Bedrock event-stream encoder +) +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import TypeAdapter + +import litellm + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +EndpointScope: TypeAlias = Literal["streaming", "non_streaming"] +ModelKind: TypeAlias = Literal["router", "direct"] +StreamingAction: TypeAlias = Literal["converse-stream", "invoke-with-response-stream"] +NonStreamingAction: TypeAlias = Literal["converse", "invoke"] +BedrockAction: TypeAlias = StreamingAction | NonStreamingAction +BEDROCK_MODEL_ID: Final = "anthropic.claude-sonnet-5-v1:0" +BEDROCK_FALSE_POSITIVE_MODEL_ID: Final = "anthropic.claude-converse-stream-test-v1:0" +BEDROCK_EVENT_STREAM: Final = "application/vnd.amazon.eventstream" +GUARDRAIL_PATH: Final = "/beta/litellm_basic_guardrail_api" +SERVER_STREAMING_CLASSIFICATION_KEY: Final = "litellm_server_streaming_classification" +STREAMING_ACTIONS: Final[tuple[StreamingAction, ...]] = ( + "converse-stream", + "invoke-with-response-stream", +) +NON_STREAMING_ACTIONS: Final[tuple[NonStreamingAction, ...]] = ("converse", "invoke") +SCOPES: Final[tuple[EndpointScope, ...]] = ("streaming", "non_streaming") +HOSTILE_CLASSIFICATION_CASES: Final = ( + pytest.param("is_streaming_request", True, id="boolean-marker"), + pytest.param("is_streaming_request", "litellm-server-streaming", id="server-marker-string"), + pytest.param("litellm_server_streaming_classification", True, id="classification-field"), +) +WORKTREE: Final = Path(__file__).resolve().parents[3] +LITELLM_PATH: Final = Path(litellm.__file__).resolve() +assert LITELLM_PATH.is_relative_to(WORKTREE), (LITELLM_PATH, WORKTREE) +print(f"stream_scope repro litellm import: {LITELLM_PATH}") # noqa: T201 # required worktree evidence + + +def _json(value: object) -> bytes: + return json.dumps(value, separators=(",", ":")).encode() + + +def _strings(value: JsonValue) -> tuple[str, ...]: + if isinstance(value, str): + return (value,) + if isinstance(value, list): + return tuple(chain.from_iterable(_strings(item) for item in value)) + if isinstance(value, dict): + return tuple(chain.from_iterable(_strings(item) for item in value.values())) + return () + + +def _key_names(value: JsonValue) -> tuple[str, ...]: + if isinstance(value, dict): + return tuple(value) + tuple(chain.from_iterable(_key_names(item) for item in value.values())) + if isinstance(value, list): + return tuple(chain.from_iterable(_key_names(item) for item in value)) + return () + + +def _marker(body: JsonValue) -> str: + return next(value for value in _strings(body) if value.startswith("scope-")) + + +def _bedrock_converse_stream(marker: str) -> bytes: + return b"".join( + _aws_event_frame(event_type, payload, marker, marker) + for event_type, payload in ( + ("messageStart", {"role": "assistant"}), + ( + "contentBlockDelta", + {"delta": {"text": f"scripted Bedrock reply {marker}"}, "contentBlockIndex": 0}, + ), + ("messageStop", {"stopReason": "end_turn"}), + ("metadata", {"usage": {"inputTokens": 11, "outputTokens": 4, "totalTokens": 15}}), + ) + ) + + +def _invoke_chunk(payload: Mapping[str, JsonValue], marker: str) -> bytes: + encoded: Final = base64.b64encode(_json(payload)).decode() + return _aws_event_frame("chunk", {"bytes": encoded}, marker, marker) + + +def _bedrock_invoke_stream(marker: str) -> bytes: + events: Final = ( + { + "type": "message_start", + "message": { + "id": f"msg-{marker}", + "type": "message", + "role": "assistant", + "model": BEDROCK_MODEL_ID, + "content": [], + "stop_reason": None, + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 0}, + }, + }, + {"type": "content_block_start", "index": 0, "content_block": {"type": "text", "text": ""}}, + { + "type": "content_block_delta", + "index": 0, + "delta": {"type": "text_delta", "text": f"scripted Bedrock reply {marker}"}, + }, + {"type": "content_block_stop", "index": 0}, + { + "type": "message_delta", + "delta": {"stop_reason": "end_turn"}, + "usage": {"input_tokens": 11, "output_tokens": 4}, + }, + {"type": "message_stop"}, + ) + return b"".join(_invoke_chunk(event, marker) for event in events) + + +def _provider(request: Request) -> Reply: + if not request.body: + return Reply(status=400, body=_json({"error": "empty request body"})) + body: Final = JSON_OBJECT.validate_json(request.body) + marker: Final = _marker(body) + target: Final = request.target.split("?", 1)[0] + if target.startswith("/passthrough"): + if body.get("stream") is True: + streamed_response: Final = _json({"received": body}) + return Reply( + content_type="text/event-stream", + chunks=(b"data: " + streamed_response + b"\n\n", b"data: [DONE]\n\n"), + ) + return Reply(body=_json({"received": body})) + if target.endswith("/converse-stream"): + return Reply(body=_bedrock_converse_stream(marker), content_type=BEDROCK_EVENT_STREAM) + if target.endswith("/invoke-with-response-stream"): + return Reply(body=_bedrock_invoke_stream(marker), content_type=BEDROCK_EVENT_STREAM) + if target.endswith("/converse") or target.endswith("/invoke"): + return Reply(body=_json({"output": f"scripted Bedrock reply {marker}"})) + if target == "/v1/chat/completions": + if body.get("stream") is True: + chunk: Final = { + "id": f"chatcmpl-{marker}", + "object": "chat.completion.chunk", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "delta": {"role": "assistant", "content": f"scripted chat reply {marker}"}, + "finish_reason": None, + } + ], + } + final_chunk: Final = { + **chunk, + "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], + } + return Reply( + content_type="text/event-stream", + chunks=( + b"data: " + _json(chunk) + b"\n\n", + b"data: " + _json(final_chunk) + b"\n\n", + b"data: [DONE]\n\n", + ), + ) + return Reply( + body=_json( + { + "id": f"chatcmpl-{marker}", + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": f"scripted chat reply {marker}"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + ) + return Reply(status=404, body=_json({"error": f"unexpected upstream path: {target}"})) + + +def _sink(request: Request) -> Reply: + assert request.target.endswith(GUARDRAIL_PATH), request.target + assert b"scope-" in request.body, request.body.decode() + return Reply(body=_json({"action": "NONE"})) + + +def _rail(name: str, sink: Wire, scope: EndpointScope) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "stream_scope": scope, + "api_base": f"{sink.url}/{name}", + "api_key": "synthetic-guardrail-key", + }, + } + + +def _chat_proxy_config(provider_url: str, guardrails: list[dict[str, JsonValue]]) -> dict[str, object]: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + return { + **config, + "guardrails": guardrails, + "model_list": [ + { + "model_name": "scope-invalid-config-chat", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider_url}/v1", + "api_key": "synthetic-provider-key", + }, + } + ], + } + + +@dataclass(frozen=True, slots=True) +class ReproRig: + candidate: Gateway + direct_candidate: Gateway + scenario: Scenario + models: Mapping[str, str] + rails: Mapping[str, str] + provider: Wire + sink: Wire + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[ReproRig]: + with httpx.Client( + base_url=os.environ["INTEGRATION_PROXY_URL"], + timeout=30, + trust_env=False, + ) as root_client: + root_gateway: Final = Gateway( + root_client, + os.environ.get("INTEGRATION_MASTER_KEY", "sk-integration-master"), + os.environ["INTEGRATION_UPSTREAM_URL"], + ) + directory: Final = tmp_path_factory.mktemp("guardrail-stream-scope-repro") + with wire_server(_provider) as provider, wire_server(_sink) as sink: + rails: Final = MappingProxyType( + { + "bedrock_streaming": "bedrock_streaming", + "bedrock_non_streaming": "bedrock_non_streaming", + "chat_streaming": "chat_streaming", + "passthrough_streaming": "passthrough_streaming", + "passthrough_non_streaming": "passthrough_non_streaming", + "passthrough_spoof_streaming": "passthrough_spoof_streaming", + } + ) + models: Final = MappingProxyType( + { + "chat": "scope-chat", + "bedrock_router": "scope-bedrock-router", + "bedrock_false_positive_router": "scope-bedrock-converse-stream-model", + } + ) + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["guardrails"] = [ + _rail(rails["bedrock_streaming"], sink, "streaming"), + _rail(rails["bedrock_non_streaming"], sink, "non_streaming"), + _rail(rails["chat_streaming"], sink, "streaming"), + _rail(rails["passthrough_streaming"], sink, "streaming"), + _rail(rails["passthrough_non_streaming"], sink, "non_streaming"), + _rail(rails["passthrough_spoof_streaming"], sink, "streaming"), + ] + config["model_list"] = [ + { + "model_name": models["chat"], + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider.url}/v1", + "api_key": "synthetic-provider-key", + }, + }, + { + "model_name": models["bedrock_router"], + "litellm_params": { + "model": f"bedrock/{BEDROCK_MODEL_ID}", + "api_base": provider.url, + "aws_access_key_id": "AKIASYNTHETICSTREAMSCOPE", + "aws_secret_access_key": "synthetic-bedrock-secret", + "aws_region_name": "us-east-1", + }, + }, + { + "model_name": models["bedrock_false_positive_router"], + "litellm_params": { + "model": f"bedrock/{BEDROCK_FALSE_POSITIVE_MODEL_ID}", + "api_base": provider.url, + "aws_access_key_id": "AKIASYNTHETICSTREAMSCOPE", + "aws_secret_access_key": "synthetic-bedrock-secret", + "aws_region_name": "us-east-1", + }, + }, + ] + config["environment_variables"] = { + "AWS_BEDROCK_RUNTIME_ENDPOINT": provider.url, + "AWS_ACCESS_KEY_ID": "AKIASYNTHETICSTREAMSCOPE", + "AWS_SECRET_ACCESS_KEY": "synthetic-bedrock-secret", + "AWS_REGION": "us-east-1", + "AWS_REGION_NAME": "us-east-1", + } + config["general_settings"]["pass_through_endpoints"] = [ + { + "path": "/pt-forward", + "target": f"{provider.url}/passthrough", + "include_subpath": True, + }, + { + "path": "/pt-spoof", + "target": f"{provider.url}/passthrough", + "include_subpath": True, + "guardrails": {rails["passthrough_spoof_streaming"]: None}, + }, + { + "path": "/pt-scope", + "target": f"{provider.url}/passthrough", + "include_subpath": True, + "guardrails": { + rails["passthrough_streaming"]: None, + rails["passthrough_non_streaming"]: None, + }, + }, + ] + config_path: Final = directory / "stream-scope-repro.yaml" + config_path.write_text(yaml.safe_dump(config)) + with owned_proxy_process(root_gateway, directory, {}, config=config_path, workers=1) as owned: + with owned.gateway.scenario() as scenario: + direct_config: Final = { + **config, + "model_list": [ + *config["model_list"], + { + "model_name": f"scope-unused-{uuid.uuid4().hex}*", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider.url}/v1", + "api_key": "synthetic-provider-key", + }, + }, + ], + } + direct_config_path: Final = directory / "stream-scope-direct.yaml" + direct_config_path.write_text(yaml.safe_dump(direct_config)) + with owned_proxy_process( + root_gateway, + directory, + {}, + config=direct_config_path, + workers=1, + ) as direct: + yield ReproRig( + owned.gateway, + direct.gateway, + scenario, + models, + rails, + provider, + sink, + ) + + +def _bedrock_request( + action: BedrockAction, + model_path: str, + marker: str, +) -> tuple[str, dict[str, JsonValue]]: + if action in ("converse", "converse-stream"): + body: Final = { + "messages": [{"role": "user", "content": [{"text": marker}]}], + "inferenceConfig": {"maxTokens": 16}, + } + else: + body = { + "anthropic_version": "bedrock-2023-05-31", + "max_tokens": 16, + "messages": [{"role": "user", "content": [{"type": "text", "text": marker}]}], + } + return f"/bedrock/model/{model_path}/{action}", body + + +def _matching_requests(wire: Wire, marker: str) -> tuple[Request, ...]: + return tuple(request for request in wire.drain() if marker.encode() in request.body) + + +def _rail_scans(rows: Sequence[Request], rail_name: str, marker: str) -> tuple[Request, ...]: + return tuple( + request for request in rows if request.target.startswith(f"/{rail_name}/") and marker.encode() in request.body + ) + + +def _chat_request_with_scans( + gateway: Gateway, + sink: Wire, + model: str, + marker: str, + streamed: bool, + rail_name: str, +) -> tuple[httpx.Response, tuple[Request, ...]]: + response: Final = gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": marker}], + "stream": streamed, + "guardrails": [rail_name], + }, + ) + return response, _rail_scans(sink.drain(), rail_name, marker) + + +def _spend_row_for_call(call_id: str, content: bytes) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + "SELECT request_id, litellm_call_id, spend, prompt_tokens, completion_tokens, metadata " + 'FROM "LiteLLM_SpendLogs" WHERE request_id=%s OR litellm_call_id=%s', + (call_id, call_id), + ), + lambda values: len(values) >= 1, + seconds=70, + ) + assert len(rows) == 1, (call_id, rows, content) + assert call_id in (rows[0]["request_id"], rows[0]["litellm_call_id"]), content + return rows[0] + + +@pytest.mark.parametrize("model_kind", ("router", "direct")) +@pytest.mark.parametrize("action", STREAMING_ACTIONS) +@pytest.mark.parametrize("scope", SCOPES) +def test_bedrock_streaming_actions_run_streaming_scoped_rails( + rig: ReproRig, + model_kind: ModelKind, + action: StreamingAction, + scope: EndpointScope, +) -> None: + marker: Final = f"scope-bedrock-stream-{uuid.uuid4().hex}" + model_path: Final = rig.models["bedrock_router"] if model_kind == "router" else BEDROCK_MODEL_ID + path, body = _bedrock_request(action, model_path, marker) + expected_body: Final = ( + _bedrock_converse_stream(marker) if action == "converse-stream" else _bedrock_invoke_stream(marker) + ) + call_id: Final = f"stream-scope-{uuid.uuid4().hex}" + key: Final = rig.scenario.key(guardrails=[rig.rails[f"bedrock_{scope}"]]) + candidate: Final = rig.candidate if model_kind == "router" else rig.direct_candidate + response: Final = candidate.request( + "POST", + path, + body, + key=key, + headers={"x-litellm-call-id": call_id}, + ) + assert response.status_code == 200, response.content + assert response.headers.get("content-type") == BEDROCK_EVENT_STREAM, dict(response.headers) + assert response.content == expected_body, response.content + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows, response.content) + provider_body: Final = JSON_OBJECT.validate_json(provider_rows[0].body) + key_names: Final = _key_names(provider_body) + assert "is_streaming_request" not in key_names, provider_body + assert not tuple(name for name in key_names if name.startswith("litellm_")), provider_body + _spend_row_for_call(call_id, response.content) + sink_rows: Final = _rail_scans(rig.sink.drain(), rig.rails[f"bedrock_{scope}"], marker) + expected_scans: Final = int(scope == "streaming") + assert len(sink_rows) == expected_scans, (marker, model_kind, action, scope, sink_rows, response.content) + + +@pytest.mark.parametrize("model_kind", ("router", "direct")) +@pytest.mark.parametrize("action", NON_STREAMING_ACTIONS) +@pytest.mark.parametrize("scope", SCOPES) +def test_bedrock_non_streaming_actions_run_non_streaming_scoped_rails( + rig: ReproRig, + model_kind: ModelKind, + action: NonStreamingAction, + scope: EndpointScope, +) -> None: + marker: Final = f"scope-bedrock-nonstream-{uuid.uuid4().hex}" + model_path: Final = rig.models["bedrock_router"] if model_kind == "router" else BEDROCK_MODEL_ID + path, body = _bedrock_request(action, model_path, marker) + key: Final = rig.scenario.key(guardrails=[rig.rails[f"bedrock_{scope}"]]) + candidate: Final = rig.candidate if model_kind == "router" else rig.direct_candidate + response: Final = candidate.request("POST", path, body, key=key) + assert response.status_code == 200, response.text + assert marker in response.text, response.text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows, response.text) + provider_body: Final = JSON_OBJECT.validate_json(provider_rows[0].body) + key_names: Final = _key_names(provider_body) + assert "is_streaming_request" not in key_names, provider_body + assert not tuple(name for name in key_names if name.startswith("litellm_")), provider_body + sink_rows: Final = _rail_scans(rig.sink.drain(), rig.rails[f"bedrock_{scope}"], marker) + expected_scans: Final = int(scope == "non_streaming") + assert len(sink_rows) == expected_scans, (marker, model_kind, action, scope, sink_rows, response.text) + + +@pytest.mark.parametrize("model_kind", ("router", "direct")) +def test_bedrock_model_id_streaming_action_text_on_converse_is_non_streaming( + rig: ReproRig, + model_kind: ModelKind, +) -> None: + marker: Final = f"scope-bedrock-converse-model-{uuid.uuid4().hex}" + model_path: Final = ( + rig.models["bedrock_false_positive_router"] if model_kind == "router" else BEDROCK_FALSE_POSITIVE_MODEL_ID + ) + path, body = _bedrock_request("converse", model_path, marker) + call_id: Final = f"stream-scope-bedrock-{uuid.uuid4().hex}" + key: Final = rig.scenario.key( + guardrails=[ + rig.rails["bedrock_streaming"], + rig.rails["bedrock_non_streaming"], + ] + ) + candidate: Final = rig.candidate if model_kind == "router" else rig.direct_candidate + response: Final = candidate.request( + "POST", + path, + body, + key=key, + headers={"x-litellm-call-id": call_id}, + ) + assert response.status_code == 200, response.text + assert marker in response.text, response.text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, model_kind, provider_rows, response.text) + sink_rows: Final = rig.sink.drain() + streaming_rows: Final = _rail_scans(sink_rows, rig.rails["bedrock_streaming"], marker) + non_streaming_rows: Final = _rail_scans(sink_rows, rig.rails["bedrock_non_streaming"], marker) + assert streaming_rows == (), (marker, model_kind, streaming_rows, response.text) + assert len(non_streaming_rows) == 1, (marker, model_kind, non_streaming_rows, response.text) + + +@pytest.mark.parametrize("streamed", (False, True), ids=("stream-absent", "stream-true")) +def test_configured_passthrough_forwards_caller_is_streaming_request_field( + rig: ReproRig, + streamed: bool, +) -> None: + marker: Final = f"scope-passthrough-forward-{uuid.uuid4().hex}" + caller_value: Final = f"caller-{uuid.uuid4().hex}" + body: Final = { + "marker": marker, + "is_streaming_request": caller_value, + **({"stream": True} if streamed else {}), + } + response: Final = rig.candidate.request("POST", "/pt-forward", body) + assert response.status_code == 200, response.text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows, response.text) + upstream_body: Final = JSON_OBJECT.validate_json(provider_rows[0].body) + assert upstream_body.get("is_streaming_request") == caller_value, ( + marker, + caller_value, + upstream_body, + response.text, + ) + assert upstream_body == body, (marker, body, upstream_body, response.text) + if streamed: + assert response.headers.get("content-type", "").lower().startswith("text/event-stream"), dict(response.headers) + event_body: Final = response.text.removeprefix("data: ").split("\n", maxsplit=1)[0] + response_body: Final = JSON_OBJECT.validate_json(event_body) + else: + response_body = JSON_OBJECT.validate_json(response.content) + assert response_body == {"received": body}, response.text + + +@pytest.mark.parametrize(("hostile_field", "hostile_value"), HOSTILE_CLASSIFICATION_CASES) +def test_client_cannot_spoof_server_stream_classification( + rig: ReproRig, + hostile_field: str, + hostile_value: JsonValue, +) -> None: + chat_marker: Final = f"scope-chat-spoof-{uuid.uuid4().hex}" + chat_body: Final = { + "model": rig.models["chat"], + "messages": [{"role": "user", "content": chat_marker}], + "stream": False, + hostile_field: hostile_value, + } + chat_key: Final = rig.scenario.key(guardrails=[rig.rails["chat_streaming"]]) + chat_response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + chat_body, + key=chat_key, + ) + chat_provider_rows: Final = _matching_requests(rig.provider, chat_marker) + chat_upstream_body: Final = chat_provider_rows[0].body.decode() if chat_provider_rows else "" + print( # noqa: T201 # required chat hostile-body observation + f"chat hostile {hostile_field}={hostile_value!r}: " + f"status={chat_response.status_code}, response={chat_response.text!r}, upstream={chat_upstream_body}" + ) + assert chat_response.status_code == 200, chat_response.text + assert len(chat_provider_rows) == 1, (chat_marker, chat_provider_rows, chat_response.text) + assert JSON_OBJECT.validate_json(chat_provider_rows[0].body) == { + "messages": [{"role": "user", "content": chat_marker}], + "model": "gpt-4o-mini", + hostile_field: hostile_value, + }, (hostile_field, hostile_value, chat_upstream_body) + assert _rail_scans(rig.sink.drain(), rig.rails["chat_streaming"], chat_marker) == (), ( + chat_marker, + hostile_field, + hostile_value, + chat_response.text, + ) + + +@pytest.mark.parametrize(("hostile_field", "hostile_value"), HOSTILE_CLASSIFICATION_CASES) +def test_configured_passthrough_cannot_spoof_server_stream_classification( + rig: ReproRig, + hostile_field: str, + hostile_value: JsonValue, +) -> None: + passthrough_marker: Final = f"scope-passthrough-spoof-{uuid.uuid4().hex}" + passthrough_body: Final = { + "marker": passthrough_marker, + "stream": False, + hostile_field: hostile_value, + } + passthrough_response: Final = rig.candidate.request("POST", "/pt-spoof", passthrough_body) + assert passthrough_response.status_code == 200, passthrough_response.text + passthrough_provider_rows: Final = _matching_requests(rig.provider, passthrough_marker) + assert len(passthrough_provider_rows) == 1, ( + passthrough_marker, + passthrough_provider_rows, + passthrough_response.text, + ) + assert _rail_scans(rig.sink.drain(), rig.rails["passthrough_spoof_streaming"], passthrough_marker) == (), ( + passthrough_marker, + hostile_field, + hostile_value, + passthrough_response.text, + ) + + +@pytest.mark.parametrize( + ("streamed", "request_fields"), + ((True, {"stream": True}), (False, {"stream": False}), (False, {})), + ids=("stream-true", "stream-false", "stream-absent"), +) +def test_passthrough_scope_follows_proxy_stream_decision( + rig: ReproRig, + streamed: bool, + request_fields: dict[str, JsonValue], +) -> None: + marker: Final = f"scope-passthrough-scope-{uuid.uuid4().hex}" + body: Final = {"marker": marker, **request_fields} + response: Final = rig.candidate.request("POST", "/pt-scope", body) + assert response.status_code == 200, response.text + if streamed: + expected_frame: Final = f"data: {_json({'received': body}).decode()}\n\ndata: [DONE]\n\n" + assert response.headers.get("content-type", "").lower().startswith("text/event-stream"), dict(response.headers) + assert response.text == expected_frame, response.text + assert response.headers.get("transfer-encoding", "").lower() == "chunked", dict(response.headers) + else: + response_body: Final = JSON_OBJECT.validate_json(response.content) + assert response_body == {"received": body}, response.text + assert "content-length" in response.headers, dict(response.headers) + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows, response.text) + upstream_body: Final = JSON_OBJECT.validate_json(provider_rows[0].body) + assert upstream_body == body, (marker, body, upstream_body, response.text) + assert SERVER_STREAMING_CLASSIFICATION_KEY not in upstream_body, upstream_body + sink_rows: Final = rig.sink.drain() + streaming_rows: Final = _rail_scans(sink_rows, rig.rails["passthrough_streaming"], marker) + non_streaming_rows: Final = _rail_scans(sink_rows, rig.rails["passthrough_non_streaming"], marker) + assert len(streaming_rows) == int(streamed), (marker, streamed, streaming_rows, response.text) + assert len(non_streaming_rows) == int(not streamed), (marker, streamed, non_streaming_rows, response.text) + + +def test_invalid_yaml_stream_scope_keeps_rail_running_on_both_shapes(rig: ReproRig, tmp_path: Path) -> None: + name: Final = f"scope-invalid-yaml-{uuid.uuid4().hex}" + invalid_rail: Final = { + "guardrail_name": name, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "stream_scope": "sometimes", + "api_base": f"{rig.sink.url}/{name}", + "api_key": "synthetic-guardrail-key", + }, + } + config: Final = _chat_proxy_config(rig.provider.url, [invalid_rail]) + config_path: Final = tmp_path / "invalid-stream-scope.yaml" + config_path.write_text(yaml.safe_dump(config)) + with owned_proxy_process(rig.candidate, tmp_path, {}, config=config_path, workers=1) as owned: + markers: Final = tuple(f"scope-invalid-yaml-{int(streamed)}-{uuid.uuid4().hex}" for streamed in (False, True)) + observations: Final = tuple( + _chat_request_with_scans( + owned.gateway, + rig.sink, + "scope-invalid-config-chat", + marker, + streamed, + name, + ) + for streamed, marker in zip((False, True), markers) + ) + assert tuple(response.status_code for response, _ in observations) == (200, 200), tuple( + response.text for response, _ in observations + ) + assert tuple(len(sink_rows) for _, sink_rows in observations) == (1, 1), (markers, observations) + listed: Final = owned.gateway.request("GET", "/guardrails/list") + assert listed.status_code == 200, listed.text + list_payload: Final = JSON_OBJECT.validate_json(listed.content) + listed_guardrails: Final = list_payload.get("guardrails") + assert isinstance(listed_guardrails, list), list_payload + listed_rail: Final = next( + (row for row in listed_guardrails if isinstance(row, dict) and row.get("guardrail_name") == name), + None, + ) + assert isinstance(listed_rail, dict), list_payload + listed_params: Final = listed_rail.get("litellm_params") + assert isinstance(listed_params, dict), listed_rail + assert listed_params.get("stream_scope") is None, listed_rail + + +def test_persisted_invalid_stream_scope_row_stays_readable_and_enforced( + rig: ReproRig, + tmp_path: Path, +) -> None: + guardrail_id: Final = str(uuid.uuid4()) + guardrail_name: Final = f"scope-invalid-persisted-{uuid.uuid4().hex}" + params: Final = { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": f"{rig.sink.url}/{guardrail_name}", + "api_key": "synthetic-guardrail-key", + "stream_scope": "sometimes", + } + database_url: Final = os.environ.get("INTEGRATION_PROXY_DATABASE_URL") or os.environ["DATABASE_URL"] + write_rows( + 'INSERT INTO "LiteLLM_GuardrailsTable" ' + '("guardrail_id", "guardrail_name", "litellm_params", "created_at", "updated_at") ' + "VALUES (%s, %s, %s::jsonb, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)", + (guardrail_id, guardrail_name, json.dumps(params)), + database_url=database_url, + ) + try: + config: Final = _chat_proxy_config(rig.provider.url, []) + config_path: Final = tmp_path / "persisted-invalid-stream-scope.yaml" + config_path.write_text(yaml.safe_dump(config)) + with owned_proxy_process(rig.candidate, tmp_path, {}, config=config_path, workers=1) as owned: + info: Final = eventually( + lambda: owned.gateway.request("GET", f"/guardrails/{guardrail_id}/info"), + lambda response: response.status_code != 404, + seconds=20, + ) + listed: Final = owned.gateway.request("GET", "/v2/guardrails/list") + list_payload: Final = JSON_OBJECT.validate_json(listed.content) if listed.status_code == 200 else {} + listed_guardrails: Final = list_payload.get("guardrails") + includes_row: Final = isinstance(listed_guardrails, list) and any( + isinstance(row, dict) and row.get("guardrail_id") == guardrail_id for row in listed_guardrails + ) + markers: Final = ( + f"scope-invalid-persisted-0-{uuid.uuid4().hex}", + f"scope-invalid-persisted-1-{uuid.uuid4().hex}", + ) + observations: Final = tuple( + _chat_request_with_scans( + owned.gateway, + rig.sink, + "scope-invalid-config-chat", + marker, + streamed, + guardrail_name, + ) + for streamed, marker in zip((False, True), markers) + ) + assert ( + info.status_code == 200 + and listed.status_code == 200 + and includes_row + and tuple(response.status_code for response, _ in observations) == (200, 200) + and tuple(len(sink_rows) for _, sink_rows in observations) == (1, 1) + ), { + "info": (info.status_code, info.text), + "list": (listed.status_code, listed.text), + "includes_row": includes_row, + "responses": tuple((response.status_code, response.text) for response, _ in observations), + "scan_counts": tuple(len(sink_rows) for _, sink_rows in observations), + } + finally: + write_rows( + 'DELETE FROM "LiteLLM_GuardrailsTable" WHERE guardrail_id=%s', + (guardrail_id,), + database_url=database_url, + ) + + +def test_management_rejects_invalid_stream_scope(rig: ReproRig) -> None: + name: Final = f"scope-invalid-management-{uuid.uuid4().hex}" + invalid_params: Final = { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": f"{rig.sink.url}/{name}", + "api_key": "synthetic-guardrail-key", + "stream_scope": "sometimes", + } + created_invalid: Final = rig.candidate.request( + "POST", + "/guardrails", + {"guardrail": {"guardrail_name": name, "litellm_params": invalid_params}}, + ) + assert created_invalid.status_code == 422, created_invalid.text + + valid_params: Final = {**invalid_params, "stream_scope": "both"} + created: Final = rig.candidate.request( + "POST", + "/guardrails", + {"guardrail": {"guardrail_name": name, "litellm_params": valid_params}}, + ) + assert created.status_code == 200, created.text + guardrail_id: Final = str(created.json()["guardrail_id"]) + try: + put_response: Final = rig.candidate.request( + "PUT", + f"/guardrails/{guardrail_id}", + {"guardrail": {"guardrail_name": name, "litellm_params": invalid_params}}, + ) + patch_response: Final = rig.candidate.request( + "PATCH", + f"/guardrails/{guardrail_id}", + {"litellm_params": {"stream_scope": "sometimes"}}, + ) + assert put_response.status_code == 422, put_response.text + assert patch_response.status_code == 422, patch_response.text + finally: + deleted: Final = rig.candidate.request("DELETE", f"/guardrails/{guardrail_id}") + assert deleted.status_code == 200, deleted.text diff --git a/tests/integration/observability/test_guardrail_stream_scope_chaos.py b/tests/integration/observability/test_guardrail_stream_scope_chaos.py new file mode 100644 index 00000000000..f8a9d0a4bba --- /dev/null +++ b/tests/integration/observability/test_guardrail_stream_scope_chaos.py @@ -0,0 +1,1079 @@ +from __future__ import annotations + +import json +import os +import shutil +import signal +import socket +import subprocess +import uuid +from collections.abc import Iterator, Mapping, Sequence +from concurrent.futures import Future, ThreadPoolExecutor +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from itertools import chain +from pathlib import Path +from threading import Barrier, Event +from types import MappingProxyType +from typing import Final, Literal, TypeAlias, cast + +import httpx +import psutil +import pytest +import yaml +from integration._support.client import Gateway, JsonValue, eventually +from integration._support.database import read_rows +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import TypeAdapter + +ChaosEndpoint: TypeAlias = Literal["chat", "messages", "responses"] +CHAOS_ENDPOINTS: Final[tuple[ChaosEndpoint, ...]] = ("chat", "messages", "responses") +CHAOS_MODELS: Final = MappingProxyType( + {"chat": "chaos-chat", "messages": "chaos-messages", "responses": "chaos-responses"} +) +GUARDRAIL_PATH: Final = "/beta/litellm_basic_guardrail_api" +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +POSTGRES_IMAGE: Final = "postgres:16@sha256:e17e86066e5ef83e0952a9347f5c792b7ece00972e2aa787a6986f471b3dd3d5" + + +def _json(value: object) -> bytes: + return json.dumps(value, separators=(",", ":")).encode() + + +def _texts(value: JsonValue) -> tuple[str, ...]: + if isinstance(value, str): + return (value,) + if isinstance(value, list): + return tuple(chain.from_iterable(_texts(item) for item in value)) + if isinstance(value, dict): + return tuple(chain.from_iterable(_texts(item) for item in value.values())) + return () + + +def _marker(body: Mapping[str, JsonValue]) -> str: + return next((text for text in _texts(dict(body)) if text.startswith("audit-")), "audit-chaos") + + +def _sse(events: Sequence[Mapping[str, JsonValue]]) -> tuple[bytes, ...]: + return tuple(f"data: {json.dumps(event, separators=(',', ':'))}\n\n".encode() for event in events) + ( + b"data: [DONE]\n\n", + ) + + +def _messages_stream(message: Mapping[str, JsonValue]) -> tuple[bytes, ...]: + content: Final = cast(list[JsonValue], message["content"]) + text: Final = cast(dict[str, JsonValue], content[0])["text"] + assert isinstance(text, str) + return ( + f"event: message_start\ndata: {json.dumps({**message, 'content': [], 'stop_reason': None, 'usage': {'input_tokens': 2, 'output_tokens': 0}})}\n\n".encode(), + b'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n\n', + f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': text}})}\n\n".encode(), + b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', + f"event: message_delta\ndata: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn', 'stop_sequence': None}, 'usage': {'output_tokens': 2}})}\n\n".encode(), + b'event: message_stop\ndata: {"type":"message_stop"}\n\n', + ) + + +def _responses_stream( + response: Mapping[str, JsonValue], + output: Mapping[str, JsonValue], + marker: str, +) -> tuple[bytes, ...]: + events: Final[tuple[dict[str, JsonValue], ...]] = ( + {"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}, + {"type": "response.in_progress", "response": {**response, "status": "in_progress", "output": []}}, + {"type": "response.output_item.added", "item": dict(output), "output_index": 0}, + { + "type": "response.content_part.added", + "item_id": f"msg-{marker}", + "output_index": 0, + "content_index": 0, + "part": {"type": "output_text", "text": "", "annotations": []}, + }, + { + "type": "response.output_text.delta", + "item_id": f"msg-{marker}", + "output_index": 0, + "content_index": 0, + "delta": marker, + }, + { + "type": "response.output_text.done", + "item_id": f"msg-{marker}", + "output_index": 0, + "content_index": 0, + "text": marker, + }, + { + "type": "response.content_part.done", + "item_id": f"msg-{marker}", + "output_index": 0, + "content_index": 0, + "part": cast(list[JsonValue], output["content"])[0], + }, + {"type": "response.output_item.done", "item": dict(output), "output_index": 0}, + {"type": "response.completed", "response": dict(response)}, + ) + return tuple( + f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': index}, separators=(',', ':'))}\n\n".encode() + for index, event in enumerate(events) + ) + + +def _provider(request: Request) -> Reply: + if request.method == "GET" and request.target.partition("?")[0] == "/v1/models": + return Reply(body=_json({"object": "list", "data": [{"id": "gpt-4o-mini", "object": "model"}]})) + if not request.body: + return Reply(status=400, body=_json({"error": "request body is required"})) + body: Final = JSON_OBJECT.validate_json(request.body) + marker: Final = _marker(body) + streamed: Final = bool(body.get("stream")) + if request.target == "/v1/chat/completions": + if streamed: + return Reply( + content_type="text/event-stream", + chunks=_sse( + ( + { + "id": f"chatcmpl-{marker}", + "object": "chat.completion.chunk", + "choices": [{"index": 0, "delta": {"content": marker}, "finish_reason": None}], + }, + ) + ), + ) + return Reply( + body=_json( + { + "id": f"chatcmpl-{marker}", + "object": "chat.completion", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": marker}, "finish_reason": "stop"} + ], + "usage": {"prompt_tokens": 2, "completion_tokens": 2, "total_tokens": 4}, + } + ) + ) + if request.target == "/v1/messages": + message: Final = { + "id": f"msg-{marker}", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5-20250929", + "content": [{"type": "text", "text": marker}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 2, "output_tokens": 2}, + } + return ( + Reply( + content_type="text/event-stream", + chunks=_messages_stream(message), + ) + if streamed + else Reply(body=_json(message)) + ) + if request.target == "/v1/responses": + response: Final = { + "id": f"resp-{marker}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "type": "message", + "id": f"msg-{marker}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": marker, "annotations": []}], + } + ], + "usage": {"input_tokens": 2, "output_tokens": 2, "total_tokens": 4}, + } + return ( + Reply( + content_type="text/event-stream", + chunks=_responses_stream(response, cast(dict[str, JsonValue], response["output"][0]), marker), + ) + if streamed + else Reply(body=_json(response)) + ) + return Reply(status=404, body=_json({"error": f"unexpected provider target {request.target}"})) + + +def _sink(request: Request) -> Reply: + assert request.target.endswith(GUARDRAIL_PATH), request.target + body: Final = JSON_OBJECT.validate_json(request.body) + assert body.get("litellm_call_id") is not None or any("audit-" in text for text in _texts(body)), ( + request.body.decode() + ) + return Reply(body=_json({"action": "NONE"})) + + +def _rail( + name: str, + sink_url: str, + scope: Literal["streaming", "non_streaming"], + *, + default_on: bool = False, +) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": default_on, + "stream_scope": scope, + "api_base": f"{sink_url}/{name}", + "api_key": "synthetic-chaos-key", + }, + } + + +def _config(provider_url: str, rails: Sequence[dict[str, JsonValue]]) -> dict[str, JsonValue]: + base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + return cast( + dict[str, JsonValue], + { + **base, + "guardrails": list(rails), + "model_list": [ + { + "model_name": CHAOS_MODELS["chat"], + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider_url}/v1", + "api_key": "synthetic-provider-key", + "num_retries": 0, + }, + }, + { + "model_name": CHAOS_MODELS["messages"], + "litellm_params": { + "model": "anthropic/claude-sonnet-4-5-20250929", + "api_base": provider_url, + "api_key": "synthetic-provider-key", + "num_retries": 0, + }, + }, + { + "model_name": CHAOS_MODELS["responses"], + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider_url}/v1", + "api_key": "synthetic-provider-key", + "num_retries": 0, + }, + }, + ], + "environment_variables": { + **base.get("environment_variables", {}), + "OPENAI_API_BASE": provider_url, + "OPENAI_API_KEY": "synthetic-provider-key", + "ANTHROPIC_API_BASE": provider_url, + "ANTHROPIC_API_KEY": "synthetic-provider-key", + }, + }, + ) + + +@dataclass(frozen=True, slots=True) +class ChaosRig: + gateway: Gateway + provider: Wire + directory: Path + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[ChaosRig]: + with ( + httpx.Client( + base_url=os.environ["INTEGRATION_PROXY_URL"], + timeout=30, + trust_env=False, + ) as root_client, + wire_server(_provider) as provider, + ): + root_gateway: Final = Gateway( + root_client, + os.environ.get("INTEGRATION_MASTER_KEY", "sk-integration-master"), + os.environ["INTEGRATION_UPSTREAM_URL"], + ) + yield ChaosRig(root_gateway, provider, tmp_path_factory.mktemp("stream-scope-chaos")) + + +@dataclass(frozen=True, slots=True) +class CallPlan: + marker: str + call_id: str + endpoint: ChaosEndpoint + streamed: bool + + +def _plans(prefix: str, count: int) -> tuple[CallPlan, ...]: + return tuple( + CallPlan( + f"audit-{prefix}-{index}-{uuid.uuid4().hex}", + f"{prefix}-{uuid.uuid4().hex}", + CHAOS_ENDPOINTS[index % len(CHAOS_ENDPOINTS)], + index % 2 == 1, + ) + for index in range(count) + ) + + +def _request(gateway: Gateway, plan: CallPlan, rails: Sequence[str]) -> httpx.Response: + guardrail_field: Final[dict[str, JsonValue]] = {"guardrails": list(rails)} if rails else {} + match plan.endpoint: + case "chat": + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": CHAOS_MODELS["chat"], + "messages": [{"role": "user", "content": plan.marker}], + **guardrail_field, + **({"stream": True} if plan.streamed else {}), + }, + headers={"x-litellm-call-id": plan.call_id}, + ) + case "messages": + return gateway.request( + "POST", + "/v1/messages", + { + "model": CHAOS_MODELS["messages"], + "max_tokens": 32, + "messages": [{"role": "user", "content": plan.marker}], + **guardrail_field, + **({"stream": True} if plan.streamed else {}), + }, + headers={"x-litellm-call-id": plan.call_id}, + ) + case "responses": + return gateway.request( + "POST", + "/v1/responses", + { + "model": CHAOS_MODELS["responses"], + "input": plan.marker, + **guardrail_field, + **({"stream": True} if plan.streamed else {}), + }, + headers={"x-litellm-call-id": plan.call_id}, + ) + raise AssertionError(plan.endpoint) + + +def _rows_for_marker(rows: Sequence[Request], marker: str) -> tuple[Request, ...]: + return tuple(request for request in rows if marker.encode() in request.body) + + +def _rail_scans(rows: Sequence[Request], rail_name: str, marker: str) -> tuple[Request, ...]: + return tuple( + request for request in rows if request.target.startswith(f"/{rail_name}/") and marker.encode() in request.body + ) + + +def _spend_rows(call_id: str, database_url: str | None = None) -> list[dict[str, JsonValue]]: + return read_rows( + 'SELECT request_id, litellm_call_id FROM "LiteLLM_SpendLogs" WHERE request_id=%s OR litellm_call_id=%s', + (call_id, call_id), + database_url=database_url, + ) + + +def _one_spend_row( + call_id: str, + database_url: str | None = None, + *, + seconds: float = 70, +) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: _spend_rows(call_id, database_url), + lambda values: len(values) >= 1, + seconds=seconds, + ) + assert len(rows) == 1, (call_id, rows) + return rows[0] + + +def _expected_in_scope(plan: CallPlan, sink_a_name: str, sink_b_name: str) -> tuple[str, str]: + return (sink_a_name, sink_b_name) if plan.streamed else (sink_b_name, sink_a_name) + + +def _assert_successful_calls( + plans: Sequence[CallPlan], + responses: Sequence[httpx.Response], + provider_rows: Sequence[Request], + sink_a_rows: Sequence[Request], + sink_b_rows: Sequence[Request], + sink_a_name: str, + sink_b_name: str, +) -> None: + for plan, response in zip(plans, responses): + if response.status_code != 200: + continue + assert plan.marker in response.text, (plan, response.text) + provider_match: Final = _rows_for_marker(provider_rows, plan.marker) + assert len(provider_match) == 1, (plan, provider_match) + in_sink, out_sink = _expected_in_scope(plan, sink_a_name, sink_b_name) + in_rows: Final = _rail_scans( + sink_a_rows if in_sink == sink_a_name else sink_b_rows, + in_sink, + plan.marker, + ) + out_rows: Final = _rail_scans( + sink_a_rows if out_sink == sink_a_name else sink_b_rows, + out_sink, + plan.marker, + ) + assert len(in_rows) == 1 and len(out_rows) == 0, (plan, in_rows, out_rows) + spend: Final = _one_spend_row(plan.call_id) + assert plan.call_id in (spend.get("request_id"), spend.get("litellm_call_id")), (plan, spend) + + +def _call_wave(gateway: Gateway, plans: Sequence[CallPlan], rails: Sequence[str]) -> tuple[httpx.Response, ...]: + with ThreadPoolExecutor(max_workers=20) as pool: + futures: Final[tuple[Future[httpx.Response], ...]] = tuple( + pool.submit(_request, gateway, plan, rails) for plan in plans + ) + return tuple(future.result() for future in futures) + + +@contextmanager +def _owned_proxy( + rig: ChaosRig, + directory: Path, + rails: Sequence[dict[str, JsonValue]], + *, + workers: int = 1, +) -> Iterator[OwnedProxy]: + config_path: Final = directory / f"chaos-{uuid.uuid4().hex}.yaml" + config_path.write_text(yaml.safe_dump(_config(rig.provider.url, rails))) + with owned_proxy_process(rig.gateway, directory, {}, config=config_path, workers=workers) as owned: + yield owned + + +def _free_port() -> int: + with socket.socket() as reserve: + reserve.bind(("127.0.0.1", 0)) + return int(reserve.getsockname()[1]) + + +def _assert_down_wave( + plans: Sequence[CallPlan], + responses: Sequence[httpx.Response], + provider_rows: Sequence[Request], + sink_a_rows: Sequence[Request], + sink_b_rows: Sequence[Request], + sink_a_name: str, + sink_b_name: str, +) -> None: + for plan, response in zip(plans, responses): + if plan.streamed: + assert response.status_code >= 500 and response.content, (plan, response.status_code, response.text) + assert _rows_for_marker(provider_rows, plan.marker) == (), (plan, provider_rows) + assert _rail_scans(sink_b_rows, sink_b_name, plan.marker) == (), (plan, sink_b_name) + continue + assert response.status_code == 200 and plan.marker in response.text, (plan, response.status_code, response.text) + assert len(_rows_for_marker(provider_rows, plan.marker)) == 1, (plan, provider_rows) + assert _rail_scans(sink_a_rows, sink_a_name, plan.marker) == (), (plan, sink_a_name) + assert len(_rail_scans(sink_b_rows, sink_b_name, plan.marker)) == 1, (plan, sink_b_name) + spend: Final = _one_spend_row(plan.call_id) + assert plan.call_id in (spend.get("request_id"), spend.get("litellm_call_id")), (plan, spend) + + +def test_h1_sink_outage_keeps_scope_isolated_through_recovery(rig: ChaosRig, tmp_path: Path) -> None: + port_a: Final = _free_port() + started: Final = Event() + unavailable: Final = Event() + release: Final = Event() + + def gated_sink(request: Request) -> Reply: + started.set() + assert release.wait(timeout=45), "sink outage gate was not released" + if unavailable.is_set(): + return Reply(status=503, body=_json({"error": "synthetic sink outage"})) + return _sink(request) + + with wire_server(_sink) as sink_b, ExitStack() as sink_a_stack: + sink_a: Final = sink_a_stack.enter_context(wire_server(gated_sink, port=port_a)) + name_a: Final = f"h1-stream-{uuid.uuid4().hex}" + name_b: Final = f"h1-non-stream-{uuid.uuid4().hex}" + rails: Final = ( + _rail(name_a, sink_a.url, "streaming"), + _rail(name_b, sink_b.url, "non_streaming"), + ) + with _owned_proxy(rig, tmp_path, rails) as owned, ThreadPoolExecutor(max_workers=30) as pool: + outage_plans: Final = _plans("h1-burst", 30) + futures: Final[tuple[Future[httpx.Response], ...]] = tuple( + pool.submit(_request, owned.gateway, plan, (name_a, name_b)) for plan in outage_plans + ) + try: + assert started.wait(timeout=30), "streaming rail did not reach sink A" + unavailable.set() + finally: + release.set() + sink_a_stack.close() + outage_responses: Final = tuple(future.result(timeout=70) for future in futures) + outage_provider: Final = rig.provider.drain() + outage_a: Final = sink_a.drain() + outage_b: Final = sink_b.drain() + _assert_down_wave(outage_plans, outage_responses, outage_provider, outage_a, outage_b, name_a, name_b) + + with wire_server(_sink, port=port_a) as recovered_sink_a: + recovery_plans: Final = _plans("h1-recovery", 20) + recovery_responses: Final = _call_wave(owned.gateway, recovery_plans, (name_a, name_b)) + recovery_provider: Final = rig.provider.drain() + recovery_a: Final = recovered_sink_a.drain() + recovery_b: Final = sink_b.drain() + assert tuple(response.status_code for response in recovery_responses) == (200,) * 20, recovery_responses + _assert_successful_calls( + recovery_plans, + recovery_responses, + recovery_provider, + recovery_a, + recovery_b, + name_a, + name_b, + ) + + +def test_h2_stream_sink_gate_does_not_block_out_of_scope_calls(rig: ChaosRig, tmp_path: Path) -> None: + started: Final = Event() + release: Final = Event() + blocked_marker: Final = f"audit-h2-stream-{uuid.uuid4().hex}" + + def gated_sink(request: Request) -> Reply: + if blocked_marker.encode() in request.body: + started.set() + assert release.wait(timeout=45), "stream sink gate was not released" + return _sink(request) + + with wire_server(gated_sink) as sink_a, wire_server(_sink) as sink_b: + name_a: Final = f"h2-stream-{uuid.uuid4().hex}" + name_b: Final = f"h2-non-stream-{uuid.uuid4().hex}" + rails: Final = (_rail(name_a, sink_a.url, "streaming"), _rail(name_b, sink_b.url, "non_streaming")) + with _owned_proxy(rig, tmp_path, rails) as owned, ThreadPoolExecutor(max_workers=2) as pool: + streaming_plan: Final = CallPlan(blocked_marker, f"h2-stream-{uuid.uuid4().hex}", "chat", True) + non_streaming_plan: Final = CallPlan( + f"audit-h2-non-stream-{uuid.uuid4().hex}", + f"h2-non-stream-{uuid.uuid4().hex}", + "messages", + False, + ) + streaming_future: Final = pool.submit(_request, owned.gateway, streaming_plan, (name_a, name_b)) + assert started.wait(timeout=30), "stream request did not reach the gated sink" + try: + non_streaming_future: Final = pool.submit( + _request, + owned.gateway, + non_streaming_plan, + (name_a, name_b), + ) + non_streaming_response: Final = non_streaming_future.result(timeout=15) + assert non_streaming_response.status_code == 200, non_streaming_response.text + assert non_streaming_plan.marker in non_streaming_response.text, non_streaming_response.text + finally: + release.set() + streaming_response: Final = streaming_future.result(timeout=30) + assert streaming_response.status_code == 200, streaming_response.text + assert streaming_plan.marker in streaming_response.text, streaming_response.text + provider_rows: Final = rig.provider.drain() + sink_a_rows: Final = sink_a.drain() + sink_b_rows: Final = sink_b.drain() + _assert_successful_calls( + (streaming_plan, non_streaming_plan), + (streaming_response, non_streaming_response), + provider_rows, + sink_a_rows, + sink_b_rows, + name_a, + name_b, + ) + + +def _worker_processes(owned: OwnedProxy) -> tuple[psutil.Process, ...]: + return tuple( + child for child in psutil.Process(owned.process.pid).children() if "spawn_main" in _process_command(child) + ) + + +def _process_command(process: psutil.Process) -> str: + try: + return " ".join(process.cmdline()) + except psutil.Error: + return "" + + +def _safe_response(future: Future[httpx.Response]) -> httpx.Response | None: + try: + return future.result(timeout=70) + except (httpx.HTTPError, TimeoutError): + return None + + +def test_h3_worker_kill_mid_burst_keeps_remaining_worker_serving(rig: ChaosRig, tmp_path: Path) -> None: + plans: Final = _plans("h3-burst", 30) + gate_markers: Final = frozenset(plan.marker for plan in plans if plan.streamed) + started: Final = Event() + release: Final = Event() + + def gated_sink(request: Request) -> Reply: + if any(marker.encode() in request.body for marker in gate_markers): + started.set() + assert release.wait(timeout=60), "worker-kill sink gate was not released" + return _sink(request) + + with wire_server(gated_sink) as sink_a, wire_server(_sink) as sink_b: + name_a: Final = f"h3-stream-{uuid.uuid4().hex}" + name_b: Final = f"h3-non-stream-{uuid.uuid4().hex}" + rails: Final = ( + _rail(name_a, sink_a.url, "streaming", default_on=True), + _rail(name_b, sink_b.url, "non_streaming", default_on=True), + ) + with _owned_proxy(rig, tmp_path, rails, workers=2) as owned, ThreadPoolExecutor(max_workers=30) as pool: + workers: Final = _worker_processes(owned) + assert len(workers) == 2, tuple(worker.pid for worker in workers) + futures: Final[tuple[Future[httpx.Response], ...]] = tuple( + pool.submit(_request, owned.gateway, plan, ()) for plan in plans + ) + assert started.wait(timeout=30), "stream requests did not reach the owned sink" + victim: Final = workers[0] + survivor: Final = workers[1] + survivor_plan: Final = CallPlan( + f"audit-h3-survivor-{uuid.uuid4().hex}", + f"h3-survivor-{uuid.uuid4().hex}", + "messages", + False, + ) + try: + victim.send_signal(signal.SIGKILL) + survivor_response: Final = _request(owned.gateway, survivor_plan, ()) + assert survivor_response.status_code == 200 and survivor_plan.marker in survivor_response.text, ( + survivor_response.status_code, + survivor_response.text, + ) + assert survivor.is_running(), survivor.pid + finally: + release.set() + responses: Final = tuple(_safe_response(future) for future in futures) + assert owned.gateway.request("GET", "/health/liveliness").status_code == 200 + successful: Final = tuple( + (plan, response) + for plan, response in zip(plans, responses) + if response is not None and response.status_code == 200 + ) + provider_rows: Final = rig.provider.drain() + sink_a_rows: Final = sink_a.drain() + sink_b_rows: Final = sink_b.drain() + successful_plans: Final = (*tuple(plan for plan, _ in successful), survivor_plan) + successful_responses: Final = (*tuple(response for _, response in successful), survivor_response) + _assert_successful_calls( + successful_plans, + successful_responses, + provider_rows, + sink_a_rows, + sink_b_rows, + name_a, + name_b, + ) + + +def _create_stored_rail( + gateway: Gateway, + name: str, + sink_url: str, + stream_scope: Literal["streaming"] | None = "streaming", +) -> str: + response: Final = gateway.request( + "POST", + "/guardrails", + { + "guardrail": { + "guardrail_name": name, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": False, + "api_base": f"{sink_url}/{name}", + "api_key": "synthetic-chaos-key", + **({"stream_scope": stream_scope} if stream_scope is not None else {}), + }, + } + }, + ) + assert response.status_code == 200, response.text + identity: Final = JSON_OBJECT.validate_json(response.content).get("guardrail_id") + assert isinstance(identity, str), response.text + return identity + + +def _assert_one_scope_wave( + plans: Sequence[CallPlan], + responses: Sequence[httpx.Response], + provider_rows: Sequence[Request], + sink_rows: Sequence[Request], + sink_name: str, + *, + database_url: str | None = None, +) -> None: + for plan, response in zip(plans, responses): + assert response.status_code == 200 and plan.marker in response.text, (plan, response.status_code, response.text) + provider_match: Final = _rows_for_marker(provider_rows, plan.marker) + sink_match: Final = _rail_scans(sink_rows, sink_name, plan.marker) + assert len(provider_match) == 1, (plan, provider_match) + assert len(sink_match) == int(plan.streamed), (plan, sink_match) + spend: Final = _one_spend_row(plan.call_id, database_url) + assert plan.call_id in (spend.get("request_id"), spend.get("litellm_call_id")), (plan, spend) + + +def test_h4_stored_scope_survives_owned_proxy_restart(rig: ChaosRig, tmp_path: Path) -> None: + with wire_server(_sink) as sink: + name: Final = f"h4-stored-{uuid.uuid4().hex}" + identity: Final = _create_stored_rail(rig.gateway, name, sink.url) + try: + with ExitStack() as first_stack: + first: Final = first_stack.enter_context(_owned_proxy(rig, tmp_path, ())) + first_plans: Final = _plans("h4-before", 20) + first_responses: Final = _call_wave(first.gateway, first_plans, (name,)) + first_provider: Final = rig.provider.drain() + first_sink: Final = sink.drain() + assert tuple(response.status_code for response in first_responses) == (200,) * 20, first_responses + _assert_one_scope_wave(first_plans, first_responses, first_provider, first_sink, name) + first_stack.close() + with _owned_proxy(rig, tmp_path, ()) as restarted: + recovery_plans: Final = _plans("h4-after", 20) + recovery_responses: Final = _call_wave(restarted.gateway, recovery_plans, (name,)) + recovery_provider: Final = rig.provider.drain() + recovery_sink: Final = sink.drain() + assert tuple(response.status_code for response in recovery_responses) == (200,) * 20, ( + recovery_responses, + ) + _assert_one_scope_wave( + recovery_plans, + recovery_responses, + recovery_provider, + recovery_sink, + name, + ) + finally: + deleted: Final = rig.gateway.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +@contextmanager +def _owned_postgres(directory: Path) -> Iterator[PostgresCluster]: + docker: Final = shutil.which("docker") + assert docker is not None, "Docker CLI is required for the H5 PostgreSQL outage test" + docker_info: Final = subprocess.run( + [docker, "info", "--format", "{{.ServerVersion}}"], + capture_output=True, + text=True, + check=False, + ) + assert docker_info.returncode == 0, docker_info.stderr + directory.mkdir(parents=True, exist_ok=True) + port: Final = _free_port() + container_name: Final = f"litellm-stream-scope-h5-{uuid.uuid4().hex}" + password: Final = uuid.uuid4().hex + created: Final = subprocess.run( + [ + docker, + "create", + "--name", + container_name, + "--env", + "POSTGRES_USER=postgres", + "--env", + "POSTGRES_PASSWORD", + "--env", + "POSTGRES_DB=postgres", + "--publish", + f"127.0.0.1:{port}:5432/tcp", + POSTGRES_IMAGE, + ], + env=os.environ | {"POSTGRES_PASSWORD": password}, + capture_output=True, + text=True, + check=False, + ) + assert created.returncode == 0, created.stderr + cluster: Final = PostgresCluster( + f"postgresql://postgres:{password}@127.0.0.1:{port}/postgres?sslmode=disable", + container_name, + docker, + directory / "postgres.log", + ) + try: + started: Final = _start_postgres(cluster) + assert started.returncode == 0, started.stderr + assert eventually(lambda: _postgres_is_ready(cluster), bool, seconds=70) + yield cluster + finally: + logs: Final = subprocess.run( + [docker, "logs", container_name], + capture_output=True, + text=True, + check=False, + ) + cluster.log_path.write_text(logs.stdout + logs.stderr) + removed: Final = subprocess.run( + [docker, "rm", "-f", container_name], + capture_output=True, + text=True, + check=False, + ) + assert removed.returncode == 0, removed.stderr + remaining: Final = subprocess.run( + [docker, "ps", "--all", "--quiet", "--filter", f"name={container_name}"], + capture_output=True, + text=True, + check=False, + ) + assert remaining.returncode == 0, remaining.stderr + assert not remaining.stdout.strip(), remaining.stdout + + +@dataclass(frozen=True, slots=True) +class PostgresCluster: + database_url: str + container_name: str + docker: str + log_path: Path + + +def _start_postgres(cluster: PostgresCluster) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [cluster.docker, "start", cluster.container_name], + capture_output=True, + text=True, + check=False, + ) + + +def _postgres_is_ready(cluster: PostgresCluster) -> bool: + readiness: Final = subprocess.run( + [ + cluster.docker, + "exec", + cluster.container_name, + "pg_isready", + "-h", + "127.0.0.1", + "-p", + "5432", + "-d", + "postgres", + "-U", + "postgres", + ], + capture_output=True, + text=True, + check=False, + ) + return readiness.returncode == 0 + + +def _postgres_is_running(cluster: PostgresCluster) -> bool: + state: Final = subprocess.run( + [cluster.docker, "inspect", "--format", "{{.State.Running}}", cluster.container_name], + capture_output=True, + text=True, + check=False, + ) + return state.returncode == 0 and state.stdout.strip() == "true" + + +@pytest.mark.timeout(180) +def test_h5_stored_scope_survives_owned_postgres_outage_and_recovers_once( + rig: ChaosRig, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + postgres_directory: Final = Path(os.environ["INTEGRATION_RESULTS_DIR"]) / f"owned-postgres-{uuid.uuid4().hex}" + with _owned_postgres(postgres_directory) as database, wire_server(_sink) as sink: + monkeypatch.setenv("INTEGRATION_PROXY_DATABASE_URL", database.database_url) + name: Final = f"h5-stored-{uuid.uuid4().hex}" + with _owned_proxy(rig, tmp_path, ()) as registrar: + _create_stored_rail(registrar.gateway, name, sink.url) + with _owned_proxy(rig, tmp_path, ()) as owned: + try: + preflight_plans: Final = ( + CallPlan( + f"audit-h5-preflight-{uuid.uuid4().hex}", + f"h5-preflight-{uuid.uuid4().hex}", + "chat", + True, + ), + ) + preflight_responses: Final = _call_wave(owned.gateway, preflight_plans, (name,)) + preflight_provider: Final = rig.provider.drain() + preflight_sink: Final = sink.drain() + assert tuple(response.status_code for response in preflight_responses) == (200,), (preflight_responses,) + _assert_one_scope_wave( + preflight_plans, + preflight_responses, + preflight_provider, + preflight_sink, + name, + database_url=database.database_url, + ) + outage_result: Final = subprocess.run( + [database.docker, "stop", database.container_name], + capture_output=True, + text=True, + check=False, + ) + assert outage_result.returncode == 0, outage_result.stderr + outage_plans: Final = _plans("h5-outage", 20) + outage_responses: Final = _call_wave(owned.gateway, outage_plans, (name,)) + outage_provider: Final = rig.provider.drain() + outage_sink: Final = sink.drain() + for plan, response in zip(outage_plans, outage_responses): + assert response.status_code == 200 and plan.marker in response.text, ( + plan, + response.status_code, + response.text, + ) + assert len(_rows_for_marker(outage_provider, plan.marker)) == 1, (plan, outage_provider) + sink_match: Final = _rail_scans(outage_sink, name, plan.marker) + assert len(sink_match) == int(plan.streamed), (plan, sink_match) + + recovered_database: Final = _start_postgres(database) + assert recovered_database.returncode == 0, recovered_database.stderr + postgres_ready: Final = eventually(lambda: _postgres_is_ready(database), bool, seconds=70) + assert postgres_ready + readiness: Final = eventually( + lambda: owned.gateway.client.get("/health/readiness"), + lambda response: response.status_code == 200 and response.json().get("db") == "connected", + seconds=70, + ) + assert readiness.json().get("db") == "connected", readiness.text + recovery_plans: Final = _plans("h5-recovery", 20) + recovery_responses: Final = _call_wave(owned.gateway, recovery_plans, (name,)) + recovery_provider: Final = rig.provider.drain() + recovery_sink: Final = sink.drain() + assert tuple(response.status_code for response in recovery_responses) == (200,) * 20, recovery_responses + _assert_one_scope_wave( + recovery_plans, + recovery_responses, + recovery_provider, + recovery_sink, + name, + database_url=database.database_url, + ) + finally: + if not _postgres_is_running(database): + restarted: Final = _start_postgres(database) + assert restarted.returncode == 0, restarted.stderr + assert eventually(lambda: _postgres_is_ready(database), bool, seconds=70) + + +@pytest.mark.timeout(360) +def test_h6_spend_rows_for_requests_served_during_postgres_restart_land_once( + rig: ChaosRig, + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + pytest.skip("BUG: LIT-9050 spend rows for requests served during a Postgres restart are dropped") + postgres_directory: Final = Path(os.environ["INTEGRATION_RESULTS_DIR"]) / f"owned-postgres-{uuid.uuid4().hex}" + with _owned_postgres(postgres_directory) as database, wire_server(_sink) as sink: + monkeypatch.setenv("INTEGRATION_PROXY_DATABASE_URL", database.database_url) + name: Final = f"h6-stored-{uuid.uuid4().hex}" + plans: Final = _plans("h6-postgres-restart", 20) + outage_markers: Final = frozenset(plan.marker for plan in plans[:10]) + arrivals: Final = Barrier(len(plans) + 1) + during_outage: Final = Event() + after_restart: Final = Event() + + def _gated_provider(request: Request) -> Reply: + if request.method == "GET" and request.target.partition("?")[0] == "/v1/models": + return _provider(request) + body: Final = JSON_OBJECT.validate_json(request.body) + marker: Final = _marker(body) + arrivals.wait(timeout=70) + gate: Final = during_outage if marker in outage_markers else after_restart + assert gate.wait(timeout=70), marker + return _provider(request) + + with wire_server(_gated_provider) as provider: + h6_rig: Final = ChaosRig(rig.gateway, provider, rig.directory) + with _owned_proxy(h6_rig, tmp_path, ()) as registrar: + _create_stored_rail(registrar.gateway, name, sink.url, stream_scope=None) + with _owned_proxy(h6_rig, tmp_path, ()) as owned, ThreadPoolExecutor(max_workers=len(plans)) as pool: + futures: Final = tuple( + pool.submit(_request, owned.gateway, plan, (name,)) for plan in plans + ) + try: + arrivals.wait(timeout=70) + stopped: Final = subprocess.run( + [database.docker, "stop", database.container_name], + capture_output=True, + text=True, + check=False, + ) + assert stopped.returncode == 0, stopped.stderr + during_outage.set() + outage_responses: Final = tuple(future.result(timeout=70) for future in futures[:10]) + for plan, response in zip(plans[:10], outage_responses): + assert response.status_code == 200 and plan.marker in response.text, ( + plan, + response.status_code, + response.text, + ) + + restarted: Final = _start_postgres(database) + assert restarted.returncode == 0, restarted.stderr + postgres_ready: Final = eventually(lambda: _postgres_is_ready(database), bool, seconds=70) + assert postgres_ready + readiness: Final = eventually( + lambda: owned.gateway.client.get("/health/readiness"), + lambda response: response.status_code == 200 + and JSON_OBJECT.validate_python(cast(object, response.json())).get("db") == "connected", + seconds=70, + ) + readiness_body: Final = JSON_OBJECT.validate_python(cast(object, readiness.json())) + assert readiness_body.get("db") == "connected", readiness.text + after_restart.set() + responses: Final = tuple(future.result(timeout=70) for future in futures) + finally: + during_outage.set() + after_restart.set() + if not _postgres_is_running(database): + recovered: Final = _start_postgres(database) + assert recovered.returncode == 0, recovered.stderr + assert eventually(lambda: _postgres_is_ready(database), bool, seconds=70) + + provider_rows: Final = provider.drain() + sink_rows: Final = sink.drain() + for plan, response in zip(plans, responses): + assert response.status_code == 200 and plan.marker in response.text, ( + plan, + response.status_code, + response.text, + ) + assert len(_rows_for_marker(provider_rows, plan.marker)) == 1, (plan, provider_rows) + assert len(_rail_scans(sink_rows, name, plan.marker)) == 1, (plan, sink_rows) + + spend_rows: Final = eventually( + lambda: tuple(_spend_rows(plan.call_id, database.database_url) for plan in plans), + lambda values: all(len(rows) == 1 for rows in values), + seconds=70, + return_last_on_timeout=True, + ) + counts: Final = tuple(len(rows) for rows in spend_rows) + missing_ids: Final = tuple( + plan.call_id for plan, rows in zip(plans, spend_rows) if not rows + ) + duplicate_ids: Final = tuple( + plan.call_id for plan, rows in zip(plans, spend_rows) if len(rows) > 1 + ) + assert counts == (1,) * len(plans), { + "missing_ids": missing_ids, + "duplicate_ids": duplicate_ids, + "counts": counts, + } diff --git a/tests/integration/observability/test_guardrail_stream_scope_matrix.py b/tests/integration/observability/test_guardrail_stream_scope_matrix.py new file mode 100644 index 00000000000..b5c66b202cc --- /dev/null +++ b/tests/integration/observability/test_guardrail_stream_scope_matrix.py @@ -0,0 +1,2076 @@ +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +import socket +import socketserver +import ssl +import threading +import uuid +from asyncio import run, wait_for +from collections.abc import Generator, Iterable, Iterator, Mapping, Sequence +from contextlib import contextmanager +from dataclasses import dataclass +from itertools import chain +from pathlib import Path +from queue import SimpleQueue +from types import MappingProxyType +from typing import Final, Literal, TypeAlias, cast + +import anthropic +import httpx +import openai +import pytest +import websockets +import yaml +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from integration._support.client import Gateway, JsonValue, Scenario, eventually +from integration._support.database import read_rows +from integration._support.mcp import McpCaller, echo_tool, register_mcp, scripted_peer +from integration._support.process import owned_proxy_process +from integration._support.tls import server_context, write_self_signed_cert +from integration._support.upstream import delete_scenario, register_scenario +from integration._support.wire import Reply, Request, Wire, wire_server +from integration.cost_calculation.cost_tracking_case import RealtimeResponse +from pydantic import TypeAdapter +from websockets.asyncio.server import ServerConnection, serve + +Endpoint: TypeAlias = Literal["chat", "messages", "responses"] +Mode: TypeAlias = Literal["pre_call", "during_call", "post_call", "logging_only"] +Scope: TypeAlias = Literal["streaming", "non_streaming"] +MatrixScope: TypeAlias = Scope | None +LoggingPhase: TypeAlias = Literal["request", "response"] +Endpoints: Final[tuple[Endpoint, ...]] = ("chat", "messages", "responses") +Modes: Final[tuple[Mode, ...]] = ("pre_call", "during_call", "post_call", "logging_only") +Scopes: Final[tuple[MatrixScope, ...]] = ("streaming", "non_streaming", None) +PROVIDER_TEXT: Final = "provider stream_scope control" +GUARDRAIL_PATH: Final = "/beta/litellm_basic_guardrail_api" +F1_BLOCKED_WORD: Final = "streamscope-mcp-blocked-word" +F3_STREAM_BLOCKED_WORD: Final = "matrix-f3-stream-block" +F3_NON_STREAM_BLOCKED_WORD: Final = "matrix-f3-non-stream-block" +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +LOGGING_PHASE: Final = TypeAdapter(LoggingPhase) + + +def _json(value: object) -> bytes: + return json.dumps(value, separators=(",", ":")).encode() + + +def _strings(value: JsonValue) -> tuple[str, ...]: + if isinstance(value, str): + return (value,) + if isinstance(value, list): + return tuple(chain.from_iterable(_strings(item) for item in value)) + if isinstance(value, dict): + return tuple(chain.from_iterable(_strings(item) for item in value.values())) + return () + + +def _marker(body: Mapping[str, JsonValue]) -> str: + return next((text for text in _strings(dict(body)) if text.startswith("audit-")), "audit-provider") + + +def _sse(events: Iterable[Mapping[str, JsonValue]]) -> tuple[bytes, ...]: + return tuple(f"data: {json.dumps(event, separators=(',', ':'))}\n\n".encode() for event in events) + ( + b"data: [DONE]\n\n", + ) + + +def _messages_stream(message: Mapping[str, JsonValue]) -> tuple[bytes, ...]: + content: Final = cast(list[JsonValue], message["content"]) + text: Final = cast(dict[str, JsonValue], content[0])["text"] + return ( + f"event: message_start\ndata: {json.dumps({**message, 'content': [], 'stop_reason': None, 'usage': {'input_tokens': 11, 'output_tokens': 0}})}\n\n".encode(), + b'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}\n\n', + f"event: content_block_delta\ndata: {json.dumps({'type': 'content_block_delta', 'index': 0, 'delta': {'type': 'text_delta', 'text': text}})}\n\n".encode(), + b'event: content_block_stop\ndata: {"type":"content_block_stop","index":0}\n\n', + f"event: message_delta\ndata: {json.dumps({'type': 'message_delta', 'delta': {'stop_reason': 'end_turn', 'stop_sequence': None}, 'usage': {'output_tokens': 4}})}\n\n".encode(), + b'event: message_stop\ndata: {"type":"message_stop"}\n\n', + ) + + +def _responses_stream( + response: Mapping[str, JsonValue], output: Mapping[str, JsonValue], marker: str +) -> tuple[bytes, ...]: + events: Final[tuple[dict[str, JsonValue], ...]] = ( + {"type": "response.created", "response": {**response, "status": "in_progress", "output": []}}, + {"type": "response.in_progress", "response": {**response, "status": "in_progress", "output": []}}, + {"type": "response.output_item.added", "item": output, "output_index": 0}, + { + "type": "response.content_part.added", + "item_id": f"msg-{marker}", + "output_index": 0, + "content_index": 0, + "part": {"type": "output_text", "text": "", "annotations": []}, + }, + { + "type": "response.output_text.delta", + "item_id": f"msg-{marker}", + "output_index": 0, + "content_index": 0, + "delta": f"{PROVIDER_TEXT} {marker}", + }, + { + "type": "response.output_text.done", + "item_id": f"msg-{marker}", + "output_index": 0, + "content_index": 0, + "text": f"{PROVIDER_TEXT} {marker}", + }, + { + "type": "response.content_part.done", + "item_id": f"msg-{marker}", + "output_index": 0, + "content_index": 0, + "part": cast(list[JsonValue], output["content"])[0], + }, + {"type": "response.output_item.done", "item": output, "output_index": 0}, + {"type": "response.completed", "response": response}, + ) + return tuple( + f"event: {event['type']}\ndata: {json.dumps({**event, 'sequence_number': index}, separators=(',', ':'))}\n\n".encode() + for index, event in enumerate(events) + ) + + +def _provider_reply(target: str, body: Mapping[str, JsonValue]) -> Reply: + marker: Final = _marker(body) + streamed: Final = bool(body.get("stream")) + if target == "/v1/chat/completions": + if streamed: + common: Final = {"id": f"chatcmpl-{marker}", "object": "chat.completion.chunk", "created": 1} + return Reply( + content_type="text/event-stream", + chunks=_sse( + ( + { + **common, + "choices": [{"index": 0, "delta": {"role": "assistant"}, "finish_reason": None}], + }, + { + **common, + "choices": [ + {"index": 0, "delta": {"content": f"{PROVIDER_TEXT} {marker}"}, "finish_reason": None} + ], + }, + {**common, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + ) + ), + ) + return Reply( + body=_json( + { + "id": f"chatcmpl-{marker}", + "object": "chat.completion", + "created": 1, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": f"{PROVIDER_TEXT} {marker}"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 11, "completion_tokens": 4, "total_tokens": 15}, + } + ) + ) + if target == "/v1/messages": + message: Final = { + "id": f"msg-{marker}", + "type": "message", + "role": "assistant", + "model": "synthetic-anthropic-model", + "content": [{"type": "text", "text": f"{PROVIDER_TEXT} {marker}"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 11, "output_tokens": 4}, + } + return ( + Reply(content_type="text/event-stream", chunks=_messages_stream(message)) + if streamed + else Reply(body=_json(message)) + ) + if target == "/v1/responses": + output: Final = { + "type": "message", + "id": f"msg-{marker}", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": f"{PROVIDER_TEXT} {marker}", "annotations": []}], + } + response: Final = { + "id": f"resp-{marker}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [output], + "usage": {"input_tokens": 11, "output_tokens": 4, "total_tokens": 15}, + } + return ( + Reply(content_type="text/event-stream", chunks=_responses_stream(response, output, marker)) + if streamed + else Reply(body=_json(response)) + ) + if target.endswith(":generateContent"): + return Reply( + body=_json( + {"candidates": [{"content": {"role": "model", "parts": [{"text": f"{PROVIDER_TEXT} {marker}"}]}}]} + ) + ) + if target.endswith(":streamGenerateContent"): + return Reply( + content_type="text/event-stream", + chunks=( + f"data: {_json({'candidates': [{'content': {'role': 'model', 'parts': [{'text': f'{PROVIDER_TEXT} {marker}'}]}}]}).decode()}\n\n".encode(), + ), + ) + return Reply(status=404, body=_json({"error": {"message": f"unexpected target {target}"}})) + + +def _provider(request: Request) -> Reply: + if not request.body: + return Reply(status=400, body=_json({"error": "empty request body"})) + body: Final = JSON_OBJECT.validate_json(request.body) + if request.target.startswith("/passthrough"): + marker: Final = _marker(body) + if bool(body.get("stream")): + return Reply( + content_type="text/event-stream", + chunks=_sse(({"text": f"{PROVIDER_TEXT} {marker}"},)), + ) + return Reply(body=_json({"received": body})) + if "audit-g3-401-" in request.body.decode(): + return Reply(status=401, body=_json({"error": {"message": "synthetic provider unauthorized"}})) + target: Final = request.target.split("?", 1)[0] + provider_target: Final = "/v1/messages" if target.endswith("/anthropic/v1/messages") else target + return _provider_reply(provider_target, body) + + +def _sink(request: Request) -> Reply: + assert request.target.endswith(GUARDRAIL_PATH), request.target + JSON_OBJECT.validate_json(request.body) + if "/g1-" in request.target: + return Reply(status=500, body=_json({"error": "synthetic sink failure"})) + if "/g2-" in request.target or "/f2-block-" in request.target: + return Reply(body=_json({"action": "BLOCKED", "blocked_reason": "synthetic policy block"})) + return Reply(body=_json({"action": "NONE"})) + + +def _rail( + name: str, + sink: Wire, + *, + mode: str | list[str], + scope: str | Mapping[str, str] | None = None, + default_on: bool = False, +) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": mode, + "default_on": default_on, + **({"stream_scope": dict(scope)} if isinstance(scope, Mapping) else {}), + **({"stream_scope": scope} if isinstance(scope, str) else {}), + "api_base": f"{sink.url}/{name}", + "api_key": "synthetic-guardrail-key", + }, + } + + +def _realtime_filter_rail(name: str, scope: Scope, keyword: str) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "realtime_input_transcription", + "default_on": True, + "stream_scope": scope, + "blocked_words": [{"keyword": keyword, "action": "BLOCK"}], + }, + } + + +def _mcp_filter_rail(name: str, scope: MatrixScope) -> dict[str, JsonValue]: + return { + "guardrail_name": name, + "litellm_params": { + "guardrail": "litellm_content_filter", + "mode": "pre_mcp_call", + "default_on": False, + **({"stream_scope": scope} if scope is not None else {}), + "blocked_words": [{"keyword": F1_BLOCKED_WORD, "action": "BLOCK"}], + }, + } + + +def _mode_scope_pairs() -> tuple[tuple[Mode, MatrixScope], ...]: + return tuple(chain.from_iterable(tuple((mode, scope) for scope in Scopes) for mode in Modes)) + + +def _rail_names() -> Mapping[str, str]: + return MappingProxyType( + { + **{f"a_{mode}_{scope or 'unset'}": f"a_{mode}_{scope or 'unset'}" for mode, scope in _mode_scope_pairs()}, + "c1_default": "c1_default", + "c2_key": "c2_key", + "c3_team": "c3_team", + "c4_modes": "c4_modes", + "c5_omitted": "c5_omitted", + "c6_both": "c6_both", + "c6_unset": "c6_unset", + "e_stream": "e_stream", + "e_non_stream": "e_non_stream", + "f1_stream": "f1_stream", + "f1_non_stream": "f1_non_stream", + "f1_unset": "f1_unset", + "f2_stream": "f2_stream", + "f2_non_stream": "f2-block-non-stream", + "f3_stream": "f3_stream", + "f3_non_stream": "f3_non_stream", + "g1_failure": "g1-failure", + "g2_block": "g2-block", + "g3_provider": "g3-provider", + "z_logging_only_barrier": "z_logging_only_barrier", + } + ) + + +def _configured_rails(names: Mapping[str, str], sink: Wire) -> tuple[dict[str, JsonValue], ...]: + return tuple( + _rail(names[f"a_{mode}_{scope or 'unset'}"], sink, mode=mode, scope=scope) + for mode, scope in _mode_scope_pairs() + ) + ( + _rail(names["c1_default"], sink, mode="post_call", scope="streaming"), + _rail(names["c2_key"], sink, mode="post_call", scope="streaming"), + _rail(names["c3_team"], sink, mode="post_call", scope="streaming"), + _rail( + names["c4_modes"], + sink, + mode=["pre_call", "post_call"], + scope={"pre_call": "non_streaming", "post_call": "streaming"}, + ), + _rail( + names["c5_omitted"], + sink, + mode=["pre_call", "post_call"], + scope={"post_call": "streaming"}, + ), + _rail(names["c6_both"], sink, mode="post_call", scope="both"), + _rail(names["c6_unset"], sink, mode="post_call"), + _rail(names["e_stream"], sink, mode="pre_call", scope="streaming"), + _rail(names["e_non_stream"], sink, mode="pre_call", scope="non_streaming"), + _mcp_filter_rail(names["f1_stream"], "streaming"), + _mcp_filter_rail(names["f1_non_stream"], "non_streaming"), + _mcp_filter_rail(names["f1_unset"], None), + _rail(names["f2_stream"], sink, mode="post_call", scope="streaming"), + _rail(names["f2_non_stream"], sink, mode="post_call", scope="non_streaming"), + _realtime_filter_rail(names["f3_stream"], "streaming", F3_STREAM_BLOCKED_WORD), + _realtime_filter_rail(names["f3_non_stream"], "non_streaming", F3_NON_STREAM_BLOCKED_WORD), + _rail(names["g1_failure"], sink, mode="pre_call", scope="streaming"), + _rail(names["g2_block"], sink, mode="pre_call", scope="streaming"), + _rail(names["g3_provider"], sink, mode="pre_call", scope="both"), + _rail(names["z_logging_only_barrier"], sink, mode="logging_only"), + ) + + +def _realtime_transcription_response(transcript: str) -> RealtimeResponse: + return RealtimeResponse( + content_type="application/x-realtime", + events=( + { + "type": "conversation.item.input_audio_transcription.completed", + "event_id": "evt_$REQUEST_ID", + "item_id": "item_$REQUEST_ID", + "content_index": 0, + "transcript": transcript, + }, + { + "type": "response.done", + "event_id": "evt_$REQUEST_ID", + "response": { + "id": "resp_$REQUEST_ID", + "object": "realtime.response", + "status": "completed", + "output": [], + "usage": { + "total_tokens": 0, + "input_tokens": 0, + "output_tokens": 0, + "input_token_details": { + "text_tokens": 0, + "audio_tokens": 0, + "cached_tokens": 0, + "cached_tokens_details": {"text_tokens": 0, "audio_tokens": 0}, + }, + "output_token_details": {"text_tokens": 0, "audio_tokens": 0}, + }, + }, + }, + ), + ) + + +async def _collect_realtime_events(websocket: websockets.ClientConnection) -> tuple[dict[str, JsonValue], ...]: + event: Final = JSON_OBJECT.validate_json(await websocket.recv()) + if event.get("type") == "response.done": + return (event,) + return (event, *await _collect_realtime_events(websocket)) + + +async def _realtime_transcription_events( + url: str, + key: str, + model: str, +) -> tuple[dict[str, JsonValue], ...]: + websocket_url: Final = ( + f"{url.replace('http://', 'ws://').replace('https://', 'wss://').rstrip('/')}/v1/realtime?model={model}" + ) + async with websockets.connect(websocket_url, additional_headers={"Authorization": f"Bearer {key}"}) as websocket: + session: Final = JSON_OBJECT.validate_json(await websocket.recv()) + assert session.get("type") == "session.created", session + await websocket.send(json.dumps({"type": "response.create"})) + return await _collect_realtime_events(websocket) + + +@dataclass(frozen=True, slots=True) +class MatrixRig: + candidate: Gateway + scenario: Scenario + models: Mapping[str, str] + rails: Mapping[str, str] + key: str + provider: Wire + sink: Wire + + +@pytest.fixture(scope="module") +def rig(tmp_path_factory: pytest.TempPathFactory) -> Iterator[MatrixRig]: + with httpx.Client( + base_url=os.environ["INTEGRATION_PROXY_URL"], + timeout=30, + trust_env=False, + ) as root_client: + root_gateway: Final = Gateway( + root_client, + os.environ.get("INTEGRATION_MASTER_KEY", "sk-integration-master"), + os.environ["INTEGRATION_UPSTREAM_URL"], + ) + directory: Final = tmp_path_factory.mktemp("guardrail-stream-scope-matrix") + with wire_server(_provider) as provider, wire_server(_sink) as sink: + names: Final = _rail_names() + config_base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + pass_through_paths: Final = ( + { + "path": "/pt", + "target": f"{provider.url}/passthrough", + "include_subpath": True, + "guardrails": {names["e_stream"]: None, names["e_non_stream"]: None}, + }, + { + "path": "/anthropic/v1/messages", + "target": f"{provider.url}/v1/messages", + "guardrails": {names["e_stream"]: None, names["e_non_stream"]: None}, + }, + { + "path": "/gemini/v1beta/models", + "target": "", + "include_subpath": True, + "guardrails": {names["e_stream"]: None, names["e_non_stream"]: None}, + }, + ) + config: Final = { + **config_base, + "guardrails": _configured_rails(names, sink), + "model_list": [ + { + "model_name": "matrix-pipeline-model", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{provider.url}/v1", + "api_key": "synthetic-provider-key", + }, + } + ], + "environment_variables": { + **config_base.get("environment_variables", {}), + "ANTHROPIC_API_BASE": provider.url, + "ANTHROPIC_API_KEY": "synthetic-anthropic-key", + "GEMINI_API_BASE": provider.url, + "GEMINI_API_KEY": "synthetic-gemini-key", + }, + "general_settings": { + **config_base["general_settings"], + "pass_through_endpoints": pass_through_paths, + }, + "policies": { + "matrix-f2": { + "guardrails": {"add": [names["f2_stream"], names["f2_non_stream"]]}, + "pipeline": { + "mode": "post_call", + "steps": [ + {"guardrail": names["f2_stream"], "on_pass": "next", "on_fail": "block"}, + {"guardrail": names["f2_non_stream"], "on_pass": "next", "on_fail": "block"}, + ], + }, + } + }, + "policy_attachments": [{"policy": "matrix-f2", "models": ["matrix-pipeline-model"]}], + } + config_path: Final = directory / "stream-scope-matrix.yaml" + config_path.write_text(yaml.safe_dump(config)) + with owned_proxy_process(root_gateway, directory, {}, config=config_path, workers=1) as owned: + with owned.gateway.scenario() as scenario: + models: Final = MappingProxyType( + { + "chat": scenario.model( + model="openai/gpt-4o-mini", + api_base=f"{provider.url}/v1", + api_key="synthetic-provider-key", + ), + "messages": scenario.model( + model="anthropic/claude-sonnet-4-5-20250929", + api_base=provider.url, + api_key="synthetic-provider-key", + ), + "responses": scenario.model( + model="openai/gpt-4o-mini", + api_base=f"{provider.url}/v1", + api_key="synthetic-provider-key", + ), + } + ) + key: Final = scenario.key(guardrails=[names["c2_key"]]) + yield MatrixRig(owned.gateway, scenario, models, names, key, provider, sink) + + +def _cell_body( + rig: MatrixRig, + endpoint: Endpoint, + marker: str, + streamed: bool, + rail: str | None, +) -> tuple[str, dict[str, JsonValue]]: + selected: Final = [] if rail is None else [rail] + if endpoint == "chat": + body: Final = { + "model": rig.models["chat"], + "messages": [{"role": "user", "content": marker}], + "guardrails": selected, + **({"stream": True} if streamed else {}), + } + return "/v1/chat/completions", body + if endpoint == "messages": + body = { + "model": rig.models["messages"], + "max_tokens": 32, + "messages": [{"role": "user", "content": marker}], + "guardrails": selected, + **({"stream": True} if streamed else {}), + } + return "/v1/messages", body + body = { + "model": rig.models["responses"], + "input": marker, + "guardrails": selected, + **({"stream": True} if streamed else {}), + } + return "/v1/responses", body + + +def _matching_requests(wire: Wire, marker: str) -> tuple[Request, ...]: + return tuple(request for request in wire.drain() if marker.encode() in request.body) + + +def _rail_scans(rows: Sequence[Request], rail_name: str, marker: str) -> tuple[Request, ...]: + return tuple( + request for request in rows if request.target.startswith(f"/{rail_name}/") and marker.encode() in request.body + ) + + +def _logging_phases(rows: Sequence[Request]) -> tuple[str, ...]: + return tuple( + sorted(LOGGING_PHASE.validate_python(JSON_OBJECT.validate_json(request.body)["input_type"]) for request in rows) + ) + + +@dataclass(slots=True) +class _SinkRowsAccumulator: + sink: Wire + rows: tuple[Request, ...] = () + + def drain(self) -> tuple[Request, ...]: + self.rows = (*self.rows, *self.sink.drain()) + return self.rows + + +def _logging_only_scans( + sink: Wire, + marker: str, + rail: str, + barrier: str, +) -> tuple[Request, ...]: + accumulator: Final = _SinkRowsAccumulator(sink) + eventually( + accumulator.drain, + lambda rows: bool(_rail_scans(rows, barrier, marker)), + seconds=70, + ) + return _rail_scans(accumulator.rows, rail, marker) + + +def _scope_scan_count(scope: MatrixScope, streamed: bool) -> int: + if scope is None or scope == "both": + return 1 + return int((scope == "streaming") == streamed) + + +def _expected_logging_phases(scope: MatrixScope, streamed: bool) -> tuple[str, ...]: + return ("request", "response") if _scope_scan_count(scope, streamed) else () + + +def _spend_row_for_call(call_id: str, content: bytes) -> dict[str, JsonValue]: + rows: Final = eventually( + lambda: read_rows( + "SELECT request_id, litellm_call_id, spend, prompt_tokens, completion_tokens, metadata " + 'FROM "LiteLLM_SpendLogs" WHERE request_id=%s OR litellm_call_id=%s', + (call_id, call_id), + ), + lambda values: len(values) >= 1, + seconds=70, + ) + assert len(rows) == 1, (call_id, rows, content) + assert call_id in (rows[0]["request_id"], rows[0]["litellm_call_id"]), content + return rows[0] + + +def _cell_rows( + rig: MatrixRig, + marker: str, + response: httpx.Response, + mode: Mode, + streamed: bool, + scope: MatrixScope, + call_id: str, + rail: str, +) -> tuple[tuple[Request, ...], tuple[Request, ...]]: + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows, response.text) + _spend_row_for_call(call_id, response.content) + sink_rows: Final = ( + _logging_only_scans( + rig.sink, + marker, + rail, + rig.rails["z_logging_only_barrier"], + ) + if mode == "logging_only" + else _rail_scans(rig.sink.drain(), rail, marker) + ) + return provider_rows, sink_rows + + +def _run_matrix_cell( + rig: MatrixRig, + endpoint: Endpoint, + streamed: bool, + mode: Mode, + scope: MatrixScope, +) -> None: + marker: Final = f"audit-a-{endpoint}-{int(streamed)}-{mode}-{scope or 'unset'}-{uuid.uuid4().hex}" + call_id: Final = f"matrix-a-{uuid.uuid4().hex}" + rail: Final = rig.rails[f"a_{mode}_{scope or 'unset'}"] + path, body = _cell_body(rig, endpoint, marker, streamed, rail) + response: Final = rig.candidate.request( + "POST", + path, + body, + headers={"x-litellm-call-id": call_id}, + ) + assert response.status_code == 200, response.text + assert PROVIDER_TEXT in response.text and marker in response.text, response.text + provider_rows, sink_rows = _cell_rows(rig, marker, response, mode, streamed, scope, call_id, rail) + assert len(provider_rows) == 1, (marker, provider_rows, response.text) + if mode == "logging_only": + assert _logging_phases(sink_rows) == _expected_logging_phases(scope, streamed), ( + marker, + sink_rows, + response.text, + ) + else: + expected_scans: Final = _scope_scan_count(scope, streamed) + assert len(sink_rows) == expected_scans, (marker, sink_rows, response.text) + + +@pytest.mark.parametrize("endpoint", Endpoints) +@pytest.mark.parametrize("streamed", (False, True), ids=("S0", "S1")) +@pytest.mark.parametrize("mode", Modes) +@pytest.mark.parametrize("scope", Scopes, ids=("streaming", "non_streaming", "unset")) +def test_a_stream_scope_matrix( + rig: MatrixRig, + endpoint: Endpoint, + streamed: bool, + mode: Mode, + scope: MatrixScope, +) -> None: + _run_matrix_cell(rig, endpoint, streamed, mode, scope) + + +SDK_CASES: Final[tuple[str, ...]] = ( + "openai-chat-sync", + "openai-chat-async", + "openai-responses-sync", + "openai-responses-async", + "anthropic-messages-sync", + "anthropic-messages-async", +) + + +def _openai_sdk_base_url(rig: MatrixRig) -> str: + return f"{str(rig.candidate.client.base_url).rstrip('/')}/v1" + + +def _anthropic_sdk_base_url(rig: MatrixRig) -> str: + return str(rig.candidate.client.base_url).rstrip("/") + + +def _verify_sdk_call(rig: MatrixRig, marker: str, call_id: str, text: str, streamed: bool) -> None: + assert marker in text and PROVIDER_TEXT in text, text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows) + _spend_row_for_call(call_id, text.encode()) + sink_rows: Final = _rail_scans(rig.sink.drain(), rig.rails["c2_key"], marker) + assert len(sink_rows) == int(streamed), (marker, streamed, sink_rows) + + +def _run_sync_sdk(rig: MatrixRig, sdk: str, marker: str, call_id: str, streamed: bool) -> str: + if sdk == "openai-chat-sync": + with openai.OpenAI( + api_key=rig.key, + base_url=_openai_sdk_base_url(rig), + max_retries=0, + http_client=httpx.Client(trust_env=False, timeout=30), + ) as client: + response: Final = client.chat.completions.create( + model=rig.models["chat"], + messages=[{"role": "user", "content": marker}], + stream=streamed, + extra_headers={"x-litellm-call-id": call_id}, + ) + if streamed: + return "".join( + chunk.choices[0].delta.content or "" for chunk in response if chunk.choices[0].delta.content + ) + return cast(str, response.choices[0].message.content) + if sdk == "openai-responses-sync": + with openai.OpenAI( + api_key=rig.key, + base_url=_openai_sdk_base_url(rig), + max_retries=0, + http_client=httpx.Client(trust_env=False, timeout=30), + ) as client: + response = client.responses.create( + model=rig.models["responses"], + input=marker, + stream=streamed, + extra_headers={"x-litellm-call-id": call_id}, + ) + if streamed: + return "".join(event.delta for event in response if event.type == "response.output_text.delta") + return response.output_text + if sdk == "anthropic-messages-sync": + with anthropic.Anthropic( + api_key=rig.key, + base_url=_anthropic_sdk_base_url(rig), + max_retries=0, + http_client=httpx.Client(trust_env=False, timeout=30), + ) as client: + response = client.messages.create( + model=rig.models["messages"], + max_tokens=32, + messages=[{"role": "user", "content": marker}], + stream=streamed, + extra_headers={"x-litellm-call-id": call_id}, + ) + if streamed: + return "".join( + event.delta.text + for event in response + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ) + return "".join(block.text for block in response.content if block.type == "text") + raise AssertionError(sdk) + + +async def _run_async_sdk(rig: MatrixRig, sdk: str, marker: str, call_id: str, streamed: bool) -> str: + if sdk == "openai-chat-async": + async with openai.AsyncOpenAI( + api_key=rig.key, + base_url=_openai_sdk_base_url(rig), + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False, timeout=30), + ) as client: + response = await client.chat.completions.create( + model=rig.models["chat"], + messages=[{"role": "user", "content": marker}], + stream=streamed, + extra_headers={"x-litellm-call-id": call_id}, + ) + if streamed: + chunks: Final = [ + chunk.choices[0].delta.content or "" async for chunk in response if chunk.choices[0].delta.content + ] + return "".join(chunks) + return cast(str, response.choices[0].message.content) + if sdk == "openai-responses-async": + async with openai.AsyncOpenAI( + api_key=rig.key, + base_url=_openai_sdk_base_url(rig), + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False, timeout=30), + ) as client: + response = await client.responses.create( + model=rig.models["responses"], + input=marker, + stream=streamed, + extra_headers={"x-litellm-call-id": call_id}, + ) + if streamed: + events: Final = [event.delta async for event in response if event.type == "response.output_text.delta"] + return "".join(events) + return response.output_text + if sdk == "anthropic-messages-async": + async with anthropic.AsyncAnthropic( + api_key=rig.key, + base_url=_anthropic_sdk_base_url(rig), + max_retries=0, + http_client=httpx.AsyncClient(trust_env=False, timeout=30), + ) as client: + response = await client.messages.create( + model=rig.models["messages"], + max_tokens=32, + messages=[{"role": "user", "content": marker}], + stream=streamed, + extra_headers={"x-litellm-call-id": call_id}, + ) + if streamed: + events: Final = [ + event.delta.text + async for event in response + if event.type == "content_block_delta" and event.delta.type == "text_delta" + ] + return "".join(events) + return "".join(block.text for block in response.content if block.type == "text") + raise AssertionError(sdk) + + +@pytest.mark.parametrize("sdk", SDK_CASES) +@pytest.mark.parametrize("streamed", (False, True), ids=("S0", "S1")) +def test_b_streaming_scope_classifies_sdk_streams(rig: MatrixRig, sdk: str, streamed: bool) -> None: + marker: Final = f"audit-b-{sdk}-{int(streamed)}-{uuid.uuid4().hex}" + call_id: Final = f"matrix-b-{uuid.uuid4().hex}" + if sdk.endswith("-async"): + text: Final = run(_run_async_sdk(rig, sdk, marker, call_id, streamed)) + else: + text = _run_sync_sdk(rig, sdk, marker, call_id, streamed) + _verify_sdk_call(rig, marker, call_id, text, streamed) + + +def _yaml_proxy_config( + rig: MatrixRig, + guardrail_name: str, + parameters: Mapping[str, JsonValue], +) -> dict[str, JsonValue]: + config_base: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + return cast( + dict[str, JsonValue], + { + **config_base, + "guardrails": [{"guardrail_name": guardrail_name, "litellm_params": dict(parameters)}], + "model_list": [ + { + "model_name": "scope-yaml-chat", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_base": f"{rig.provider.url}/v1", + "api_key": "synthetic-provider-key", + }, + } + ], + "environment_variables": { + **config_base.get("environment_variables", {}), + "OPENAI_API_BASE": rig.provider.url, + "OPENAI_API_KEY": "synthetic-provider-key", + }, + }, + ) + + +def _management_params( + name: str, + rig: MatrixRig, + *, + mode: str | list[str] = "pre_call", + scope: JsonValue = "streaming", + default_on: bool = False, +) -> dict[str, JsonValue]: + return { + "guardrail": "generic_guardrail_api", + "mode": mode, + "default_on": default_on, + "api_base": f"{rig.sink.url}/{name}", + "api_key": "synthetic-guardrail-key", + "stream_scope": scope, + } + + +def _raw_chat( + gateway: Gateway, + marker: str, + streamed: bool, + model: str, + *, + rails: Sequence[str] = (), + key: str | None = None, + call_id: str | None = None, +) -> httpx.Response: + return gateway.request( + "POST", + "/v1/chat/completions", + { + "model": model, + "messages": [{"role": "user", "content": marker}], + "guardrails": list(rails), + **({"stream": True} if streamed else {}), + }, + key=key, + headers={} if call_id is None else {"x-litellm-call-id": call_id}, + ) + + +def _assert_raw_call( + rig: MatrixRig, + marker: str, + response: httpx.Response, + streamed: bool, + expected_scans: int, + rail: str, + call_id: str | None = None, +) -> tuple[Request, ...]: + assert response.status_code == 200, response.text + assert PROVIDER_TEXT in response.text and marker in response.text, response.text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows) + if call_id is not None: + _spend_row_for_call(call_id, response.content) + sink_rows: Final = _rail_scans(rig.sink.drain(), rail, marker) + assert len(sink_rows) == expected_scans, (marker, streamed, expected_scans, sink_rows) + return sink_rows + + +def test_c1_default_on_rail_respects_stream_scope(rig: MatrixRig, tmp_path: Path) -> None: + name: Final = f"c1-default-{uuid.uuid4().hex}" + parameters: Final = { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": True, + "stream_scope": "streaming", + "api_base": f"{rig.sink.url}/{name}", + "api_key": "synthetic-guardrail-key", + } + config_path: Final = tmp_path / "default-on-stream-scope.yaml" + config_path.write_text(yaml.safe_dump(_yaml_proxy_config(rig, name, parameters))) + with owned_proxy_process(rig.candidate, tmp_path, {}, config=config_path, workers=1) as owned: + for streamed in (False, True): + marker: Final = f"audit-c1-{int(streamed)}-{uuid.uuid4().hex}" + response: Final = _raw_chat(owned.gateway, marker, streamed, "scope-yaml-chat") + _assert_raw_call(rig, marker, response, streamed, int(streamed), name) + + +def test_c2_key_attached_rail_respects_stream_scope(rig: MatrixRig) -> None: + marker0: Final = f"audit-c2-0-{uuid.uuid4().hex}" + response0: Final = _raw_chat(rig.candidate, marker0, False, rig.models["chat"], key=rig.key) + _assert_raw_call(rig, marker0, response0, False, 0, rig.rails["c2_key"]) + marker1: Final = f"audit-c2-1-{uuid.uuid4().hex}" + response1: Final = _raw_chat(rig.candidate, marker1, True, rig.models["chat"], key=rig.key) + _assert_raw_call(rig, marker1, response1, True, 1, rig.rails["c2_key"]) + + +def test_c3_team_attached_rail_respects_stream_scope(rig: MatrixRig) -> None: + team: Final = rig.scenario.team(guardrails=[rig.rails["c3_team"]]) + key: Final = rig.scenario.key(team_id=team) + marker0: Final = f"audit-c3-0-{uuid.uuid4().hex}" + response0: Final = _raw_chat(rig.candidate, marker0, False, rig.models["chat"], key=key) + _assert_raw_call(rig, marker0, response0, False, 0, rig.rails["c3_team"]) + marker1: Final = f"audit-c3-1-{uuid.uuid4().hex}" + response1: Final = _raw_chat(rig.candidate, marker1, True, rig.models["chat"], key=key) + _assert_raw_call(rig, marker1, response1, True, 1, rig.rails["c3_team"]) + + +@pytest.mark.parametrize("streamed", (False, True), ids=("S0", "S1")) +def test_c4_per_mode_scope_map_selects_each_mode(rig: MatrixRig, streamed: bool) -> None: + marker: Final = f"audit-c4-{int(streamed)}-{uuid.uuid4().hex}" + response: Final = _raw_chat( + rig.candidate, + marker, + streamed, + rig.models["chat"], + rails=(rig.rails["c4_modes"],), + ) + assert response.status_code == 200, response.text + assert PROVIDER_TEXT in response.text and marker in response.text, response.text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows) + sink_rows: Final = _rail_scans(rig.sink.drain(), rig.rails["c4_modes"], marker) + request_bodies: Final = tuple(JSON_OBJECT.validate_json(row.body) for row in sink_rows) + request_texts: Final = tuple(chain.from_iterable(_strings(body.get("texts", [])) for body in request_bodies)) + assert len(sink_rows) == 1, (marker, streamed, sink_rows) + assert any(marker in text for text in request_texts), (marker, request_texts) + assert (any(PROVIDER_TEXT in text for text in request_texts)) == streamed, ( + marker, + streamed, + request_texts, + ) + + +@pytest.mark.parametrize("streamed", (False, True), ids=("S0", "S1")) +def test_c5_mode_omitted_from_scope_map_means_both(rig: MatrixRig, streamed: bool) -> None: + marker: Final = f"audit-c5-{int(streamed)}-{uuid.uuid4().hex}" + response: Final = _raw_chat( + rig.candidate, + marker, + streamed, + rig.models["chat"], + rails=(rig.rails["c5_omitted"],), + ) + sink_rows: Final = _assert_raw_call( + rig, + marker, + response, + streamed, + 1 + int(streamed), + rig.rails["c5_omitted"], + ) + request_bodies: Final = tuple(JSON_OBJECT.validate_json(row.body) for row in sink_rows) + request_texts: Final = tuple(chain.from_iterable(_strings(body.get("texts", [])) for body in request_bodies)) + assert sum(marker in text and PROVIDER_TEXT not in text for text in request_texts) == 1, (marker, request_texts) + assert sum(PROVIDER_TEXT in text for text in request_texts) == int(streamed), (marker, request_texts) + + +@pytest.mark.parametrize("streamed", (False, True), ids=("S0", "S1")) +def test_c6_explicit_both_matches_unset_scope(rig: MatrixRig, streamed: bool) -> None: + marker: Final = f"audit-c6-{int(streamed)}-{uuid.uuid4().hex}" + response: Final = _raw_chat( + rig.candidate, + marker, + streamed, + rig.models["chat"], + rails=(rig.rails["c6_both"], rig.rails["c6_unset"]), + ) + assert response.status_code == 200, response.text + assert PROVIDER_TEXT in response.text and marker in response.text, response.text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows) + sink_rows: Final = rig.sink.drain() + both_rows: Final = _rail_scans(sink_rows, rig.rails["c6_both"], marker) + unset_rows: Final = _rail_scans(sink_rows, rig.rails["c6_unset"], marker) + assert (len(both_rows), len(unset_rows)) == (1, 1), (marker, streamed, both_rows, unset_rows) + + +@pytest.mark.parametrize( + ("case", "scope"), + ( + ("missing", None), + ("null", None), + ("empty", ""), + ("invalid-scalar", "sometimes"), + ("invalid-map", {"unknown_mode": "streaming"}), + ), + ids=("missing", "null", "empty", "invalid-scalar", "invalid-map"), +) +def test_c7_yaml_unset_and_invalid_scope_values_are_tolerated( + rig: MatrixRig, + tmp_path: Path, + case: str, + scope: JsonValue | None, +) -> None: + name: Final = f"c7-yaml-{case}-{uuid.uuid4().hex}" + parameters: Final = { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "default_on": True, + "api_base": f"{rig.sink.url}/{name}", + "api_key": "synthetic-guardrail-key", + **({} if case == "missing" else {"stream_scope": scope}), + } + config_path: Final = tmp_path / f"{case}-stream-scope.yaml" + config_path.write_text(yaml.safe_dump(_yaml_proxy_config(rig, name, parameters))) + with owned_proxy_process(rig.candidate, tmp_path, {}, config=config_path, workers=1) as owned: + for streamed in (False, True): + marker: Final = f"audit-c7-{case}-{int(streamed)}-{uuid.uuid4().hex}" + response: Final = _raw_chat(owned.gateway, marker, streamed, "scope-yaml-chat") + _assert_raw_call(rig, marker, response, streamed, 1, name) + if case == "empty": + assert "Ignoring invalid stored stream_scope value of type str" in owned.log.read_text() + + +def test_c8_identical_requests_have_one_scan_and_spend_each(rig: MatrixRig) -> None: + marker: Final = f"audit-c8-identical-{uuid.uuid4().hex}" + call_ids: Final = tuple(f"matrix-c8-{uuid.uuid4().hex}" for _ in range(3)) + responses: Final = tuple( + _raw_chat( + rig.candidate, + marker, + True, + rig.models["chat"], + rails=(rig.rails["a_post_call_streaming"],), + call_id=call_id, + ) + for call_id in call_ids + ) + assert tuple(response.status_code for response in responses) == (200, 200, 200), tuple( + response.text for response in responses + ) + for call_id, response in zip(call_ids, responses): + _spend_row_for_call(call_id, response.content) + sink_rows: Final = _rail_scans(rig.sink.drain(), rig.rails["a_post_call_streaming"], marker) + assert len(sink_rows) == 3, (marker, sink_rows) + guardrail_call_ids: Final = tuple( + cast(str, JSON_OBJECT.validate_json(row.body).get("litellm_call_id")) for row in sink_rows + ) + assert set(guardrail_call_ids) == set(call_ids), (call_ids, guardrail_call_ids) + + +def _create_management_rail(rig: MatrixRig, name: str, params: Mapping[str, JsonValue]) -> str: + created: Final = rig.candidate.request( + "POST", + "/guardrails", + {"guardrail": {"guardrail_name": name, "litellm_params": dict(params)}}, + ) + assert created.status_code == 200, created.text + payload: Final = JSON_OBJECT.validate_json(created.content) + identity: Final = payload.get("guardrail_id") + assert isinstance(identity, str), payload + return identity + + +def _management_observation( + rig: MatrixRig, + name: str, + streamed: bool, +) -> tuple[int, int, int]: + marker: Final = f"audit-d-management-{int(streamed)}-{uuid.uuid4().hex}" + call_id: Final = f"matrix-d1-{uuid.uuid4().hex}" + response: Final = _raw_chat( + rig.candidate, + marker, + streamed, + rig.models["chat"], + rails=(name,), + call_id=call_id, + ) + provider_rows: Final = _matching_requests(rig.provider, marker) + sink_rows: Final = _rail_scans(rig.sink.drain(), name, marker) + if response.status_code == 200 and len(provider_rows) == 1: + assert marker in response.text, response.text + _spend_row_for_call(call_id, response.content) + return response.status_code, len(provider_rows), len(sink_rows) + + +def _eventually_management_scope( + rig: MatrixRig, + name: str, + expected: tuple[int, int], +) -> tuple[tuple[int, int, int], tuple[int, int, int]]: + return eventually( + lambda: ( + _management_observation(rig, name, False), + _management_observation(rig, name, True), + ), + lambda values: ( + (values[0][0], values[0][2]) == (200, expected[0]) + and (values[1][0], values[1][2]) == (200, expected[1]) + and values[0][1] == values[1][1] == 1 + ), + seconds=30, + ) + + +def test_d1_management_create_read_and_runtime_scope(rig: MatrixRig) -> None: + name: Final = f"d1-management-{uuid.uuid4().hex}" + params: Final = _management_params(name, rig) + identity: Final = _create_management_rail(rig, name, params) + try: + info: Final = rig.candidate.request("GET", f"/guardrails/{identity}/info") + listing: Final = rig.candidate.request("GET", "/v2/guardrails/list") + info_payload: Final = JSON_OBJECT.validate_json(info.content) + list_payload: Final = JSON_OBJECT.validate_json(listing.content) + list_rows: Final = list_payload.get("guardrails") + listed: Final = ( + tuple(row for row in list_rows if isinstance(row, dict) and row.get("guardrail_id") == identity) + if isinstance(list_rows, list) + else () + ) + info_params: Final = info_payload.get("litellm_params") + list_params: Final = listed[0].get("litellm_params") if listed else None + assert ( + info.status_code == 200 + and listing.status_code == 200 + and isinstance(info_params, dict) + and info_params.get("stream_scope") == "streaming" + and len(listed) == 1 + and isinstance(list_params, dict) + and list_params.get("stream_scope") == "streaming" + ), (info.status_code, info.text, listing.status_code, listing.text) + observed: Final = _eventually_management_scope(rig, name, (0, 1)) + assert observed[0][2] == 0 and observed[1][2] == 1, observed + finally: + deleted: Final = rig.candidate.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +def test_d2_patch_then_put_updates_runtime_scope(rig: MatrixRig) -> None: + name: Final = f"d2-management-{uuid.uuid4().hex}" + identity: Final = _create_management_rail(rig, name, _management_params(name, rig)) + try: + patched: Final = rig.candidate.request( + "PATCH", + f"/guardrails/{identity}", + {"litellm_params": {"stream_scope": "non_streaming"}}, + ) + assert patched.status_code == 200, patched.text + after_patch: Final = _eventually_management_scope(rig, name, (1, 0)) + assert after_patch[0][2] == 1 and after_patch[1][2] == 0, after_patch + params: Final = _management_params(name, rig) + updated: Final = rig.candidate.request( + "PUT", + f"/guardrails/{identity}", + {"guardrail": {"guardrail_name": name, "litellm_params": params}}, + ) + assert updated.status_code == 200, updated.text + after_put: Final = _eventually_management_scope(rig, name, (0, 1)) + assert after_put[0][2] == 0 and after_put[1][2] == 1, after_put + finally: + deleted: Final = rig.candidate.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +HOSTILE_SCOPE_VALUES: Final[tuple[JsonValue, ...]] = ( + 1, + [], + "", + "x" * 5000, + {"unknown_mode": "streaming"}, + {"pre_call": 1}, +) + + +@pytest.mark.parametrize("operation", ("POST", "PUT", "PATCH")) +@pytest.mark.parametrize( + "scope", HOSTILE_SCOPE_VALUES, ids=("integer", "list", "empty", "long", "unknown-key", "wrong-value") +) +@pytest.mark.parametrize("repeat", (1, 2), ids=("first", "second")) +def test_d3_management_rejects_invalid_scope_values( + rig: MatrixRig, + operation: str, + scope: JsonValue, + repeat: int, +) -> None: + name: Final = f"d3-management-{operation.lower()}-{repeat}-{uuid.uuid4().hex}" + identity: Final = _create_management_rail(rig, name, _management_params(name, rig)) if operation != "POST" else None + payload: Final = ( + {"litellm_params": {"stream_scope": scope}} + if operation == "PATCH" + else { + "guardrail": { + "guardrail_name": name, + "litellm_params": _management_params(name, rig, scope=scope), + } + } + ) + path: Final = "/guardrails" if operation == "POST" else f"/guardrails/{identity}" + try: + response: Final = rig.candidate.request(operation, path, payload) + if operation == "POST" and response.status_code == 200: + created_payload: Final = JSON_OBJECT.validate_json(response.content) + created_identity: Final = created_payload.get("guardrail_id") + if isinstance(created_identity, str): + rig.candidate.request("DELETE", f"/guardrails/{created_identity}") + assert response.status_code == 422 and "stream_scope" in response.text, ( + operation, + scope, + repeat, + response.status_code, + response.text, + ) + if operation == "POST": + stored: Final = read_rows( + 'SELECT guardrail_id FROM "LiteLLM_GuardrailsTable" WHERE guardrail_name=%s', + (name,), + ) + assert stored == [], (operation, scope, stored) + else: + existing: Final = rig.candidate.request("GET", f"/guardrails/{identity}/info") + existing_payload: Final = JSON_OBJECT.validate_json(existing.content) + existing_params: Final = existing_payload.get("litellm_params") + assert ( + existing.status_code == 200 + and isinstance(existing_params, dict) + and existing_params.get("stream_scope") == "streaming" + ), (operation, scope, existing.status_code, existing.text) + finally: + if isinstance(identity, str): + deleted: Final = rig.candidate.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +def test_d3_management_normalizes_uppercase_scope(rig: MatrixRig) -> None: + name: Final = f"d3-uppercase-{uuid.uuid4().hex}" + identity: Final = _create_management_rail(rig, name, _management_params(name, rig, scope="STREAMING")) + try: + info: Final = rig.candidate.request("GET", f"/guardrails/{identity}/info") + payload: Final = JSON_OBJECT.validate_json(info.content) + parameters: Final = payload.get("litellm_params") + assert ( + info.status_code == 200 and isinstance(parameters, dict) and parameters.get("stream_scope") == "streaming" + ), (info.status_code, info.text) + finally: + deleted: Final = rig.candidate.request("DELETE", f"/guardrails/{identity}") + assert deleted.status_code == 200, deleted.text + + +def test_d4_unauthenticated_management_create_is_rejected(rig: MatrixRig) -> None: + name: Final = f"d4-unauthenticated-{uuid.uuid4().hex}" + response: Final = rig.candidate.client.request( + "POST", + "/guardrails", + json={"guardrail": {"guardrail_name": name, "litellm_params": _management_params(name, rig)}}, + ) + assert response.status_code == 401, response.text + + +def _scoped_rows( + rows: Sequence[Request], + marker: str, + streaming_name: str, + non_streaming_name: str, +) -> tuple[tuple[Request, ...], tuple[Request, ...]]: + return ( + _rail_scans(rows, streaming_name, marker), + _rail_scans(rows, non_streaming_name, marker), + ) + + +HOSTILE_CLASSIFICATION_VALUES: Final[tuple[JsonValue, ...]] = ( + True, + 1, + [], + "", + "litellm-server-streaming", + "x" * 5000, +) + + +@pytest.mark.parametrize( + "value", + HOSTILE_CLASSIFICATION_VALUES, + ids=("true", "integer", "list", "empty", "marker", "long"), +) +def test_ea_is_streaming_request_body_does_not_change_chat_classification( + rig: MatrixRig, + value: JsonValue, +) -> None: + marker: Final = f"audit-ea-{uuid.uuid4().hex}" + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": rig.models["chat"], + "messages": [{"role": "user", "content": marker}], + "guardrails": [rig.rails["e_stream"], rig.rails["e_non_stream"]], + "is_streaming_request": value, + }, + ) + assert response.status_code == 200, response.text + assert PROVIDER_TEXT in response.text and marker in response.text, response.text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows) + provider_body: Final = JSON_OBJECT.validate_json(provider_rows[0].body) + assert provider_body.get("is_streaming_request") == value, provider_rows[0].body.decode() + sink_rows: Final = rig.sink.drain() + streaming_rows, non_streaming_rows = _scoped_rows( + sink_rows, + marker, + rig.rails["e_stream"], + rig.rails["e_non_stream"], + ) + assert (len(streaming_rows), len(non_streaming_rows)) == (0, 1), (marker, streaming_rows, non_streaming_rows) + + +@pytest.mark.parametrize("endpoint", ("chat", "passthrough"), ids=("chat", "configured-pass-through")) +@pytest.mark.parametrize("value", (True, "litellm-server-streaming"), ids=("true", "marker")) +def test_eb_namespaced_caller_body_field_cannot_flip_classification( + rig: MatrixRig, + endpoint: str, + value: JsonValue, +) -> None: + marker: Final = f"audit-eb-{endpoint}-{uuid.uuid4().hex}" + body: Final = ( + { + "model": rig.models["chat"], + "messages": [{"role": "user", "content": marker}], + "guardrails": [rig.rails["e_stream"], rig.rails["e_non_stream"]], + "litellm_server_streaming_classification": value, + } + if endpoint == "chat" + else { + "marker": marker, + "litellm_server_streaming_classification": value, + } + ) + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions" if endpoint == "chat" else "/pt", + body, + ) + assert response.status_code == 200 and marker in response.text, response.text + actual_streamed: Final = response.headers.get("content-type", "").lower().startswith("text/event-stream") + assert not actual_streamed, (endpoint, response.headers, response.text) + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows) + provider_body: Final = JSON_OBJECT.validate_json(provider_rows[0].body) + if endpoint == "passthrough" and value == "litellm-server-streaming": + assert "litellm_server_streaming_classification" not in provider_body, provider_rows[0].body.decode() + else: + assert provider_body.get("litellm_server_streaming_classification") == value, provider_rows[0].body.decode() + sink_rows: Final = rig.sink.drain() + streaming_rows, non_streaming_rows = _scoped_rows( + sink_rows, + marker, + rig.rails["e_stream"], + rig.rails["e_non_stream"], + ) + assert (len(streaming_rows), len(non_streaming_rows)) == (0, 1), (marker, streaming_rows, non_streaming_rows) + + +HOSTILE_STREAM_VALUES: Final[tuple[JsonValue, ...]] = ("true", 1, [], "") + + +@pytest.mark.parametrize("value", HOSTILE_STREAM_VALUES, ids=("string", "integer", "list", "empty")) +def test_ec_hostile_stream_values_follow_observed_response_shape(rig: MatrixRig, value: JsonValue) -> None: + marker: Final = f"audit-ec-{uuid.uuid4().hex}" + response: Final = rig.candidate.request( + "POST", + "/v1/chat/completions", + { + "model": rig.models["chat"], + "messages": [{"role": "user", "content": marker}], + "guardrails": [rig.rails["e_stream"], rig.rails["e_non_stream"]], + "stream": value, + }, + ) + assert response.status_code == 200, response.text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows) + observed_stream: Final = response.headers.get("content-type", "").startswith("text/event-stream") + assert marker in response.text, response.text + sink_rows: Final = rig.sink.drain() + streaming_rows, non_streaming_rows = _scoped_rows( + sink_rows, + marker, + rig.rails["e_stream"], + rig.rails["e_non_stream"], + ) + assert (len(streaming_rows), len(non_streaming_rows)) == ( + int(observed_stream), + int(not observed_stream), + ), (marker, value, response.headers, streaming_rows, non_streaming_rows) + + +@pytest.mark.parametrize("streamed", (False, True), ids=("stream-absent", "stream-true")) +def test_ed_configured_passthrough_forwards_caller_flag_and_uses_route_body_stream( + rig: MatrixRig, + streamed: bool, +) -> None: + marker: Final = f"audit-ed-{int(streamed)}-{uuid.uuid4().hex}" + body: Final = { + "marker": marker, + "is_streaming_request": True, + **({"stream": True} if streamed else {}), + } + response: Final = rig.candidate.request("POST", "/pt", body) + assert response.status_code == 200 and marker in response.text, response.text + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows) + provider_body: Final = JSON_OBJECT.validate_json(provider_rows[0].body) + assert provider_body == body, (body, provider_body) + + +@pytest.mark.parametrize( + ("route", "body", "streamed"), + ( + ( + "/anthropic/v1/messages", + { + "model": "synthetic-anthropic-model", + "max_tokens": 32, + "messages": [{"role": "user", "content": "MARKER"}], + }, + False, + ), + ( + "/anthropic/v1/messages", + { + "model": "synthetic-anthropic-model", + "max_tokens": 32, + "messages": [{"role": "user", "content": "MARKER"}], + "stream": True, + }, + True, + ), + ( + "/gemini/v1beta/models/audit-model:generateContent", + {"contents": [{"parts": [{"text": "MARKER"}]}]}, + False, + ), + ( + "/gemini/v1beta/models/audit-model:streamGenerateContent?alt=sse", + {"contents": [{"parts": [{"text": "MARKER"}]}]}, + True, + ), + ), + ids=("anthropic-S0", "anthropic-S1", "gemini-generate", "gemini-stream"), +) +def test_eh_provider_passthrough_routes_classify_effective_streaming( + rig: MatrixRig, + route: str, + body: dict[str, JsonValue], + streamed: bool, +) -> None: + marker: Final = f"audit-eh-{uuid.uuid4().hex}" + request_body: Final = JSON_OBJECT.validate_python(json.loads(json.dumps(body).replace("MARKER", marker))) + team: Final = ( + rig.scenario.team( + metadata={ + "allowed_passthrough_routes": [ + "/gemini/v1beta/models/audit-model:generateContent", + "/gemini/v1beta/models/audit-model:streamGenerateContent", + ] + } + ) + if route.startswith("/gemini/") + else None + ) + key: Final = ( + rig.scenario.key(team_id=team, guardrails=[rig.rails["e_stream"], rig.rails["e_non_stream"]]) + if team is not None + else rig.scenario.key(guardrails=[rig.rails["e_stream"], rig.rails["e_non_stream"]]) + ) + headers: Final = {"x-goog-api-key": key} if route.startswith("/gemini/") else None + response: Final = rig.candidate.request( + "POST", + route, + request_body, + headers=headers, + ) + assert response.status_code == 200 and marker in response.text, response.text + actual_streamed: Final = response.headers.get("content-type", "").lower().startswith("text/event-stream") + assert actual_streamed is streamed, (route, streamed, response.headers, response.text) + provider_rows: Final = _matching_requests(rig.provider, marker) + assert len(provider_rows) == 1, (marker, provider_rows) + sink_rows: Final = rig.sink.drain() + streaming_rows, non_streaming_rows = _scoped_rows( + sink_rows, + marker, + rig.rails["e_stream"], + rig.rails["e_non_stream"], + ) + assert (len(streaming_rows), len(non_streaming_rows)) == ((1, 0) if streamed else (0, 1)), ( + marker, + route, + streaming_rows, + non_streaming_rows, + ) + + +def test_ei_unauthenticated_scoped_request_has_no_upstream_or_guardrail_call(rig: MatrixRig) -> None: + marker: Final = f"audit-ei-{uuid.uuid4().hex}" + response: Final = rig.candidate.client.request( + "POST", + "/v1/chat/completions", + json={ + "model": rig.models["chat"], + "messages": [{"role": "user", "content": marker}], + "stream": True, + "guardrails": [rig.rails["e_stream"]], + }, + ) + assert response.status_code == 401, response.text + assert _matching_requests(rig.provider, marker) == () + assert _rail_scans(rig.sink.drain(), rig.rails["e_stream"], marker) == () + + +@pytest.mark.parametrize( + ("rail_key", "blocked"), + (("f1_stream", False), ("f1_non_stream", True), ("f1_unset", True)), + ids=("streaming", "non_streaming", "unset"), +) +def test_f1_mcp_content_filter_respects_stream_scope( + rig: MatrixRig, + rail_key: Literal["f1_stream", "f1_non_stream", "f1_unset"], + blocked: bool, +) -> None: + with scripted_peer(echo_tool("echo")) as peer: + alias: Final = f"streamscope{uuid.uuid4().hex}" + identity: Final = register_mcp(rig.scenario, peer, alias) + key: Final = rig.scenario.key( + object_permission={"mcp_servers": [identity]}, + guardrails=[rig.rails[rail_key]], + ) + marker: Final = f"audit-f1-{uuid.uuid4().hex}" + caller: Final = McpCaller(rig.candidate, key, "rest", alias) + outcome: Final = caller.call( + f"{alias}-echo", + {"text": f"{F1_BLOCKED_WORD} {marker}"}, + identity, + ) + peer_requests: Final = peer.drain() + marker_requests: Final = tuple(request for request in peer_requests if marker in json.dumps(request)) + if blocked: + assert outcome.error is not None, outcome.raw + assert "block" in outcome.raw.casefold() or F1_BLOCKED_WORD in outcome.raw.casefold(), outcome.raw + assert marker_requests == (), (marker, marker_requests) + return + assert outcome.ok and marker in (outcome.text or ""), outcome.raw + assert len(marker_requests) == 1, (marker, marker_requests) + + +@pytest.mark.parametrize("streamed", (False, True), ids=("S0", "S1")) +def test_f2_mismatched_policy_step_skips_matching_sibling_still_enforces( + rig: MatrixRig, + streamed: bool, +) -> None: + marker: Final = f"audit-f2-{int(streamed)}-{uuid.uuid4().hex}" + response: Final = _raw_chat( + rig.candidate, + marker, + streamed, + "matrix-pipeline-model", + ) + provider_rows: Final = _matching_requests(rig.provider, marker) + sink_rows: Final = rig.sink.drain() + streaming_rows, non_streaming_rows = _scoped_rows( + sink_rows, + marker, + rig.rails["f2_stream"], + rig.rails["f2_non_stream"], + ) + assert len(provider_rows) == 1, (marker, provider_rows) + if streamed: + assert response.status_code == 200 and marker in response.text, response.text + assert (len(streaming_rows), len(non_streaming_rows)) == (1, 0), (marker, streaming_rows, non_streaming_rows) + return + assert response.status_code == 400 and "synthetic policy block" in response.text, response.text + assert (len(streaming_rows), len(non_streaming_rows)) == (0, 1), (marker, streaming_rows, non_streaming_rows) + + +def test_f3_realtime_transcription_uses_streaming_scope(rig: MatrixRig) -> None: + streaming_scenario_id: Final = f"matrix-f3-stream-{uuid.uuid4().hex}" + streaming_upstream: Final = register_scenario( + streaming_scenario_id, + _realtime_transcription_response(F3_STREAM_BLOCKED_WORD), + ) + rig.scenario.cleanups.callback(delete_scenario, streaming_upstream) + streaming_model: Final = rig.scenario.model( + model="openai/gpt-realtime-2", + api_key=streaming_scenario_id, + api_base=rig.candidate.upstream_url, + ) + non_streaming_scenario_id: Final = f"matrix-f3-non-stream-{uuid.uuid4().hex}" + non_streaming_upstream: Final = register_scenario( + non_streaming_scenario_id, + _realtime_transcription_response(F3_NON_STREAM_BLOCKED_WORD), + ) + rig.scenario.cleanups.callback(delete_scenario, non_streaming_upstream) + non_streaming_model: Final = rig.scenario.model( + model="openai/gpt-realtime-2", + api_key=non_streaming_scenario_id, + api_base=rig.candidate.upstream_url, + ) + key: Final = rig.scenario.key() + proxy_url: Final = str(rig.candidate.client.base_url).rstrip("/") + + streaming_events: Final = run(_realtime_transcription_events(proxy_url, key, streaming_model)) + streaming_transcriptions: Final = tuple( + event + for event in streaming_events + if event.get("type") == "conversation.item.input_audio_transcription.completed" + ) + streaming_errors: Final = tuple(event for event in streaming_events if event.get("type") == "error") + assert len(streaming_transcriptions) == 1, streaming_events + assert streaming_transcriptions[0].get("transcript") == F3_STREAM_BLOCKED_WORD, streaming_events + assert len(streaming_errors) == 1, streaming_events + streaming_error: Final = streaming_errors[0].get("error") + assert isinstance(streaming_error, dict) and streaming_error.get("type") == "guardrail_violation", streaming_events + + non_streaming_events: Final = run(_realtime_transcription_events(proxy_url, key, non_streaming_model)) + non_streaming_transcriptions: Final = tuple( + event + for event in non_streaming_events + if event.get("type") == "conversation.item.input_audio_transcription.completed" + ) + non_streaming_errors: Final = tuple(event for event in non_streaming_events if event.get("type") == "error") + completions: Final = tuple(event for event in non_streaming_events if event.get("type") == "response.done") + assert len(non_streaming_transcriptions) == 1, non_streaming_events + assert non_streaming_transcriptions[0].get("transcript") == F3_NON_STREAM_BLOCKED_WORD, non_streaming_events + assert non_streaming_errors == (), non_streaming_events + assert len(completions) == 1, non_streaming_events + + +def test_g1_sink_failure_fails_closed_only_when_rail_is_in_scope(rig: MatrixRig) -> None: + marker_s0: Final = f"audit-g1-S0-{uuid.uuid4().hex}" + response_s0: Final = _raw_chat( + rig.candidate, + marker_s0, + False, + rig.models["chat"], + rails=(rig.rails["g1_failure"],), + ) + assert response_s0.status_code == 200 and marker_s0 in response_s0.text, response_s0.text + provider_rows_s0: Final = _matching_requests(rig.provider, marker_s0) + sink_rows_s0: Final = _rail_scans(rig.sink.drain(), rig.rails["g1_failure"], marker_s0) + assert len(provider_rows_s0) == 1 and sink_rows_s0 == (), (marker_s0, provider_rows_s0, sink_rows_s0) + marker_s1: Final = f"audit-g1-S1-{uuid.uuid4().hex}" + response_s1: Final = _raw_chat( + rig.candidate, + marker_s1, + True, + rig.models["chat"], + rails=(rig.rails["g1_failure"],), + ) + assert response_s1.status_code == 500 and "Generic Guardrail API failed" in response_s1.text, response_s1.text + assert _matching_requests(rig.provider, marker_s1) == () + sink_rows: Final = _rail_scans(rig.sink.drain(), rig.rails["g1_failure"], marker_s1) + assert len(sink_rows) == 1, (marker_s1, sink_rows) + + +def test_g2_blocked_verdict_blocks_only_when_rail_is_in_scope(rig: MatrixRig) -> None: + marker_s0: Final = f"audit-g2-S0-{uuid.uuid4().hex}" + response_s0: Final = _raw_chat( + rig.candidate, + marker_s0, + False, + rig.models["chat"], + rails=(rig.rails["g2_block"],), + ) + assert response_s0.status_code == 200 and marker_s0 in response_s0.text, response_s0.text + assert len(_matching_requests(rig.provider, marker_s0)) == 1 + assert _rail_scans(rig.sink.drain(), rig.rails["g2_block"], marker_s0) == () + marker_s1: Final = f"audit-g2-S1-{uuid.uuid4().hex}" + response_s1: Final = _raw_chat( + rig.candidate, + marker_s1, + True, + rig.models["chat"], + rails=(rig.rails["g2_block"],), + ) + assert response_s1.status_code == 400 and "synthetic policy block" in response_s1.text, response_s1.text + assert _matching_requests(rig.provider, marker_s1) == () + sink_rows: Final = _rail_scans(rig.sink.drain(), rig.rails["g2_block"], marker_s1) + assert len(sink_rows) == 1, (marker_s1, sink_rows) + + +def test_g3_provider_errors_reach_caller_and_proxy_remains_usable(rig: MatrixRig) -> None: + marker: Final = f"audit-g3-401-{uuid.uuid4().hex}" + unauthorized: Final = _raw_chat( + rig.candidate, + marker, + False, + rig.models["chat"], + rails=(rig.rails["g3_provider"],), + ) + assert unauthorized.status_code == 401 and "synthetic provider unauthorized" in unauthorized.text, unauthorized.text + assert len(_matching_requests(rig.provider, marker)) == 1 + assert len(_rail_scans(rig.sink.drain(), rig.rails["g3_provider"], marker)) == 1 + unknown_marker: Final = f"audit-g3-unknown-{uuid.uuid4().hex}" + unknown: Final = _raw_chat( + rig.candidate, + unknown_marker, + False, + "audit-unknown-model", + rails=(rig.rails["g3_provider"],), + ) + assert unknown.status_code in (400, 404) and "model" in unknown.text.lower(), unknown.text + assert _matching_requests(rig.provider, unknown_marker) == () + healthy_marker: Final = f"audit-g3-healthy-{uuid.uuid4().hex}" + healthy: Final = _raw_chat(rig.candidate, healthy_marker, False, rig.models["chat"], key=rig.key) + assert healthy.status_code == 200 and healthy_marker in healthy.text, healthy.text + assert len(_matching_requests(rig.provider, healthy_marker)) == 1 + + +VERTEX_LIVE_HOST: Final = "us-central1-aiplatform.googleapis.com" +VERTEX_LIVE_PATH: Final = "/ws/google.cloud.aiplatform.v1.LlmBidiService/BidiGenerateContent" +VERTEX_LIVE_AUTHORITY: Final = f"{VERTEX_LIVE_HOST}:443" + + +@dataclass(frozen=True, slots=True) +class _VertexLivePeer: + port: int + paths: SimpleQueue[str] + authorizations: SimpleQueue[str] + frames: SimpleQueue[dict[str, JsonValue]] + + +@dataclass(frozen=True, slots=True) +class _VertexConnectTunnel: + url: str + authorities: SimpleQueue[str] + + +async def _answer_vertex_live_peer( + connection: ServerConnection, + paths: SimpleQueue[str], + authorizations: SimpleQueue[str], + frames: SimpleQueue[dict[str, JsonValue]], +) -> None: + request: Final = connection.request + assert request is not None + paths.put(request.path) + authorizations.put(request.headers.get("Authorization", "")) + async for raw_frame in connection: + frame: Final = JSON_OBJECT.validate_python(json.loads(raw_frame)) + frames.put(frame) + if frames.qsize() == 1: + await connection.send(json.dumps({"setupComplete": {}})) + + +async def _serve_vertex_live_peer( + tls: ssl.SSLContext, + paths: SimpleQueue[str], + authorizations: SimpleQueue[str], + frames: SimpleQueue[dict[str, JsonValue]], + ports: SimpleQueue[int], + stop: asyncio.Event, +) -> None: + async with serve( + lambda connection: _answer_vertex_live_peer(connection, paths, authorizations, frames), + "127.0.0.1", + 0, + ssl=tls, + ) as server: + ports.put(next(iter(server.sockets)).getsockname()[1]) + await stop.wait() + + +@contextmanager +def _vertex_live_peer(cert: tuple[Path, Path]) -> Iterator[_VertexLivePeer]: + loop: Final = asyncio.new_event_loop() + stop: Final = asyncio.Event() + paths: Final = SimpleQueue[str]() + authorizations: Final = SimpleQueue[str]() + frames: Final = SimpleQueue[dict[str, JsonValue]]() + ports: Final = SimpleQueue[int]() + thread: Final = threading.Thread( + target=loop.run_until_complete, + args=(_serve_vertex_live_peer(server_context(*cert), paths, authorizations, frames, ports, stop),), + daemon=True, + ) + thread.start() + try: + yield _VertexLivePeer(ports.get(timeout=10), paths, authorizations, frames) + finally: + loop.call_soon_threadsafe(stop.set) + thread.join(timeout=10) + loop.close() + + +def _pipe_vertex_socket(source: socket.socket, sink: socket.socket) -> None: + with contextlib.suppress(OSError): + for chunk in iter(lambda: source.recv(65536), b""): + sink.sendall(chunk) + with contextlib.suppress(OSError): + sink.shutdown(socket.SHUT_WR) + + +@contextmanager +def _vertex_connect_tunnel(peer: _VertexLivePeer) -> Generator[_VertexConnectTunnel, None, None]: + authorities: Final = SimpleQueue[str]() + + class ConnectHandler(socketserver.StreamRequestHandler): + rbufsize = 0 + request: socket.socket + + def handle(self) -> None: + request_line: Final = self.rfile.readline().decode().split() + authority: Final = request_line[1] if len(request_line) > 1 else "" + while self.rfile.readline() not in (b"\r\n", b""): + pass + authorities.put(authority) + if authority != VERTEX_LIVE_AUTHORITY: + self.wfile.write(b"HTTP/1.1 403 Forbidden\r\ncontent-length: 0\r\n\r\n") + return + self.wfile.write(b"HTTP/1.1 200 Connection established\r\n\r\n") + self.request.settimeout(10) + with socket.create_connection(("127.0.0.1", peer.port), timeout=10) as upstream: + outbound: Final = threading.Thread(target=_pipe_vertex_socket, args=(self.request, upstream)) + outbound.start() + _pipe_vertex_socket(upstream, self.request) + outbound.join(timeout=12) + + with socketserver.ThreadingTCPServer(("127.0.0.1", 0), ConnectHandler) as server: + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + thread.start() + try: + yield _VertexConnectTunnel(f"http://127.0.0.1:{server.server_address[1]}", authorities) + finally: + server.shutdown() + thread.join(timeout=6) + + +async def _exchange_vertex_live_frame( + url: str, key: str, vertex_project: str, vertex_location: str, frame: str +) -> str | bytes: + websocket_url: Final = ( + f"{url.replace('http://', 'ws://', 1).rstrip('/')}/vertex_ai/live" + f"?vertex_project={vertex_project}&vertex_location={vertex_location}" + ) + async with websockets.connect( + websocket_url, + additional_headers={"Authorization": f"Bearer {key}"}, + open_timeout=10, + ) as websocket: + await websocket.send(frame) + reply: Final = await wait_for(websocket.recv(), timeout=10) + await websocket.close() + return reply + + +def test_ej_websocket_passthrough_runs_only_streaming_scoped_rails(rig: MatrixRig, tmp_path: Path) -> None: + vertex_project: Final = f"matrix-ej-{uuid.uuid4().hex}" + vertex_location: Final = "us-central1" + streaming_name: Final = f"ej-streaming-{uuid.uuid4().hex}" + non_streaming_name: Final = f"ej-non-streaming-{uuid.uuid4().hex}" + + def _token_response(_request: Request) -> Reply: + return Reply( + body=_json( + { + "access_token": "synthetic-vertex-access-token", + "expires_in": 3600, + "token_type": "Bearer", + } + ) + ) + + private_key: Final = rsa.generate_private_key(public_exponent=65537, key_size=2048) + private_key_pem: Final = private_key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.PKCS8, + serialization.NoEncryption(), + ).decode("utf-8") + credentials_path: Final = tmp_path / "vertex-service-account.json" + + with ( + wire_server(_token_response) as token_double, + wire_server(lambda _request: Reply(body=_json({"action": "NONE"}))) as sink, + ): + credentials_path.write_text( + json.dumps( + { + "type": "service_account", + "project_id": vertex_project, + "private_key_id": uuid.uuid4().hex, + "private_key": private_key_pem, + "client_email": f"integration-test@{vertex_project}.iam.gserviceaccount.com", + "client_id": "123456789012345678901", + "token_uri": f"{token_double.url}/token", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/integration-test", + } + ) + ) + rails: Final = ( + _rail(streaming_name, sink, mode="pre_call", scope="streaming", default_on=True), + _rail(non_streaming_name, sink, mode="pre_call", scope="non_streaming", default_on=True), + ) + config_data: Final = cast(object, yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())) + config_base: Final = JSON_OBJECT.validate_python(config_data) + config: Final = cast(dict[str, JsonValue], {**config_base, "guardrails": list(rails)}) + config_path: Final = tmp_path / "ej-vertex-live-stream-scope.yaml" + config_path.write_text(yaml.safe_dump(config)) + certificate: Final = write_self_signed_cert(tmp_path, (VERTEX_LIVE_HOST,)) + with _vertex_live_peer(certificate) as peer, _vertex_connect_tunnel(peer) as tunnel: + with owned_proxy_process( + rig.candidate, + tmp_path, + { + "DEFAULT_VERTEXAI_PROJECT": vertex_project, + "DEFAULT_VERTEXAI_LOCATION": vertex_location, + "DEFAULT_GOOGLE_APPLICATION_CREDENTIALS": str(credentials_path), + "HTTPS_PROXY": tunnel.url, + "https_proxy": tunnel.url, + "NO_PROXY": "127.0.0.1,localhost", + "no_proxy": "127.0.0.1,localhost", + "SSL_CERT_FILE": str(certificate[0]), + }, + config=config_path, + workers=1, + ) as owned: + with owned.gateway.scenario() as scenario: + key: Final = scenario.key() + marker: Final = f"vertex-live-{uuid.uuid4().hex}" + frame: Final = json.dumps( + { + "clientContent": { + "turns": [{"role": "user", "parts": [{"text": marker}]}], + "turnComplete": True, + } + } + ) + client_reply_raw: Final = run( + _exchange_vertex_live_frame( + str(owned.gateway.client.base_url), + key, + vertex_project, + vertex_location, + frame, + ) + ) + proxy_log: Final = owned.log + + tunnel_authorities: Final = tuple( + tunnel.authorities.get_nowait() for _ in range(tunnel.authorities.qsize()) + ) + peer_path: Final = peer.paths.get_nowait() + peer_authorization: Final = peer.authorizations.get_nowait() + peer_frames: Final = tuple(peer.frames.get_nowait() for _ in range(peer.frames.qsize())) + client_reply: Final = JSON_OBJECT.validate_python(json.loads(client_reply_raw)) + expected_frame: Final = JSON_OBJECT.validate_python(json.loads(frame)) + token_requests: Final = token_double.drain() + sink_rows: Final = sink.drain() + streaming_rows: Final = tuple(row for row in sink_rows if row.target.startswith(f"/{streaming_name}/")) + non_streaming_rows: Final = tuple( + row for row in sink_rows if row.target.startswith(f"/{non_streaming_name}/") + ) + token_request_count: Final = len(token_requests) + streaming_count: Final = len(streaming_rows) + non_streaming_count: Final = len(non_streaming_rows) + assert tunnel_authorities == (VERTEX_LIVE_AUTHORITY,), ( + tunnel_authorities, + proxy_log, + ) + assert peer_path == VERTEX_LIVE_PATH, ( + peer_path, + proxy_log, + ) + assert peer_authorization == "Bearer synthetic-vertex-access-token", (peer_authorization, proxy_log) + assert peer_frames == (expected_frame,), (marker, peer_frames, proxy_log) + assert client_reply == {"setupComplete": {}}, (client_reply, proxy_log) + assert token_request_count == 1, (token_request_count, proxy_log) + assert streaming_count == 1, (streaming_count, non_streaming_count, proxy_log) + assert non_streaming_count == 0, (streaming_count, non_streaming_count, proxy_log) diff --git a/tests/unit/experimental_mcp_client/test_mcp_client.py b/tests/unit/experimental_mcp_client/test_mcp_client.py index b06ed9468b0..9a30e1990b1 100644 --- a/tests/unit/experimental_mcp_client/test_mcp_client.py +++ b/tests/unit/experimental_mcp_client/test_mcp_client.py @@ -2936,8 +2936,11 @@ async def test_cancellation_delivers_termination_over_tcp( listener: Final = await asyncio.start_server(handle_connection, "127.0.0.1", 0) port: Final = listener.sockets[0].getsockname()[1] + client_timeout: Final = 2 if cancel_mode == "read_timeout" else 30 client: Final = MCPClient( - server_url=f"http://127.0.0.1:{port}/mcp", protocol_version=protocol_version, timeout=2 if cancel_mode == "read_timeout" else 30 + server_url=f"http://127.0.0.1:{port}/mcp", + protocol_version=protocol_version, + timeout=client_timeout, ) async def calls(): diff --git a/tests/unit/integrations/test_custom_guardrail.py b/tests/unit/integrations/test_custom_guardrail.py index 72c362425bb..e04f8af5a66 100644 --- a/tests/unit/integrations/test_custom_guardrail.py +++ b/tests/unit/integrations/test_custom_guardrail.py @@ -1,5 +1,9 @@ import asyncio +import copy import datetime as dt +import json +import pickle +import threading from typing import TYPE_CHECKING, ClassVar, Final, Literal, Optional from unittest.mock import AsyncMock @@ -8,7 +12,10 @@ import pytest from litellm.integrations.custom_guardrail import ( DEFAULT_ADVISORY_MESSAGE, CustomGuardrail, + _request_is_streaming, + guardrail_request_data_with_streaming, log_guardrail_information, + without_server_streaming_classification, ) from litellm.litellm_core_utils.litellm_logging import Logging from litellm.proxy._types import CallTypes, UserAPIKeyAuth @@ -531,6 +538,233 @@ class TestCustomGuardrailShouldRunGuardrail: assert always_on.should_run_guardrail(data=forged, event_type=GuardrailEventHooks.pre_call) is True +_STREAM_SCOPE_HOOKS: Final = ( + GuardrailEventHooks.pre_call, + GuardrailEventHooks.during_call, + GuardrailEventHooks.post_call, +) + + +class TestCustomGuardrailStreamScope: + @pytest.mark.parametrize("event_type", _STREAM_SCOPE_HOOKS) + @pytest.mark.parametrize("stream", [True, False]) + @pytest.mark.parametrize("stream_scope", [None, "both"]) + def test_both_and_omitted_run_on_streaming_and_non_streaming( + self, + event_type: GuardrailEventHooks, + stream: bool, + stream_scope: str | None, + ): + guardrail = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=event_type, + stream_scope=stream_scope, + ) + assert guardrail.should_run_guardrail({"stream": stream}, event_type) is True + + @pytest.mark.parametrize("event_type", _STREAM_SCOPE_HOOKS) + def test_scalar_streaming_skips_non_streaming(self, event_type: GuardrailEventHooks): + guardrail = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=event_type, + stream_scope="streaming", + ) + assert guardrail.should_run_guardrail({"stream": True}, event_type) is True + assert guardrail.should_run_guardrail({"stream": False}, event_type) is False + assert guardrail.should_run_guardrail({}, event_type) is False + + @pytest.mark.parametrize("event_type", _STREAM_SCOPE_HOOKS) + def test_scalar_non_streaming_skips_streaming(self, event_type: GuardrailEventHooks): + guardrail = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=event_type, + stream_scope="non_streaming", + ) + assert guardrail.should_run_guardrail({"stream": False}, event_type) is True + assert guardrail.should_run_guardrail({}, event_type) is True + assert guardrail.should_run_guardrail({"stream": True}, event_type) is False + + def test_per_mode_map_applies_to_named_hooks_only(self): + guardrail = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=[GuardrailEventHooks.pre_call, GuardrailEventHooks.post_call], + stream_scope={"pre_call": "both", "post_call": "streaming"}, + ) + assert guardrail.should_run_guardrail({"stream": True}, GuardrailEventHooks.pre_call) is True + assert guardrail.should_run_guardrail({"stream": False}, GuardrailEventHooks.pre_call) is True + assert guardrail.should_run_guardrail({"stream": True}, GuardrailEventHooks.post_call) is True + assert guardrail.should_run_guardrail({"stream": False}, GuardrailEventHooks.post_call) is False + + def test_default_on_early_return_still_honors_stream_scope(self): + guardrail = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + stream_scope="non_streaming", + ) + assert guardrail.should_run_guardrail({"stream": False}, GuardrailEventHooks.post_call) is True + assert guardrail.should_run_guardrail({"stream": True}, GuardrailEventHooks.post_call) is False + + def test_apply_stream_scope_overwrites_constructor_default(self): + guardrail = CustomGuardrail( + guardrail_name="scoped", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + ) + assert guardrail.should_run_guardrail({"stream": False}, GuardrailEventHooks.post_call) is True + guardrail.apply_stream_scope("streaming") + assert guardrail.should_run_guardrail({"stream": True}, GuardrailEventHooks.post_call) is True + assert guardrail.should_run_guardrail({"stream": False}, GuardrailEventHooks.post_call) is False + + def test_direct_constructor_normalizes_mixed_case_map_keys(self): + guardrail = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + stream_scope={"Pre_Call": "streaming"}, + ) + assert guardrail.should_run_guardrail({"stream": True}, GuardrailEventHooks.pre_call) is True + assert guardrail.should_run_guardrail({"stream": False}, GuardrailEventHooks.pre_call) is False + + def test_realtime_transcription_counts_as_streaming(self): + streaming_only = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=GuardrailEventHooks.realtime_input_transcription, + stream_scope="streaming", + ) + assert ( + streaming_only.should_run_guardrail( + {"litellm_metadata": {}}, GuardrailEventHooks.realtime_input_transcription + ) + is True + ) + non_streaming_only = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=GuardrailEventHooks.realtime_input_transcription, + stream_scope="non_streaming", + ) + assert ( + non_streaming_only.should_run_guardrail( + {"litellm_metadata": {}}, GuardrailEventHooks.realtime_input_transcription + ) + is False + ) + + def test_path_defined_streaming_classification_cannot_be_spoofed(self): + generate_content_body: Final = {"contents": [{"parts": [{"text": "hi"}]}]} + streaming_only = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + stream_scope="streaming", + ) + assert streaming_only.should_run_guardrail(generate_content_body, GuardrailEventHooks.pre_call) is False + assert ( + streaming_only.should_run_guardrail( + {**generate_content_body, "is_streaming_request": True}, + GuardrailEventHooks.pre_call, + ) + is False + ) + assert ( + streaming_only.should_run_guardrail( + {**generate_content_body, "is_streaming_request": "litellm-server-streaming"}, + GuardrailEventHooks.pre_call, + ) + is False + ) + assert ( + streaming_only.should_run_guardrail( + {**generate_content_body, "litellm_server_streaming_classification": True}, + GuardrailEventHooks.pre_call, + ) + is False + ) + server_streaming_data: Final = guardrail_request_data_with_streaming( + generate_content_body, + is_streaming=True, + ) + assert ( + streaming_only.should_run_guardrail( + server_streaming_data, + GuardrailEventHooks.pre_call, + ) + is True + ) + non_streaming_only = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + stream_scope="non_streaming", + ) + assert ( + non_streaming_only.should_run_guardrail( + server_streaming_data, + GuardrailEventHooks.pre_call, + ) + is False + ) + + def test_streaming_classification_is_json_serializable_without_spoofing(self): + d: Final = guardrail_request_data_with_streaming({}, is_streaming=True) + serialized: Final = json.dumps(d) + assert _request_is_streaming(d) is True + + round_tripped: Final = json.loads(serialized) + assert _request_is_streaming(round_tripped) is False + assert round_tripped["litellm_server_streaming_classification"] == "litellm-server-streaming" + assert isinstance(round_tripped["litellm_server_streaming_classification"], str) + assert "litellm_server_streaming_classification" not in without_server_streaming_classification(round_tripped) + + def test_streaming_classification_preserves_caller_fields_and_removes_only_server_marker(self): + caller_data: Final = { + "contents": [{"parts": [{"text": "hi"}]}], + "is_streaming_request": "caller-value", + "litellm_server_streaming_classification": True, + } + non_streaming_data: Final = guardrail_request_data_with_streaming(caller_data, is_streaming=False) + server_streaming_data: Final = guardrail_request_data_with_streaming(caller_data, is_streaming=True) + + assert non_streaming_data is not caller_data + assert server_streaming_data is not caller_data + assert non_streaming_data == caller_data + assert server_streaming_data["is_streaming_request"] == "caller-value" + assert server_streaming_data["litellm_server_streaming_classification"] is not True + assert without_server_streaming_classification(caller_data) == caller_data + assert "litellm_server_streaming_classification" not in without_server_streaming_classification( + server_streaming_data + ) + + def test_server_streaming_classification_survives_scan_raw_request_snapshot(self): + from litellm.litellm_core_utils.core_helpers import independent_snapshot + + generate_content_body: Final = {"contents": [{"parts": [{"text": "hi"}]}]} + snapshot: Final = independent_snapshot( + guardrail_request_data_with_streaming(generate_content_body, is_streaming=True) + ) + streaming_only = CustomGuardrail( + guardrail_name="test_guardrail", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + stream_scope="streaming", + scan_raw_request=True, + ) + assert streaming_only.should_run_guardrail(snapshot, GuardrailEventHooks.pre_call) is True + assert ( + streaming_only.should_run_guardrail( + independent_snapshot({**generate_content_body, "is_streaming_request": True}), + GuardrailEventHooks.pre_call, + ) + is False + ) + + class TestApplyGuardrailCheck: def test_apply_guardrail_check_only_on_direct_implementation(self): """ @@ -3307,3 +3541,113 @@ class TestCustomGuardrailTimeout: ) assert guardrail.timeout == 7.0 + + +@pytest.mark.parametrize("stream_scope", [None, "both", {"post_call": "streaming"}]) +def test_guardrail_survives_deepcopy_and_pickle_with_its_stream_scope(stream_scope): + guardrail: Final = CustomGuardrail( + guardrail_name="copyable", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + stream_scope=stream_scope, + ) + expected: Final = guardrail.should_run_guardrail({"stream": False}, GuardrailEventHooks.post_call) + for clone in (copy.deepcopy(guardrail), pickle.loads(pickle.dumps(guardrail))): + assert dict(clone.stream_scope_by_hook) == dict(guardrail.stream_scope_by_hook) + assert clone.should_run_guardrail({"stream": False}, GuardrailEventHooks.post_call) is expected + + +class _LockHoldingGuardrail(CustomGuardrail): + def __init__(self, **kwargs): + self.lock = threading.Lock() + super().__init__(**kwargs) + + def __getstate__(self): + state: Final = dict(self.__dict__) + state.pop("lock") + return state + + def __setstate__(self, state): + self.__dict__.update(state) + self.lock = threading.Lock() + + +class _SetstateOnlyGuardrail(CustomGuardrail): + def __setstate__(self, state): + self.__dict__.update(state) + self.restored = True + + +class _SlotsGuardrail(CustomGuardrail): + __slots__ = ("vendor_client",) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.vendor_client = "vendor-client-object" + + +def _subclass_guardrails(): + kwargs: Final = { + "guardrail_name": "vendor", + "default_on": True, + "event_hook": GuardrailEventHooks.post_call, + "stream_scope": {"post_call": "streaming"}, + } + return ( + pytest.param(_LockHoldingGuardrail(**kwargs), id="dict-getstate"), + pytest.param(_SetstateOnlyGuardrail(**kwargs), id="setstate-only"), + pytest.param(_SlotsGuardrail(**kwargs), id="slots"), + ) + + +@pytest.mark.parametrize("guardrail", _subclass_guardrails()) +@pytest.mark.parametrize( + "cloner", + [copy.copy, copy.deepcopy, lambda g: pickle.loads(pickle.dumps(g))], + ids=["copy", "deepcopy", "pickle"], +) +def test_out_of_tree_guardrail_subclasses_survive_copy_and_pickle(guardrail, cloner): + clone = cloner(guardrail) + + if isinstance(clone, _LockHoldingGuardrail): + assert isinstance(clone.lock, type(threading.Lock())) + elif isinstance(clone, _SetstateOnlyGuardrail): + assert clone.restored is True + else: + assert clone.vendor_client == "vendor-client-object" + assert clone.should_run_guardrail({"stream": False}, GuardrailEventHooks.post_call) is False + assert clone.should_run_guardrail({"stream": True}, GuardrailEventHooks.post_call) is True + + +def test_router_constructs_with_a_dict_getstate_guardrail_in_deployment_callbacks(): + import litellm + + litellm.Router( + model_list=[ + { + "model_name": "m", + "litellm_params": { + "model": "openai/gpt-5.4-mini", + "api_key": "sk-test", + "callbacks": [ + _LockHoldingGuardrail( + guardrail_name="vendor", + default_on=True, + event_hook=GuardrailEventHooks.post_call, + stream_scope={"post_call": "streaming"}, + ) + ], + }, + } + ] + ) + + +def test_subclass_that_skips_super_init_still_runs_with_default_scope(): + class NoSuperInit(CustomGuardrail): + def __init__(self) -> None: + self.guardrail_name = "no-super" + self.event_hook = None + self.default_on = True + + assert NoSuperInit().should_run_guardrail({"stream": True}, GuardrailEventHooks.pre_call) is True diff --git a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py index fbcbda183bc..875a2fc1b1a 100644 --- a/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py +++ b/tests/unit/llms/bedrock/passthrough/test_bedrock_passthrough_transformation.py @@ -10,9 +10,14 @@ from datetime import datetime from typing import Final from unittest.mock import patch +import pytest + from litellm.litellm_core_utils.litellm_logging import Logging from litellm.llms.base_llm.passthrough.transformation import PassthroughStreamCollector -from litellm.llms.bedrock.passthrough.transformation import BedrockPassthroughConfig +from litellm.llms.bedrock.passthrough.transformation import ( + BedrockPassthroughConfig, + is_bedrock_streaming_endpoint, +) from litellm.types.utils import ModelResponse CONVERSE_MODEL = "anthropic.claude-sonnet-4-5-20250929-v1:0" @@ -20,6 +25,22 @@ CONVERSE_STREAM_ENDPOINT = f"/model/{CONVERSE_MODEL}/converse-stream" INVOKE_STREAM_ENDPOINT = f"/model/{CONVERSE_MODEL}/invoke-with-response-stream" +@pytest.mark.parametrize( + ("endpoint", "expected"), + [ + ("converse-stream", True), + ("invoke-with-response-stream", True), + ("converse", False), + ("invoke", False), + ("model/my-converse-stream-model/converse", False), + ("model/x/converse-stream?foo=1", True), + ("model/x/converse-stream/", True), + ], +) +def test_is_bedrock_streaming_endpoint_matches_final_action_segment(endpoint: str, expected: bool) -> None: + assert is_bedrock_streaming_endpoint(endpoint) is expected + + def test_bedrock_passthrough_get_complete_url_default_endpoint(): """Test get_complete_url with default AWS endpoint (no override)""" config = BedrockPassthroughConfig() diff --git a/tests/unit/llms/pass_through/guardrail_translation/test_handler.py b/tests/unit/llms/pass_through/guardrail_translation/test_handler.py new file mode 100644 index 00000000000..82ad7702052 --- /dev/null +++ b/tests/unit/llms/pass_through/guardrail_translation/test_handler.py @@ -0,0 +1,52 @@ +import json +from typing import Final + +import pytest + +from litellm.constants import SERVER_STREAMING_CLASSIFICATION_KEY, SERVER_STREAMING_CLASSIFICATION_MARKER +from litellm.llms.pass_through.guardrail_translation.handler import PassThroughEndpointHandler + + +def test_full_payload_guardrail_text_excludes_the_server_streaming_marker(): + body: Final = {"model": "m", "stream": True, "messages": [{"role": "user", "content": "hi"}]} + + text: Final = PassThroughEndpointHandler()._extract_text_for_guardrail( + {**body, SERVER_STREAMING_CLASSIFICATION_KEY: SERVER_STREAMING_CLASSIFICATION_MARKER}, + None, + ) + + assert json.loads(text) == body, text + + +@pytest.mark.parametrize( + "marker", + [ + SERVER_STREAMING_CLASSIFICATION_MARKER, + json.loads(json.dumps(SERVER_STREAMING_CLASSIFICATION_MARKER)), + ], + ids=["enum", "json-string"], +) +def test_full_payload_guardrail_text_scans_caller_value_but_not_the_marker(marker: str): + text: Final = PassThroughEndpointHandler()._extract_text_for_guardrail( + { + "model": "m", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + SERVER_STREAMING_CLASSIFICATION_KEY: "BLOCKME caller content", + }, + None, + ) + + assert "BLOCKME caller content" in text, text + + marker_text: Final = PassThroughEndpointHandler()._extract_text_for_guardrail( + { + "model": "m", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + SERVER_STREAMING_CLASSIFICATION_KEY: marker, + }, + None, + ) + + assert SERVER_STREAMING_CLASSIFICATION_KEY not in json.loads(marker_text), marker_text diff --git a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py index 23defa6f513..5ae3c595c53 100644 --- a/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py +++ b/tests/unit/proxy/_experimental/mcp_server/test_mcp_server.py @@ -464,7 +464,7 @@ async def test_sse_mcp_handler_mock(): mock_sse = MagicMock() mock_sse.connect_sse.side_effect = connect_sse - run = AsyncMock() + serve = AsyncMock() # Mock scope, receive, send with proper ASGI scope format mock_scope = { @@ -489,7 +489,7 @@ async def test_sse_mcp_handler_mock(): ) with ( - patch("litellm.proxy._experimental.mcp_server.server.serve_loop", run), + patch("litellm.proxy._experimental.mcp_server.server.serve_loop", serve), patch( "litellm.proxy._experimental.mcp_server.server._SESSION_MANAGERS_INITIALIZED", True, @@ -505,13 +505,21 @@ async def test_sse_mcp_handler_mock(): patch( "litellm.proxy._experimental.mcp_server.server.set_auth_context", ), + patch( + "litellm.proxy._experimental.mcp_server.server._raise_preemptive_401_for_unauthenticated_servers", + new=AsyncMock(), + ), + patch( + "litellm.proxy._experimental.mcp_server.server._check_passthrough_upstream_auth", + new=AsyncMock(), + ), ): from litellm.proxy._experimental.mcp_server.server import handle_sse_mcp # Call the handler await handle_sse_mcp(mock_scope, mock_receive, mock_send) - assert run.await_args.args[1:3] == (read_stream, write_stream) + assert serve.await_args.args[1:3] == (read_stream, write_stream) assert mock_sse.connect_sse.call_args.args[0]["path"] == "/mcp/sse" diff --git a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py index fa33c2d462b..b7bc5f3f8ec 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_endpoints.py +++ b/tests/unit/proxy/guardrails/test_guardrail_endpoints.py @@ -575,7 +575,7 @@ async def test_get_guardrail_info_from_db(mocker, mock_prisma_client): """Test getting guardrail info from DB""" mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) - response = await get_guardrail_info("test-db-guardrail") + response: Final = await get_guardrail_info("test-db-guardrail") assert response.guardrail_id == "test-db-guardrail" assert response.guardrail_name == "Test DB Guardrail" @@ -584,6 +584,21 @@ async def test_get_guardrail_info_from_db(mocker, mock_prisma_client): @pytest.mark.asyncio +async def test_get_guardrail_info_tolerates_invalid_stored_stream_scope(mocker, mock_prisma_client): + mocker.patch("litellm.proxy.proxy_server.prisma_client", mock_prisma_client) + mock_prisma_client.db.litellm_guardrailstable.find_unique = AsyncMock( + return_value={ + **MOCK_DB_GUARDRAIL, + "litellm_params": { + **MOCK_DB_GUARDRAIL["litellm_params"], + "stream_scope": "sometimes", + }, + } + ) + + response = await get_guardrail_info("test-db-guardrail") + + assert response.litellm_params.stream_scope is None async def test_get_guardrail_info_normalizes_invalid_scope_from_db( mocker, mock_guardrail_registry, mock_in_memory_handler ): @@ -745,6 +760,40 @@ def test_get_guardrails_list_response_includes_guardrail_id(): assert response.guardrails[0].guardrail_id == "stable-config-id" +def test_get_guardrails_list_response_tolerates_invalid_config_stream_scope(): + from litellm.proxy.guardrails.guardrail_endpoints import ( + _get_guardrails_list_response, + ) + + response = _get_guardrails_list_response( + [ + { + "guardrail_id": "invalid-scope", + "guardrail_name": "invalid-scope", + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "stream_scope": "sometimes", + }, + }, + { + "guardrail_id": "valid-scope", + "guardrail_name": "valid-scope", + "litellm_params": { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "stream_scope": "STREAMING", + }, + }, + ] + ) + + assert response.guardrails[0].litellm_params is not None + assert response.guardrails[0].litellm_params.stream_scope is None + assert response.guardrails[1].litellm_params is not None + assert response.guardrails[1].litellm_params.stream_scope == "streaming" + + def test_get_provider_specific_params(): """Test getting provider-specific parameters""" from litellm.proxy.guardrails.guardrail_endpoints import _get_fields_from_model diff --git a/tests/unit/proxy/guardrails/test_guardrail_registry.py b/tests/unit/proxy/guardrails/test_guardrail_registry.py index c7b13f0bd00..3bc6d163e5d 100644 --- a/tests/unit/proxy/guardrails/test_guardrail_registry.py +++ b/tests/unit/proxy/guardrails/test_guardrail_registry.py @@ -1,7 +1,7 @@ import json from collections.abc import Iterable, Iterator from typing import ClassVar, Final -from unittest.mock import AsyncMock, MagicMock +from unittest.mock import AsyncMock, MagicMock, patch import pytest from pydantic import ValidationError @@ -160,6 +160,43 @@ def test_duplicate_config_guardrail_names_get_distinct_stable_ids(): registry_module.guardrail_initializer_registry.pop("dup_name_test", None) +def test_initialize_guardrail_treats_invalid_stored_scope_as_both(): + from litellm.proxy.guardrails import guardrail_registry as registry_module + + guardrail_type: Final = "invalid_stored_scope_test" + + def _initializer(litellm_params: LitellmParams, guardrail: Guardrail) -> CustomGuardrail: + return CustomGuardrail( + guardrail_name=guardrail["guardrail_name"], + event_hook=GuardrailEventHooks(litellm_params.mode), + default_on=True, + ) + + registry_module.guardrail_initializer_registry[guardrail_type] = _initializer + try: + handler: Final = InMemoryGuardrailHandler() + guardrail: Final = Guardrail( + guardrail_id="invalid-stored-scope", + guardrail_name="invalid-stored-scope", + litellm_params={ + "guardrail": guardrail_type, + "mode": "pre_call", + "default_on": True, + "stream_scope": "sometimes", + }, + ) + + parsed_guardrail: Final = handler.initialize_guardrail(guardrail=guardrail, source="db") + callback: Final = handler.guardrail_id_to_custom_guardrail["invalid-stored-scope"] + + assert parsed_guardrail["litellm_params"].stream_scope is None + assert callback is not None + assert callback.should_run_guardrail(data={}, event_type=GuardrailEventHooks.pre_call) is True + assert callback.should_run_guardrail(data={"stream": True}, event_type=GuardrailEventHooks.pre_call) is True + finally: + registry_module.guardrail_initializer_registry.pop(guardrail_type, None) + + def _register_mode_following_initializer(guardrail_type: str): """Registers like the shipped initializers do: construct, then add the instance to litellm's callbacks.""" import litellm @@ -1605,6 +1642,35 @@ def test_sync_guardrail_from_db_applies_db_dict_params_to_live_instance(): cb_list[:] = snapshot +def test_configure_callback_scoping_copies_stream_scope_when_constructor_omits_it(): + from litellm.proxy.guardrails.guardrail_registry import _configure_callback_scoping + + class _CtorWithoutStreamScope(CustomGuardrail): + def __init__(self) -> None: + super().__init__( + guardrail_name="scoped", + event_hook=GuardrailEventHooks.post_call, + default_on=True, + ) + + instance = _CtorWithoutStreamScope() + params = LitellmParams(guardrail="bedrock", mode="post_call", stream_scope="streaming") + _configure_callback_scoping(instance, "scoped", params) + + assert instance.stream_scope_default == "streaming" + assert instance.should_run_guardrail({"stream": True}, GuardrailEventHooks.post_call) is True + assert instance.should_run_guardrail({"stream": False}, GuardrailEventHooks.post_call) is False + + +def test_configure_callback_scoping_tolerates_a_custom_logger_callback(): + from litellm.integrations.custom_logger import CustomLogger + from litellm.proxy.guardrails.guardrail_registry import _configure_callback_scoping + + callback: Final = CustomLogger() + _configure_callback_scoping(callback, "logger-backed", LitellmParams(guardrail="custom", mode="pre_call")) # pyright: ignore[reportArgumentType] # module-path guardrails may be plain CustomLogger + assert "stream_scope_by_hook" not in vars(callback) + + _ENCRYPTED_PREFIX = "litellm_enc::" diff --git a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py index 1887021a53a..d329db75fb4 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_llm_pass_through_endpoints.py @@ -23,6 +23,7 @@ from starlette.datastructures import FormData import litellm +from litellm.integrations.custom_guardrail import CustomGuardrail from tests._master_key import MASTER_KEY as SHARED_MASTER_KEY from litellm.caching.caching import DualCache from litellm.types.utils import CallTypesLiteral @@ -47,6 +48,7 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( get_vertex_base_url, is_azure_ai_search_service_level_index_create, gigachat_proxy_route, + handle_bedrock_passthrough_router_model, llm_passthrough_factory_proxy_route, milvus_proxy_route, mistral_proxy_route, @@ -61,9 +63,32 @@ from litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints import ( from litellm.proxy._types import LitellmUserRoles, SpecialHeaders, UserAPIKeyAuth from litellm.proxy.auth.handle_jwt import JWTHandler from litellm.proxy.auth.user_api_key_auth import user_api_key_auth +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.passthrough_endpoints.vertex_ai import VertexPassThroughCredentials +def _assert_bedrock_processing_data_classification(data: dict[str, object], is_streaming: bool) -> None: + streaming_guardrail: Final = CustomGuardrail( + guardrail_name="streaming-only", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + stream_scope="streaming", + ) + non_streaming_guardrail: Final = CustomGuardrail( + guardrail_name="non-streaming-only", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + stream_scope="non_streaming", + ) + provider_body: Final = data["data"] + + assert streaming_guardrail.should_run_guardrail(data, GuardrailEventHooks.pre_call) is is_streaming + assert non_streaming_guardrail.should_run_guardrail(data, GuardrailEventHooks.pre_call) is not is_streaming + assert isinstance(provider_body, dict) + assert "is_streaming_request" not in provider_body + assert "litellm_server_streaming_classification" not in provider_body + + class TestVertexPassthroughGetVertexBaseUrl: """Module-local get_vertex_base_url (trailing slash); rules match common_utils.""" @@ -413,9 +438,7 @@ class TestVertexAIPassThroughHandler: # Mock the vertex handler for global location mock_handler = Mock() - mock_handler.get_default_base_target_url.return_value = ( - "https://aiplatform.googleapis.com/" - ) + mock_handler.get_default_base_target_url.return_value = "https://aiplatform.googleapis.com/" mock_get_handler.return_value = mock_handler # Mock create_pass_through_route to return a function that returns a mock response @@ -1237,9 +1260,7 @@ class TestVertexAIDiscoveryPassThroughHandler: # Mock the discovery handler mock_handler = Mock() - mock_handler.get_default_base_target_url.return_value = ( - "https://discoveryengine.googleapis.com" - ) + mock_handler.get_default_base_target_url.return_value = "https://discoveryengine.googleapis.com" mock_get_handler.return_value = mock_handler # Mock create_pass_through_route to return a function that returns a mock response @@ -1459,6 +1480,170 @@ class TestBedrockLLMProxyRoute: assert call_kwargs["model"] == "anthropic.claude-3-sonnet-20240229-v1:0" assert result == "success" + @pytest.mark.asyncio + @pytest.mark.parametrize( + "action, is_streaming", + [ + ("converse-stream", True), + ("invoke-with-response-stream", True), + ("converse", False), + ("invoke", False), + ], + ) + async def test_bedrock_direct_actions_classify_guardrail_stream_scope( + self, action: str, is_streaming: bool + ) -> None: + mock_request: Final = Mock() + mock_request.method = "POST" + mock_processor: Final = Mock() + mock_processor.base_passthrough_process_llm_request = AsyncMock(return_value="success") + request_body: Final = {"messages": [{"role": "user", "content": "test"}]} + + with ( + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints._read_request_body", + return_value=request_body, + ), + patch( + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing", + return_value=mock_processor, + ) as processor_constructor, + ): + result: Final = await bedrock_llm_proxy_route( + endpoint=f"model/test-model/{action}", + request=mock_request, + fastapi_response=Mock(), + user_api_key_dict=Mock(), + ) + + assert result == "success" + processing_data: Final = processor_constructor.call_args.kwargs["data"] + _assert_bedrock_processing_data_classification(processing_data, is_streaming) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + "action, is_streaming", + [ + ("converse-stream", True), + ("invoke-with-response-stream", True), + ("converse", False), + ("invoke", False), + ], + ) + async def test_bedrock_router_actions_classify_guardrail_stream_scope( + self, action: str, is_streaming: bool + ) -> None: + mock_request: Final = Mock() + mock_request.method = "POST" + mock_processor: Final = Mock() + mock_processor.base_passthrough_process_llm_request = AsyncMock(return_value="success") + request_body: Final = {"messages": [{"role": "user", "content": "test"}]} + + with patch( + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing", + return_value=mock_processor, + ) as processor_constructor: + result: Final = await handle_bedrock_passthrough_router_model( + model="test-model", + endpoint=f"model/test-model/{action}", + request=mock_request, + request_body=request_body, + llm_router=Mock(), + user_api_key_dict=Mock(), + proxy_logging_obj=Mock(), + general_settings={}, + proxy_config=None, + select_data_generator=None, + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + version=None, + ) + + assert result == "success" + processing_data: Final = processor_constructor.call_args.kwargs["data"] + _assert_bedrock_processing_data_classification(processing_data, is_streaming) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("model_id", "action"), + [ + ("my-converse-stream-model", "converse"), + ("my-invoke-with-response-stream-model", "invoke"), + ], + ) + async def test_bedrock_direct_model_id_does_not_imply_streaming(self, model_id: str, action: str) -> None: + mock_request: Final = Mock() + mock_request.method = "POST" + mock_processor: Final = Mock() + mock_processor.base_passthrough_process_llm_request = AsyncMock(return_value="success") + request_body: Final = {"messages": [{"role": "user", "content": "test"}]} + + with ( + patch( + "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints._read_request_body", + return_value=request_body, + ), + patch( + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing", + return_value=mock_processor, + ) as processor_constructor, + ): + result: Final = await bedrock_llm_proxy_route( + endpoint=f"/model/{model_id}/{action}", + request=mock_request, + fastapi_response=Mock(), + user_api_key_dict=Mock(), + ) + + assert result == "success" + processing_data: Final = processor_constructor.call_args.kwargs["data"] + _assert_bedrock_processing_data_classification(processing_data, is_streaming=False) + + @pytest.mark.asyncio + @pytest.mark.parametrize( + ("model_id", "action"), + [ + ("my-converse-stream-model", "converse"), + ("my-invoke-with-response-stream-model", "invoke"), + ], + ) + async def test_bedrock_router_model_id_does_not_imply_streaming(self, model_id: str, action: str) -> None: + mock_request: Final = Mock() + mock_request.method = "POST" + mock_processor: Final = Mock() + mock_processor.base_passthrough_process_llm_request = AsyncMock(return_value="success") + request_body: Final = {"messages": [{"role": "user", "content": "test"}]} + + with patch( + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing", + return_value=mock_processor, + ) as processor_constructor: + result: Final = await handle_bedrock_passthrough_router_model( + model=model_id, + endpoint=f"/model/{model_id}/{action}", + request=mock_request, + request_body=request_body, + llm_router=Mock(), + user_api_key_dict=Mock(), + proxy_logging_obj=Mock(), + general_settings={}, + proxy_config=None, + select_data_generator=None, + user_model=None, + user_temperature=None, + user_request_timeout=None, + user_max_tokens=None, + user_api_base=None, + version=None, + ) + + assert result == "success" + processing_data: Final = processor_constructor.call_args.kwargs["data"] + _assert_bedrock_processing_data_classification(processing_data, is_streaming=False) + @pytest.mark.asyncio async def test_bedrock_error_handling_returns_actual_error(self): """ @@ -1879,7 +2064,6 @@ class TestBedrockAgentRuntimePassthroughToggle: class TestBedrockAgentRuntimePassthroughVirtualKeyLeak: - VKEY: Final = "sk-litellm-victim-key" MASTER_KEY: Final = SHARED_MASTER_KEY ENDPOINT: Final = "knowledgebases/KB1234567/retrieve" @@ -2075,10 +2259,10 @@ class TestVLLMProxyRoute: "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_passthrough_request_using_router_model", return_value=True, ) - @patch("litellm.proxy.proxy_server.llm_router") # test-quality-ok: patching litellm internal for unit test isolation - async def test_vllm_proxy_route_with_router_model( - self, mock_llm_router, mock_is_router, mock_get_body - ): + @patch( + "litellm.proxy.proxy_server.llm_router" + ) # test-quality-ok: patching litellm internal for unit test isolation + async def test_vllm_proxy_route_with_router_model(self, mock_llm_router, mock_is_router, mock_get_body): mock_request = MagicMock(spec=Request) mock_request.method = "POST" mock_request.headers = {"content-type": "application/json"} @@ -2111,9 +2295,7 @@ class TestVLLMProxyRoute: @patch( # test-quality-ok: patching litellm internal for unit test isolation "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.llm_passthrough_factory_proxy_route" ) - async def test_vllm_proxy_route_fallback_to_factory( - self, mock_factory_route, mock_is_router, mock_get_body - ): + async def test_vllm_proxy_route_fallback_to_factory(self, mock_factory_route, mock_is_router, mock_get_body): mock_request = MagicMock(spec=Request) mock_fastapi_response = MagicMock(spec=Response) mock_user_api_key_dict = MagicMock() @@ -2140,10 +2322,10 @@ class TestGigachatProxyRoute: "litellm.proxy.pass_through_endpoints.llm_passthrough_endpoints.is_passthrough_request_using_router_model", return_value=True, ) - @patch("litellm.proxy.proxy_server.llm_router") # test-quality-ok: patching litellm internal for unit test isolation - async def test_gigachat_proxy_route_with_router_model( - self, mock_llm_router, mock_is_router, mock_get_body - ): + @patch( + "litellm.proxy.proxy_server.llm_router" + ) # test-quality-ok: patching litellm internal for unit test isolation + async def test_gigachat_proxy_route_with_router_model(self, mock_llm_router, mock_is_router, mock_get_body): mock_request = MagicMock(spec=Request) mock_request.method = "POST" mock_request.headers = {"content-type": "application/json"} @@ -2396,21 +2578,25 @@ class TestGigachatProxyRoute: return _inner() - with patch.object( - processor, - "common_processing_pre_call_logic", - new=AsyncMock( - return_value=( - processor.data, - processor.data["litellm_logging_obj"], - ) + with ( + patch.object( + processor, + "common_processing_pre_call_logic", + new=AsyncMock( + return_value=( + processor.data, + processor.data["litellm_logging_obj"], + ) + ), + ), + patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.common_request_processing.route_request", + new=_fake_route_request, + ), + patch( # test-quality-ok: patching litellm internal for unit test isolation + "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing.get_custom_headers", + return_value={"x-litellm-call-id": "call-123"}, ), - ), patch( # test-quality-ok: patching litellm internal for unit test isolation - "litellm.proxy.common_request_processing.route_request", - new=_fake_route_request, - ), patch( # test-quality-ok: patching litellm internal for unit test isolation - "litellm.proxy.common_request_processing.ProxyBaseLLMRequestProcessing.get_custom_headers", - return_value={"x-litellm-call-id": "call-123"}, ): result = await processor.base_passthrough_process_llm_request( request=mock_request, @@ -4223,7 +4409,9 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: @pytest.mark.parametrize( ("credential", "authenticated"), [ - pytest.param("modified_key", UserAPIKeyAuth(api_key="modified_key"), id="custom-auth-echoing-opaque-credential"), + pytest.param( + "modified_key", UserAPIKeyAuth(api_key="modified_key"), id="custom-auth-echoing-opaque-credential" + ), pytest.param( LITELLM_JWT, UserAPIKeyAuth(api_key=LITELLM_JWT, user_id="jwt-subject"), @@ -4298,7 +4486,9 @@ class TestVertexCredentiallessPassthroughVirtualKeyLeak: (b"x-goog-api-key", b"AIza-real-google-api-key"), (b"content-type", b"application/json"), ], - authenticated=UserAPIKeyAuth(api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, user_role=LitellmUserRoles.PROXY_ADMIN), + authenticated=UserAPIKeyAuth( + api_key=LITELLM_PROXY_MASTER_KEY_ALIAS, user_role=LitellmUserRoles.PROXY_ADMIN + ), ) assert raised is None assert forwarded is not None @@ -4440,10 +4630,14 @@ class TestAnthropicPassthroughVirtualKeyLeak: raised, forwarded = await self._run( monkeypatch, [(header, value), (b"anthropic-version", b"2023-06-01"), (b"content-type", b"application/json")], - authenticated=UserAPIKeyAuth(api_key="sk-ant-api03-callers-own-key", user_role=LitellmUserRoles.INTERNAL_USER), + authenticated=UserAPIKeyAuth( + api_key="sk-ant-api03-callers-own-key", user_role=LitellmUserRoles.INTERNAL_USER + ), master_key=None, ) - assert raised is None, "with no master key the proxy authenticated nothing, so nothing of the caller's is a LiteLLM secret" + assert raised is None, ( + "with no master key the proxy authenticated nothing, so nothing of the caller's is a LiteLLM secret" + ) assert forwarded is not None assert forwarded.get(header.decode()) == value.decode() @@ -5231,7 +5425,9 @@ class TestTranscribeProxyRoute: ) -> None: with respx.mock(assert_all_called=False) as upstream: route = upstream.post(TRANSCRIBE_UPSTREAM) - response = transcribe_client.post("/transcribe/StartTranscriptionJob", json={**dict(self.START_JOB_BODY), **body}) + response = transcribe_client.post( + "/transcribe/StartTranscriptionJob", json={**dict(self.START_JOB_BODY), **body} + ) assert response.status_code == 403 assert member in response.json()["detail"] @@ -6873,9 +7069,7 @@ class TestAzureBodyModelGroupRelay: AZURE_SPEECH_SHORT_AUDIO_ENDPOINT: Final = "/speech/recognition/conversation/cognitiveservices/v1" AZURE_SPEECH_BATCH_ENDPOINT: Final = "/speechtotext/v3.2/transcriptions" AZURE_SPEECH_FAST_ENDPOINT: Final = "/speechtotext/transcriptions:transcribe" -AZURE_SPEECH_PCM16_HEADER: Final = ( - b"RIFF\x24\x0c\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80\x3e\x00\x00\x00\x7d\x00\x00\x02\x00\x10\x00data\x00\x0c\x00\x00" -) +AZURE_SPEECH_PCM16_HEADER: Final = b"RIFF\x24\x0c\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80\x3e\x00\x00\x00\x7d\x00\x00\x02\x00\x10\x00data\x00\x0c\x00\x00" AZURE_SPEECH_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + b"\x00" * 3072 AZURE_SPEECH_WAV_SECONDS: Final = 3072 / (16000 * 2) AZURE_SPEECH_NON_UTF8_WAV_BYTES: Final = AZURE_SPEECH_PCM16_HEADER + bytes(range(256)) * 12 @@ -6912,9 +7106,9 @@ class TestAzureSpeechProxyRoute: def test_short_audio_forwards_raw_wav_bytes_with_server_key(self, azure_speech_client: TestClient) -> None: with respx.mock(assert_all_called=True) as upstream: - route = upstream.post( - f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}" - ).mock(return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT)) + route = upstream.post(f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}").mock( + return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT) + ) response = azure_speech_client.post( f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", @@ -7390,9 +7584,9 @@ class TestAzureSpeechRawBodyThroughRealAuth: self, monkeypatch: pytest.MonkeyPatch, caplog: pytest.LogCaptureFixture, body: bytes ) -> None: with respx.mock(assert_all_called=True) as upstream, caplog.at_level(logging.ERROR, logger="LiteLLM Proxy"): - route = upstream.post( - f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}" - ).mock(return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT)) + route = upstream.post(f"https://eastus.stt.speech.microsoft.com{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}").mock( + return_value=httpx.Response(200, json=AZURE_SPEECH_TRANSCRIPT) + ) response = self._post_wav( monkeypatch, f"/azure_speech{AZURE_SPEECH_SHORT_AUDIO_ENDPOINT}", "sk-master-key", body=body @@ -7416,9 +7610,9 @@ class TestAzureSpeechRawBodyThroughRealAuth: ) -> None: boundary: Final = "lit7939boundary" multipart_body: Final = ( - f"--{boundary}\r\nContent-Disposition: form-data; name=\"definition\"\r\n\r\n".encode() + f'--{boundary}\r\nContent-Disposition: form-data; name="definition"\r\n\r\n'.encode() + json.dumps({"locales": ["en-US"]}).encode() - + f"\r\n--{boundary}\r\nContent-Disposition: form-data; name=\"audio\"; filename=\"eagle.wav\"\r\n" + + f'\r\n--{boundary}\r\nContent-Disposition: form-data; name="audio"; filename="eagle.wav"\r\n' "Content-Type: audio/wav\r\n\r\n".encode() + AZURE_SPEECH_NON_UTF8_WAV_BYTES + f"\r\n--{boundary}--\r\n".encode() @@ -8170,9 +8364,7 @@ class TestTinyFishProxyRoute: assert response.status_code == 200 assert json.loads(route.calls.last.request.content)["use_vault"] is True - def test_returns_401_on_missing_api_key( - self, tinyfish_client: TestClient, monkeypatch: pytest.MonkeyPatch - ) -> None: + def test_returns_401_on_missing_api_key(self, tinyfish_client: TestClient, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("TINYFISH_API_KEY") with respx.mock: diff --git a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py index cd279a6ed4b..a6ce8ee6fbc 100644 --- a/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py +++ b/tests/unit/proxy/pass_through_endpoints/test_pass_through_endpoints.py @@ -5,7 +5,7 @@ import logging import os import sys import zlib -from collections.abc import Callable, Mapping +from collections.abc import AsyncIterator, Callable, Mapping from contextlib import ExitStack, contextmanager from dataclasses import dataclass from io import BytesIO @@ -27,6 +27,7 @@ from starlette.datastructures import UploadFile as StarletteUploadFile import litellm from litellm._logging import verbose_proxy_logger from litellm.constants import DEFAULT_REQUEST_TIMEOUT_SECONDS +from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.proxy._lazy_features import LazyFeature, attach_lazy_features @@ -55,6 +56,7 @@ from litellm.proxy.pass_through_endpoints.success_handler import ( ) from litellm.proxy.route_llm_request import ProxyModelNotFoundError from litellm.types import utils as types_utils +from litellm.types.guardrails import GuardrailEventHooks from litellm.types.passthrough_endpoints.pass_through_endpoints import ( LITELLM_PASS_THROUGH_DEPLOYMENT_MODEL_INFO_STATE_KEY, LITELLM_PASS_THROUGH_RAW_BODY_STATE_KEY, @@ -1638,6 +1640,223 @@ async def test_pass_through_request_streaming_marks_logging_obj_as_stream(): assert logging_obj.model_call_details["stream"] is True +@pytest.mark.asyncio +@pytest.mark.parametrize("body_stream", [None, True], ids=["stream-absent", "stream-true"]) +async def test_pass_through_request_preserves_caller_streaming_request_field(body_stream): + captured_hook_data: dict[str, object] = {} + + async def capture_pre_call(user_api_key_dict, data, call_type, endpoint_type: EndpointType): + captured_hook_data.update(data) + return data + + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client" + ) as mock_get_client: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.PassThroughStreamingHandler.chunk_processor" + ) as mock_chunk_processor: + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=capture_pre_call) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={}) + + upstream_response = MagicMock() + upstream_response.status_code = 200 + upstream_response.headers = {} + upstream_response.raise_for_status = MagicMock() + + async_client = MagicMock() + async_client.build_request = MagicMock(return_value=MagicMock()) + async_client.send = AsyncMock(return_value=upstream_response) + mock_get_client.return_value = MagicMock(client=async_client) + + async def _empty_chunks(*args, **kwargs): + return + yield # pragma: no cover + + mock_chunk_processor.return_value = _empty_chunks() + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = httpx.URL( + "http://test-proxy.com/gemini/v1beta/models/gemini-pro:streamGenerateContent" + ) + mock_request.scope = {"path": "/gemini/v1beta/models/gemini-pro:streamGenerateContent"} + request_body: Final = { + "contents": [{"parts": [{"text": "hi"}]}], + "is_streaming_request": "caller-value", + **({"stream": True} if body_stream is True else {}), + } + mock_request.body = AsyncMock(return_value=json.dumps(request_body).encode()) + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + + await pass_through_request( + request=mock_request, + target="https://generativelanguage.googleapis.com/v1beta/models/gemini-pro:streamGenerateContent", + custom_headers={}, + user_api_key_dict=MagicMock(), + stream=True, + ) + + assert captured_hook_data.get("is_streaming_request") == "caller-value" + assert captured_hook_data.get("stream") is body_stream + + upstream_json = async_client.build_request.call_args.kwargs["json"] + assert upstream_json["is_streaming_request"] == "caller-value" + assert "litellm_server_streaming_classification" not in upstream_json + assert upstream_json["contents"] == request_body["contents"] + + +@pytest.mark.asyncio +async def test_streaming_pass_through_drops_marker_after_hook_rebuilds_body_from_json(): + async def json_rebuilding_pre_call(user_api_key_dict, data, call_type, endpoint_type: EndpointType): + rebuilt = json.loads(json.dumps({k: v for k, v in data.items() if k != "litellm_logging_obj"})) + return {**rebuilt, "litellm_logging_obj": data["litellm_logging_obj"]} + + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client" + ) as mock_get_client: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.PassThroughStreamingHandler.chunk_processor" + ) as mock_chunk_processor: + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=json_rebuilding_pre_call) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={}) + + upstream_response = MagicMock() + upstream_response.status_code = 200 + upstream_response.headers = {} + upstream_response.raise_for_status = MagicMock() + + async_client = MagicMock() + async_client.build_request = MagicMock(return_value=MagicMock()) + async_client.send = AsyncMock(return_value=upstream_response) + mock_get_client.return_value = MagicMock(client=async_client) + + async def _empty_chunks(*args, **kwargs): + return + yield # pragma: no cover + + mock_chunk_processor.return_value = _empty_chunks() + + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = httpx.URL("http://test-proxy.com/openai/v1/chat/completions") + mock_request.scope = {"path": "/openai/v1/chat/completions"} + request_body: Final = { + "model": "gpt-5-mini", + "stream": True, + "messages": [{"role": "user", "content": "hi"}], + } + mock_request.body = AsyncMock(return_value=json.dumps(request_body).encode()) + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + + await pass_through_request( + request=mock_request, + target="https://api.openai.com/v1/chat/completions", + custom_headers={}, + user_api_key_dict=MagicMock(), + ) + + upstream_json = async_client.build_request.call_args.kwargs["json"] + assert upstream_json == request_body, upstream_json + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("route_stream", "body_stream", "expected_streaming"), + [ + (True, False, False), + (False, True, True), + (True, None, True), + (None, None, False), + ], + ids=["body-disables-route-stream", "body-enables-streaming", "route-enables-absent-body", "both-absent"], +) +async def test_passthrough_guardrails_follow_effective_relay_stream_decision( + route_stream: bool | None, + body_stream: bool | None, + expected_streaming: bool, +): + async def return_pre_call_data( + user_api_key_dict: UserAPIKeyAuth, + data: dict[str, object], + call_type: str, + endpoint_type: EndpointType, + ) -> dict[str, object]: + return dict(data) + + with patch("litellm.proxy.proxy_server.proxy_logging_obj") as mock_proxy_logging: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.get_async_httpx_client" + ) as mock_get_client: + with patch( + "litellm.proxy.pass_through_endpoints.pass_through_endpoints.PassThroughStreamingHandler.chunk_processor" + ) as mock_chunk_processor: + mock_proxy_logging.pre_call_hook = AsyncMock(side_effect=return_pre_call_data) + mock_proxy_logging.post_call_failure_hook = AsyncMock() + mock_proxy_logging.post_call_response_headers_hook = AsyncMock(return_value={}) + + upstream_response = MagicMock() + upstream_response.status_code = 200 + upstream_response.headers = {} + upstream_response.aread = AsyncMock(return_value=b"{}") + upstream_response.text = "{}" + upstream_response.raise_for_status = MagicMock() + + async_client = MagicMock() + async_client.build_request = MagicMock(return_value=MagicMock()) + async_client.send = AsyncMock(return_value=upstream_response) + async_client.request = AsyncMock(return_value=upstream_response) + mock_get_client.return_value = MagicMock(client=async_client) + + async def empty_chunks() -> AsyncIterator[bytes]: + yield b"" + + mock_chunk_processor.return_value = empty_chunks() + + request_body: Final = { + "message": "hello", + **({"stream": body_stream} if body_stream is not None else {}), + } + mock_request = MagicMock(spec=Request) + mock_request.method = "POST" + mock_request.url = httpx.URL("http://test-proxy.com/guardrail-stream-scope") + mock_request.scope = {"path": "/guardrail-stream-scope"} + mock_request.body = AsyncMock(return_value=json.dumps(request_body).encode()) + mock_request.headers = Headers({"content-type": "application/json"}) + mock_request.query_params = QueryParams({}) + + await pass_through_request( + request=mock_request, + target="http://upstream.test/guardrail-stream-scope", + custom_headers={}, + user_api_key_dict=MagicMock(), + stream=route_stream, + ) + hook_data: Final[dict[str, object]] = mock_proxy_logging.pre_call_hook.call_args.kwargs["data"] + + streaming_guardrail: Final = CustomGuardrail( + guardrail_name="streaming-only", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + stream_scope="streaming", + ) + non_streaming_guardrail: Final = CustomGuardrail( + guardrail_name="non-streaming-only", + default_on=True, + event_hook=GuardrailEventHooks.pre_call, + stream_scope="non_streaming", + ) + assert streaming_guardrail.should_run_guardrail(hook_data, GuardrailEventHooks.pre_call) is expected_streaming + assert ( + non_streaming_guardrail.should_run_guardrail(hook_data, GuardrailEventHooks.pre_call) is not expected_streaming + ) + + @pytest.mark.asyncio async def test_pass_through_request_sse_response_marks_logging_obj_as_stream(): """ diff --git a/tests/unit/proxy/policy_engine/test_pipeline_executor.py b/tests/unit/proxy/policy_engine/test_pipeline_executor.py index 6aa7eca0f15..d13abbd379b 100644 --- a/tests/unit/proxy/policy_engine/test_pipeline_executor.py +++ b/tests/unit/proxy/policy_engine/test_pipeline_executor.py @@ -96,6 +96,20 @@ class AlwaysPassGuardrail(CustomGuardrail): return None +class StreamScopedPassGuardrail(CustomGuardrail): + def __init__(self, guardrail_name: str, stream_scope: object): + super().__init__( + guardrail_name=guardrail_name, + event_hook="pre_call", + default_on=True, + stream_scope=stream_scope, + ) + self.calls = 0 + + async def async_pre_call_hook(self, user_api_key_dict, cache, data, call_type): + self.calls += 1 + + class PassthroughBlockGuardrail(CustomGuardrail): """Mock guardrail that blocks using the legacy passthrough contract.""" @@ -378,6 +392,46 @@ async def test_block_carries_original_guardrail_exception(monkeypatch): assert result.original_exception.detail == "Content policy violation" +@pytest.mark.asyncio +async def test_pipeline_step_honors_stream_scope(monkeypatch): + stream_only = StreamScopedPassGuardrail(guardrail_name="stream-only", stream_scope="streaming") + later = AlwaysFailGuardrail(guardrail_name="later-block") + monkeypatch.setattr(litellm, "callbacks", [stream_only, later]) + steps = [ + PipelineStep(guardrail="stream-only", on_fail="block", on_pass="allow"), + PipelineStep(guardrail="later-block", on_fail="block", on_pass="allow"), + ] + + skipped = await PipelineExecutor.execute_steps( + steps=steps, + mode="pre_call", + data={"messages": [{"role": "user", "content": "hi"}]}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="stream-scope", + ) + assert stream_only.calls == 0 + assert later.calls == 1 + assert skipped.step_results[0].outcome == "skip" + assert skipped.step_results[0].action_taken == "next" + assert skipped.terminal_action == "block" + + stream_only.calls = 0 + later.calls = 0 + ran = await PipelineExecutor.execute_steps( + steps=steps, + mode="pre_call", + data={"messages": [{"role": "user", "content": "hi"}], "stream": True}, + user_api_key_dict=MagicMock(), + call_type="completion", + policy_name="stream-scope", + ) + assert stream_only.calls == 1 + assert later.calls == 0 + assert ran.step_results[0].outcome == "pass" + assert ran.terminal_action == "allow" + + @pytest.mark.asyncio async def test_unsupported_mode_yields_error_outcome_without_exception(monkeypatch): """An unexpected hook mode must surface as an error outcome (carrying no diff --git a/tests/unit/proxy/test_litellm_pre_call_utils.py b/tests/unit/proxy/test_litellm_pre_call_utils.py index c6c473ae71e..818ca50fed9 100644 --- a/tests/unit/proxy/test_litellm_pre_call_utils.py +++ b/tests/unit/proxy/test_litellm_pre_call_utils.py @@ -16,6 +16,7 @@ from pydantic import ValidationError as PydanticValidationError from starlette.datastructures import Headers import litellm +from litellm.constants import SERVER_STREAMING_CLASSIFICATION_KEY, SERVER_STREAMING_CLASSIFICATION_MARKER from litellm.proxy._types import AddTeamCallback, ProxyException, TeamCallbackMetadata, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( KeyAndTeamLoggingSettings, @@ -850,8 +851,13 @@ def test_initial_snapshot_refresh_clears_a_previous_guardrail_checkpoint() -> No from litellm.proxy.litellm_pre_call_utils import refresh_proxy_server_request_body_snapshot logging_obj: Final = Logging( - model="test-model", messages=[], stream=False, call_type="acompletion", - start_time=datetime.now(), litellm_call_id="new-request", function_id="new-request", + model="test-model", + messages=[], + stream=False, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="new-request", + function_id="new-request", ) logging_obj.shadow_eval_request_snapshot = GuardrailRequestSnapshot.capture( {"messages": [{"role": "user", "content": "previous request"}]}, @@ -913,8 +919,14 @@ async def test_post_guardrail_snapshot_preserves_logging_only_masking_in_spend_l } data: Final = {"messages": messages, "metadata": metadata, "proxy_server_request": {}} logging_obj: Final = Logging( - model="test-model", messages=messages, stream=False, call_type="acompletion", - start_time=datetime.now(), litellm_call_id="mask-spend", function_id="mask-spend", kwargs=data, + model="test-model", + messages=messages, + stream=False, + call_type="acompletion", + start_time=datetime.now(), + litellm_call_id="mask-spend", + function_id="mask-spend", + kwargs=data, ) data["litellm_logging_obj"] = logging_obj refresh_proxy_server_request_body_snapshot(data, guardrails_applied=True) @@ -928,9 +940,13 @@ async def test_post_guardrail_snapshot_preserves_logging_only_masking_in_spend_l kwargs, _ = await guardrail.async_logging_hook( kwargs=logging_obj.model_call_details, result=None, call_type="acompletion" ) - stored: Final = json.loads(_get_proxy_server_request_for_spend_logs_payload( - metadata={}, litellm_params=kwargs["litellm_params"], kwargs=kwargs, - )) + stored: Final = json.loads( + _get_proxy_server_request_for_spend_logs_payload( + metadata={}, + litellm_params=kwargs["litellm_params"], + kwargs=kwargs, + ) + ) assert kwargs["messages"] == [{"role": "user", "content": "email [EMAIL]"}] assert stored["messages"] == kwargs["messages"] @@ -1089,6 +1105,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): "messages": [{"role": "user", "content": "hello"}], "mock_response": "free response", "mock_tool_calls": [{"id": "call_1"}], + "is_streaming_request": "caller-value", "disable_global_guardrails": True, "enable_prompt_caching": True, "routing_decision": {"cause": "forged", "routed_model": "spoofed"}, @@ -1114,6 +1131,7 @@ async def test_add_litellm_data_to_request_strips_user_control_fields(): assert "enable_prompt_caching" not in updated assert "routing_decision" not in updated assert "litellm_gateway_injected_cache" not in updated + assert updated["is_streaming_request"] == "caller-value" assert "weights" not in updated assert "_router_weights" not in updated assert "weights" not in updated["proxy_server_request"]["body"] @@ -8803,20 +8821,27 @@ async def test_mcp_credentials_only_removed_from_logging_copies(path: str, custo request.headers = Headers(request.headers) settings: Final = {"mcp_client_side_auth_header_name": custom_auth, "user_header_name": "x-user-id"} server: Final = MCPServer( - server_id="header-test", name="header-test", transport="http", url="https://example.com/mcp", + server_id="header-test", + name="header-test", + transport="http", + url="https://example.com/mcp", extra_headers=["x-service-token", "x-user-id"], ) with ( patch("litellm.proxy.proxy_server.general_settings", settings), patch.dict( "litellm.proxy._experimental.mcp_server.mcp_server_manager.global_mcp_server_manager.config_mcp_servers", - {"header-test": server}, clear=True, + {"header-test": server}, + clear=True, ), ): updated: Final = await add_litellm_data_to_request( data={"model": "test-model", "messages": [{"role": "user", "content": "hello"}]}, - request=request, user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), - proxy_config=MagicMock(), general_settings=settings, version="test", + request=request, + user_api_key_dict=UserAPIKeyAuth(api_key="hashed-key"), + proxy_config=MagicMock(), + general_settings=settings, + version="test", ) for header_dict in _all_header_dicts(updated, metadata_name): assert not any(value in json.dumps(header_dict) for value in secrets.values()) @@ -8836,7 +8861,10 @@ def test_signoz_callback_vars_are_scoped_to_the_signoz_callback(): data=AddTeamCallback( callback_name="signoz", callback_type="success", - callback_vars={"signoz_ingestion_key": "team-key", "signoz_ingestion_endpoint": "https://ingest.eu.signoz.cloud:443"}, + callback_vars={ + "signoz_ingestion_key": "team-key", + "signoz_ingestion_endpoint": "https://ingest.eu.signoz.cloud:443", + }, ), team_callback_settings_obj=None, ) @@ -8856,6 +8884,54 @@ def test_signoz_callback_vars_are_scoped_to_the_signoz_callback(): assert under_other.callback_vars == {"langfuse_host": "https://cloud.langfuse.com"} +def test_body_snapshot_excludes_the_server_streaming_marker() -> None: + from litellm.constants import SERVER_STREAMING_CLASSIFICATION_KEY, SERVER_STREAMING_CLASSIFICATION_MARKER + from litellm.proxy.litellm_pre_call_utils import refresh_proxy_server_request_body_snapshot + + proxy_request: Final = {"body": {}} + data: Final = { + "messages": [{"role": "user", "content": "hi"}], + SERVER_STREAMING_CLASSIFICATION_KEY: SERVER_STREAMING_CLASSIFICATION_MARKER, + "proxy_server_request": proxy_request, + } + + refresh_proxy_server_request_body_snapshot(data) + + assert proxy_request == {"body": {"messages": [{"role": "user", "content": "hi"}]}} + + +@pytest.mark.parametrize( + "marker", + [ + SERVER_STREAMING_CLASSIFICATION_MARKER, + json.loads(json.dumps(SERVER_STREAMING_CLASSIFICATION_MARKER)), + ], + ids=["enum", "json-string"], +) +def test_body_snapshot_drops_only_the_marker_and_keeps_caller_value(marker: str) -> None: + from litellm.proxy.litellm_pre_call_utils import refresh_proxy_server_request_body_snapshot + + marker_request: Final = {"body": {}} + marker_data: Final = { + "messages": [{"role": "user", "content": "hi"}], + SERVER_STREAMING_CLASSIFICATION_KEY: marker, + "proxy_server_request": marker_request, + } + + refresh_proxy_server_request_body_snapshot(marker_data) + + assert SERVER_STREAMING_CLASSIFICATION_KEY not in marker_request["body"], marker_request + + caller_request: Final = {"body": {}} + caller_data: Final = { + "messages": [{"role": "user", "content": "hi"}], + SERVER_STREAMING_CLASSIFICATION_KEY: "caller-value", + "proxy_server_request": caller_request, + } + + refresh_proxy_server_request_body_snapshot(caller_data) + + assert caller_request["body"][SERVER_STREAMING_CLASSIFICATION_KEY] == "caller-value", caller_request def test_arize_otlp_protocol_on_a_key_logging_entry_reaches_the_destination(monkeypatch): from litellm.integrations.otel.model.config import is_otel_v2_enabled from litellm.proxy.litellm_pre_call_utils import resolve_tenant_otel_destinations diff --git a/tests/unit/proxy/test_pricing_field_strip.py b/tests/unit/proxy/test_pricing_field_strip.py index a0e25e91f37..395fdc3a16e 100644 --- a/tests/unit/proxy/test_pricing_field_strip.py +++ b/tests/unit/proxy/test_pricing_field_strip.py @@ -26,7 +26,6 @@ from litellm.proxy.litellm_pre_call_utils import ( from litellm.types.utils import CustomPricingLiteLLMParams - def _make_request_mock() -> Request: request_mock = MagicMock(spec=Request) request_mock.url.path = "/v1/chat/completions" @@ -58,9 +57,7 @@ class TestStripClientPricingOverrides: # The strip set is built from the model so additions are picked up # automatically — this test guards against the model and the strip # set drifting apart if someone replaces the auto-derivation later. - assert _CLIENT_PRICING_CONTROL_FIELDS == frozenset( - CustomPricingLiteLLMParams.model_fields.keys() - ) + assert _CLIENT_PRICING_CONTROL_FIELDS == frozenset(CustomPricingLiteLLMParams.model_fields.keys()) # Sanity: the obvious top-level pricing fields are in the set. for field in ( "input_cost_per_token", @@ -184,9 +181,7 @@ class TestStripClientPricingOverrides: verbose_proxy_logger.setLevel(logging.DEBUG) with caplog.at_level(logging.DEBUG, logger=verbose_proxy_logger.name): _strip_client_pricing_overrides({"model": "gpt-4", "temperature": 0.7}) - assert not any( - "pricing" in record.getMessage().lower() for record in caplog.records - ) + assert not any("pricing" in record.getMessage().lower() for record in caplog.records) @pytest.mark.asyncio @@ -211,6 +206,26 @@ async def test_add_litellm_data_to_request_strips_root_pricing_fields(): assert "output_cost_per_token" not in updated +@pytest.mark.asyncio +async def test_add_litellm_data_to_request_preserves_caller_streaming_request(): + data = { + "model": "gpt-4", + "messages": [{"role": "user", "content": "hi"}], + "is_streaming_request": True, + } + + updated = await add_litellm_data_to_request( + data=data, + request=_make_request_mock(), + user_api_key_dict=_user_api_key_auth(), + proxy_config=MagicMock(), + general_settings={}, + version="test-version", + ) + + assert updated["is_streaming_request"] is True + + @pytest.mark.asyncio async def test_add_litellm_data_to_request_strips_client_disconnect_metadata(): data = { @@ -318,9 +333,7 @@ async def test_add_litellm_data_to_request_skips_strip_with_team_opt_in(): "input_cost_per_token": 0.0001, } - user_auth = _user_api_key_auth( - team_metadata={"allow_client_pricing_override": True} - ) + user_auth = _user_api_key_auth(team_metadata={"allow_client_pricing_override": True}) updated = await add_litellm_data_to_request( data=data, request=_make_request_mock(), diff --git a/tests/unit/types/test_guardrails_case_normalization.py b/tests/unit/types/test_guardrails_case_normalization.py index 26c1d395320..8c8192be3be 100644 --- a/tests/unit/types/test_guardrails_case_normalization.py +++ b/tests/unit/types/test_guardrails_case_normalization.py @@ -2,12 +2,19 @@ Test case normalization in LitellmParams for all guardrail types """ -from typing import Literal +import logging +from typing import Final, Literal import pytest from pydantic import ValidationError -from litellm.types.guardrails import BaseLitellmParams, LitellmParams +from litellm.types.guardrails import ( + BaseLitellmParams, + LitellmParams, + runtime_stream_scope, + stored_stream_scope, + with_tolerated_stream_scope, +) class TestLitellmParamsCaseNormalization: @@ -184,3 +191,80 @@ class TestSensitiveDataRoutingValidation: on_sensitive_data="BLOCK", ) assert params.on_sensitive_data == "block" + + +class TestStreamScopeValidation: + def test_scalar_is_case_normalized(self): + params = LitellmParams(guardrail="bedrock", mode="post_call", stream_scope="Streaming") + assert params.stream_scope == "streaming" + + def test_map_keys_and_values_are_normalized(self): + params = LitellmParams( + guardrail="bedrock", + mode=["pre_call", "post_call"], + stream_scope={"Pre_Call": "Both", "POST_CALL": "Non_Streaming"}, + ) + assert params.stream_scope == {"pre_call": "both", "post_call": "non_streaming"} + + def test_invalid_scalar_is_rejected(self): + with pytest.raises(ValidationError, match="stream_scope must be one of"): + LitellmParams(guardrail="bedrock", mode="post_call", stream_scope="chunks") + + def test_invalid_map_key_is_rejected(self): + with pytest.raises(ValidationError, match="stream_scope keys must be guardrail modes"): + LitellmParams(guardrail="bedrock", mode="post_call", stream_scope={"not_a_mode": "both"}) + + def test_invalid_map_value_is_rejected(self): + with pytest.raises(ValidationError, match="stream_scope must be one of"): + LitellmParams(guardrail="bedrock", mode="post_call", stream_scope={"post_call": "sometimes"}) + + def test_runtime_stream_scope_normalizes_direct_constructor_maps(self): + default, by_hook = runtime_stream_scope({"Pre_Call": "streaming"}) + assert default == "both" + assert dict(by_hook) == {"pre_call": "streaming"} + + def test_runtime_stream_scope_rejects_invalid_direct_input(self): + with pytest.raises(ValueError, match="stream_scope must be one of"): + runtime_stream_scope("chunks") + + @pytest.mark.parametrize( + "value, expected", + [ + ("streaming", "streaming"), + ("non_streaming", "non_streaming"), + ("both", "both"), + ({"pre_call": "streaming"}, {"pre_call": "streaming"}), + ("sometimes", None), + ({"pre_call": "sometimes"}, None), + ], + ) + def test_stored_stream_scope_tolerates_invalid_values( + self, + value: object, + expected: object, + caplog: pytest.LogCaptureFixture, + ) -> None: + caplog.set_level(logging.WARNING) + + result: Final = stored_stream_scope(value) + + assert result == expected + if expected is None: + assert f"Ignoring invalid stored stream_scope value of type {type(value).__name__}" in caplog.text + assert "sometimes" not in caplog.text + + def test_tolerated_stream_scope_rewrites_only_the_scope_field(self) -> None: + params: Final = { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "stream_scope": "sometimes", + } + + tolerated: Final = with_tolerated_stream_scope(params) + + assert tolerated == { + "guardrail": "generic_guardrail_api", + "mode": "pre_call", + "stream_scope": None, + } + assert params["stream_scope"] == "sometimes" diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailReadOnlyDetails.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailReadOnlyDetails.tsx new file mode 100644 index 00000000000..5fe88b4870c --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/GuardrailReadOnlyDetails.tsx @@ -0,0 +1,68 @@ +import { Badge } from "@/components/ui/badge"; +import { GuardrailModeRows } from "./GuardrailModeDisplay"; +import { GuardrailStreamScopeDetail } from "./StreamScopeFields"; +import ToolPermissionRulesEditor, { type ToolPermissionConfig } from "./tool_permission/ToolPermissionRulesEditor"; + +export const GuardrailReadOnlyDetails = ({ + guardrailId, + guardrailName, + displayName, + litellmParams, + streamScope, + defaultOn, + piiEntityCount, + createdAt, + updatedAt, + showToolPermission, + toolPermissionConfig, +}: { + guardrailId: string; + guardrailName: string; + displayName: string; + litellmParams: { mode?: unknown; logging_only_scope?: string | null }; + streamScope: unknown; + defaultOn: boolean | undefined; + piiEntityCount: number; + createdAt: string; + updatedAt: string; + showToolPermission: boolean; + toolPermissionConfig: ToolPermissionConfig; +}) => ( +
    +
    +

    Guardrail ID

    +
    {guardrailId}
    +
    +
    +

    Guardrail Name

    +
    {guardrailName || "Unnamed Guardrail"}
    +
    +
    +

    Provider

    +
    {displayName}
    +
    + + +
    +

    Default On

    + {defaultOn ? "Yes" : "No"} +
    + {piiEntityCount > 0 && ( +
    +

    PII Protection

    +
    + {piiEntityCount} PII entities configured +
    +
    + )} +
    +

    Created At

    +
    {createdAt}
    +
    +
    +

    Last Updated

    +
    {updatedAt}
    +
    + {showToolPermission && } +
    +); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/StreamScopeFields.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/StreamScopeFields.tsx new file mode 100644 index 00000000000..1a8f55ce477 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/StreamScopeFields.tsx @@ -0,0 +1,95 @@ +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { GuardrailField, labelWithHint, type GuardrailFormControl } from "./GuardrailFormField"; +import { + STREAM_SCOPE_OPTIONS, + formatGuardrailStreamScope, + type GuardrailStreamScope, + isGuardrailStreamScope, +} from "./guardrail_info_helpers"; + +const STREAM_SCOPE_ITEMS = STREAM_SCOPE_OPTIONS.map((option) => ({ + label: option.label, + value: option.value, +})); + +const REQUEST_SHAPE_HINT = + "Run this guardrail on streaming requests, non-streaming requests, or both, for each selected mode."; + +export const StreamScopeFields = ({ + modes, + value, + onChange, +}: { + modes: string[]; + value: Record; + onChange: (next: Record) => void; +}) => { + if (modes.length === 0) return null; + + return ( +
    + {modes.map((mode) => { + const selected = value[mode] ?? "both"; + return ( +
    + + +
    + ); + })} +
    + ); +}; + +export const StreamScopeFormField = ({ control, modes }: { control: GuardrailFormControl; modes: string[] }) => ( + + {({ value, onChange }) => ( + | undefined) ?? {}} + onChange={onChange} + /> + )} + +); + +export const GuardrailStreamScopeCaption = ({ raw }: { raw: unknown }) => { + const label = formatGuardrailStreamScope(raw); + if (!label) return null; + return

    {label}

    ; +}; + +export const GuardrailStreamScopeDetail = ({ raw }: { raw: unknown }) => { + const label = formatGuardrailStreamScope(raw); + if (!label) return null; + return ( +
    +

    Request shape

    +
    {label}
    +
    + ); +}; diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx index 230be13ef33..9628c284bd2 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.characterization.test.tsx @@ -89,6 +89,23 @@ describe("AddGuardrailForm create payload characterization", () => { }); }); + it("sends stream_scope when a mode is restricted to streaming requests", async () => { + const user = userEvent.setup({ delay: null }); + renderForm(); + + await user.type(await screen.findByLabelText("Guardrail Name"), "my-bedrock"); + await pickProvider(user, "Bedrock Guardrail"); + await chooseSelectOption(user, screen.getByLabelText("pre_call applies to"), "Streaming only"); + await user.type(await screen.findByPlaceholderText("The guardrail id on Bedrock"), "gr-123"); + await user.click(screen.getByRole("button", { name: "Next" })); + await user.click(await screen.findByRole("button", { name: "Create Guardrail" })); + + await waitFor(() => expect(networking.createGuardrailCall).toHaveBeenCalledTimes(1)); + expect(payload()).toMatchObject({ + litellm_params: { stream_scope: "streaming" }, + }); + }); + it("switches mode from the seeded string to an array once the user touches the multi select", async () => { const user = userEvent.setup({ delay: null }); renderForm(); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx index 43a542289f6..098245fa34b 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/add_guardrail_form.tsx @@ -23,11 +23,14 @@ import { shouldRenderContentFilterConfigSettings, shouldRenderLLMJudgeFields, shouldRenderPIIConfigSettings, - supportsDirectionalLoggingOnlyScope, + streamScopePayload, toModeArray, + type GuardrailStreamScope, + supportsDirectionalLoggingOnlyScope, type LoggingOnlyScope, type LoggingOnlyScopeChoice, } from "./guardrail_info_helpers"; +import { StreamScopeFormField } from "./StreamScopeFields"; import { Logo } from "@/components/molecules/logo/Logo"; import { MultiSelect } from "@/components/shared/MultiSelect"; import { FieldGroup } from "@/components/ui/field"; @@ -172,6 +175,7 @@ const INITIAL_VALUES: GuardrailFormValues = { logging_only_scope_choice: "default", skip_system_message_choice: "inherit", skip_tool_message_choice: "inherit", + stream_scope_by_mode: {}, }; const ALWAYS_ON_ITEMS = [ @@ -466,6 +470,14 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a guardrail_info: {}, }; + const streamScope = streamScopePayload( + toModeArray(values.mode), + (values.stream_scope_by_mode as Record | undefined) ?? {}, + ); + if (streamScope !== undefined) { + guardrailData.litellm_params.stream_scope = streamScope; + } + const skipForCreate = choiceToSkipSystemForCreate(asSkipChoice(values.skip_system_message_choice)); if (skipForCreate !== undefined) { guardrailData.litellm_params.skip_system_message_in_guardrail = skipForCreate; @@ -773,6 +785,8 @@ const AddGuardrailForm: React.FC = ({ visible, onClose, a )} + + { expect(screen.getByRole("button", { name: /update guardrail/i })).toBeInTheDocument(); }); + it("should open without crashing for a tag-scoped mode and show it read-only", async () => { + renderModal({ + editData: { + guardrail_id: "g-tag", + guardrail_name: "tag-mode-guardrail", + litellm_params: { + mode: { tags: { "team-a": "pre_call" }, default: "post_call" }, + default_on: true, + custom_code: "def apply_guardrail(): pass", + }, + }, + }); + + expect(await screen.findByText("Edit Custom Guardrail")).toBeInTheDocument(); + const modeInput = screen.getByLabelText("Mode (tag-scoped)"); + expect(modeInput).toBeDisabled(); + expect(modeInput).toHaveValue("post_call, pre_call (tag-based)"); + expect(screen.getByText("Mode (tag-scoped, read-only)")).toBeInTheDocument(); + expect(screen.queryByText(/applies to/)).not.toBeInTheDocument(); + }); + + it("should omit mode and stream_scope from the update payload for a tag-scoped guardrail", async () => { + const user = userEvent.setup(); + renderModal({ + editData: { + guardrail_id: "g-tag", + guardrail_name: "tag-mode-guardrail", + litellm_params: { + mode: { tags: { "team-a": "pre_call" }, default: "post_call" }, + default_on: true, + custom_code: "def apply_guardrail(): pass", + }, + }, + }); + + const nameInput = await screen.findByDisplayValue("tag-mode-guardrail"); + await user.clear(nameInput); + await user.type(nameInput, "renamed-guardrail"); + await user.click(screen.getByRole("button", { name: /update guardrail/i })); + + await waitFor(() => { + expect(mockUpdate).toHaveBeenCalledTimes(1); + }); + const [token, guardrailId, payload] = mockUpdate.mock.calls[0] as [ + string, + string, + Record>, + ]; + expect(token).toBe("test-token"); + expect(guardrailId).toBe("g-tag"); + expect(payload.guardrail_name).toBe("renamed-guardrail"); + expect(payload.litellm_params.custom_code).toBe("def apply_guardrail(): pass"); + expect(payload.litellm_params).not.toHaveProperty("mode"); + expect(payload.litellm_params).not.toHaveProperty("stream_scope"); + }); + it("should keep save disabled until a guardrail name is entered", async () => { const user = userEvent.setup(); renderModal(); @@ -111,8 +167,7 @@ describe("CustomCodeModal", () => { expect(await screen.findByDisplayValue(/async def apply_guardrail/)).toBeInTheDocument(); - const comboboxes = screen.getAllByRole("combobox"); - await user.click(comboboxes[comboboxes.length - 1]); + await user.click(screen.getByRole("combobox", { name: "Template" })); const options = await screen.findAllByText("Block SSN"); await user.click(options[options.length - 1]); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx index 9deef4aeee3..c89540596bd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/CustomCodeModal.tsx @@ -37,152 +37,30 @@ import { import { Switch } from "@/components/ui/switch"; import { Textarea } from "@/components/ui/textarea"; import { UiLoadingSpinner } from "@/components/ui/ui-loading-spinner"; - -// Code templates -const CODE_TEMPLATES = { - empty: { - name: "Empty Template", - code: `async def apply_guardrail(inputs, request_data, input_type): - # inputs: {texts, images, tools, tool_calls, structured_messages, model} - # request_data: {model, user_id, team_id, end_user_id, metadata} - # input_type: "request" or "response" - return allow()`, - }, - blockSSN: { - name: "Block SSN", - code: `def apply_guardrail(inputs, request_data, input_type): - for text in inputs["texts"]: - if regex_match(text, r"\\d{3}-\\d{2}-\\d{4}"): - return block("SSN detected") - return allow()`, - }, - redactEmail: { - name: "Redact Emails", - code: `def apply_guardrail(inputs, request_data, input_type): - pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}" - modified = [] - for text in inputs["texts"]: - modified.append(regex_replace(text, pattern, "[EMAIL REDACTED]")) - return modify(texts=modified)`, - }, - blockSQL: { - name: "Block SQL Injection", - code: `def apply_guardrail(inputs, request_data, input_type): - if input_type != "request": - return allow() - for text in inputs["texts"]: - if contains_code_language(text, ["sql"]): - return block("SQL code not allowed") - return allow()`, - }, - validateJSON: { - name: "Validate JSON", - code: `def apply_guardrail(inputs, request_data, input_type): - if input_type != "response": - return allow() - - schema = {"type": "object", "required": ["name", "value"]} - - for text in inputs["texts"]: - obj = json_parse(text) - if obj is None: - return block("Invalid JSON response") - if not json_schema_valid(obj, schema): - return block("Response missing required fields") - return allow()`, - }, - externalAPI: { - name: "External API Check (async)", - code: `async def apply_guardrail(inputs, request_data, input_type): - # Call an external moderation API (async for non-blocking) - for text in inputs["texts"]: - response = await http_post( - "https://api.example.com/moderate", - body={"text": text, "user_id": request_data["user_id"]}, - headers={"Authorization": "Bearer YOUR_API_KEY"}, - timeout=10 - ) - - if not response["success"]: - # API call failed, allow by default or block - return allow() - - if response["body"].get("flagged"): - return block(response["body"].get("reason", "Content flagged")) - - return allow()`, - }, -}; - -// Available primitives organized by category -const PRIMITIVES = { - "Return Values": [ - { name: "allow()", desc: "Let request/response through" }, - { name: "block(reason)", desc: "Reject with message" }, - { name: "flag(reason, metadata={})", desc: "Let through, record a non-blocking violation" }, - { name: "modify(texts=[], images=[], tool_calls=[])", desc: "Transform content" }, - ], - "HTTP Requests (async)": [ - { name: "await http_request(url, method, headers, body)", desc: "Make async HTTP request" }, - { name: "await http_get(url, headers)", desc: "Async GET request" }, - { name: "await http_post(url, body, headers)", desc: "Async POST request" }, - ], - "Regex Functions": [ - { name: "regex_match(text, pattern)", desc: "Returns True if pattern found" }, - { name: "regex_replace(text, pattern, replacement)", desc: "Replace all matches" }, - { name: "regex_find_all(text, pattern)", desc: "Return list of matches" }, - ], - "JSON Functions": [ - { name: "json_parse(text)", desc: "Parse JSON string, returns None on error" }, - { name: "json_stringify(obj)", desc: "Convert to JSON string" }, - { name: "json_schema_valid(obj, schema)", desc: "Validate against JSON schema" }, - ], - "URL Functions": [ - { name: "extract_urls(text)", desc: "Extract all URLs from text" }, - { name: "is_valid_url(url)", desc: "Check if URL is valid" }, - { name: "all_urls_valid(text)", desc: "Check all URLs in text are valid" }, - ], - "Code Detection": [ - { name: "detect_code(text)", desc: "Returns True if code detected" }, - { name: "detect_code_languages(text)", desc: "Returns list of detected languages" }, - { name: 'contains_code_language(text, ["sql"])', desc: "Check for specific languages" }, - ], - "Text Utilities": [ - { name: "contains(text, substring)", desc: "Check if substring exists" }, - { name: "contains_any(text, [substr1, substr2])", desc: "Check if any substring exists" }, - { name: "word_count(text)", desc: "Count words" }, - { name: "char_count(text)", desc: "Count characters" }, - { name: "lower(text) / upper(text) / trim(text)", desc: "String transforms" }, - ], -}; - -const MODE_OPTIONS = [ - { value: "pre_call", label: "pre_call (Request)" }, - { value: "post_call", label: "post_call (Response)" }, - { value: "during_call", label: "during_call (Parallel)" }, - { value: "logging_only", label: "logging_only" }, - { value: "pre_mcp_call", label: "pre_mcp_call (Before MCP Tool Call)" }, - { value: "post_mcp_call", label: "post_mcp_call (After MCP Tool Call)" }, - { value: "during_mcp_call", label: "during_mcp_call (During MCP Tool Call)" }, -]; - -const TEMPLATE_ITEMS = Object.entries(CODE_TEMPLATES).map(([key, template]) => ({ - value: key, - label: template.name, -})); - -type ModeOption = (typeof MODE_OPTIONS)[number]; - -const MODE_OPTION_BY_VALUE: Record = Object.fromEntries( - MODE_OPTIONS.map((option) => [option.value, option]), -); +import { StreamScopeFields } from "../StreamScopeFields"; +import { + formatGuardrailMode, + streamScopeByModeFromConfig, + streamScopeForUpdate, + streamScopePayload, + type GuardrailStreamScope, +} from "../guardrail_info_helpers"; +import { + CODE_TEMPLATES, + MODE_OPTION_BY_VALUE, + MODE_OPTIONS, + PRIMITIVES, + TEMPLATE_ITEMS, + type ModeOption, +} from "./custom_code_catalog"; // Data for editing an existing guardrail + export interface EditGuardrailData { guardrail_id: string; guardrail_name: string; litellm_params: { - mode?: string | string[]; + mode?: string | string[] | Record; default_on?: boolean; custom_code?: string; logging_only_scope?: LoggingOnlyScope | null; @@ -204,6 +82,7 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS const isEditMode = !!editData; const [guardrailName, setGuardrailName] = useState(""); const [mode, setMode] = useState(["pre_call"]); + const [streamScopeByMode, setStreamScopeByMode] = useState>({}); const [loggingOnlyScopeChoice, setLoggingOnlyScopeChoice] = useState("default"); const [defaultOn, setDefaultOn] = useState(false); const [selectedTemplate, setSelectedTemplate] = useState("empty"); @@ -315,11 +194,14 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS setCode(CODE_TEMPLATES[templateKey as keyof typeof CODE_TEMPLATES].code); }; - // Normalize mode from API (string or string[]) to string[] - const normalizeMode = (m: string | string[] | undefined): string[] => { + // Normalize mode from API (string or string[]) to string[]. + // A tag-scoped mode dict ({ tags, default }) is managed outside this editor, so it + // contributes no editable modes and is displayed read-only instead. + const normalizeMode = (m: string | string[] | Record | undefined): string[] => { if (m === undefined || m === null) return ["pre_call"]; if (Array.isArray(m)) return m.length ? m : ["pre_call"]; - return [m]; + if (typeof m === "string") return [m]; + return []; }; // Reset form when modal opens or editData changes @@ -329,6 +211,12 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS // Edit mode: populate with existing data setGuardrailName(editData.guardrail_name || ""); setMode(normalizeMode(editData.litellm_params?.mode)); + setStreamScopeByMode( + streamScopeByModeFromConfig( + editData.litellm_params?.stream_scope, + normalizeMode(editData.litellm_params?.mode), + ), + ); setLoggingOnlyScopeChoice(loggingOnlyScopeToChoice(editData.litellm_params?.logging_only_scope)); setDefaultOn(editData.litellm_params?.default_on || false); setCode(editData.litellm_params?.custom_code || CODE_TEMPLATES.empty.code); @@ -337,6 +225,7 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS // Create mode: reset to defaults setGuardrailName(""); setMode(["pre_call"]); + setStreamScopeByMode({}); setLoggingOnlyScopeChoice("default"); setDefaultOn(false); setSelectedTemplate("empty"); @@ -411,11 +300,21 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS if (defaultOn !== editData.litellm_params?.default_on) { updateData.litellm_params.default_on = defaultOn; } + const nextStreamScope = streamScopeForUpdate( + mode, + streamScopeByMode, + editData.litellm_params?.stream_scope, + existingMode, + ); + if (nextStreamScope !== undefined) { + updateData.litellm_params.stream_scope = nextStreamScope; + } await updateGuardrailCall(accessToken, editData.guardrail_id, updateData); toast.success("Custom code guardrail updated successfully"); } else { // Create new guardrail + const streamScope = streamScopePayload(mode, streamScopeByMode); const guardrailData = { guardrail_name: guardrailName, litellm_params: { @@ -423,6 +322,7 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS mode: mode, default_on: defaultOn, custom_code: code, + ...(streamScope !== undefined ? { stream_scope: streamScope } : {}), ...getCustomCodeLoggingOnlyScopeCreate(mode, loggingOnlyScopeChoice), }, guardrail_info: {}, @@ -511,6 +411,11 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS const lineCount = code.split("\n").length; const selectedModeOptions = mode.map((value) => MODE_OPTION_BY_VALUE[value]).filter(Boolean); + const rawEditMode = editData?.litellm_params?.mode; + const tagScopedModeLabel = + rawEditMode !== null && typeof rawEditMode === "object" && !Array.isArray(rawEditMode) + ? formatGuardrailMode(rawEditMode) || "-" + : null; return ( !open && onClose()}> @@ -533,32 +438,38 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS />
    - - setMode(options.map((option) => option.value))} - multiple - > - } className="w-full"> - {selectedModeOptions.map((option) => ( - - {option.label} - - ))} - - - - No matching modes - - {(option: ModeOption) => ( - + + {tagScopedModeLabel ? ( + + ) : ( + setMode(options.map((option) => option.value))} + multiple + > + } className="w-full"> + {selectedModeOptions.map((option) => ( + {option.label} - - )} - - - + + ))} + + + + No matching modes + + {(option: ModeOption) => ( + + {option.label} + + )} + + + + )}
    {mode.includes("logging_only") && ( @@ -600,6 +511,11 @@ const CustomCodeModal: React.FC = ({ visible, onClose, onS

    ATdn&eCTgasHa1yEfYFPS@cn@tzbSDPnnYk6)rgekT{Th1-JRub+J&4($2gpymFas_<)7D~*^t|E`L`6~5Y5LH|GnQw1ifuT|6J^9B_NFfD zAWJJwRr4GleNIgwkuHWT*ux~wVyv%o30|SAJIwuij-ZG1DCl>RaS@^mW)CDS2 zE6#@Pji&yZg+720bh+H7-y=x;$erGZHb7-8@-mrA-K;Ny0Q^EMHr=eTEJ!th9u$B2}o!+J@l-=A1x^j!jhyCE)E8r** z^bP$X!dpq_m5(dXo^#pSj5EF)-RXgInBR#`bD*G?qNjrO*N2^vUq43vd>ib2D{RLm z`Wfa^9yM0c|5#TZA!n2qfhOm)^-;1oH#pxpv!kAv&;7|g(62No%nSEPc(_|!FHlpg z$?U`FPOl@83cv&TF}iERh^7b8H*pw7+923alfbn)>5rIHx(@Zg>8Pik@eWh7QM0qb@Rs(e}A zj832+_k#K4f#vj$7;6FE>k|Dw?@Vwq=0;v-qUsTH<@scf{a_dF<0C()2lb#fx{jQB zEe!F2uwl>R4@7QOH1^~kmN_@NR?T4e?;=7-N8Pm?igIn?{x6Vjqj9$qcDhd+%p(?K zie5%k=0c@=U@(oizaPK=Hd0Ifh)r9?^Q^%G?!eDoCL_;@5?L@^>`j@CoCQYBYtWQl z)bqwuO=@Ojg`@GB+O6-c9Q7+FYW7E%sak|-m}m8MdLQ_bx2ZX)u)t@5W%T0f71YwN z>!*3HEqZUg1I(M!R0Qv$-hT`p)O+x7g^KAYb%?fE4>5-c8R+R9f;ROMF}vij4O0w9 z1ao=at`yfNcMrc=ekIx8*X|td+^)3FXh#{xX-9WwE~f`v#*%NqFBnWe(G5_pbH;YK zgpV~pR&_5-&ih&)aF_yQ;F21z_S4dX_uT}MD1nE1haC;0i{To4-U>vxZ-wRJC=@Lt zK5f~+vJ-{0x0m8qUL<}O50`>ajHdKsv6*Sy_sAH^OINVJ#ptuE z2P3{db|np6MJeq49x|En#0Vxa_g=b&chg_>k-0Q*0i+Bta?hfa7!Ue4M_Mh7l6pu@ zrQc}6HAl_92GQ9OJhPWPW)obxl5iKVlYQ@i1GgH>6l1S(lb?78CQVClpEoEv zrqc&-9s-ym(M3>Daa{xkeL{XC&$OMfA5t3boxfJet0NKJqGV@ZVMIXd;?C*YT(aY!OkwB+;c6#*+5f9L1PsDJT*s~12eiC&)pYmT5BCRsq?@rWXcXK-};;M?EZo%Sb z{-1)X+Hqp1bHq(dDIwN+%SS`#yibeQoK6ho#AkdWlKX^(KTTA7j|kMKUecOODVmt4 zG4}Z!NJm#XRUdGxW#)RSU_qbh8*ng7D>JjPz7t9GAd9$92akp2?~mVnO%8ID*_4sw z*?G)wc%HTFQb$gvPgSBJJbgQu`ay2)O7?LP4BWg}-b38&CE)l&sXex37aRXQ*OQnc z{*KN8$xH#ADL{`%bI_Zebd;3iv=8Df3bl&Ed`%dumX2u4pPMomuQH7;o?hrTw~+m? zi&3`8wq>^0wjuU{%5L!gp^jmW5suD|ct?KcB9q>qIJ?XP@5P5 zmhNN%bpTbS{>%)yt%YETM&OT+p`LmbwptW9+e)eyX^1G_5tU3Jhx$%L6Gk1TAssLi zxS^+s&%9vnnb8idNqx?OFuKtK9uM>8{2XNM2LvFgT5$ryh6T6pf2 zsSYh9`hQOD@*E$1kn{fu9B&-nbU(YjnrP!OyQVN(ISuoof9caYNp zHx;d3VA6G1iFw)??H0Q3C*h=i)^dZxcVhx`X}+T<^SoE-pQz|mV0RmF5?XO;CNXD9 zq$^|`-0o&{O|7P1@+haV8}*%bXaf8|Bi4gTYzummY)l%?=iKW|bXv|-=QU;!Z*)Fk zR`g5f0u(XYI+ns%JwZQH6Lz=1?HEcRYamk#V8y@_|VFCY-n5Vjwrq z6jd}HE)%~GX1_|XUkTJ>P7?D^;C1&TLvDao*vjrt<@`4yUrSH#T?g{7L-YgKi05^z zz!@;CO<+|+h`fBcT2@d89|Qac(dtyrM^kzgD`9cGU?0=Xp0Ji$plp(YN14g=;7sTg zyv3(Z*9S7kyA}5Of7F_@=nAv6vop)#xONsbn-C^y59401V7B=kcGFJ8F_f5M3jI3+ zVTAYKo?pa922nR@&0AG`{}0$X{jnN}rcCx!oA}@oyrmA{Q+e?vHKl*0LUM|H$=1|< zg`LiUqV#qp12d)V&JNt%6vuAv^Dsv&|6X-Wb39eLD{Jhb=uN+{eX}iM()4WkBz(X) zv`KqX8T${E;SM?8DR9!7<|}OX2)yMjo@)qIo5IA*IrO)j*x8)b^>kt!(crBgPheO;=Ti3*9cn`X#D{_e*}g&2b?$$ z$odQH&rZ63Iub#@!k0cn2kASy2)~$!@)lj9m;C#Xk6qEXVWZdTmv}FU=_S5N&5eyn zy5P34Y8U9z%Fh#zr$_oV9#01;bAm>F;QwjKl%|m<%qBPAORV=0TNH!`EeBpx1DvHM z^^&P%4>8<^Sa8&yyz*zD09)}XrKLtp^VrMO;>EJtHpsTxcFK0!_RLnAUX8c*3e-vl zDkV^^IAh<1D*FTKyIt+U_8Ye0wp_%&6NrAZ%E#$RyZ~D?1Rmca*5nsejrvr~u7UD) zrM~c)smf`LZQR*bdP8{GPNrVm$1-ivmTSv+JIG!y$Bv)ZzG+^1E2?0**Ai3yz&e#9 zlbVa3j|3;P0Sb|C&5BgLACY;M;Wi$Dxta-UJPMui^Y~4l)^clUg0!1me8k?}!fqXr zwzAWk*xO-LRk}&7q_V`zzc~*Fu?CB<{N1q0f51G)_ zG64*HiMfLA$NQj4zt}?s{NX02_&8WeC6J~5c&7oN4KY}-heWIU$P|5@wco&9d^t!58k(OZFdG!vRplIG%Ad(d7j0VmRu#`G{H~ zK=sdKo8su3jwWLq09qHzv&V8iCh`6>bZT~kBi+R2`7}JUU<3QnDYFAiQp7j9Kr;Nu zv2xI^zpsuT)XO(8J1nnXy0z`K_*67~&TGJ>Y_P1KfRMZ+Qu_reK92jjfw*rDi1T&gh1Rf=3)30Cn!Eb|gs?k! z?BQIi0#>dYh-n+^n3)yiArfq-1su>%W^*_exA1Lg zdDhi*8C)|rGC_GEr#itb3Hv<_7)}Cc?NTiDPF_JIc*G%|FoIVeB(}jt7(7`T< zS-|R|m|A`g_ZC+9JA9cg;D`0C9aP{C;ssaIk^O=!qL*39I!BFOvIc?|$yA4$ zTKmm<@Q~(OYs{TIeH{C^k{uZc;uQxjSed7+3-0dgjBgAZq$XT)6C`T{=wn)**J39y!PGzz9lkZ{f{N;kWg z`*c)zF4i-@z+bB@c7ayeH5}k(C=^nfZi9=0%tXbInyCd4+@&Dk+PwbN;Y^>T&x{(fNOZ@9ROI*)DLA ztYR^4Mx(#k@JQlDm-JZ>ta?o3Z6w#AhY7tzfv#!p-ysym4qI!@j^YWSI=?f4N^5Cm zKi9${HDfYx4p6hzRw}P9I~?l!+_|x^V{X9)y34(5hwUFF(A&>i<`D{W?hi7nu`;Kq zF*mF(td&aO2$#v`b}+T3rP!PNt&6ygyIKuS^hc24iQMj2+}}VkopD46lbVCntuh`< zm#si!7E$AJc$%xsTtuXk@GnDPML(h@GXORQ>i*(Cpg=9TufK#PW~#6hHf9(&Uv5$0 zR-ZGQ;JN3(SKMIE5sQG6m!cZ7h}-xCe%NwzGS!`MaNclpAy2oLSNNSOb)b}Aj5gxT zNx~f|lle?Pg0-$~Z8rWe(S1KWx~)V{v%v@(dp;K-d2deJ#Q2mbE?A=Qk9tD8=A z!{D_Ehs0m(+hfr7{oul_tSRPdaMVs@<{iam)L-+O>&ap?vjiuggLF!`Vk~F+Y*pdD z(Ne6;bkHW&b8@Yw#(pca_z5pIoU>j)Z0f75@_WNYhh>A+nub1xdEAif#&5BsG>*8n z4=On=`F)R!%0gAChFB4sdyY6G3;S4&c>FY#2E%&8t=M5?6LLtEh4)5jvo)u3HC`hp z6$yL=6Qi+;@bz}l@l==I>rwo!ci4kV?BXl)7jxWqGrK!4aeq4Mj)ORBy9D%4#52M? zQsdFZuF}))NgleidnX_lX(5(T({VIKJx=;t!UTr4F z?uE%)my;DKg|PBFnJ679FvnC%v^p}+tD)4xe1OhCNh=>*&mGiP@>?b4S)k}wO(&Ss zY5ge5C5xp5W)z(g_gg>3gT@^C4X+z_g^Ct>Y&?l>{)auyC|nU28q-igm}@rT9yG(^ z78iF~Hw>5AN@^%JG*@!3J9B$$z!1M;ZWr2!$MF@>)+1>KUCq-u=bt#~qnPkLT3X2D z_;px}=kN(fh@*@))@C78K4?BOo{1r3-B*~`+lK0Y5Ag7j;%4~F&**@7DWVlADliYr zVMz|c9BxA8DlL<-i-C0(f<1E)^`%cjZRwyjgrDG&=FmU9mZ{|`6{p7ByJ+((UDoZz z7G{``SpC58)`RD3h0b7U>xXcL`?gnTWj%$rS{hE)G`xB` zCQ=Wv$oJu&_JS8%P~_GNPtC5r+Om)$_Aq|KN^UMbgx$Ck25uR$?M700VXxlbdMlMP zdl2zu6E_J(jW}V2lvSu~Ocx5l2VQ7wGWSa5zy!8)dsZ;DTL}I1eJD#*rYh5x3d<4V z{;t+(X^#2E@E5v^w}fE5m^GhF%tn-}izw97O+Si>-lxcv8dE>3ZOju3a8eJ$Qz<09 zrcb^+`*s1`VXd_l+h5LfNpqF6*_bG9w@$(Uh%t{?)1@$Bx0#H6>xZ?eNVFe>XFn|NloHL}*rtP09!p?u zc~v3S^rG<5N;qqvlq~fYs-pVvgDC7RHu3>h?TmFmTtGx$3mY}riZ%>5$OFw6_z&5N z65>qfM%TDF}{oc03`onaD)<(J3)r7%6X}!|JBg zfZpU3PSGc^1W&gDUU><=b1dijk?9d?i5tll-By&e6MIlWIKT;QX1ym~@YUPT5Gme) zjT;6h{2LZ`xU_*ie{FuYE{ccX0q@}4sMZ>z8gXPSXTKX>u|M~H7Iv~6?DofapbpY} zYcjZCC^=R=BRzI2RLm?aHg^+;%_B!^O)tk5@gQuBSHw+mL}ts;*jaA%r()z&J^rcx zv38J6A2rJvOk#jXyN;&3K7fqlrv%|<`EkY+5gY@ zgR=C09OUL_Afo^m#jmF!2YQ328A#MQlKN0M3Jn8@GkU|BUC4LkBxW?%Kp6^94K8i@p&NG$9HePUg_iT8llL!1*eKmL7LTYzc?2Jr(Z}*rHy%*TMb zmSb-APBeV-5sy;IMU`bTu|@^((qUMT7(C1xUVS>O`v5qi53z(Lh&tm?pU^>Ju*dA~ zEn?d1<`sCKRmDt1OA(yJ@%-c_WN=S;r9sp;@PXt>#aVA(zs5b3i7bB4JWkXLo*!AA}v2J1#=OB-I-QX=k|+~91iQX1-K(fGD9 z8aPM6b8GNkdG+5vxi731-S@93_1Lri|1%;E+bTvy3LM{whksgFD+ zCs~3|_{3BGB=RZFF3H4Vqge+hv43Z>%@<@QFUgAiIZ5@{dtdeADp}$ctp93m@*b+% zgL%FL(5eWS(9P*%s`R&~%z<_2zzQb80YA=9Y0Z;FVh> zh0^h5P@22+8d&@(NKD^>=rNQSdm0~cutUS}{WbBaBgrVeC~CZ+TF^u~%pBy6QX{#C zZH=uAEWw)g4E7Lv5qlf>yc6vU>_2VOY>jNax=qm(jTfd?*ftb`{u2Z!oW3VE(7cPkZqHVO0LVf=ZNR4JVOh|DbXe$^A=1w%D4S zyc#FdhwHkq^}oQC7J^#kA!b?y_wbLA!`uueyo^fa2oUBS^d>|bMT~mj{=rnsK2y`H z36j1Ygk~Kb!*jqn3enNAocGFu-6m80Nd%#stw-x`P!w$mN9&t06iA_XK<-}bm4fN#h;x$o) z+{Dja&-}m`DqXu(hq@V7OdU@FR_ROX^3;i`+foOoo=fSQ;!24~nU^9kQMgv>`qW*i zIXw}~UC*u$HfM>2?dzNy{XPd23wj-tB~9_P&(oF2@Hs>OjF&UKNdG3?rgZbu)=g79 zxNG1z|EjK>%4#`G=xKPh3+ip}K2OfnFUhNt1ZFPAB%VxspLi~DX`(wRTXLS1WvTyo zcW5a_b)kkdLf*+-`Uu7C$V9y_pR0_kg6oSbi@O6;FcVx`Tz8#`j%P}O?J0TuduFZ& z>#tx|XY$tayhx2tU6?vB^=!(pl=dkx{M|5R8eFI_&tjC>cj+C?^W=J;#0GL#+jm=I z`(RjOeU)X3p$uWq8#@m;MOUCJ(z(>}kMhM9YCA8*fJywO3tBcRz*MV6kH7+EPT%o{ zc`JAhr>;xwo%%j?il?Esu^ObgK%6Ua>T`lH&Sn>;F#q(BlqmIsiK>8(Z-=RN%~l!B z$+U7gX{k5@2Gvk=I+L#h;Cqxq@pTw{vp5v`&v}-?sI29w=ZW+*_YCo@^N8Ns-cm3F zf1=gB&3nNc3SX|SnnR6;DLslQ*@@~G?YLgZ>MgH#B>80t${)Nk=uqIupm}LFr?X`U z$Z$BFH%(}oAHi3H3j_}e?BKuM-O^RkvCAflul1u|HFYT}_T5r{Bs-JxCmIPy5*H<9 zO>UGNn!F=fOZk=hz*B&e-N^_MD$ALaKaPy92Ci1FwC*VP7k9j0H@}}$<}bQBy4E{G zoLd|kJc&&97VD6 zL}E|Ev?=i1ezP`nyp6mSy;;3uypO%Fy?>Z_aN7GF9^5tct`=;Bn2UfXE>e?-)5d83Xiv~u|K#o9je~bK);m-?XBLqS#|=Lv z@M&P0pawzxgX^ZXr!SM?X!@S%9x?y=Sdf2E@xXimNBwrXySrLBrrWj%eRYTTLrR*I zqsf<&MkSv3)9QDQ-vfUS{xdP5Wn$K(V#%jd@_HJ;6q}&sHJHXHJ-3&2E_2mn@@7Fl zKmRZOQv+|YNWQ|<}C9Bd)Kn-8{nq*s=ldhTmcLF8`b_i+|{3uPmbZOJaq$`v5EPIn7Xj0&Z0I&Z6 zzX9%|u6>UA_D<3nbAi^<8{~PAGB(*Ssa1mipL4&n{W=D`b?RXL6<>F$tltJyHpCtiYBP zbe3|Kbjr?ej<H!Zh!wg0rvyW1ojI4n5Ja9 zp6QOHy_DuvaJ}IFg3<@22AKXK{?j<^nOxhHy7F`Dmu{-dyrVsznZ(>DF@m|p)e}Z1 zY)rVG5S{oTsb)&|R0S)0#Pi(yS!)5VG#WMj-T&w4EWo1Lx-dL*&Y5KBuscw(ySuKf z*xh1dcVf4{-Hol-b?xrBcDEu(OxKzD-^0Vhy?}6H@3mKc>syjd(^dCZ-;^mRADE7P z#q`89-?TNLx3RHdwEn41W=?V^>4M;ZC8vz<7c#gTn-JmZg{HIFhuST+=eFARANF&O z9nP7qV)!^mJZauo-&wy+^-6tAh=aAe1NPuAb@D@+n%cYCFkPhX1U;|aG)w7lZwViI zeWtCqh9R#ZeOFtVi5o6|qmHfdZlhc5DZOGhT#wK~7WOIy{omAm!aW|2iX-xHUrTC6 zSI9zL0WdVKsZNLa{|0C?fEKL>>$10?xFG% z)j_G0exmV2z(7-cVC&$zp^d{UMp(j2hPMeT9acEZ99lor7hE%_exTFT+BDLbQ#V3% z`Pa(>y^B0i?(L2!TW!lI^BD6!^HQ_PoRnFNiQEgUU2PL>J#0^G`5lW~vEG@!2V|>W zt2>K5HN$lVgWs^+ST&%8={DWd3rv!!PQYnnEVF(D!y4UBOvi-EKLz9oU zjVGpY*tlE}0)Q zUuV|Llrr~bZm>+X%g)>GxgL>8&9C4gt0tyMm6!u?PhZ=Z6tL3t(6q)>!&J^R)HKnQ z$MnhgkKvTAt#%_l`VOhCWERG$163bD2jB9p@^o=magBElaCX8IN_X^j6m(2+41uel ziMt~lF$u0|E)#Zrh)3{lrh7ZVcibNZ>ed8{!&&C2&!bcK2I#><+LmC5j%m7R&Dw@K zmo{8G5KHx$&d)C7%KnqVHF~1mC0#k3xg5{!6Y)ysxn{U8xd*Uw!AMJCcQ5xG_Y8N0 zCjp=Pu&g0pvRZvncq_W3wM1!gQX8?h(2+S>6|sD|)lbPnGzUNH2WK$Ge}KO9Io<=F zVeTTX6^=C9Nox-4H%pMUyfxCg+|u9j#4^a*#J1n=b-wa6^Sgvt&2a5Rn97&x3K&xZ zV?t|($A|wH_D|U9u=MaJ5%!4k5ouwL(5s;XLkoml2%KshtXrYU34-RNlqwwb2YRcy zRyl6ktJ??J3R`Vvjd@+>{>)#Qe=|R`M?K6VEE}vpZCCB39RnTj9D%Mc?l$rb|30;v z$s^Nwo3Gju`cT6Fe7o$1NJAIHKEnw^f+5BrGA|?55N=##$fnPwts%VxQ8|GB7g3T4|s_)$IXza-E7>jo3=o$eoRLf+PG2pG5QF}W?o}nl4 zI7dR4r+x_J*fUS0=a4dSG;jS*5?zpyv_Jbx< zDlHyYU6G4;W@qt|Y&rY+bY@~+8)_69Z~R>)c4Hu zew9nmEtRfn2U?>6NQK4fM&Rj&NOd)9H48LzHA$LQS}jZoOLZpQ56xw%h15y9Ar(PS z_10)$xvwW_q&wn7@g_U~=fV5u!8hCNn?`-l4K6o<`384juqXniF9$Ja0ncUkakmas z&0uitD?orfVw%=V*4_vneunQI+|rw|FqNn@UIs(Zh5eaIopA(Ngl_a&OosERD7ck4 zdLcVAqrfA-1AjE1Oo1Mznr<*g9rg_-m-CnGN-`PF9Atbxk|X*`rX?R$h-RFe6=ahB zM;2x;d9KxDOUCj2cyb+9y7}gl)0jwxp(A+R+;j%$=s4(34`V&@H!D=5)B}W6a67e# zufo7tnP3Tj0CM^)-QF|k)%_{%W!A-35Lx@FKlkTz5fO7qw09?XXnuq3>q2FvJN@J_ zbZnHRzP$)L`WgJn3i2-d)nQoE8q6xOf(U#G(yt_3F@4C&XoOFoZ|{N?`^x?}s6DI5 z44xz-KZl%83R#)+AXe7GLNJTI<*9VnJ_WU}Ohz6K{^KRNy1r!FHj=%KAxHL+oL~#G zdHwildA{GwHA1OWo&y#38T9cY*0UW92a#l8n!rxJ4qV%45MOn{)NcUqB(gT$$xEH^ zePUf!fDh1t<7!Ris~I(cl4J!pf^&OIkH>R5VDEwj89^>N4UP4k>i;M@!H$p}?Ze-c zBdZokrT7k4=}-S)Zb;%TkTUm~=G&Yaq7R0dH9XlL@{nFQCG#Qk zqu9ev!U=F$xx_)7zHIQFg-KfJ9laGQ__dl#t)<3NG(Ulqn(rft-HSS<~f6Z$Hk&d*2pjBq%%!*+E|E!O89cp}z)RpQGfr zic=e#0n&09Xrq!;IocuH&FM^;jGWivT&O{o&qc;7qD$%$%?x4A*&KNFI)Lq(!2aB% zH|7_$%sk*@oAd4`naa?DyYD3z_nMl2B>+INA`QmG5Vwh|1g-$Tz$B3Yc# zRJ^K?&2_Ruk?f071sKIgU$X8C=d{I;e4Y0cAQE@9qmE$R$1}m6$*bttP0C5-`bzxFlT zd<44tp|pgYrb^l;_7^LQ+39(HLRZ3Mp%q!M06J05!|ITYirs$R>k3$&FW_evlcjH* z)rT8OCU-NowHAI>Iue#aPE+~nI|%KybhAhJlEBebgdM8_JRcSKD!-fyv~xbN`))7L zB)qTOV3rrk7v!sO^c6&(+#?rSiEOAct#~UsX9k!nFiY^WO&~Y_FL<_>)QdGBi1IQ= z?-+Kg8#T4fWb0S3%JacTE+K>66K!VU>?%{rZoI6xFca1J(u8oa^pd?Yt-1D8(fnvW3HUC*1zpw5L?tU8HmAC9S8{st@4% zX0l$B(G?|mo{r?BtAG}hz))&I*#-0evvg=EQ+$e}F&2@V-T_MXI@ah3dH3s}>t3OC z)+3W~AObs+>s$;{?*+)a`(QBdAtm8t@8Rx+%Tr6m^d5hAA6~?8q-Z=ysids;)zAqX ziD!_O^<6R6`xZ#aR&?MD$>3JC<5AxLlow*H3Xu_LKKnw0?8uC@5Ya_U?rc_A% z$nzpl|8}rux8Ob zCy3R(^v~9W^`;HpVNPsSeYu`o6P$jA_cDm=KE(RN;5gkwoK=jvOfoC53eS3#x*D7a z5yAv=*W=awg!Z7x)5z3Wgc(@qE?})kB45)$#WX;ZZK48po743_>N>MQ*XE_0EZy6d z*;>1}R|rqlp+kT&#G*t*aKL8VQ@ z116cO297Z$8rSPT=zcIm>a#f7SHbngzQI}Fn@N|$Ro6UgYQ}^Nf94O%XzOarYjb`} zlC`@%!Lf_`UU4jOz4SCkGOoio*1}&*tfAXt_+VJ1&!yY0?WQ}We_>c<%*9OC#sOxd z3SP|Rx~oj3Y6eH;3bC_#H}V;XL{;@ah8fQ66dX5g9c)*L9Je^;!^h@uE^;OlE4Oj( zam9NI%5!{!;3!_BE-1c~g0(A|Id+bzV;8j+?N;4({Zqpl<5=TR!z-OedqH}J6)r3; z7Pd2`hUo^rf4$Q^1>N_Y=NuM$oIQ^{%~sw1*?!s4-dWvw*-_M?cMNk3a^82vdTx4O z!&ToGjk=Fo^lthUW9fklhPP+ExQWS4=fqi}QOqa!@lb4dJl|jyjh5?pqdYxaM;+1j zrPh^}?dFQ+66VI{f6eF3c5^k$O3QHTE!$>ie>qAF*55UB&}WciwCUo4enyth)i3w5 zoIkSdiyDwEe-0sMM2^oLLUe~PQ0byDW_%p;aR)>76H znIHeEQ__=1{JE2o$YEZ3z41HpQtW@_c>U4pvkW{1-uP>7`!s% zpODtUErV(V#+yC_To3pfa4z73F0Yd_x23-w$6}ZXt zgR9S^FY>vew&9Jwxc;!VjMQFz!}pA=Pfhn7XGO;)+Y53A8?Do<&n$Y&60^ZF#OkuO zcT91aTQS<*Y) zKTGgSt#tzosFP4t*ZnDNt*?tc{k+3i(x=xgztr1ofaJaqbIrYnodrVz}t_Ej> zlnkpMo*14tJT^2VsH>@kv7Eu7zoV_}9WqglE) zZC!em%z>7>Hk)In>l*9$(ibCyYR4PYrrv?i0uKkq2KEcQ8F(Y;Y;cW`Bf*7(-UZY& z{Lp@r@`};mE3gm(`h*}dYN6;nGTn$k9QGh z$eH3UO>y06{e7csIutY}cx7;#;EKT`g55!<13Lt?(O1T6L7__Bd}NGygXvT+68LmxsrWF+j(s`}~~<p!|3sLgl?|Du-$6M-UW9ZF4GGH^-X)?{Wb4S{5l2IvfvXKgHA6w4 zr+G%X?${-3wanvb%Tmw&shzy$chc{KDH`#YD@+XT+oenNuOP8V&=XZp{SXsTqo98f>trqO7;t#75XX|$RT zqE@)<*UJyx1zh3Iv5tfGBev((Cf3H**VZ34$+6IJ!tvf=bdGlZbRLJPx3I_Ox$G?< z@AgjfymA$AwxgC^*}2#G$&uUs*Rs^y#WKqw_$TVuhn|Uwj+z}3ZMZFkYr_MVM&-+U zF@K-D&2m(TkV4agPMPW&wn=~dPR}#vPn+9(J|ik)VrEg`rY@-CkqQAY*8ruo0nGgFS(h zOq~Mq8;9tNX|IS2z^)IKqdaq*!FIiMs5vlmdwR~a=c$uZhyFdD{@&ccKH8P(`7Xnv zq8hJG5++FJwJiCzo4rZ*v+QOKiuji>-gHS8QMH6`dof^;Y)0@f3&S*Ww*PRNod1 zXJz8W4DedbKu5jvU6gltH+#~_AWw76ayD~h*p689S{`Rs%ee>W@&IAziW_Awa(1{yx;Zfjp^4x?$C zs+WR{TMQC3(!Wez>RIRN<{a$o#8l@3?sA?J-m+vFs*>?2Nw?Axe9oSdA9j^)n%SD$ zniy>l?HSDiX`c92XeNB2{&))?bF23v^9wGxJGik62`Ji)}bBgn(bF6Ee zyCKZ9y*x>tP2Q#0JstXVx2y&qvsKPV4q-fYvy!~g`=954`;@bzeUEjFrKF{XwYELV zRnXT)^VC!`Y*~1@kio_=nq`_C0XM_?$$78tFN<|bHD44ceDDS_O`J_;Hkjvrpw01`pP=Jc9y0UvzZH1SE?)Z6$=Xc!KOa+ z&U3$WXl&cfT{F96t}-vSF0|{Nsm{DEuk#vfcEj1)HOD>Co9sKJo+ioKaKka<%YbF3 zO{U_eCIPvO4%kzw==L(-d8Dqa-lw~$Ehc4Cckul~4QCMfdG3cct}{L-a>cA4NwG?y&6w(cf4ymeDAC6 zi8h06we^42@77+n5c_+3nj_u0#wEIoxQD{1dES%m)sdM@AunkJpI!v4dUv1MYxLNi zZ|!PZOY0=-ciSpwKKYH%N59c@IPk17S#wdnQ+-lX$22qSY-D_7uJDqfA)%R}Tf(%V z>82yPg+f#RFEDRvc{136Ekq;^-y832=U?-Kw1mH5>FJplt<~*j`)kJ>_hjEvVS(nX z)~Q)7b%d#Wig-b6C~eU6(AP2L4k;GaGBhnP*04m=Nt_AiN(H8={ZC#?_im!Rg#1N{ zJkfj4UC*`8@!M7ti5X`ZY^&`oVE2`>o#g@YPN}GsTT&)-`Sqt&G(=?_4wm?yhC93e#Df6%fwbzXCqkIW8KR=b>+PN z!>Shi_E?a%d13qMDda*=7S@&2cQ)iQRx*AubgWM!rkMO8x0dEzPk(t~&;xy#sC?C10q%!o^fSRv%+!&|a)kFK z=$Hf4@V2{a(Xl-a{(z*`MlQLS_oGP)oc^hT$_EZ>z{v{vvmA z491e&CH@61sq1Xzktw_HKe_hrEY* zrBQTN)CE(T7aZ{=kk3(ItM95x&;va`tDiX&4ilBwMKCk{E4!QqpM>n`!@3-U$@nBY zebSrD+ZMLhxp0cNf|-A^*Nf*(4u`&t>uAeqbO0{yN7z!#1HX8=>-t10(~Ss>oK8_Y%vcp<7|6D9i1<9toPzyINMuO`>mg|4AF!g6sG z=(mS(iRTl?!8fE4o*|v1!7@K38)fo6f=QCmv&rqJz!y-Q zyxJbRmAX>9DupfjAN_KosYr zD%=d&2?Kv$4TSp&o-7y~cMEXoi(!w*4r|MKbqBbh4wCJj&$CP=58NLt_H@3#%zI`M z|23p*p=j2uvDz?n4FjLxRU4UEu@T(OV{-l@$W0HXDpv+#}7yNXFE~nC%4W^(du^7A?FTgf*;H{p+Nq2*f z6?BR>hMA!>7^1#R18oGa$_75kw0V;~?>%e1gPD?Df!%8hkI+>r42f`^iR78@gUjCu zi@-qY98EYq*T}{e<1D|XPPLVa!UXUTC1IAz&5FN(g`zfb=LP1Ugi?Q)O5XCL>}3AM zENWcs!Y-|De2pUK|_!9a=ik zzhEOD2>HY&VnulQ&%$vtg3P)RzMJZN_km1kCVBpjV6Coka(c_9<>%hXoX1?=Xzay4 z*tS$mQrECzU8%h1Q>l55*;yR4qR^0*2(!4K>U$xLPue5(f|Wjxv_)(zhKm{8bvkT1 z4$i?dIy0Wbq0>rU=rwq6dhSwXf8+7Pk`>{F7T)9bgnRQc)nX8=S(m)Y-aOP_PRk`> zD>%lPEah{-`d^NUZGUq2HIa-y)IZN6cV&EY<^SL&>EoFL4$bc@;LPVpv=6sewujg& z+8f(9*+)C~d6U%Hwfpd34{3IDR@ZA(0b4^>M&ym!7r8d#On8s5x*>^y)dQS{gN7tS zHbXXD38^z3ORae3Wj@K<-Z|d7AhS$H@eD)eFmo>JZ`&8gAXk1E_U|&YMC~f&Ddf|t zr;DvXXXnwC*4H)+g-J{YI;Bg1!!Sax*LBve(GJvEbkB4zv~Q(s!dIWubIH};>2b7z zIB*ja270-MJ4ZYI!yD=4hydRh&ism_t_|*R??+IXqo8>$0@`bl6o4eu6$?nGG^2Eh zdW~V3eu1u%_Jg!coFHsgD;+1LRqM!~pOkNUo1k0U!DTbYGuhqVwbX zO=}A^s}O1=o9Uk}qHZIMCbp@lZKZ9aovr<=Eez(M3-WzYTMDe_O|UM0>M+%W^XgLS zN-DYzLEnhj%2288)@Zq$kVmu(PS~hMcF$ z@_`Q1O7P5_fGI0Ueg)2bGbeSSd|S@WxlID2c8yGt5A>J?{8(Lgq`QOb1qi-bjwnYd z`!8EDdo@QDS4D4t`mScYp;JI)z(swM=CqVkH!t8rNRLQW_IcU!WP2MSgt>xrrZa}4 z`sw;E`Ww2c+9x1ZzxsxeduQH{tDd8uEx{UVy>8uNJ8s|aC(^!1kG zz2Tl#u5yn2wwabK=IiFymgTl34xjUaYm3X}G&kfJ?8W0z$y2>X?-1`LIXB(O&DDQ|@35e$U^z4CQjo$k@X@~oA9Mq}`f6P{U844Z z<`8etS}id1I|H4xjh$IYy=Atyho^$Of@`Joq_e2&F>}WngPL0C-UDJP!(9gEHVv3d zKiJ-J@(v=N|KVqxq=vYVo~nPTx))WaQgt3pMbkr7P7BAuJ2*#z;cmF=OP2Rg|JVjr z-R>Ifayw<-pdJz*e%A=>FDNs<$UP=&-Yr$r~Oyw)ZUdIP$Aqc{m_pHY!>=6 zyiLTVuE`K}kWC0=pUKYO{$+U{o$M)7Xmt7OWcR>+Q|$e(i{} zFSTv44W!<2n=_Z;D&;QauH`Pl#FK~M6olz!QF!S zIDMEvbH>wF-r`GuSL1P3-7X2d%1`=^BGkEW-hMW>e1Z&wamU7oqGgIVc|AT^GuhA{R02mS7Q zvEw?J8m@ESbnHPf_=x&|;dw_?Q`NT(oIyD63L!Rk?sw8(R z4ku49YNB63C}amUoPf7@pQ^WvFDUrG`AWl*To%ki4rMkbQP*TNen~2FJ(SypBj8RV%BBgzrt<;D{YJ;>-HE)R8C7n~ZK`-l{>$Q5X_L}A$lY?GLOEp!reYHn4 z4W;?yO56G8vL9`W8ALU7)Et;VFHGdCm(bdo2F407vE{_;3~?ady1T zLezd=sq&IXzmCVUj?ee_vrI3i2J#0EuxRq@e^iZ_Akb1cCD`fB+5(@=dRFWUOd?I_ znmfncdclFxpS3EVWj&`-0J#KvX$%#~Hu(FZ|35tZM)<4)eP4V7$smou11U}1)<@k2 zUbz<3$QMvAzYnem&Ia`(vLB(~buWSzO~|@-YZx&efk_GTJ;WxogQxL^_dj^9j={ti zjXwT~E@%KosJeVZu8l7?jQ+U~eD0vu`;rQ08Bh!FU;|2p+hsCuX=aw}a@c>4z-Kv< zDXuH=iN=9SjZk-kA!r(0K0m;h6hS5pbQlg`4b$n5tfwvwC))$AP?LVyH!$|t=`l@$ zP33?1j7j{r2m0hY9xXZzrkPsm9sGPMygTvqT6P9^Vx;capLbfsdi`Q1(^jH@DzGEX z1fTI;e#?4of|0C0c$7-`lS7fvRPbWEv-o{Zu$y+(Sy&ZzGJowzp< z4kn=&$S5_~jA@*(f3e&N^oGvo_j+;erx4v;r{-d#(rE;1QHajQJlKohyyahh7DYEo zH&7uv>5SfkhhLWy=!fUm#e~=9^iB5X*>q@-7I4eF%JR9Nfqi%eH{3g44(@n_=jcKl zV&(gfAU#~ZhD0$FRR(Z2Gmx?>X!{pnhSq`>8i=g>RBhSq9<0=SqLp1dbtqcv9Vm)u zP9h!+%oypu_t=Yyy!8#9qbLZDzR0~Y!EPwA&np;tTHt9`1TAxxzPZlyD-P%CD_|C? zM&Ici&V<5VwVulSMq>Fi_;VB%a1Hk53bAi-g#(vZq&6$>V-;Q_RX@R>lw;k8u@8at zb`Awm7Ef&WBWop6`1cP)r*G&vRoLo(!FOGPh-)H~F}`EbauKKf;0fi#}%yU-&Ula!dXE&?7mEW7j(~d@Flfm&Na+>#Z-8XdIKH0i2>=T~)6%3+=EcsH=E4+`$uPs^EFEDDI@ICf9K>_3k zd(#AYsX_iMFl&~@5G3ddea+3tO*Cg`E)dII10{5u$p9C@_8kI)k&Asf1eVZ3jCug% zmJ+XiMJwE4wZEx!$XFjZ@G251TfkBsN3upB5pzHq9!D-B+0WBtKmG?Aaw44F|6|3E zB58v`XB6h1J?PqvXAN&43mQ0i`f^4`vvQ?amkf5|Ga1D+kWF8Bo;{qs0sMXkVt`KQ zm$Bd)*72T`*oB=u^)FDTm66BF?06B*S`4eJobaNYvIzF90eh!m*OkcuZ&*?Dq}!e8cIgd0v4| zf5y43#7S(-lja4BWzITlZ;|sfdj9o1O(pbSCp^;_@HzKb|6}aJAAe1rXFMLtG&G__ zKff~Lt_XYCk^9%;jf?Z{?b(ZotacYF2}YO$E~BFrTlkgiOfnxAIps6SP;NrT=kZJr zIqj#B#U*%V%0!RxNL4csV~w-WY03piNt1cX z)9lk4{*A0LXXQ&4HV-%>EBGGmr(>0yaPFsyuebXiXJ&Bxz_APxCBGqqXA1t1jL zf^;;Yi=$ZQp5RQGrH-WSX8)HVZ>!jiUPxbl^l(+U7)tTmZgkWi?){$q`NG#>+{wcK zf1uf1?5=^k=0;8;KqqSXJLQ}}5`A5Rg z5()Coro;z6n>+zL;Y3z{D7}}@b?k6gBs_|4ABD4f4N?`&Px^D_Y^>~8 zxUc>}_T$N@Rp+M({C!T4gj@J62a-9G-ycLw5P;sB3eV6%{OFe8J&V9PGL`*qhb->R z!im;LkHzzbAJ}b8mgQ5iGBONeQ_(9E*zxI{y>0wUdu&)k{>=`rM=KZ&V&Hq&%;`{A z<*y;D%AArW3KI_!R)D!wWV(>sQ%KcjPR~K^k%zZw1RC`obL5AQhkDPB?KO!5HxIg)^s)b`t^A6KHnf@=P{AZ zCHx&7AHRI%c?yZ^4TAZw9oPH8xp(vV9&Ip#b5It$P?&7?3oz;m`nfWm=ycYsE2sAl z*5)$n^U`0Me=7-}%sA{+Bdl)@-f$e88pZf0cuhu&z&oc1BSU0dY4EYGIqsd6%pB+a)74dr1S#_~1~ z$yHmB-Ce9)J*4b4YxtQR4&Vfp;tZ$o2D`Cgq3{GeLmu;RqF0g=mx+R^!YMQ#87z+O zSb(kyhjplak2JJJraz!l?txmW#d-N3Kgo`L+``H% zK#Mj+Mt5LYmEF>^t3B|)Y7@(T!$YdT%Kw7b($8mJyy1bwtS>;~NZES4rWdeX$duH@eW(WC`=voKEDMC7|9r+gGV zze?m5}> z_C0oYKX20(8LY{|wQGz9?ZH_n#QVhXRE>$d@^YGDS@T-NT?Xtze(oE9)>UR%9%fHR za)y)8cRA1t|FBv*WP1@S7zQ@FDR0z+Rhmr4N~8L*F!F&FkaSzvayb z@g_6zg1_M#*XJn({?5T9@N`b$4OagH>#OHECvy5{v0m?y*){BSE-Xt9-a84IxrQBA zJokU`2o@nB|6wZ&;YS&HlLp8_MRq?3Ng2u6eMe49=E|$^HTz?Wl?h^BxZ49H-G&Bw z!>KEZ2NlZdedfd}6OxwWKXgL_^<%{~FgA_D&pC)Mt7V<@qq%m`7tw-!9gno;;4GBj zJwBjWw{q$(a?OHhs94s=&brUy9aoyqEX)Sv0Jy&_e->%`=qlq9^V>@&4oXt6x&-jga{_+gNnrB#6X_aR$ql^A3wHDuS~4B)2d)V`+Aheq z!nj)+-$(I`O#D}wv~!Ac6^GTDiWTTh+^k2_tLWG`2BXhNsw&;+DVq=9PeWgMs-Fdz z+|e1f_Az)aF;rHSuE}U-w`^jf(;FgoFHznX&=C<>%*M!4Zn_tVKr!)H>D6bg$=#7Sh&PO0mQ#WX*(xecw1 z!;LA1Vfv!Fx%4(R(62GHH@-7;*I(9l*OZgii1AF_ybiV^kN8-<7|yW{bSvGX%R%Wy z{RPKOORtlu2GO1SgMl1UWhxZ8sUR{3*w-H${y*$fPIWah0kL>1Qr4W5 zkHTxRd2N^sy8yO{c&cS1$OB$NBShkz74>hWviy^~?V;nV2NMvQP${WH^_pZRUti&^ zN>N>1O#UUDoYGSEFPf~EGG%rnoZ%PMf#jmr2veBsxP+;pB3-UI1wWnoYsvZ@!*XY$ zsm~H~j9~TBK(DBL2bh*K9**!O-1QgzeM+AH5}mX|kSUKZ50-8M`xbz&HXJ$LgdR-9 zswJRr3nE4Tankm2iU$(W|D>iqfiASC@F`OHfn{Yg$*1p03bIg>NouQJgBa0B$uVmjB_5wXoeMk;4zy3QfD z52*5Oq`p~@^=e1ISX(sk3-tp!SVd7o_sB?LgrwIEGF&$vMS@dP;h%Zk0#J^o|%0_zcew&pUSww;FbY zukLf6Ti$(48|g&XhlX7D8M;pUsz$?Ix=vWh#KeNEdM=>|GdNR)9^w${t4yg7F2Qqt z7oD&QpL#id(+aZOu~cmP($|+yE&{KOH~lt-Js#q#XK3yD$b{G-SQ zZ}8P3OZ1icuNvEQniKcUHw5m)mT;Wv=rGDop1L-Ft9TlEByOld+$?oZtpGNwR-N;rQa50PEIf(W0?5T5PZO35REOk ze>wI!1`DFFyAQw{%T!fIeh)Ey@;kNTvE*|vkR6+ftU1X5M{x!{h>x6CnsVDO2KVa+xLN)RsGNn6F3={@C$3H_NWcg5bEbAG&#k5APjCv zdP8T^l)zOcjj@L=4lGp}&2{Z%eIDZPhmhl(oo1JGvaLzN^^b{!E__5|4^b@h+2x zYS1e>f~j17u>u{W$xQ9~&2-=mFyDVsrI97Sh?nsIUwAJvQJ7OykF4%&GMIBgQS?V6 zj^?ba^WFAISil#=eG8GJ-FU<=VFryuj~trr(7d91CvT4wuAN z&UI#J6&XxqrAKn62HOs!=pEZ}gs7?gV!| z&mnJ5-!-__FOfa6AoWF=&NN9lCC-O?sG#OAEH~e<{ezg_mO>S|339PRyurl!7i2}w zbJ~@eU>ThI-K=S(e40t-ci|sM@qPrk@|ph|S!o~t4MoeQ%S(N=k$DFm&U<{A6GX*{ zs_E)rG+QA724uR;H!^j>L0z#XA13y$3;Bx6=-K-Cc;|@MJCT#k02{UrJjNH+au+?h zn?S;hqUxh)p_;z4avYeFF1|@HMsH*;)H`(-;S5apUBGsY5{in^(m3fK>62JZOb}ip zm8VeOUg}K0h{D3LZo}{+KG9>|6`kz^(RH0laXxRTH`1F5KD{lvPb>FC?+u~zpb99M zKKK)3h#wjd`F3 z;?vM=G88@iQhttQ>I;r#64v|?A0o3Eldx6IsR{MLZb#$y(Nc8|1H*rC27T+);@oC>f7gd6?ZhEgu(l(g4$(~Z~1>l^3_Y2FCG z;m(Z4XD>~Lp&A$flkW_CU*$bvo|o?B?jGQWpVB8Y1sqLT?+DLhn0%&t?s$j$o{&8_ zi*2pW$%w%puc2NcOr=xtHItkVV^zYK74lW+EAFCGtudCRo^TQ#=x5X(TI1gpAwRj; z@1cjHGS=r6(9*f_CK~gu9kBQ><+^D8E%ay(!MM#*h2WMFq zI3ddj|5BNIt?nz_=B)3h()ECx$wd6ny+m`<$y&EUTBg&{eVTfjhZ#wI`Iy1P7!9^; zF}>|cXv7xS#}H(v3q1X^)V1ie)C*6L#FfD9pmoCrY-mcy{)Sr)mY?zG=&{C&_j~sT0b-tDG?*`&; zH^zS+hW9%k`;x961Li10ctr28PMCt0$&SsrPtWaMaw1zc`DqP2RS_*sbJWMwasL4 z<`MgRBu~=>A8-y>#iQbNskbItQ%`CthKiM>OwDTD41F*CNL_94G?g{YH9a)*K%TFn z!$VKk+W|5hgNag8$r%^*pJ7)06W02=H^4jJQ^E7goeoD$LDnOu{GGW)My6AB^;QIP zVIr2(5TB-!YpljIWLF>Hq&^nR0=-9cOsd38)?$?~75rUiVJJ)!I$@xC1sJl?$bUSM z;%c%qC-Lzo$#0o`rS#N>b4He6H@46TIgtA4QM~`@zCzro5_WtHQDA+%nmBa#FeX?P zLtYxddZGBa${fKqRJPBc$-&sG*O3n;ONP&`u!-&?ewl?{I4$R3#S@sH^%hB;f*w!7 znm?06KoNwJ6_`$xnF59NiA&>UosKtfpAPbCRNesH5w_EnTdm_zZGNl z&Lk%AsMW1iQ~e7;ryc-NbOC>*I-SxFutZn*_}9CieLdlQ=Z)gsqIsg};1&jeA4|cD zXobHc`==58KcM>bib`L7I)~@euRUEoPd!sznJM(k$E@O?8Wo>igtvb`u{fj2a!t zi^P)UJ)kP6u8uU<;C^+O9VIYrv9VMTey9UtRbjCDm3pkWMpIe$PFG!bMUxKZFj9Cf z)RT5<+G}%a{h&m02-(zEh}4@h&#fmJ!BQ|jeZ-^4OQbXbTj z`+T(W7cBNmZ1o~4C-_uvPl5)V{G@&d^R zg%izA!rN@lPbA_#CZ3QZ7=$z{EJ#MO2N$y9h1TfTMO;0Ci2en+|GVVy&XB{h!K8eg z?9yUlmlU!C*T_1ML+4o)6v9k6Sl;mOSMiOF)M-W77fj%Kb|K43O(_pL@ga9yNj_m& zR@557EZK(ON0aGtAA>h~i50ox8w;mJZcw~m_~}+Sa6y&xPFkwiapVqiVEGS`e@P%$ zeHuhad-RPH-n3kB?LOj6uFqms{S2Pv0F#fafvA2*>{Ekm^;OmGi3iu~yrD)hTpv03a@D>5hf$%CchO&q5hqQ|ZdU@bkY z@MWIQ4!`nZys|0CXm4u!{m2W=Cr`8u>ve}~-X}Aa!sjEh^S{9}KLqXZ0zAcU&hlt{ zQi+UEVfIt0<~8D-+mlI#{EPh4cs$#Ee^}O zmm~P}bJ(rp{LV?%QsG}IKrY=$E+-Nl(~%YK$6CdZS1Q71Q?hMc@ZTD);^*ZiN=!Y`Ufef%e|YBnO5R7C0ZNKcXiQOamd3YGRs~0O(jch zAbvMd1=QlTtFY)r`0gF+w16jC&gnYDYVIYob%JMHOci1U=WRVdy~yu8BnvH)vv@>S zZY{d_Fj@N$B_^S>5y0uV(Tao$(;{;DNnmLe+FuVSy?eD8JTZau>3X zIq45+i|5e}zhwefNvBsZmTNT6s@7b>t{TbeB@_P(XvUr7cQ5c=4&)>U>(UchzJ&}M zk@S9?#tX>B5~{=d@d)Q3hkbZQJ$lGX42~j~pPLNe1wPDJ_wM}7CU*D$ zCrHuhp`2|qSJq;wx+3*sS^oyqC)%P9_VNT>IZMmHv@B!iZgQr+f(ILdf3*l?*;p*} zR3Z)^{8`n6szf^N;Pxmkd}EU4EBwHpAnlfu*QkyiTt)U_BC=YXER%=d!o)TYta$3<$kp`>g9}Z5lDcPjf zbZyKhN7_JMN@vI_Q08AftzakFMqcqOEM{Y1gX#@nCDw8L6(KEH-7ZkiCoJiuTF z4t+2V3v>;pA!WktPpVQ+(NzVAxkiyUstfPYM)H>@IX%0O&u`Q{67d$kkQEw@r@w&r z-3My9A6``v@^=YvI=y5^Yw!`_FNxKB$XyEK`E0@tD3c`qN9D^x?jwR2qXHjA!1?7P z=lTJ=y&4Va!~3$3$5tj!WN=RAaFuMF;Bd}d4Es?sYhNn!EbY)360H0c(Y)30&o?rw z`Z7{8L^#W}jl?f*^-!|A0bQ|AI z=I4r?&||Zo;QQP~?_5TcD>kG*=E2t;h)D5Uz#E|usVad9QeY2s{R-!GJP~*LbgpA>@4&qOhg3F8E zWqG1JfPB|TdT$odTQix=-V}1^`-thM^8H$((kR~VAV|&rXqeo*^97>6O6d3>Nb6c+ znQ7?subiS#yskLx!!S-n%PhZd5OyvI|9BCecs8U&VQ%b#?&*z|+C-JGHE%Wtzg$bM zWfN z1`e@kX`hrVJ&;C7vba+8pmUz1p=!geS&houV{GpuJp7Mna2s!0k0^XO{eya+kIti8 zXq7HhURq-%p4wdA(!tt=;So#3u%+>& zwiAE0<+(?aDH%-0z9*P>wfZu8W)A+&KNnY#crxr3 z`f|$%X-wp`5gYCx@=!1kKk)-PP#?OE2ebfLKa7kHB>v1sJ}(Eha4fdsHIliRlh_3B zPT>Qcgx~&%TD~2xF*|-$DLxgnK^df`EY_|A^3s!7<^-Nb8RSb@k$s$iL3o9^m>P2s z2G+{-UB?o;)g?R9hOdUf$hv{MtmY{e^WI8@IUYYFmFPf0K+or{E4izl41k{;u^n&Y z2m0zh_?J*Tt`JahKG0IrIFnIKVcV_tQ<-UiC0-(Y6v~N<(J`swC2@l|9tmrX))^>v zh1)Y&d@gJP%O{gto`zN^gMTm&FZ2}|nrX;zP5h^8JX=?ELP6>e$`tI;RHmnp(;3YZ zKc=TC7H{ed8Z-#+?YZUUvEzK83(@$s8rqnqLx58Uicojb%Pi$ETtNfM-PwC~_$(5>L=>y@KY{GRXU?)4`<%e*hzLV#B$$Ne!W3!9A zPApQdWN~`ooh_oyw~+5!gJ+mZ0c`ZEcfqW=BY9aZY1Jnsuv5%|CEBwTUMBuMTM7kf4kA1wyBanJga2h{F z9(M6Qikyr@iUuP?W69Mh$m`o^Nm^UCvJy@B=F_68qekIHxuCxhZ?9Ctqh1jnYFj#-6 zitFh(2~}IT!g%r=ZOFTqC(6vuo8}}cH{!J%AudypO)GfrT}0O7S)1(mWDX)dBh@Vf zHEu69=^dT7(~!O4ph`xfwI-lrSNV<*|6V{gzY-ZLGrMw;X>j3xOL*mZxQ-Gz=H!gV zu@i%k>-xm^YI3wD{xt>!qQZ?*h}9X&37v+9KaNLiCR$vBemKH@-oStGP@iCa6YH=G z89a=wKg*u%<>N5_KZ~s0AcJ@rH1rEHG6qm{Pm#Gz*xbRaV=1Iv04cVaGccD3w*pVl zBx}vb;kR^U{Y&$l#dwaYWVFhHY^slxC|v+!*}rB)z@4!5Yk8u1$W9wjz03*a%|Mi7R81Y3r*OCru>G|?Gs9ME!tm+9qLW& zx*N%y&exS#_a0n-Gd6q}a->L8PxeO9mt*;zd+5<;cp`@$9mJ+O0MOLL5TCq94+BCHG19s;$tFj(Rd&()Yp$9)_lY^LS`R;_GP%x%T)jM%L!~R=82ge! zWO*LjF_TF19J^;jhrP!hD3&^zPcNU@@liBnnp+@1+)u^d-# z#apyumqYl;8?tMk@vD?f+9A&ML9E?2R$({mwiWw-6|1Kp@z3KK+#|=cA?ws7WLdD; zS+V~{G|)`+<_JFL@>j}C#lu*ug4fqu!_@)A*dqLo&~-bd0@a?{<dXB`q5W3z(@l6Eld$en zvb>P(ypzJfy&k<3$5Zy;_qTF|8`!W*Tv^E$DjmkFCMlBsBOT{EHRD7ay@ve!PNA^qmI%XUFf{h;83Z zrbCH~lkpRBsU7&WH}GzmW6e{|!Qx&b-~JTe;w?}7nJ2i-^WRkEA-{B#XVa)JVI$tF z@)JwvQvc#^gNTW;b0+JsW-oZ6|Fd)!U{aM|1E0Bbr#D%arD5srSh_(%Lg@wpL9i)7 zP(fM*0TEE7B&EAUI+pJ4USf0R)_lJ+|L>V+VRvTkz3+Rv-gA<{`avlFH*Cbe;gzFs zo*xL72C~C72(PhJklHB zY8_A5&7DVr_5NYKJ`9X^MU!;py(zpVpOjnZILch!r}X`~3#LWN=#Me}2t7ATM_A3o!Bk z{*zwq5BOWoOWuQ?g6vsNRAsU1O+5c2UY|qb<9XI!Q1)t8HH$kpg?EzRxKH4<-pJdE z$n2a%2dkkmKZLtvKE@m@tshx|aL8v!>sh?UvWpqOVIC6iP?#b_HaU@AC5U9iVt2Mf zDt5ynk(u)jWLj}DkRJABva%TX{}I>iU~LCz7)7E2M;BL?Vvi`r9E=G zD3-zlIPWR7&~@P>iPDzgo<;brJ=Wmo{Cvmra+0I+a%Dy2dv!j~ft-GVCOnK3dxz{! z;F+cI+?kIJ#FrqCFY>9xz;D4Ba^C1Q`mHs*SdLh)X49Z&pwVh`V=}2me^}Zx; z=nYFBfl@Bk;{xzJV5KWGPz9RE z2Pf48YR!-{_0S{9?EQ8)JE|hv-msUaK=%>wkTW`Ofya7Y|52+i(-Br-?F|Lza<1(z z_Q+FgXwH5?O}E(p6X?ecuB)Qy>wx{jVE3ai-N@O{jy$g#bkc$+%UUE3A(M~qA!kd5 zGLlK8w5dv3v<%Jc0~VBf~mz2K!GQbbO^4hv@t znuPUuVV)=FWP0=TkD)%9uG|c$)nf-SoXd^pIypn<2FH=$wHEx+md{<}*dKx)i#v-C zmw+X97nr;OQVLhfDdL>u#xKDwOW>A`Xt0gp3fFDGYCG89jU8|ntc!j;1!S`5ztEYL zv_)&zhNAO|hW!!E?^)1_9eQqr1wI_VXau@MqWt;bifXK?Gv412{%r*(M1z-1 z?20Z4v>oxK1^cLOJKhQAKqs}J^z06RI)cai6&0mV%4bPoRf6F!x*{cBk5dGP)t z*2zr%I*Xi-<0-|UMd{*jgC{>`brpbjWv~^5R>zu@O{?1b<6rkt0NoL?kXpPZs1h-9Dd z;jHI;GnQ{A@gsimAvobYoOTDuCPRapprt8z4M(8kRKEQZoZJT5H=sF@q9Vz2b5F71 zGq~;*aEjzF7Z$Yi#}FLZkwB05w;Gzc73*+<<2J~+)w==cB6iknRIEzC93oMdE^;xn~q`ErERP3$)}Md=6;T&wC!A$jA)nxGeW8 z0H4M4$t!4V5BCv&>H*k&2ko8*rw_ngi24l4x0L034z83w01?2k9GDmdq^t3}8n#jH zb6pCUKF6xIA~k=-!x5>v9Qu8S9C!~6TxMq?xl>rP^sXoYP3c^53YysnO-_b}7lkc_ zZD^Ub(DyHJ;wWt0zme4!p(J{=vb#KJlXLLG!tiT8&)5ldFU5Y8`l?}EwU@Q$2V(j8 zwg>LZ;Mv=t%VX?Ms+Dd*8Ta5x6M78tX(k+a4!NcRW6_+su_=ZEy(-AFBH+3?6jd9o zA4FzL?bSNgcbGAOQRlX&Do64RQypJR7EtxyY84EdO{+d(jjyOJUbg1aRhw% zd0HCx-N%}Cu(D&UOXkRUcs~mHTM)Sr&%d!y)hnKUf_+>7jym-rIiRmPP<(0bFQ=@_ z!C_5#r#e(5=ZXaz&-ihp4XXUVmJ@qds7fNC{owP~Xb7n;FUkKEx#J_`mCSRx6@G>t zIKKjho4~u&&$-z}O?bL0_fLRIr7lgwdMg57^g+@JS2W=+8Q@!Fm*@>2lqdeoUHhOv5xp<6ZdJDE`I|shwd8Whm7zyWPtu3IH zR#-xffPh$H2}qXg?4tq{^da_ISF~hzR#S^lCAL!fVPhTKrufI;mr)l~u_ZTLMWFqU{u z4t8CTy_ZDFK8G8ZpyhVMx6;q#KF{0)=S)MNAH#aN!uzL?G*ToLdg)k1G%N8c&P%$#G-PE`wdZDUG8^O_@W}+5f$)9{fNML~I)RT9T_M`ejkGBo zrppo7Cu72rx+?38MJK6Hlvw%F%VaBjxed;|jVws#jww9jt4J7}l{@J{5}5(_SY4Iwyt0+hwVb3mEVtW+jE zNzKGFWP#4}Zu2?|)c!_B?MJE~05j4_L~ykg8E}z3%Ngv8Jn<-yTMixV0@hF2hgcbJ zfM5iapAGtxJ|!jjKRftPc~4f53%Cha7`zh+G^N+t3i3Bgi3PQR>ip;enTC0b-9HV> zdFku)8tB@A_XBWujuix1??W(}%O;op5k5k{j)W+9a% zzxs|H{Ks!s!1uGTJ`rzp6Oez&JzcC+D(h}?rRXuC>;O_;=4zeh4u60pv6}uwJ|5#) z4y1VslJy;svsgtL=&=%pb1;6MRf&ha72I54?=mYhnZK{_4e`dV z0-@{RI~`dpR`zS&6=^Q`-T~iFXXj7BmP*b0Ht;O)NeD|^(G-dN7RU1vSYK7>z5?&W zgFCTBW1-%X$erBKVL^5x-_wXzh_*C%UNkHHfE|B<9rYzPM8)uToLE!Gps9Q0xbnbD z@xZ~!{WuWE|90%KldSnBd)N(K-30Qor{}Dy5U^_n{Y%Cu2Xqw8)s=x!9iAewv3lqM zAJmi&xaVPwCUDolPCS2A=X-LZd?@Sa&9`M*VRK|$S7@&`{2}$BAF&%Zc+Z3a5?Omv z*r)ygJT&BcJ=sGa?opE6Nl&a+cp`nM&RK*%{Uy4l9{OQ4_y3&Lw1+37p#6M&*FpEF zN7$@lflGEodoW}8~$bnACZ;291+}H`$uL2Q?_ne1TZ@{nDS*J+e5S%1dLjeB21SC$f zUXk3=<7)vcIRM=*#HJa|YSyreRjm64*QB6{vVfx06pCcO&K3XiogY}OVe-lk_lVUZRPGA%O%d*%i}z}WCA9c322$%W(oA^w z0T74)#}4i(5jHQ>Ecq9e-}1vrWq8#D`vww6V_gY6zbCZa5xVY%e38CwF-Q-wHp)WZ z?V#>DtgAJVpK-*0<`UWdmhZRW(>z!Qjp2}nTqhNio$)N&^HZ0*#PgDRYUxNNejP2T zS>qw7WEYr|999O}#*IJs8a)1mY+MHn_d{3vp`$BMmqe9kV26&!nw|mvB+e)lZiH*P zgZNb0*roV(5{Z#sRw_^m@)QO56y#Hd{fiA2fsV5w;iV_L~w7B`2_v*lRr8*a4__0J`UFH~+zF6;g8-ob@-~7(vEo7P@aEe6j|f7CkO@u4L<)LbLhUg%9k#02?`= zC+RyBjcm=wnq}=nh^36DSJ5bBrr3j3!k%79WMEBbs|*xX0?1oP@+9yhwx&d;Lg=p& zd|nfplKyF8CrLcxJ(Bzm>wE|`iMM2h3UbFH;PVyFf6dNsK^Z5Zt@B8a$KXZkD1`qd zp0)?B-@wl2!qF3vLfheYiPcAOe;;%sJ;bELrF5E;x&pCc3-Wx4waB*uVEQ)SaYHo) zS$lcDA-h$I~~2S1(Zl>6p?HKB|ZnU7DTV5AR2E zhB44#8t`3=y}6E6CjhY@kq$+}9=7B@S3!4IS%Jp0_OOf3p~?x!pZ8pC5&4v}DNfcI zfXZ{judVo7ddt-b%Z}naPhgxqjGabE`3B(X1r%{9dwPkNHm{Fj|8u_M@7F7@Fr$!nB{%Zfuc-MDfvaBdC$D)8L~NZ8K6p*(cflE1_%@^aTG=&=;{?7`ED z<6|WAdd6NQGEtH)AG3k_di2{NUXOU!gvwHQNmk-2a<~b1lM0kv(5qO~g?L_fR+pQV zRSeT+d9e6BkrlC~rNiHM$j#+cp?%M_BAX=ggQusxq!bttOeyMS9t1Uu)Yj^ zB|Q-z0L@C=U$R|6XzC<9{SdAYJy3}IJmUKS*5wW3Cl7M>EL1-OPXCR29Ot`2JsOx1 z85zxrb0FnaDEJmMlN|{o)|vF%yT^0iLl5te(9$Pq39@-EE07G|GA!LY&_e;fuVWSF z0IJ2gx&Sz62v%#cuT-ey8IV%o9f3<#K1)ZUONUQ`Youq`|6r7hmAvJ<;vq%xl1_ob zy$|>%^VaCvQxbetf?smOA$g&Ll4zE4?D0N)E|$hsbYd<(HQ_wzhqRR|@AK^U{Cx?} zTp}bI{85W_%3W+|>ziRN^|DgY1>ytTXI**0NdaIG3sfX-B6*pW-0vE=&jy?*;fLb= z{Qm|Ge-()k>>)PF@aAUYxM$QKiHP6t@Jj( zgyi{OUNn_=B-3~R?tX!O%nmlBJCs1b2=}SUT4aVwY5s2j_8P-W1%Y#6tXt`$+W`J? za8)@lERl#zb`lNmhtMrzA4+swdQ8Uh3LrfME3#L~1d8AJ2pDBR1%kU6JlR*s7O72B z{@)%+2Sz&9O>N}CN8rbgtoS`#-6T?QKX|#xkI>Dp#MB0btIF2$9{t+E*jA+Acl>OI z9;Ft;!zwbt+eYlPsnF?B?w$dUNVZU7!|Sjgc7J?X<)5t>SYing*Y>4&_V=NxBsYoNOMXsG|qIhSmva8eK%A^8UJ zf+bri^3Q?1kkf~wh`Dw|28e}Ff)$B&6q_oFJ;^CV@sxdhCU`3Z^mFmIV96eqni3)N zphHgrCCN#j<^F}ic5&7xxE0Hs9v;A@I?}Q*@3^6#2!3)x#r@Ik?SVr{b|f*4eC)Iy zzgh5vbccLHuAvMvs}XmR$cy+Se%=x7UJZPugUg3dw#4Z2qpb>X4~g{`Wj{T!HOJv? zeH6w|E}mBc%`u*+`*iY39l@K(^lDINd8k&R0n!I|C)j$#?j-x3$djb^>Pu)y>T@KY zEBQImi)DFL2RH4Zb%8;4R^#J;iBFYewRy3iZi1WT;N=o4%*$tze^2DkITi%|WF@O9!niLANFRnPhC1y_ic ze})7}1u8zCF7;tmfmi@4eFV*l^vi-D#QrJFPkC_o0s9j_Fq5aH@u|cX5AcltS;G=N zmP%FWX(&0cYskM{!2SsLIsz?kLLy#ZN0Ntm&FXIO9O_D*l^v1DCwD#fk&m=>p%p3pApEy-15n;6dtTGehx6samYJ2KX$`-dcw( zQK?(V2|Y!#PvL;P@OINMe%d10TC;!gDGl5Knr zj3kd7fo`}8b-!fSQaPZ5wSTdJdXuA*lfB8T(GPbNbMVpA8a`XQwbo(zzO0jIc=n#o;OYiDaU?r9J zq{rt9@Ol=?dm7f~QSj_F-kXPRl&J^T!RH<8UQ&g3o^Oh!ew4d0eF=z&w3Gabq#I=_|&cT)542pnGmyU)3&MAvTcjUcoX1m?Yv>d`@=jYc0>pCK@GkcPaI6T=)d8<4)XEh)1769qEBD*-U~k6 z$PN~Q!)rW$e;5`L*n{+sp7zA9~m^6>~I-8$MfApD6~9u zU!DI8aedyflzfB4d(GX`Sxbl?>FM_Xj5@-+B^p~eRYMj?)I+6)=@NT735>H?p@Yv~ z@y=7Oje$m9vC`}8`VQaO5~dc3H9bbgNG*og2xr)tbWWBk%ERnaa%^k}HI* z0A729efdu4T;b82(0Unu6A3Mv>@64&ZQubia?cF3fyB!9ai@E%S7kMs@aGnEs`M-u zUtTne3rM{JyBbog1p5(fBwfv~vVX~*%bHf=LrPVxL=r_iN)|%0f+u)tJg_c>yf4jt zoMDM5c7*J!29i>;S7M=x7imKhUSl=luir+lNlrqdQTM{ME|`b{Ub3IJNIy5Qm;93+ z#=g|fh=a&U{ILP2`cJ_W7oJzhy zGOhno5ix5E3xv@tX1sy|3&A-UM>txiQOJWW?Zba^T09_2*`TVv9Khb zBpGS3E99w_kUdS=b#|_ed@o`Fm8kcGN*nXi%?$<`AH6z8sXC`-ERN`6pga&!VF^?;XPB7vt# zoKK=OXSv5^_WJ}|xj3{Wd4UZ6J^*AMgA1v=5IZpr+RDS7BsZtATd8r00Gc9qg!WTe zyL5aNjlx(jup0@kWU7apw1^2)ekr6?e^w}dv&7WJB9z#RoL3gAO@SV7az}}i=7UxW zLnY3zB#PnLCHPycU5h9EuR^5|?@8T{^cs+yj>MDCfX!XpN36KJ>~0^QoZ(;Tk1-F4 zI-O|RcX$)u5-s@?FJvMWv{EB28T=i{m&fc)s+w*>bvk^1hWp+JQ?L2#->~dX20GVy z?=4a$2P-NLjyn)L{*L;Q{zx0C!Rt(C8{8KAMtO+9+Wt)$gf23m&Utdl}aC&1ME*CwVT5K zN*{!I>}@R^FZrQlIBPAkeHGGuBYTkS(o5czIV)0KnasZ@S&PgAITw~)674t%{G>8c zEQuRo&OF4r1lqa#e{NPfuKw`C5fsUGoD_nfBh+AasM zD#|t4kW=?rO}GF58)??V>JS&HCTsu)-|98p-_n<6#@q0 zm&%-mT0lUcQW4BaUA@FQ-mnr6*GpfEK}e~b>@o@5+KHjxWt~sJ#dTIEH3?GZ{1k}C z@D$OiURIF>?nB@}`jyCjb8<3N{I}dZwFERMvofN1Z#Po^A`sgj#{WtvVI@|@vT)w# zUnt@LSBV5T0mhGnDb5EUrvXQ?Q+3uTkp$89QcHac8M%`+slco#P?t&~$@T>3R-c#8 z?}vui5*#@c{OZsuPFA;L+Fg6PxJKCVQ8?kX*J_D8ky_Te*eT~ZwNryQLkH^q#{lJc z*x)&Vqy=n!*w(Sg$o^!ng>$R2chORf ziSCU<&Ypwb&jG0iKu@as?;(RyxlVH0|C0<0kSmfES-`z_bH#kP#)ka8d<$?=Cx*eadXFCdkHtVJ&6jZx=d8@Ij?z&3{z|W8ZPvBoSv|ox0bO|)qQ7SDLu+;ZcR=VnXdw?={g~){ zHRz`er{3!Elxj?t_?+{(muz*FJ+^Md)PCaha52_2m9vIFb9G&6H6?%67^>`sEOGH~ z2_Sch6DyxWflRLj>T*hGH)q#c(T`&R_(+8-Kcqrfs)X|cfgZdnlPO$?UYY~W*TGjU zIDgiP^MMW7+gVQKl&2rR51LCuA`V7I*X13J*j)qiZ8{jZ$l7P%R~Mxl#ZY!zh<@Rb z%!GSk)#NPWX-FC?Yna4o27H7X@f=*3h?X8fSO_BQ((R0d^ z5ZTcVI+(+05b5aI1sojU_Z8@wy!3G}=;pDMouo69vM7|ej=$uTMkl<^+3cn>@^=7| zVmtS*#k>8`w!_ggQ-RbAaCZ}))95VP53Mth6(k^m^F!zJ$nVF~UuO{bnHGlgKTyG8 zQ#^r%lbENnOoEf?c3`bGjKN-pz$(crt_u3-=ngPQH++XUoA3z1d zLFKqs&INqK$-|1!hL=6x;(X#eItav}uk*qK%qMQ@B9pMO0wr_mS6g zcvm7UjnVe!z(s5FfV_a+0JRnp?s$x=VP;TcDev_DCTj}@@MRkZ^LyIzoNl2f#xo(Q?Y)&Vs)p$ z=}@SvHSm87Ee+;LqtSAcSao-s^oiSxNV8>2(qT#^Cr0;H1IB+jqo=n~Q zXY9Bj`L5CY{w2EXEZ5Wpnq}dSj=-iHy0$j&q`~9Ik&E}>!WcL}{N3_Evo5=;hBTc7 zK3DOq;(SxiwRAw@t^~h%nZPpzsQio0nS^Z22@Y%V)XqeQCvp0#4&8KR*1-X^Z*5>H z=Y&6nJMG}<9(G!9=<+{kunUrHHZnO4&iW8Zejv>EC$T|)M63MO7`H#bq&>GTW5|Hagw}h@?SvZkXj&Z~=)0r#w2wU$S zy73{hsUNbaJzD4@>m1J>(%_3)R5(B9Y@FY|0&aSZEXdEvJ|7*jYGBzaU@`$17s3)) zz-hA}9W^_n4S#~C9-@;kAPsASw=+E9L$qxieJFO&FTA?_OK`W2v;5ELGFQP)6*Zj- zX49K)8?sY+8!m%ZOF(svprr(OG6Knd0Qy=+RkdWwVvu0%z|$tEBN|CKjh+i-$%bA+ zawmpl2Jj|&JIxLCu^j}%E5Jcpw7>V3t-lEr9V)@F<#4d2c zN66o$V7Mgb3-3cUo#5Px;OI|kIOMct8tWej?zdtaEW~n~1GH+>$>kn>w`w9?rP}%k z=gr^J;bJQGRGRe@ny3!aZvb?35-RSGEp?dFm+d&qpO5Ya-R;Yu%Z8kA^w3}62)S&T z{ulx`DywV*+?GQjauTf~`ga`9xJ+lwo3@?yyL1IDVoyS&Zo|@S#)<8o^i60_AEANR zB-4>7=XvrpPEKBgb5?W0xwd^iy|`bpuR7dsIQIQzPBqVgdn0URLIvp4UKmTppr`F| zba4>ryb`&QY$eg%=N?@`MxhVlLVwalSvteEW&(F5x_I2SzM>yR3wqiaNM@egytXthO@F0$_vLqHNx44&Se)Jebof#K<8R#F4uGC zF14ehgVKpJuJ?fXDLBY$AH@y|VY!Szdi)4q9pTi%b6aux1Yf1!U?ckyp4t+wXooE7 z4F0F0sVanIhQeR;63mNaIgX4<#M%q6le2VSh=n>{FcoD4`Mv>koVY`mhMM%Bt4v?= z$Mk1v2HrlVDn-toPNyztg$60M{hwZx@dz`C1d;NPOPWs<@*LsS1x_Ec^Cq{i3{eATO$nCyl zS3Spz&=LCm%`rdY=@qR`<~bwX$O(-WVCG5zYhh@x-RYRFUUmNNn#t_)a_TGSaUhd1 z+R+=T67xWI+wzCnn)#T%-C4`8_0jD5L4C9FgZYN8R|m|&W?lMQyQ~jF7j4ay=FDn{ zaa87n^g8G<2FY=p^C!E|PTkSc{h2;di+(nPtuu75%8Tag2i47nr$+$8lThbqPCs`+ zuI`{O@*w+V@^saqfZJHCBk+_i(QzajXPMukr^-OPKVsKE!_FCn1oqg5hECB5a5A_n zPbar;=`sE_Gd7ZqN%Uy`of#XcMoId0U8fkXAbpn_(kG@5eVKC5nYcXtJN8<6u-@t- zsY@fZp3%K22AMn$Ykp{0Dt`wR4Trw=a?L>c@7y!<(YrmT=`fEQZOtNdT7PRcf*Zy% z8%8tJm;|#Job|T0nw^MOScm}U-1?pRB%`_c(wu4)!G14n|4~`uD5H*1C#$>E$Err*@glXFI-JQ6 z&$!bv^vMf6l!Br4^s73{#GJD9J95LPUm;yyhdxF--Gg4bq4zh#zXmXGLFX)uuIVk1 zhZBI}$3|VFrcoz652Fa($eO`lQFO!@f(}@V<+c>;mXlE0987o~Mq)iN#2x@`yL66o)Pd46|!;nx>^gK_s_SvdBt~jfCo_ZVl zdU>;Xesa}uR#$J+U5k0<_6N3Ap?cOE;}2ccehqZXnwr@rQ_K7+@TE4(7-Ag=4Zylz zL3iZe?LC+ekW0<({KvV+Iof$$UFfK(T(AuYWwR2^*2WwCxV}lBpug4HXj_8Cg9iii z0;d9_gQK-%Jv;cwj@*~J{vp&BAHveiMJIvF_OIxDW+@94Mft$K8e4TL{ahNrryiy< z%~W-!9s63br!Tr!ut#8*uEbLkY<&nlSz9l zm}Z<>T!Y2;HQ(-yt?>*?PB~+kK1$1_ z9SS}Ob`5q9E)Jg1zBAqSIj&g|onqI-nb}@Or$)T>ed>Q1**!KrZdLZkxO>ri{j1%D z)w=d^);Yab@aL?Znf0=E2JUIEjK*{T3eh(*(Wq=pFw;X%l%vjT-p-)}g&f!HcZ6kTFdADzGqfc*b`bJu*XCB{f~2Z2o5Tz-EcDzp!sq20KnW zBGm%C+NiVD?do{-l4FkY+*TwfF~eSC9!)&as$$qvcdrq1kFWZeZ z^wM`RYhx(gD7%qd1OEhC zYPXDlt&n>{l;JI49ae#guJgUd3T zXEhG4(XX37g(jk9CWJn*=9&A)}{m@;*{jKwmay9gw+0d4lMN3)9I!=(RgS4V_vbk*c7FgBc1sp`5i5kOZG?h0m@S)pM%bw zN)vluJc?SOZ9ue?={25dnb2Uv;KtymTD1NrQe`N1M+!3!rkHol-1NBp7~8TwHaVWG zy_WK+(q6es=dA^Jk3HzCHVDh>XCr)IA1YUCY$=eu@?#QhN(%l z?aV>tUZ0q^jAcOjmQm2$Y}R5%Ty89)<@jWYL>G43iqh%X&3uS^Oo-?R)tn@Xl4Vu5 zrkTaeE6nw%Y24L+V~X2Mtv-9tuC>(uLa&X}D;XEiu^pLnkqADoGCd^#C7#5JozC2m z`}!c=u5Z$6Xf|zbFnjPq;6|WT@ItVrHeIWxeG&XIkT*~*&`M)Ao!ZA&GrE3k-NWIj`hDYL>sw-{ zxtO|p3k$rYd7d7yHRuNW*vv*>x%|Y??Dj&+6~)W!j7ViZowcSA2l@~Cj%5}@7BS=3 z^aM*b^Oz%zv-+2MydI_3)8{gurxOs!%~YV$$i7s|%eqo5Kepou=2C8^w^Ua;7w({A z%W8Y79ez_(M;S+Hhu=}i@rmQUqmw!v-p{M{aExR&TMYi{EWEEAwi94Xdbw@D%aL4% zR4WCDO4Y|p%4^?j&&~v+()4GYPam6&$kx{QhS`W+ZzXP%N*Ch|*tnaFHcUS`uCLWM z>z8$xQHZHRY3RAG`UiR|eI&16^?&rEdQsyOW2&*%IAdIcZe|%-ukj!+}s^ z6{czYYfds3n}5^Yx{7%L%~;eZ$ZVfq;kbNeWxU6DGstwg<=9DOc~!)M$!}ycY({o0 zs3FXc`i40`<&B_zS5ISR$qoIG-e1qqMrlc!Pm9!QYjd?2{iN%3oizxF=!Zu716H;EV?;f?s*zrgpv`;oV}w-G=4JO`N^n8nn# zj~rzkwH>7#E0{8Km-u2^G=gE3C-N(O_v6X9l%resZD!W|Y@MRR^;0m}Pc5RJr`u^JUAYe``>=~TDW#MbOeyI}$H5oGN`~SK<|Lwi zljz@%blJ3P?t{NpA`u+3)Ys0^?);v`o)?~?-m%`Cz8Ag`{)PT({$aj_-k@iV=LgRW&k|1` z&t-RYcT-nOXD9WhV+|T4nh807+LMWjOE1ZPu(Z43#dZRkCz!DL0`56MtYS9aU}GkQ zot8>h0HP57F9iNU@jhch27nJaV*@2@33V@=uP!Y+E-eXb~Lyr z`%>R#SY~dhF$PKALn-Wd;8?G=a4If``@DOs=c4C~XQStiXS#QjZ;}6;zmos0ccEvV zyM%kF`+$3iJKo*S^|A9S)vMN1`>HL~4EjF*rKCceW3kEPjMqjyt_-^H7Ny5?EdMqI zCziRzoQa^6Gtj1l%P`2(TUE(haD*n-SMAeqN4_S@+NZvMqsJr!OEIV zH&C1X9?__+=*iE>#=f|`60^l(R(E~T@cu&{& zyU^KuqHs%~+Xm(XV+4}(EL!qceTM$C-d!)CU(hCKZM8;P3#|$A@?>ya@WWu0U=GIe z<_<0oF4gYn`OT7+4=r`bp4$ zK?nRnFY*KCKg{QQi2vgu24S#|`{=P!Xs_yUTmdGgUSnRt0pwi__>*a(Rf)dNBzBS; zk1K<`PDA+RW1{RofVI4#`$*y|*eiGFSl<~cxCx#AjwEWqOoeXD{%Z`q#}fs~FdW#0 z&6r!;4U2s%k&l_y8><`f*gE7A?$9B(8#6Tm_Sjw;M-{7|i~)>M+(#rd<%`P1s0=-N_`Da4krp!$qZnx#2M%GefmZwUJTYNGH3FXwobdMDcEQcdbA$U z4r<@w3tnYP>F2?^!36yCTY=+&xZsOmXT6%)oKm=NiEFG=S34KG{&4s6qp(6I@&13r zHuhq7mLz_5g6gYg=&ldQD`vCp$AfM~Y_c@*--Te$Z5>BK_C@csGrQBxei+jt>oNs1 zlb-(-&_JV@99h?lLhICnI~$;@@?v4_qh_rck>N&k*B*v1aECc~WgUYZ2OSA&TeZ8| zLmi|JR)0``RtKs*!Du@$8li5(Yn-4wq#v)-zK7UcA*gF5lAtO2xDa&J4V(1}nY$Cj zYm%U{HP&?Os*z~M;$ZL{ytgK-oj3FEQsOnuv2Hq}`O9IMZiM%WVr8z<7HRvnTTFsX z)pF?-cvV8K{-A%4#VQ_Ry0$?ZsWsQiXce>~V77qP4!^3ZG2Sd3+GC4B4^~neI6Jx) zxQ}~|co+LR`!D%pBXUF(iFogyjV>(ZZ{Tn4PxMcOC%^G5a!0u*x!kTE&X3i?j#tF3 zrx3?_O6K}Lv5Zkn9mv7-Y0IpQRy&D(UmQ9;O*~*QcH=mDP;bKeSdFEagRI2>WTjLH zv?GSn0@-u|I1hok2az>PBLyi_?o&hpN zqfFE-r*=|@s^6<4)Nj=$Ob5ttc(89;W2a69yM@&gOwN3xOjLAwou6l|6Y%8M&{sZ* ziNS5r51)}+_!6IH9~5d3?aqf}dcyBd>5e>{_+AJbdpeU6x?$ziz$VRaJ~37p{qQm_ z=;P6bHhr%)NgJpAL)4-bR!yd!!$@Lg;xOiM{9(-HXNB>XG02ECe$(^mSMadcV4qgd z-Ug2aUj|ESd$nbHsxim<%vK2QG}QX85$@9-hcD89-k%gPHezGMy9j^e%ZM)`&iE($ zfA){`$NBU7)_SVC4QB`EMX2_WQ@+;z)_sIIy0#=FEZ{dkLoz04- z+k9o%nZmOe9h!lk;XoheM=pn;bD7au7TzlZ2P*JkJLbn`5ZU|-XzF;et;ldoAJ*H{ z7!C^eX}UoMyA}54dwUgSD)X}YFasn7$vRw}t1eZStNYb=>POD0&R?BFoOPYC&YR4V zX+cc)Uq@ZXF{T!Lk8RT(&XkG~jjrmmq1HY`NmoECQp@loQB*mLzk=$9M)*_C&^4^r z&e#H5(TSDKbYh|%@Hw*Z8#ZBu{;ki{r-IdA^;!J>pMFulqZ@j(QJ6@{G%Q<%Yq}$M zIzkB%NFFy+-**}_@ft7bFc*dXu{U-sRG&GEx&QRs z^;Y(`k9ZPMEV4pmeB`l+QV~lyj6Kqq*SE#n*}KTI$GzIM#2Mv0iG^<9J!ey*h<@j! zS|wX(0Q&DM^i*!5$p0CC8;i&Xd;!0VfHU`F2?(c+#{>8Y+p#S^NlRc}3r_YCi7Y^j z^7C-U=O*y^DJ-|IATgchNoHX#WP)gW;{*K5zlnK7qY(=-gMO~5o1O614$~9-KfH^x z{67QVtqxX3SuCrnOjurv9{YnTkJ{wePm%fPX}@BRQfex#m1asMB|*tgq^O=UKv_dX zAeW;NHe5$0%(i19(l{bL=N)sHj(v|Qv%fLH{2kGQ{>&q|PekDxJbmf$Rv?^N{hNH3 zg${3rE$7Ety@@sVoYmQ^YhinD9+RK0g*DDuvX0}?@ZHc8J<;dYnQ{3FXzd_-av!d^ z!z}7#LUI`c_?VIpiN zBcHKNZ$}KMmG(S1GdNzWVBEBdDs!EQo-N)%zD<62)asZPaZ|E)$v!dL>gXoB49 zQ{`vn2C`0b3`Iv3R`WR?+ILWQ^2oYk{%YjVX9Oz+s%D+abY*SG+7s|-W%VQ@J67BL z&^uc}c&diO?HH^)klCU3@#L7_;fcnX1BpE6G#Z2LL@X~SQ!EQH<*Eo7zYmdlZ^^VZ zG@D>4#em=LQYA;v;EIj{YNS(lo^#cA|KuL;PIS+59fv0!YH3GfrJa2ocW+8B*+lAy z){$qfj{kT9kL@6nYW`GCDSs&{v+$llCC7;I*CWoe0{Sc!axiNtf{7q)=@WhjDm#w$ zc!agk5PRVds?Fpy!#Bhd_ENFXnkb%39-fE%UWfPeo1roPs}G)I4AQlc{+WJ8uWf8K zDq%}z$G*6TAJvX5Z=MiwWW2RENTR!N%1Ly}2&lI&^B_j!168!nV>9F@uegB-!!ErE z`gs!^+*vEHtqpz=tQAZN?8GWg(sCJBtVl<1_i+E9s1-5(Y&+um=Sa)hH@A_ieZru) z`%yLg<2*Z@wUty;4K7aK`TqBJ%ifiGAAJ8bbwGOdtOwc>>$QED`mgJjC(V}}u|Bd& zg`%7@Uk{y<@Orw#$GPoiTA2^y-C#!pA`;61+*V5Z(l*%lc zH7d|a+hz0&9ku`K(AA=@j_!$`&fY=Z@t&Ek;c91Py6uV8$XuXb4~`4$%lbZRZ`Q|w zJi&rYt2d3I*6h$PWYKFXtFVP;;(>hVc%iIU_A=|apraSQ$##cR9fbs5tX5UmJKi!8 zet^9*@yMRm7-j;uhX)E7RhWl7+MHlr3u(4%_6}rQb1~0+26NlSDPP+Q*{)#^<~1XU zq)ukqe8AjBbixdE;G5@_-j1_8qm|$>aGShSmndoQC>sn0osxh8e3nsH&$HIIT{?V~EzaU#O+K4jF>-Y3$!~$>ViAd)f z#P^wfMr1fAp4f4GX_>c^!`_oFYe}K+&;}!rc594U#9(jekC6GtiA%iIpTncw^ji9L ztt_1WPtY4Y5ZD|T5p1V74YgMrdXpmtNB54c8aFS;&$+wgdzWu}p0f#Nwi8ij{Zl-x z)lJs!K=rh*-Y!m=lJZ|l(wlm32aqO*?rMj3ABtVHM!tirbBEtAoC6`3bC!d%1z zrJ77qIRlq2!aDm2ep(Uw%~qBvz>Aq?TUq&6sg6Wn<7nepPCdwTDkDC}+K;DNaWy_p zrdb0^Zg`0KA@;(=LU$^Alwxkd}ocOmA*?g0+jt-tXv z63l5tuMGVQDjLoN6ScEiHuDGjQs)ZqPXAw#YV?tqxa_YJp5?lct9!zXxH-{%BU<>T zyYo9Xo5QjiruyCI6n-_024r@k{!N^puP|S@D4#fz^Q{fp3Fjw2OKLbAy#+ zTVaoPj8%_1-@7V#T6h&-PhTG2J5OErVrO1;k@Cp4&T<(IgISrIGMbaAOUoFWnK!F+ z;BK&xvCe8?U*wqT^t$6b%RJjXO+4f96zUMO&Z{)E4X|8hb1b9>`c`bCsroyz3>qGY zjo7Y(T-;fERi!_#mP&CYPBOgA1l)oiUx&YZg2`Dq?UV55%2F{m2RXkOUtTwhV{@-% zLeF4o1J~Mqvrix*a^LZ-x=fv-Hdl`_EASPyJx4=7TFuS6Mj?HS=F!rE4}-owWd5ZR_!b1S*0de}3c7*mYhWQYbqO*@&oJs9cV zmX$~+mo0b`-KbI7O1-Si%9%jbZ+EH}kCSsRf|n;V`|?qfW@1$hA{Re}`nwL)yS-+1 z$3UVmX;5^3o|Ml>(i64qflgUpWqy>IGfNNlwW>OPaQE=tk60TuKlXBbz1%1BrsZv% zJ0i!Z*g#}e|0B1qBpZ7&p1&>p`s~ZSFM}`3y!z(VuGhEUZcQJeoeF)aCV49QFGc2w z=@2_O_WPLEQ6(cQ`rmqvIo(PlYmj~;SS=VEoEzwnm6lOFV^&7qtSfcJ4*aNVSeq*0#XhrWFfR%50O- zE#qCr&zXVDV6~0^8P`@z#=9vi`>8!WJ7h`t!E<}G=`PdAqVeAQ)E$H zw%%$+#{RPHbNg+(OL=cU3C-?d@=kB6b`otpLbI8#=D-_mK!kWCe#~TSh+n8ny=5C= zKg=}QV>AJcQ*tuxzN|8l9W@D!C(6Iwa2kiTKEb?!PqJ=i*3LYbk&^K;^GvXlr&U&@{^{AACDbG{( zyczKRLB`kmD%*3lndg`B!3I4}+(uF7kiWE;Lpp*C zbkWcxbd=p(iPv41%7x#t@rzOtLoG)4&hA8 za_rQhd`HgC3?z!RnricL)c?$+?(iA@b{*>n^9B5}PWvI)DDXP-LPoueMHy=|Uj|E9 zr5#J%b^Os$gJPP+l})&p`%T`}dB^A4pZ!|&2LG>~GETaWXkVwDO!?|%p_iRs`BH|w zY4P^ZyX9$L2Bus6)Uw{@5k_Q8v_IxSj6b$}%%Z5*{`Vfc%k5}tyJ_Ck2WrcLNx@rz zHi2KV6so4WWi<}e4=%%=bp^8m3xfeI(|B%m#YXS0oO0}T9(6DAp7Djy9reAB-NT69 zo>m6f)|r{w)WE9D+8MpmtEcZwe~~dPD_>C4@|r({x+@uKUH4WrPRLWv)5iU+YnU@Z zt*@v&Kheme@1i~;gw5$THX47JTP;pU*b|jBrM08J!|phs^aA@AnQ~PepQbz~bE{CV z6Kk(c_QGR7M(%J7yk@6X=N|Pn=aIKh%?fCalf-Gh=Bz^ze1i7)7b$k9a-4}{x$W!8 zyS$|a?f|yNEF!0$!s#>2L*`y;ulAT%!2MYw%CF7y=1FL)BeiuOGh_BoGH5~kgZsdW;}(Mhlv$(7)tDJbu*_Ld-U4+WNlOM`#_hhA(;g-Kg;|fYp+&2 z^oiQU+aNMB=03f*%JPYqRw2#u9TfHr9%gjl}*?lzcAA# zoqGhxp_U|`w+`*{9X{VSDk3|Dr?q^6maj;a?YC3|WiuyZ{l}3X@f*4E@b?;BsD@0p z)>2I`9f9gnPyH>nK}9NDUsKKXipP1H*FJh_mUxa1QpDNH^XpiUieN=s4=j_LG zVrgIFd#a&J);;E^e!&{knMK=$97O}FWjm7BIZu3hje$n5Pv!0H}my(s7 zl@!Pk{911mI^?M2x$nOgwJIhvc6Z#``1}c75<0|}i>)5f*WJu)qOv zCMiqaWP86O?NP>@thC?^)2saH8tgkAu{iRh$RiOYA_n{W`bK$TT+FS6rp=F^l_}`u;&< z^l>@V9rlkyMa@LLUGQAirm0spGIrdwO(43UeK6i8dgQ3feV0N8LY|m$}Z}?Y)V)1(Ce5|lty*X zGBCS~OyUXZ@@A4nn#V~ICDavc%re^(>G_OEZCNz*i z6(_ZPp`F$+GD)$-JLXgEdzg#XxQHE~J zG!w8h_ES5bZp%p}e<$p}iJYF?Xf?8y67PIz6ee@kin)#>sMeotD?q*Bbo-A?IIIL6 zp5Z)Ye(aZt)Uy9b=HNysi4y~VkB#JikJFl@c2C`ywk+d#ma5e^R$1TJr>i5~bG@T|pZgAYo!&N{G479Do7G!NcKg40 zL-);^R0Gc^ZoQS7@fv1|QNk#y_oYIrqc%tTMVmrgq?7)yvD!Lh+o1GNw>sas-lGB1 zJPSNOx;Huhc1*Aj41I1c(er6pfhqwt&@WIaI7Ca+51Q9Qw~;K=0Xh2+(Jk!UL;kv- zGSha?`U*d;iBX#z$snTT?~M88D(g6PP&MqsnI$GH128V&Iolw4Wer ze-~`@#Lmx6Y+(wqVuKSdlfleC7GSL1XzFUA(0UuI>_A_ypHP}gC zTFcDU#uhkaowi&%prvaqsNooHj4^i-Rmf)dD=`j-dYn@es&lD&-SJAf$&T~bT82Ki zW<%FM8EuImohEXfk2w8K&ddaD&6qkn-+tVljMq@cJ|AxR9p0X5)h9055E*@#Z@wqe zoP@Ofo%0gXO)VxI9Zn#|PzTvOm%DVbFSi$=j^#R0)ay!$a#Hz3xd#PhvWrQ08WXYR ze+Pc;;ni2B7hkkL`>D!_=T7*6TR6F~1}v1u5~vFca9~rTQWUv!>dO9)c zheTETVCUAP=Rs9Y{~W{vdBzOF`}}Ki4(2Tp(gkF5a&vO|JbRTh;#bMVOu@<=NR@H( z@QID4)CYCLHXTBQ|1r9)qD4%{Iz-HN685cSp0lom@=;Hjp#JW3yAwT`*l0DpHt!5_ z^NH@3uEx$xD(i>XyV=GPS$}1HNS>q+QCT_3QJr3tekPBuwEqchS7bKqOrrZMSXU(V zn#IDCEkm4%SV-MwUQQ&&ggOw7O*Q%&ZYm^$dTF%TM&dTx&F)rCc2E}?XMp2hsHc{S z>bIN#?ZCODjy&f%pWLNFWfk9%NpKItXD!A6{jEsL2Gk|Gu)$Q~Lg$!jy$(C@Co5`}fodA)Vk_!IA5T4p!g<-e&4=N2D2BruR=u5JCO^UNGxeHk;l?x#+Q&yEI|(RH_q$bC$AQ(BgC3Mz#->rT9(ab(=m$;dUQDkXyRNLzUJ2BK@R@bi!Ocy~F6TLFG<4@VEi z-|ElnLt^!#IjRP+f6rQEj#Mfqm!psvGJB~ec$6NbOQ5;Gp^Trg)|x~6Z8<4DlXt&| zHoBo@s`2jEXo4C1TOYX~J^f_Tb}hK%9(7#9=w zco%VMAO7rq>apjejlSh%)>Qtwh}5pl{P3TtDnHICo0oVCD(BIYkTe0Lot)&jNB-gr z)R9IOqaiyvg?>ta0_$RpE&(n&GlU=UTR*Vgf|Zr!vq(;3dWnOyXVt6N+do9CKA~^P z3GCggRzGw}2rp*J{jtl;>?ame9=26{mS zKdB>mEmd!+&((mMNrm=8^>_61(unFQ_1LxxbC@Ue7~R1 z`|}>J_xQX&9yj5_S$1C@qWh{OYT?7n=)~cPk7%;SiGS=|d`?YF3-PBMeI7=-&l)TG zgqg(dtH#;f))9O8u43(Hy5r08S4q+EYuJ(lyofOcAFAPeMbz>+Re@7gkDZGGKjGwv zJ#UcJ{?(OM(|+I5ehdJ4mgtilrCRh*vQ(;lYE-IOdRn?{ZoAy(xm9v=)aoBim(DFKdp$ATBz?P@ zw!4$n=!_4b;TkI1hm)%FXwjGXcLO}jf#BaJ!)M#ER=uzY^z@r*sPkd!HZnXNWFkA$ zALq0GAF7C~z~aRzw$EqxJU)jXqC3xGR9#B@MgNxQEOxV*4k5+w;EyTpcpG_-?uT(| z?Qdl0B=;DJl8bOw8M0Q?eH-GC+f^rg<%)57&m=p!A7^R1u(;7(`fhi<4=v)v^=>%t zbGEunqFv%<*l(ul&B(*vxk7pHE7xB$c;||1_sD;xyOj7ZT?K#95s?X`&O< z@x>{8(e-B?xqI2&>RXMaMmErjJa3G@$bGGmX&FftHnGs#$w5X`;VtyIi?lrFag97# zJ5h_CUU^2&@O`n7eB5@WM?bNsp|oV2EFEW`6=fe<(`?sg_rrFj>t3J*ql0Li>$cYD zCcswx+0^sUWEw91NY&3gJ5K*#o!-TB_p@ZP*|H7z@fY`c)mXk&HS(+s?cSVKiM|lm zH_4Zi zV~LSa&+VdxxiI|_nS);-jR&kiD_j|8KzAWo<8&HsFs1sPC%RPC^8*qbfQ6@l3iZ`Zybd;cJWw0U}IX-qm5~jo9Ne)x&uXT%Ad`A zh=^Z?V_PJ-}?vi=lmgwa}&*Qq3}p+2N4I=pE&%f%$%5vzGQO?Mgj{*+|PSEB6M zg|ndb#%d8?fXNE=D(z9;D2o*SqLQy_W{GI)H3dCXP9HAVkU1yk+r%stDhE{o^tUtf_tc8?=G^%z z8ru}DRrGrG5a;ARkZx%2?RoYNHc5Z0PWr6mlZmC!%~JEMN#CTiIuAIW0JaMj#e8A%BbiP-q`le`SK5bGQ z-soUgO&9v4O5!{IR-wB0H~}_#Ydv9|`dRy1*{GLfAEL+kPvqm9EP_62CtRE;Q4A0L z*XJ{Gp3zBZh^$8C!X>QCEhOg*nj)E5Z~wzIcxnS zstMUyLsPWyh<)tM)ppg%G__ZDnCiO6RLh>Hit=#&cLm23_DM8L-IRVUeQ9b+V!RyV z$*IY?9gEH=R#!=4V=vj+njS7>VKj3?rHTw zpB5a0yB>rAI#|CGR6F(H+3)7Z3}?}=g1x3giKmNTchb3I1Dx<6FSjk(Ib8+vIV$*v z^Y`Ai%XbFPc%4d$@%VKi`+h6@dyKlx)3W?o7mD2CeA8<5(Q_njKMS##A6Sw-sg$@W z(Ld1^AKsOi2Uq1JGW61oqR+ipk;nJ}5t;lPp4rGVTqIA@fd-sLJM={Jv+0QOP;O)6 zzmr~jp0&BacVDBCrm3SpmsBqz9YZ{x@rXV_Wl6`cto#(#<8!~?!xQ(@Cm-+`M$tt3 z@J4tp``ztpGFQd7;^e(%Jdw$yt0sMLrJPv3tj9HiB*n>`34Bq475tRfxKlLXJU-nF z{1M~qL9ZSMH|G^@B{y$FyVWw=c}({~$-k)SI0@dVl$j;UR<7XN{QFg3UL)i8P(d|g zJWyCAIV}BR-mtuNYM^H1e3w|B9#{0y5|@^2Q=(I`SBo^tOXSW@eJw`xd@7lHLf)9% zFH+SL>B75sS=XW3Sk>$o(wV>MCbE`Rdy<}fh0YiSgS0KYC1*}zP4c+Z@yT~{F4xuO z%S=_Wb``66gStf#ADr{P_~{n~5Arfb(#Gf0zT4rGl7+kJl~3T6o_3dv%&D2U-p;Ih zcuP8;pIbP< z&XV$~xeHX)eVa2!AD3$4jL#&V#nE@t9Cab@v>i_m@^R+VQsr<;oLh7i4D>ynRi|(j zZr@?NFS9diRp|Cc|9*Hnx^gXpsv^g?getpf|TQ%>j^ zHs)JgaSP1;F6-Te?f%Bt=6nBA9JzySugIS}wy-n`52ka*Lo)q6ZpvzxGOYVJ8g~`$ zT+Gf*7b(pZnQ6-M^b(^fVn0l8S%XK#0$XPK=d@10P^3%IPI-?fUzMTxEmghPypk=xpwRWp@X z_~!}_*yVR1|55uNirIOzSzNX-@osWi;(Q%2#$_JYU9M>24S0Jf&*ct?>kB9}S=b4O z&k>U>R4;u%OthtV@ar<3`Qp5#GbQ<@Kj+j)l+zFCId&(PMz6uAkBCN`RbPtl8fU`K zf^beR9HCE44fM^=#MwA!7cP*~yqKQb2+5vD8%$yQ+OZ@Zv%d1>e8msR|1bEclAJ}M z{OoSl=2>>*46ASw3sg@{$NT(-BVuh8q4ERtN;8%|mrebRt?b4c#R;(2^R8M!Dg|P? zQ^fKn;^>j#|!}9_X*>bYTTEd_>)8L;C`*;&q;mw=)F^am?dAuPNVq zGGuZe>06<9(P`q05!q=Edq(%v-Z0j4@cvfy0dc;0oQ%{NdP|6~&$1R%MRPw;-SZxQ z^A4zb1$)`d_t895}?)PBEXijH8~2L=kl#-R$O=qN2TsYWAnp z8J5kgrGrWqY|UR>@MuoS^jCQ^@;0Tm6h2?DCG%BkY0=9|+*AC#BHyO(PB+gxM=e#k zBHy!Eqf!UbGjglvKAYN`xG%9bacc7H#Pm$t{L)9a9c`C+R4u{$!msos>70K{!3VTy zi%iRcGU`+M>6UdrZGH|s5xwWW7c;!U*zd`?TGqLD;w4=eN{eZ96#=M#o|n>^SBeDm zwN{;Umgz~3pSCg){i<&z!h)<2+a6eWl=dD&9*fIh-3aAeSJ=9+JYQ!I&37_9RI%_6 zp6IDidlwpdl<3AckbPs`(NfXvm!Z?@?%zAn#`UfhEji%T-yqp56X!!IyGZYFeq43Y z!5_%&D)#0c-+IjdSK{k3aDrZLJgybSb&vIVNmQ?ai0WTv@~5BAt?HL>%nbZIP=v5C zPWf-=LT}jiBGp1y8}-xVy_Xq3u9s6EeDOrKPG=}9^$`R(5FQ-n*;H}C+PHp;oZGqX zd<|sQ8WnHLSqpK!2h~0*?pzo0K7>2(g8g^%R1S-n+`~Tp%rASK{XR_79ab;4owg}j zI0{nUOqx=KFOsxf1r=zxhe%mxwxg$UJO+)P&gW`wFXz|n;y&`y4ehQWy=P?4JQ$jD zcjBVtA{F14t8%(M*($X?)h^u;);--$`5L*Ca)09){g_UtC+od%OuAjVnZ3Fvq#lwJ zot^kfhI_oc*va_(w4C=zMomaC_Bk9^ctvKBdWTcoceNdI=jz(ET7CEFyuz&orTl*? znJVS^YM7{@C`(s$ls#G5r6SZ1*bVtq)(0LymM=p2JFVZbG=5bb=^n-{c|72U@YGTj z4STKE9u(Qai;nXThl^@Y<2BTvS#GnUH)k{5=ZZR=Xx2SMWEQYdkK(`*Jc{T(9p_cW zsSrt;CC;rYV4WJ$N^uTuS>Jg!hXq$fel{IajQ(gRGH?&9PzWj9$Sdz)EbnAF;hIRM^^9tz`e{lA$eqCNMrS{Kj&XQ18jFkcN>hD@eJ z_OG&}p(z|PgqDeH!Dz_3GMw}^-|Z=p);Lx7Noc((&#ftKQ#x^mjN;>Rd-)0-G}b9;DPA=*|n_qX(~>A9k@i}fjLoc>n@#oeh>Q#rcqT%x+_ zMfl=)+;NE<(sOJ=vhWi=acS7M5}&Uxd$7G=q8j;4tV=~c^bGZpZ`gyrO>B6)=g)Wy zm1X>{V6i>W-C&nhy!_AD^YQR*e_SzNoUA}*C(eJLLCd~|A`^4|p?&v~tYs=~C$j<1 z$ywCq8RVeDDXjS~qWj~`^IZPZbu`CPF@>~P$U_iDpwT<5@RNMa$iKaqbC89tlvs}L zZ_y}E`qX{wNf#)$t~E&07;#RB?soJ_6Y-R)aszF>e-)Hc7NT56CT8<^;CFU#7nyh( zub&NT7i4!Y?KkhI&GS39x`wI%J*4bSTMBP=Bg*Bv;&6i$z>{!4V?Jl?FUWjXsCxan3vRr z_49KtE!>%w`%Ww;;wJs@aSs--wy4I3Wb+mGOVVItS>!eL5hqhuq+UwBm-=67v(5|y z(i78Lb*!tz&eYCL<}OXoOn;evPsF}Mx^(&zJ67AJN~gBdH|He3<6EwWu5KVt)#WEA z(WRZBs>!_OF1WN29Z`;@8>4dSZuL>^3tHPL+ef8bFMmI&vTl|Ns!B$7sVL7xe(edO z5Ld9uLqr1OjQLqOqm&hh^MKCc`$S*%v!MNJ$j~h)(3{6{8_RJn`H%BW;?&nG#TtrQ zv%~aFee!jbCHPl$W=)Z&*pXJz_g=zLAL6NJ@Je)ie+Q2|k8^$!wJNO_;X3b+5;vKq zPv|>1W<1_}-}`-eU$yZ@Yu-_FS*z_(YYw#j18v?4rdY*~Sz(p`f`d2n*|*{MmuZiL z{RPpHp)|>@o;gewepD$oyFf*ZJXR}v5c)x~??SvU!?kP0?5oitIfZ*g6S}ZrU%*$bNOm5y z){c(c<9^X2?mO9|Zwh`_qd!+BeIzZ`Mg9CW@>-MR`9}ENN8WKLL|Y!N8;PPhsx&VU zL7&HBtb}BKMvracZU<=nj1_N=8@u9zW~4Z}w8zQ8rQLlC?GWeY#iF@tuxH8TT zj|j;|-p+^C?A>e>CC(BLPkHjXaNDyOEB2 zkNr5HBB?Cjehb|?1qzvoN2Z8DO~BWWlAXJ7@;}z*Whg4Tsb9x_l@oPJit5B3z3Ysu zFa6OUfBc;}lVtv)`(!g3_DAUEnZ)kI`N=m$>yIScrQS*HNHtP-e`UIR`l56XzpwLG zb-NGO+SC4)-FoMyTBpuWbxhSvElA#D7uN(la%$oz+3H44zf}JgeM+{77u~_aRkt&3 z9G-ZB)vGBnESn~!UL?!a9C#)3T$CDOW~JVevZNQ?dXU& zQ+7Chc@1~>!qeSJ$|&F?czyy zhESHsAa<7t|DIl|u1mwXL>X(nga-L1SvGZ({qn8TpRqAzbC0SQI35PMCHJA+2lVKw zBB!(3PO&S}RnvR%(i&OgyG3ee>Vt5Fm|RhNuczB{RW&gMqPw1KFEXlU&8@F-RV-Xj zqYV+|x!jJ@O0v_%?a0c<84>5X9c~)~b9@PX%;#xVh0j_;Z5Q+NuU8H80L}Q5HTj0@ z4)NCmbnRkui8CCcTW^8X&nQ3(7`^!(E&XW^eX;r29|C!F6b13wQTpXC7g%xmihm2Z9wGl1ILJvrF2+b5{ zZY(kWCM4ie2x1sS@i@yiC1<2kv$qK1%l2q=fLTk4Ty0V{Gny?O$SQZ>qn$#(R-kj&;Q5#A(zt^M`nVX& z3+!4`C?)n$=7{P?RaTm2E5;D}9VHR?xk= zf_&e_?teCXw}J&6L`pAV_kYK&^>ic{ONYHgdtE6%Q2{^9ml^-oF2aus?y*m+Ou-5} zb;sEO@IiT1xJrkp3;jPh|MUDsx_i_tXfEn=7rb3q&_ty5 z;>_J_=nLd;qL}|`(W^4%^CuLrfGk!d*B9~?d&0W^Bm2c*yUtk+(wkp15T!=a#ZzgH z6{2>FMctOD#W+rf>W--2N0wj^3?6%+{z=%&WH-wp-RP3Z0^QM8t6&`=_II(G##Vf| zz4Ah{#VbUQNJnRSC%COG{T1huwn4>C5OQ@LD`%0D0?Gv?e&@sG?YV6eti0ei3810Vl z)3gKW-s|bA6QHg*kGnE!S6n1A&M*#)e*=GNm^^epkDKA*Cp?Dw=4cphg*@0%{zM7! zr_&O*q31{T4i&Q_=sNuz9@kIix?~qWk0&40Ip^GfK{?t!Ps8Us3Ror|n z4X~5693%daA?K6mfj7PLfoDshi{&x_i}@KX@!SAW(bq)?`;)sK;-B64xgXOk?~0SY zL~9PooJcb);Gw?g^TXKTnI5xgifuGeoUb*~ryrp|-iJ6wLFRAy)KF2RCtPnh z8}^7-Mr3W;7ryzuh|4lNN9xzPe;3;UR_?*1y-}amAcKE4P|wnAPFyK zz4D>3>=+rs=y(65ccVM@pQif^S&I`C9 z&T)^k`pVcH7Pce@ zHR;>@{Ha_1YqI%|;{LW~ae6kgzQ#P4!V%lV*S2QsI9K^=5o|w=L{EUU#UaH+{W&Cs}`q?2Z)2>F9b5vdEw^A9E(3^KEi9)mdr$?UUQAW$)D7M16YvBUM9{Q+xIBtzS0gE|#%}qn&P|)l*0G zXP&3}Y+$lxae{KpP>-HiZXi)BYkS4Is%fx-tIFSN#%aZ{lkl(u-=k zzo&_PZl$ZABS)v9?J&co#;|#CA)e+A}&o`mMtNdI}=6YH8 z7Up59XUqTnyw{gG# zqidpeWD5K>oIg9-diUqmbk<+5j+|r*-c3^(l4hy}`^sqT%XFl3dgt7mc*w54`6~b0 zr)Q+E&HXF4Ltfv!F?sv*9xQUONYkQ+ii|DtQIQoz$`(DY=$AzjMSAEZ-8Xk*`keI0 z)M0(q&P`5)1WPzoY%UHy-X4Pu_G1k`dY*j-(~k7hhok9{YmVHg$Jtf7Vb#>L?tQ&I zud@reh)(oXSm7GRxixc_*wgJetQPA(-d==Wx+Z?Ezs$;HL9)KCGqsUdRfhSQ;_jbk6m{8$9;DT<%XgdA*g|oID?I9OAoORroDVtg05HXTe-uf2-s#x4Zk~quY+G zI5JP)rMr(DcVz0}{~ca(xQ@ML^N*A|dWW6J_t?MnAVhbaO2P42Nxw^^Z6$j&j7_hR zyh=4dO;NJpsqv}rQ+sqiIZ0>KL%K|LQ-k)Q_lt^|-UuoD!jg328P0+?-i0BX7yivV zy;>#j3J9@-UV=3WO4tjwG=H5w;KvuVw9oinJ9d63$P;(|iu_hYqIuKa_4`S?xTbwA{@icX31HK)_s!&r@0G{bs1 zjp_8>tD*@#$?550Q5|HY`iek34Sl{V8udAUb2XivQMV73VHdoXka`A^UXV64l z$o&=ccMsh6HSabmMwg3DKTiWShZUmYaVD(z8XNI=7FCX9^JcMmah~H&6-H}$!g)|y z^i+5n_Kzyxw;;s(AiOxIJcn#eH0qmp@4;zDaL8Vx*`KLu9({4)8hf;c^AYb){HD+A z<+_O^t-)woC6hW%B>QGn=;PCq(<|&$nwS1I{gy6otwgHQ>196EBb6)raaXdwo&1BL zx~Y7r4DK!tv2+)uTcgtYVHE&fRlT2Umqsi5%bV&H^&)AiBzM@8rdw-;T8cZp%?8%s zgFPaKd_>*X3-uYJU0>v(!0B$96z*;Slye}VkhGa%X1n-GldXiQl@e$9P%>+H60?qLlh{^-z~r+AFw#lck&@o7U$ae z7yaL;e~94-WcL@1Fd2$iAu6$!)c%!SkDM$vDMbTCe5EIz=;rtRP~~-EUgP1+wd}`g z)+J7Wic{Y6#m83oYdKC>n~ke%5`Bq?*;3YL5yUxGobqLU$20Kh!>(AzJ@&faE?Qz1 z-8PWsIs@0XWK~j`A4S-|jvXV7mj4v-qeA5gTD8tF3s0hl^j1Hqgm$Xa24B{5BVTuM9CoU)KuI z>eVtkbHrcH<|{mF|Jr0#;AQ3C-cHU+{;tdVTDvHIPX4I(*?q}}?Ws7^F7=n-mou4mA=!Lcp4l% z6b0Ja7q%W34&iNe!nG&!y&A(CRmGB;@#s=Ib8Mx-zVjFdt#@OC%Ck~=su)VaK-KA) zCh$#bQqZ0hyiC*F3z#!(~JI(YynUY`GZWsnF**naXRR!`d<^W0QqWy!evNTIt+`8tHoW zQEg}UK1e;S_g#_H+~jk~Hp$A#EqsetS(VeF$3^aYFAZ75Ihx~Twu=`oP~Y`54IZ7z zGTGgpkLr5znvCs%g0swL0KL5`Q;+>#jT5^W;}RKzV`QRhiP}6SzBJn2y>a%VPxCVl zRviZ6+%4KyJn@HGjmhTqwp>&#(d~73WH233ob8_|vvHH$=gFCEVcapEoD3 zf8OfcTDiBTr=^OgE>8|I$8+VSt|xFuR7GCTq-<}c zRiy5W#0MfKWpeJ7jVoXH9-gkvuDvef_LRpB@W$nENHZrt)TPhrv2*=oTgJ*;{!vgx zu4SGmXM5hqU_RNMc&41F$AdVeAkm&RdluUHC^>|6dqMZcd=frR-0d8?WgYLRF8loo zxxA4^Xjym^E?pyf{S02bgytxbIjW*0I%`*?yKc3zE5y5s^H_WEUq5CG%F_xFC0cE- z$jLZ=fJn-#iTCXNm_rsm5Sebrh90pW^stKPLYbIvMU6iZAAB*pFXD2Zd`mgp;Mu>$ z>W`|sE`(El%W9K2L;eN!ZWJ6b8bAES!-+G#qwB^QV&*MKYMjcOl%tRDM*e*c8?{=* z-~+nmPI{*dEcF%sd4m3X6(>G~|IW`=)0LwaPqQ2OF7olXxbYx3cP5)z0+MNBhPS}E zV_CY-jOH-fri`p4&3C*k-LFv8SYCW*5%LAN-KlnTZ!6mVfIV=7WKPepYC~0DWD4q& zk5h2OEBw%5&R@79RV%FuDEI8#QMre5JLlcar>dW~J@-fZ=YN%Nc{p_VM9fpQ_&BUkJ3Q-t!;flt;6@ll$m)b|>HFTe+!vBz}+x!aH_d zzM+R_cX6mQTy+GGQ_|tyw!3W*ps&=r?IlRU;SxfUW$@>NlJ&7(KE>c-her24U&z)J3MTw&jd2<@_ zPQ8L!Cy#}#KF2FLi5c=8?a*^CtP$CsC+L)iV98!^NoQy!q7j#Qb&%Ju=Bd0cei0|y zZ*az9Ykt*_a9n#;(_?AI@16M6-pLh@LULd6#3niY={6o{A>aNN9NAS?V!f)9LG*rg zJ$%NhyE~he`b&=QJ{qZ~*lr6EhEi%qYRXep#wFdeTr+}ZTS%uZhpJ2Y#;K6=oibg6 zMRl%)HEw0W9)X1Vup$x7sjY5$PSzrSY^^R;e{mguVu*a#1aurtBgT&Bne^HSS>@q0 z=O_|)BkfpEbmP)YY5dkp_A(`!bs$**D!w##YwoJN9YvNE-Bt9pqCJatD>^`Dh8jgv zMYiYW=3bT_n0h36apEuWg$Zh-`W3vW6V0hdTOaLk^a4G_e>$?{NLBr(habJkxi6iL z-hXuU(OPmV4V(h-e!l*9eBDb){$uvRokDtd$Rp>eP zc81oVTh^-98fOjP(`n%q>(hcae{$*>7IkrInChqtM2JeHK24sQ{6A_WpfH;~-6Vu6+DjUG#?0{eRh6SteC3RV;NZ3-F^I zat)Ik(78!sHOah8=JGd@?qSeV2XUg8JByWz8a;?J?$;Jb+&OS$ygkHtJL{EZso(Q-gf_0@saQ zV$};bqxY*KOv7OR$is~#13Tf(eVHPp`5IDlK2PXs6dPtw$y#T^&dfO>@wyz^t!fsg zriQ26=l+meBCmX2v%IVG2IURNtDEegnQ zwou=?kz(DKiogz6D_2V2lD%|wz04MwuAAgn8x=I<8TaNHe=1UXq~L6p@CO;vI%e>; z?7>Vt_zR9Yfqon>?|BlGFp8dDl-MIbS2cNF@`7ZGWG!=;AXj*ftj`v4(H?B?4-iB< z9FxZ@jy~&4$;Mxq3b?I2T6Uz($B3;A_wy4QT`ZfquOtIHfM*@}{A1O*IrPb&BJs=k z!L!w|U7+$~r%xa9oo8t3bLiEIY;8FdZ-s|CYpKjrm{~{ zU9VE2kxW%bkE`W1uNVF82#u^$pVdbkpt;EPYP;hLcrZV+GMBs0JtXTQ($bqYzaN*^ zQD4!JcU_!ZJi<2JVzuL(?btI_oE>RpRiXm!S~%uh8IE$+cAHGijlS2|3f2;rkDPxa zW4+N8K1A!0DmdOjjqgRN`;x#G?9hKZ)iy$}hcZzOakihH#?gbc4DkC37P+nK)Hkl@ zAe%wFoFCcQzeJF#^UKbcC9Y&X5uL8d=h)8!_<=^bo<<3svO#>PzIf31!n@eyV|cNr zt28`8e~9WNY_m#_r_>wvPVW+Nc|JK>CTxx@NRgbo#RXqry;||*yV|2tCbO}i82_a= z526d()`|6+o+)Z%&5d%Voq3h{5PS7*JCzr)EO{0QJBfUqll;^Usx2y73Q^!1t9O>D z<66~J4O!e)kmY{1wW^3lAk0Iw-`^sTn=@st!V+F_8xQA+@>K7_lZ*LfTV&xgtbV7& zTyk+peB~0_>3e=|89NfckqDzBSX_%6?3@9ShIaFs7n*BhD95dk@FluX1D?>;DKZ<#BrTZnWzGcRVJtI}if+ zpQy=N*ElL-6n#pI@b61|R+}8JLWzpx>0;M9QLR@+eH?108|BW)ZJIYKFP9%vzQ~_> z59ck(?U?(%eYP9LHh*xQ{VYD;m+B)T?` z3fHl7>opxHz7PdFT44VTfBZPI^tT>K)oF}#*}?YqkMB#Jl&+Ots+MSj^Hh4XfR{Tt z{0|(Kmzk=2-yI%3=<{n`?|IRpMMii$-n$i_edC%1nJY-m$!ZXG<=m6V(Qo+oWa-r9 zFvG2>YjE1{x&zITZ|!eaKy*vFiWgG~N>0=0%}{;>DlcPEXUX3GirSe>_3V!6Um^Sc zIN~ekYOPN`#b3NF{iL%e_CU#ZCErebMY8)t#Dn+@-_xNTWe-lw)<=G22jLA?bCj#@ zCJ8APdn2p)o;w`K#4d%UqK-GnQm>V0aE#$PM*z5G%oxd1shq7^1S&ztn=YaKlQC&=iH)$*L#kD%bkc8A2in*#Zo z22j>YS?#OEtg49=?=GmobGi&SuMwgDg3Nu)U+)Yrp32)x!MgW}sSjf7meU9y(l`HR z+Os<&JU(T0Qr4@E%p0r#VIzH$p6#u-QuRbByR_G&2CE^?PcLx>*?YXg;>li+N_0kjl5OiM=2tXl zIZJy1Z~76QUvWL<){EUnlRREn$%(PCtmV5nCjWNsfa`k5w!9 z0G~Zy#qiQ>J<9=nF~jI?gAy9id3(v_1halcoUb#@>eTwo-LTJn@W|~rE|X{shn$`5 zFZ!@-^IK$M9#8CI($tj)Hq;uv%?>oe(OcQPKUux4ca-5|d>$O5{v~Pj7=UdO$ASAb<*x_%AHgKr_G1sGbm~iybFdvawe| zE49VVW~gJDC_;CydaB;mv!m5KKJgd&-5_r8hRDhQRTXn!%9~^-?vy_o4oQBQJ-MuW zwi|IjeDi0frF*a78O5%ccg3Ms>!nqYQ&ctIN6`F0@_)TKZ#U07;P71{8;|iv+VLF+ zS=$;$Js7@-n!acFs1Nw-L!7co-k~&){A&ocnCw$CC}^`)x=*z2Y%!D*#VOl+)ws<^MSPh2BTcCp!=?ES55>Wd-^<3xgw@Dtj*V<$9e<|6JLHxG^?|y51d_ZMN1qGh=aY($>5=Yi{UyAHX1JiF`o~*I z;V?S(Q#$ZV(Sj9ZvzGPx&3vL0)H5F3eqT6*9ecwzy(x;eY1(beou z^#_B|e5Sn!9jx))qTU~?DE|bCAIZahgBDmJI=3IkM?bHUvLqYDW-eDrSU+2tRSB(z z!$ZfYeTvsPPfr^4b)5P&{F-9tnq=#G|B@Sv{8(_v z87$rrvA)-7Eq4qmc$LJ!@ibV z8U<+%CDljC!}<1D4>g9^do+qJc$qdh8BTu-;$AO4ITBW{Te!?SBg7lOqG^{vpI4yD zR&m5FG~`1(gPkl@4$acY3Vv!e_sPb@F1n1|+aXtZjYskp-dtiwMO4kzrpYg`B0JD> z8qHJ4c673G{mpx{XzIl%b_yA+f`6Z}p0ngTnkDKbqLO$CJ@f!d-9f+I!6uAR$96z8 zGARSU1fA|B!BK~|61_{h>Tp;k!-hyR&8kJo+2S0D0%ry?Ow(zM%nMgsHElqvM3zXGy|ic&jIW>N>pl zDsOfn@2G_PS4O+atY8Kn--gz=6rRBLFNLaiWJ=Rc*OH!mp367l)79nLimO2y%<47D z?myTiFOh?)(c7>SFR!uK;X8P^WZ|l86mMUqsTDmI*G+{5!J9CsNjHyPKSDkj&E zCb$9@59GCt!SR!=-w5*D&Fkm0%pIYQ`{}b!%;{PF!wUPDj@W(s5*ePU5^;qZwU21b zbNM3wsQmd8=kG$v42jQG)n470Vi(9_YkI^8in2Xrt?f^AY)jvW+OLDW!Vb9Q9-I=n zyl!-EB{W${Z@h{kQJeO#dCf4}s;odn(!XX|Up20g=<|;8-auyGHHX{D<3JkeIg!r? zy#J~7-NrAulI#schpXU)ZfL$w4E`SL(M5c0W>ycRvNgm1kiQ?y^hzuD1nriCci$v& zwaJ&WU=!bn?f&AO&}{8k^Sf#N9fjS=@&P(3dMD7VL$XE~)9-Z6X|#Gb`aVgjA0xTld@riR?_|y&aQ_(#Za%6`xdc>`b~6r&^!nD#(gL@RJ499WVet+$DWOi#<-h~ z>152e&{xqrtO`WGH~Y>G`Y+D7%*)nL=Na8r68}A0IY%Fe4J2keZE%dN?@nv=t1GT$ zohSKOL!xGog#F&FqDCVs$q##ND(;%?vA}i9kr2Xk7f{p^xBfp2uC z`0Tlb|H}N&k&P;E#XmOcicm}!lshAXru^6|Yx@@?o)^ z488P*$8;;$g7iGju6<6%KcWYZx0C)!K0p&$m6mYFMwVn0O*qfU-tz6M@aa0=f7fG} z_5B$oZl^;>vy6Ap1ILn&L+EhS%0~6hHe=ZAGa+F&8{wnu&-F%s2c2~fT~Lp1U1wEa z!IzE1OB;~3@A*AlP=AzrPNy+GWs?W7WwHOKFN&T`-fyJIhroS(%=ZwP|3}Oz7pE4* zB>_p$j7F_$5uyB#5(?uR6b`V z=b_ntt8|L@Izj{;t#jz~z`*rfa}SD6ca<-VVH;FZ8S<%!E>+DadUwP(E3?ptvJ!m| z{R+_K2O}JXSEl*qEShRK`o$j6Y3{czyN4)g%???I+0e&FW;MyYzBcA>eQpA+{~^pc z-Zx)1&JkpK9ywoZWlO-C=Nf11t-TdLolTDF8fOY`7bUYP@_y8OE@B;DkokWV|2*u| z7Z~d)yu{N;ZcV%poucN@28DJoACD_apvoTf4C_%D!aK=lkGCRqaKRbbl?n8GHm%XX zJ&ySHTw|Ys(tGgJIwPNrPN7#L_j(+Di(bY{XuKoVrWSgY$Xcb8ZrD@k+R;{E9Quzn z>yJEE(B=s?wS=*i5K~&`yR-1fZZvOCOFxgF`j|&VR2$%f>`Sk_(3x@&a9tjYGI(J1zcmG)|`^{ZmeCHx_`&}#!-YMKD7vd0?@C9EXNNHyhu=$}QS1_nK3VURji*T1 zo8}yQ{6@kli)rzt)^x4=ta6`4cr>Eb6VdSp--7npGQ~HW zX2%pA7s}%KN+^De*;TSW=hKjF>HU51b*#!7e6z`{x1nytjH=`9>qzHw^hrO@TGK?a z2eJ`5L=U>zxb1(&w3k+?g&wh&@+{OV#~S4Mc4<<%!zkvV|6Y<4dDk`Sj>GOQfeEIP z(wXXhCYtX{#{HSUqmC?Te3gwhWsSCyxn0@yu7XyzQG31n&T-XQWB&jF+1HG?H7>92%B8c&_z(Bm>6(*W ze+_Oa1`&jQX>6@dc9r^BE7X<-C~6hs5j@h2dL?ls2$UuU-cu2J6WHLS@Mu5lH;-rnyfR;h$FJ8BMju3L)K9d*UMbi-%l zAa?c4M!%`%{j;mh#VxT{{-DQZ8hVB~9JWLD7^AAmBAn$afBViV^jksNX1dOBD>%jD zJKv7|*Wnv3z*XO(!2)`0x$A!GbJKk03*X)3+R;z9idp`P?^n_7yU_eE*ISH-mYeG~ zcMZLAg7N43dzE*h|G{x)cq}au-Dc19INeoBlBTL=7G6t9f92rDysWOQZ@jHYQzPH0 z>v614pN!I{d-pUn`OAF|*wGff-V1TqA#^_IzPa9SfzILkEikLGc-n@iGZv0HVs@Bd>>7Fz3%>Bx`C(j0eNWbR8* zemrC}27mnP&P|OgFyKZKw%fP%)3q6&`NaFbX0hgWqge0tU0&VfGlx8n3f?WQIT>%w z!7+<`vbt+EHq#U=t)yT?t&c@-^q8S3>Sai`Nj$D?<3>z`*OWAE09 zWVP#05i^Re0Y%`tUDj*~X${Rfj-T>5K3hl|Mm6pd7Gtv4hLMJmR(>)mpMXmy z(<497(+5#4EZ;x$)n>Bus~Ht{&3&+b#@$L~?;V}66Gm1@jsi=T&8}J-t5?_i!Lhqh zb`2U9H@9l8xDNOHhQlM;cFu}zTY;^<^}bJT zH=;C|XlRaec}Ne_`_GxdM|d|pr0{aq`A*uXQtV5hx+U)U84a+Iv_>U*8m*4Ek`2&2 zEPVqsJq=ac`oFamEWrwg_BiNXp=bA!qYdU*9`BVfx*Ayy3RIAAr84GK+f@pAca^P7 zCw@w2{LvDcYDfRnF!P8)U4VP8!MSI8=TbOq0BnCd`DlX|cW0$6;aO!|5dFr>qH-mp ztb;#phS|>ce5cj<&|E_2e}`@{zVLu2(Z|1|X*E<2-z<=23ICsw<(7_Q@meFg!u^{V zNo8Y>owntSyByoSk4^~Bud=(I;M+~i@)*=DZ3U0vZ&~fK z$elM^slBKjo#yiK$#U=QvSxXxvIoWgunH?^se{(%SJE)s3f)T>W+FXA?NwK&#y4X-Cq68XFl?&K-GU*=N(2K=NgoT%fcp{Ktp%U>g{u^L3_3{ zI)*f4@tUAxV4d)9Q!GXTXI3Dm%iSsZiv8|tfd?DWEj4`O3_lI6Wf?rY13&%YKKoEO zH1pqV*qbCM<^A#~av{Ds2Ce?Gh6}v6)<^@zuJ*|i?oxvOKF+DiCyFfj`aQ7x&uYbKw&O zGCg99p$9k6fYC*~w)Lz@s-osT`X1!a4ZF$2Y_$B+_fyt1hwi9p=4Cxfn|Uj9YVLP^ z+$!A#75o}{`#+x{cApmUnPcgy zH7FEXFk&s;ab_>8)YhGj&EEG6cZ@tjbv&2ru7Rl?it9TIQROTD&qCSoRS%j;8TSa! z>Y!J4dL_<%NxM=7^QcOaTN-^&eBa*hOWZHKo`_v!jAt*Nn+5ZHKsQdKS-y1riRd~T zk8VW0qE|MXW|xg^;Bk zWV4QWpJwKTzDoprBlxDkszlfJ3C8y~y>kr8l_MLw&EPk>HhP@Tb;sXv=58|jul3q) z-L~U{EoQaawfB;V_5M%!Zs6m{MeN73+kEe9*O*6&{|AXaDt7r2To$NwxVg-R7ebZ; zfiJSc%W0I~y)qlW{9vZPxOQ<=j`_q`3*9>x7gonnw8~Mx=hHSH`{okz zbHG~dCV6o>%3*w5$!aI*>(JZLXRlD)wJv>@@x3%Ds)+v^uvdNfe1SGEkqH|h$`l!^ zoABf-*{(9@v2my1j4J$zzs)dgWYicW&@8&>hp+dq(fs4Szw;lyF}Gztx!pL|(&J;v z-S6lTG0Mf>nQx^(@%YAE|LdL-_BSGwe_5mbzPHUBLweWv^pB_=cA}QY$#^z!c^VI& zU`!VobKuX$v{A$?BQsM6m5O)-syYduU6#d=O=+JxJ{2dYbY?X!_pA~r+hr9)+7G#R z*oT_vQHKNt=56eG3uvJu9&heN4+(u&qF?dr3# zQI9XPp3bLYLT})%MR;PFxvuxKz+Z*v96r@nI%JFc#;lWOU+DQ!ug|CXXS>oIvUZez z5uDtT<__BzzDy;bEa|5x`G_uaVQ;^Ol1H!^tI)a_%I;(bLwjw-;qkNC`b2bbgAp(F zxd&AM46#aMeeY{>A5rzNnR(`%V-zvtLs^N79w-G?V4v|XbIqbwuO>?Fp)tRL`DS~y z0KMwr)#_F_bn+=ys}=ipu9Y~=tJQIKc-#B&LC`B~N$Ak<1OiEvK<$&P$AwwW??IaT zlko1OrI%L%G1kdy|A^y7>>?uk(Pga>nwZL}5a<+q?QnWMd{S+esEGx%#m*crI%FjM_h%e+gqNZJc3ozV*q7uK$HEB5E`n zMI&mm2Ce6t;|}jEB15CC&`PiU0v){~ClDtxzAS@uuh{;3^z^&_dRzoN>W-J;oC$2j zG;2DZKL63*KluHa&qNi#D4DI-Y4wO^eac69)ARS;<3qWyS3RGIA13>Mq~~wC+q-Pr zqoSN+J&zvigZO7(8|xgr{fruj-ZDz<)hpaCkM*#-JSErh5iGa>pUpG7z;}zyb&|0~ z-e#Wh{Ddcqlh){nS<3o!x5^Q-?BrQj{zy+4qZb}{R^IGmaj+?>&Nu5TmnVB!SNDRZ zb^!IXbMP5^#6DE1J9*ls>b^QSDWq(oz<#yZhk2s9gRXWSx3L4YjjGEoa`;E& zfS1bcR#A0P-wH$*r!GF%&F3yqZFHe)w9t>IqMEIBMzhl>R?Da!Rx4Uht;VTpk!pC< z^4->YY_#>gD{w>b!}-S1S{CmF72Rc>b5kO5K&?ojJ;{gcKKwIhrF`~W^@T%ZvWCj@ z#EB}g8##QeC6LPJbn6_FGElzu5n8jSTJ1)%uk}$eUtQQ8*61P?Z(Xh35Or?@)cti- zmwv!bzW1G;_CLLtUUX{NDeCJh6;vxYL8bbMc2CtQD6TT{Z;##ihn()-*#3@d?emBl z{|gFkR4@OgeO7ndb$6}bW3p>~xS)}%zichzgr(@gw@m%lZ1rD1;;-_un}^lo?8vmH zBZiUk!Lqi`sK$7V2AR#;{7E~^rX8Yl>#gSVs!zSgZhTJLtfSk1r{}}djBb@Dvn`u& z$j9FQkXP^)RQMQOFos51z~Tgg2;8K;-(ctJ19n6GsG@bNz7!+8I@&SXFl?@wk!wUOoz7_>(LQ7Z5dIsT(Z{QoeNuE2Z zd?~6@CwdXCmJOb(GIK_DZ`cga*Q&=Wtq@~Nc6Zcp^?fg89#BzztDQYJ+s9koqrPg| zugL!<)w4%4B?^m>$LJN*R4qU|8QXfQymM4r9`M*|zd@c#jN|Q$IYkGC%dP7`djAdj zJo0GstXWz1t2s*+F@y}iuDs`sVWmb{v`~kAiWO}NA-3~4KkIWw58H?lm1kLd;riFD z%_H)%!(@UU@_bM>)7pZcP~0_RH~+O&V;z#^8h8-J#KkxU6 zzA7<#WNS#O>)W5B683oXi4%lss*Bx?K5@p_L6z7QQLu@sqgvjnM8`Gs*$OC|=aq(j zVwZMPRcp7fSdaLLZe<^k?7%%gEM&9cG zUvcbsH3u)K0vSk;-EOS0+UHsIY7o;wDDq^;qz)cg$hv>bO7};H7tQ}?bf3Uh-KIt+ z_AH#KBBh5s{w?Z;+Sx18TD?dwYwJ;YE;y?RqgnEU|>Vu0U{`G%YhOiC2*o_u+c6a(< zAWQwcz2Vof04M0v-+)Cq-MekriF6|7y(A|GJ&#T7RjK-Uwuba^8P=CoDO^DY8pEZ< zviLO*uC0SJE0c%FeMDAwChi*!7rn;ceGso*rGoYt)zN9Zaym|m{#2XT?UnXdd`+gm z)_Y>Qdd62(r9MyUSEvYCrIz}8`yOs*2hLYz{y2;N0(<+CeFtOgejKC5c^$n_P8Iih zc6p-Dj~{S65prl)4YeNn;iF}WXz`+A#6>`<+17gaespAI@jVn1o#&W{*Z``~DM7RQVeXV^7Hj-#C&vMh#^pub05ZozU(o zH2aw(??jP^%mpGZ@GK$;5m!6~ZmUl|a`NRQ8`=F?Eiq;OM4c*087xOk?+mt|72*D zD!8yJNvY3k4gV@S2h|Cy_6ohl-G2D z`(0+eBMTgv?uaCuYAh{1q6cD}2^~JxxklBMwu+Nu8;NFR%rSDH5eryml_IKekhKfn zEQ1CcXu@CUi0?@FAN1jFSEFeR}c>WA&zs;;}!FT=DGW7HAFo=CRO*0dE{~UIYt{AWIC>~Hl@(|5^E)5mg zpEz+evO7^7($$^o(Gbz|ZL#^5@GRdPXPSR>E&K{n`VbC!LXPKiaygm?9gy|A!f%>@ z5>tF;0zElS1nEa~`+#PB*>@w7`vcxujnh`5>wdQJPpi8Pg=V5{WO=voiB7P74U8fx zRpNZsh?GQT@oe7Fxj41~PKZu}EwigWz`RD7|J~+uH%{!DokMpNjdLun_Kl{la}v(H zJZr12z&Vi}jM~WZxO1nut+75UtXjU6-$g>F`77e_k^R}kHwa(wHIL}W`M%i-=gg*RE(;D zMQB?AAC*(vyU1%x`8RvbD(znNva}1@R&=$5pU74g^(>+jOIgE@ShR7jx5QN<3bqOV zZn3UY{mgNXs8ieLy{P!gC*3*ZAhI`U*QrX%nvssku@typL`CXa)05$aYw6u5;jGT0 zbLV6^En+_rg*eT+U1Fv^MK$lVo^SJdN6?~IW!F3;IjV_6GaPo`17uQnOV~a7H++Dz zq7p0YTXct>gwpf8I||B=v-a+%3D4!B-@@h(vr@NOsfTFb*F+JXwQkRQ{c?7`EBiN$ zWtj)hN4EYMoIQ|zAMg3weh;_0|M790&9js?pNgL&PrngOB0e0Ij*-pWfg0;k=rA-| zQ9R~vqlvnKxqOVM7^{whBaR<&ld#PZ^({jq?xQ6ls$Wip&mx+53eNl4cZ#8C6WTJe z=MiU1`U$yBI@dDdT#lEqdif|&6s4QGLLKyu^^Hux4p({$-wtxE$iN@5QjtZ>LCgKD zd~t=Wh6xWx)a9ahv@MQ-&>D_qL>PN$LY z%&u_Mciv3?E+My370}&xZlHNyQ`!5G$80EQG?X2+{MXRktz0|uDp5Dyi4@=JE`gik z^r5b_T-4QehCYYz$)l70ZQdVAtBp_-dOLl1J^gs1$Z=Oc5%sQS%neZ?bXyaT zmhY7UEAwlX4$J)huP)+sQa8_?qjGN->gT%q7P@YQ_t&#CKN`m`-reiHdggkOpXfqc zLQTR}D^nC_#?J45(LMUlTx_lv`9IDeC~M4-9}Ah?;3wk4G1k3gBSkJM4VIC}+j1VIyN0tTKhR$>mtW!Y zKE^yBDxR+1S2u|2Qo2(=C_tJ3B``!a|*hn;; z1b2Q+3P$6-UE=61aBC^_EuHfYPOD-UU{pM=MB&}&9Vdg%bFD=aY03M zpO0Vn(VzJ`a_kq;Y3TLkR&bLQK0!~3P59(Q+#Ffi-m;M*By$sc6m&ybzT&?TqAj+O<*kJ|*mYKGjvD zI;J1FYOe}C0ZFzd_dR$Zcay%i;KGigjf3Fwp)77E^tpySG-Ok6@`xyNPt@$o*7oHK zO`}^o!A8BUL0?$%7Mk^II`$Rbe1F#X4LIvN|M&2z`yuT+Vaj)TQV;R<`|$eT^^LZ! z{SaL<#CuP;M^Blz`)Hlm5&D$zHSua|ueG2dPl3of!!$Q}y&0*D8t7-xsSB;&hYpHb zotDO3pCn#NW^&Q-68baFfe3jGZLt_PMs2`ovNDb@I29L0HP>AH^qsX{h_1g`@!zan zAHGmvfq}5-aCZGAyPP_ajFAxCmp(fY7e;1afh#X|{k=G7lUMG;L9h8%RKgCxt8>U` z$if2eE%e!)Bxs#aeC)b+`_5pJF;H~nK7AP;gft$@M!+A>#vlj4Af0^bCVNkY8~G&P zEl*FhpjR?v`X4`$0ceSDqAokoev&PVij7;W^7*J2b?=$NK*t@tAEz}PAW6r0cEtXl zvUF_Jm6U@v*Wm4zvIkGIkKI|<$bh!NhZjR(^|R<;4{rXMRF+o#6OK~r$7)ylFY0zsg+atV%FuA(_xw()y@)z* zv$!k0GMV(po|aAYPMlu9lFpb&MjxT4hG%utCs}>6*=HA9%?t@BXLf;Xq9Up&`Kd+@ zS{YR(6$5`--#n5OxxN1?19xPtP}H$4VIA^~HO^R?@P9R(c~F(t8HXpHPsf4 zkp@&06afKQ1limWMG|xSJfH2{xh(g7-}%mY_w%0Ldk!4fh&|ZK|FwZFsKa{gM)!8W z(Gsp`hX(D8##1{IgJ(>F?qhkL>&D}t`b|dJjeqbIku&JNXytV1em7UA;M@GpU8}Ou ztrc+b0Y2}*)@KHu(#U=|*YyV;RO@*RSrx(QxmKP@CL;Vewl|HjhM~=0$q2fx^%k~F ztUwzSd6%m`VyxYGDrc)R?{{itDa=s2yN@()KJDW9eH?~ z9*XC&+9Zs9n~Q8XJ5NW$()c-&c;N?V++yTy06u)mh@BzDAg>ppp~%F45J7E+dwb|x zHBY;M1vLNSO5(SXI8Ou{%`^p;_=PS zVa4;0K$osV`$Sq?id9^RwYd*|iRu=0@DTT|#sbdbUSqs;+IfQLQwx;?mE0fBIZXm> zHDF(y2bV+lFnd*GKrt($2O+7G89^rPYKc5Mt`Dw>x)T@TX-Bbq>q*MJ$i94YFV;Tp zG}~~KNT`Dzi>Zi4-d!O*jf_@ujp&4SWX^LUoX1U|Cq}w)jLY+KoO5JA#WjpWA36*l zxe_~(1xG}niZ(ObVa(l$2?V!b?BU(}3Ow3Z$USrPmF)fSJm;iNippEi|2j6rYq!|6)?yTb*yQA|%V&-gX4JC$cVjE_j+Q*)i-?9>Wlygp>snuC+je#rmwEr%GDv(=$+ zEs`v>2rbte)Q4)Q;*3B<&~SPY&C^*TgObXc7Vj)=nb8!_=gc4;d5Yw_I~LJXp2Vhg zK}BPPICi8b2R!G zvGc@wwAnSN2xMI}rWFSrjK`XUc%scpM*2Dti+iqYVP)eN$nUSQ>7J2!g1eo0UE?)? z4mvAs!0w3AvHHuZq|cEDtxFs+fmX}i$M4}uF-V_#O{`}g|6^W7Smt?*EfHJ#II-P* z$c;FGD8@LSxbzvWSjosn(NANVc;rGXSsneTq*dkEJg75Z>#*Ftd=QeMok-=k_@62~ z)On;x`RhhMx8vri zTNKj2_~aD4t3>m6py8gGEiXHv$|dsY8E`ZYjYy#P_abA)vNMo}82Tl}+j&+J9CNmk ziX2A3agj6Pg2e+`6_bwjb-vsUcj|asyJDs@j%z#NmT2BKdT*Y>Q_`w=zIe4u^kGoo zC$uQ8;_YEHb&OGapze$Nb5D#pyyxQ)MB6Tb3v=PegK)(>%|@cj`RMh0^#6H2b8Ro1 zvCPCSh(CG^-Co5#t{kOfb-u%_<4K~(N0H8I@vg>KsBskFL@b_;0L)tk*o(Tmuqb;!v)+8M{Q zzRulx^&V)`51pjO0Mu@W0;TY>7MgTHC3RZ7spvzsClxu;V#tHr=!RS>g}aSNKm-(X z))|X8(brnL)y`G1#K@KCn`nM>5n3Jdn;#IJ?L>>jbvyU0=K8moGyEA%EaSiPfd^8< zwQ0RboET5Dof8>nBGO<5%r;`w_sQoRLe{icV#qUSU+k7~LK*$ok4#&Aw1bw<2ALdqxdY*n%VfWvouiMZB<=Xt^UfQgpM|~iNe*I$ewfd!D=_D6Ccq5r5qc^f zSGrpJwhRAR%rm;6fa_3K0*YCKVD*`qoO1!a%Fs!(UnR79gnPyLTkUiM+meT5WMa9@ zxSpZyPI_{cpOyT5l9sx#PGg|%_mDhSrH$$~v%3E@aZw?3%0ULkB5B5JdWsfWSN}zg z{a;r{pvANBWg)cBR~PbSMT`hupIi~ho8H3ex;R7AZuf3*V?5t zw73u*w%Wod}xX=SjxA)4>Y zXM{$c)`N*Xw%$3CD_wU>2*#RBdsbYTk#FYvJK58dw4^3lvnDFjnXVC+I$VvtH%o7Q ztFvkmvGy&Fk<7ztnN`*ITc2FWc;;}Ok&0FLXKA+?&WUE#bL~g|kMcUm+*h>WZgj;J zq>@0Uoi$q}ZMxuQBX3Uh*Wp=#b~<;yI@6?=kV$? z)9|XEC%6DQixcUT`oOJCyHx> z3u(dM&bu;sdET@u7S0ZqK-C#MaSWa!#LU1NM03o}#tP_%wH7z&ZzB|_!#-I{cpOc0 z)?C2d1<2$Lo?-Q+GLgdl=IeW))+r?4*i>0RAIzgaGF$e_a`#%pI$+J3k@@T8`u_;t-2bhgzH9OHWJ-4*e|cJN5^Z~$jF@q;mf^q}$jKBW zKwn}VnDS(`#4#*ED#3>T^d*t2v#(%<^Q2gD;PyI>I>iJ_f|YpDNh@W)R~iV&Ga+u za5;bJyN~n#zp>Eb%?>aMPf7_x>m2Nu>ypOiMle?Rq@g>;L@R=9f^`^+kqmcp7>;zh zGV?8LTMXlIW-|iWv(hw|&)nz0)t*==n+~-Hp)Dy$jysV(#GU4paNiD9gq%RrVq zW!+eBOkkT`YfYeSt%GtTsv;ajg04VoO~he{g0eQ!)hTNT#dnG6a1D1DGMr719tmQ+ zM~U^!D6gT<)?$pr0%Y_2QQXxGe_D~UR>qNx)@9R!1$=M4-+cVdIwHyCjLr@K(DfNr)3;)B=i%3^qXd!yIg?%NP!^hZb@tyF_(D?9njOiJo&15p+k!bZgC_R|*&P3mj1abRnzE8!jJ_Vg} zuuHY{xs%AD2M(3sGpvOCjQSA2@x;_BG9`u7NnXZYIRA}70DTh!GV*&5d~3vGmwEQ9ZniWggaw z)LK1j)q=5uS(3}#@0xEX-xaeKbPzxEGBUppI*Uknf&VrT;a$W=x>wKh;qRlnf8gqU zLDVLu;TSKo>CTVLSGtB*K_9im<-E-HAj_gLBJ~9{cFJLOLtNZyX`F&UDO{}Io+=VU3qYtx)WAZ^)Tt)(}a8_CoS_AiAgeJw@ zU&a+zpyl1n8*JYyZP%==`Y{J)+-t679OH1FYVOZ$%~kTJg=q9@#vn?e4vJsqvnlBJ HLiFXo&2)c3 diff --git a/tests/router_unit_tests/test_router_endpoints.py b/tests/router_unit_tests/test_router_endpoints.py deleted file mode 100644 index c4a2003237a..00000000000 --- a/tests/router_unit_tests/test_router_endpoints.py +++ /dev/null @@ -1,261 +0,0 @@ -import os -import json -import traceback -from typing import Optional -from dotenv import load_dotenv -from fastapi import Request -from datetime import datetime -from unittest.mock import AsyncMock, patch, MagicMock - -from litellm import Router, CustomLogger -from litellm.types.utils import StandardLoggingPayload - -## Get the current directory of the file being run -pwd = os.path.dirname(os.path.realpath(__file__)) -print(pwd) - -file_path = os.path.join(pwd, "gettysburg.wav") - -audio_file = open(file_path, "rb") -from pathlib import Path -import litellm -import pytest -import asyncio - - -@pytest.fixture -def model_list(): - return [ - { - "model_name": "gpt-5-mini", - "litellm_params": { - "model": "gpt-5-mini", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "gpt-5.5", - "litellm_params": { - "model": "gpt-5.5", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "gpt-image-1", - "litellm_params": { - "model": "gpt-image-1", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "cohere-rerank", - "litellm_params": { - "model": "cohere/rerank-english-v3.0", - "api_key": os.getenv("COHERE_API_KEY"), - }, - }, - { - "model_name": "claude-sonnet-4-5-20250929", - "litellm_params": { - "model": "gpt-5-mini", - "mock_response": "hi this is macintosh.", - }, - }, - ] - - -# This file includes the custom callbacks for LiteLLM Proxy -# Once defined, these can be passed in proxy_config.yaml -class MyCustomHandler(CustomLogger): - def __init__(self): - self.openai_client = None - - async def async_log_success_event(self, kwargs, response_obj, start_time, end_time): - try: - # init logging config - print("logging a transcript kwargs: ", kwargs) - print("openai client=", kwargs.get("client")) - self.openai_client = kwargs.get("client") - self.standard_logging_object: Optional[StandardLoggingPayload] = kwargs.get( - "standard_logging_object" - ) - - except Exception: - pass - - -# Set litellm.callbacks = [proxy_handler_instance] on the proxy -@pytest.mark.asyncio -@pytest.mark.flaky(retries=6, delay=10) -async def test_transcription_on_router(): - proxy_handler_instance = MyCustomHandler() - litellm.set_verbose = True - litellm.callbacks = [proxy_handler_instance] - print("\n Testing async transcription on router\n") - try: - model_list = [ - { - "model_name": "whisper", - "litellm_params": { - "model": "whisper-1", - }, - }, - { - "model_name": "whisper", - "litellm_params": { - "model": "azure/azure-whisper", - "api_base": "https://my-endpoint-europe-berri-992.openai.azure.com/", - "api_key": os.getenv("AZURE_EUROPE_API_KEY"), - "api_version": "2024-02-15-preview", - }, - }, - ] - - router = Router(model_list=model_list) - - router_level_clients = [] - for deployment in router.model_list: - _deployment_openai_client = router._get_client( - deployment=deployment, - kwargs={"model": "whisper-1"}, - client_type="async", - ) - - router_level_clients.append(str(_deployment_openai_client)) - - ## test 1: user facing function - response = await router.atranscription( - model="whisper", - file=audio_file, - ) - - ## test 2: underlying function - response = await router._atranscription( - model="whisper", - file=audio_file, - ) - print(response) - - # PROD Test - # Ensure we ONLY use OpenAI/Azure client initialized on the router level - await asyncio.sleep(5) - print("OpenAI Client used= ", proxy_handler_instance.openai_client) - print("all router level clients= ", router_level_clients) - assert proxy_handler_instance.openai_client in router_level_clients - except Exception as e: - traceback.print_exc() - pytest.fail(f"Error occurred: {e}") - - -@pytest.mark.parametrize("mode", ["iterator"]) # "file", -@pytest.mark.asyncio -async def test_audio_speech_router(mode): - litellm.set_verbose = True - test_logger = MyCustomHandler() - litellm.callbacks = [test_logger] - from litellm import Router - - client = Router( - model_list=[ - { - "model_name": "tts", - "litellm_params": { - "model": "openai/tts-1", - }, - }, - ] - ) - - response = await client.aspeech( - model="tts", - voice="alloy", - input="the quick brown fox jumped over the lazy dogs", - api_base=None, - api_key=None, - organization=None, - project=None, - max_retries=1, - timeout=600, - client=None, - optional_params={}, - ) - - await asyncio.sleep(3) - - from litellm.llms.openai.openai import HttpxBinaryResponseContent - - assert isinstance(response, HttpxBinaryResponseContent) - - assert test_logger.standard_logging_object is not None - print( - "standard_logging_object=", - json.dumps(test_logger.standard_logging_object, indent=4), - ) - assert test_logger.standard_logging_object["model_group"] == "tts" - - - - - - - - -@pytest.mark.asyncio() -async def test_rerank_endpoint(model_list): - from litellm.types.utils import RerankResponse - - router = Router(model_list=model_list) - - ## Test 1: user facing function - response = await router.arerank( - model="cohere-rerank", - query="hello", - documents=["hello", "world"], - top_n=3, - ) - - ## Test 2: underlying function - response = await router._arerank( - model="cohere-rerank", - query="hello", - documents=["hello", "world"], - top_n=3, - ) - - print("async re rank response: ", response) - - assert response.id is not None - assert response.results is not None - - RerankResponse.model_validate(response) - - -@pytest.mark.asyncio() -@pytest.mark.parametrize( - "model", ["omni-moderation-latest", "openai/omni-moderation-latest", None] -) -async def test_moderation_endpoint(model): - litellm.set_verbose = True - router = Router( - model_list=[ - { - "model_name": "openai/*", - "litellm_params": { - "model": "openai/*", - }, - }, - { - "model_name": "*", - "litellm_params": { - "model": "openai/*", - }, - }, - ] - ) - - if model is None: - response = await router.amoderation(input="hello this is a test") - else: - response = await router.amoderation(model=model, input="hello this is a test") - - print("moderation response: ", response) diff --git a/tests/router_unit_tests/test_router_helper_utils.py b/tests/router_unit_tests/test_router_helper_utils.py deleted file mode 100644 index c2ed526e4c7..00000000000 --- a/tests/router_unit_tests/test_router_helper_utils.py +++ /dev/null @@ -1,526 +0,0 @@ -import asyncio -import json -import os -import traceback -from dotenv import load_dotenv -from fastapi import Request -from datetime import datetime, timezone - -from litellm import Router -import pytest -import litellm -from unittest.mock import patch, MagicMock, AsyncMock -from litellm.types.utils import ModelResponse, StandardLoggingPayload -from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper -from litellm.caching.dual_cache import DualCache -from litellm.caching.in_memory_cache import InMemoryCache -from litellm.types.caching import RedisPipelineIncrementOperation -from litellm.router_utils.router_callbacks.track_deployment_metrics import get_deployment_successes_for_current_minute -from litellm.types.router import Deployment, DeploymentTypedDict, LiteLLM_Params, ModelInfo -from litellm.constants import DEFAULT_AUTO_ROUTER_MAX_INPUT_CHARS, ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY - - -@pytest.fixture -def model_list(): - return [ - { - "model_name": "gpt-5-mini", - "litellm_params": { - "model": "gpt-5-mini", - "api_key": os.getenv("OPENAI_API_KEY"), - "tpm": 1000, # Add TPM limit so async method doesn't return early - "rpm": 100, # Add RPM limit so async method doesn't return early - }, - "model_info": { - "access_groups": ["group1", "group2"], - }, - }, - { - "model_name": "gpt-5.5", - "litellm_params": { - "model": "gpt-5.5", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "gpt-image-1", - "litellm_params": { - "model": "gpt-image-1", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "*", - "litellm_params": { - "model": "openai/*", - "api_key": os.getenv("OPENAI_API_KEY"), - }, - }, - { - "model_name": "claude-*", - "litellm_params": { - "model": "anthropic/*", - "api_key": os.getenv("ANTHROPIC_API_KEY"), - }, - }, - ] - - -def test_validate_fallbacks(model_list): - router = Router(model_list=model_list, fallbacks=[{"gpt-5.5": "gpt-5-mini"}]) - router.validate_fallbacks(fallback_param=[{"gpt-5.5": "gpt-5-mini"}]) - - -def test_routing_strategy_init(model_list): - """Test if all routing strategies are initialized correctly""" - from litellm.types.router import RoutingStrategy - - router = Router(model_list=model_list) - for strategy in RoutingStrategy: - router.routing_strategy_init( - routing_strategy=strategy, routing_strategy_args={} - ) - - - - -def test_routing_strategy_init_valid_string_strategies(model_list): - """Test that all valid string routing strategies work without error. - - Valid strategies are derived from RoutingStrategy enum values plus 'simple-shuffle'. - """ - from litellm.types.router import RoutingStrategy - - router = Router(model_list=model_list) - - # All strategies from enum + simple-shuffle (default, not in enum) - valid_strategies = ["simple-shuffle"] + [s.value for s in RoutingStrategy] - - for strategy in valid_strategies: - # Should not raise - router.routing_strategy_init( - routing_strategy=strategy, routing_strategy_args={} - ) - - - - - - - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.flaky(retries=6, delay=1) -@pytest.mark.asyncio -async def test_image_generation(model_list, sync_mode): - """Test if the underlying '_image_generation' function is working correctly""" - from litellm.types.utils import ImageResponse - - router = Router(model_list=model_list) - if sync_mode: - response = router._image_generation( - model="gpt-image-1", - prompt="A cute baby sea otter", - ) - else: - response = await router._aimage_generation( - model="gpt-image-1", - prompt="A cute baby sea otter", - ) - - ImageResponse.model_validate(response) - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -def _rpm_tpm_router(model_id: str) -> Router: - return Router( - model_list=[ - { - "model_name": "gpt-5-mini", - "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-fake", "tpm": 1000, "rpm": 100}, - "model_info": {"id": model_id}, - } - ] - ) - - -@pytest.fixture -def router_minute_pinned(monkeypatch): - pinned = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) - monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) - - -def _ratelimit_headers(response: ModelResponse | CustomStreamWrapper) -> dict[str, int]: - return {k: v for k, v in response._hidden_params["additional_headers"].items() if k.startswith("x-ratelimit-")} - - -@pytest.mark.asyncio -@pytest.mark.usefixtures("router_minute_pinned") -async def test_acompletion_headers_read_post_increment_counter_and_count_once(): - router = _rpm_tpm_router("lit-3058-async") - - response = await router.acompletion( - model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], mock_response="pong" - ) - total_tokens = response.usage.total_tokens - assert total_tokens > 0 - - headers = _ratelimit_headers(response) - assert headers["x-ratelimit-remaining-tokens"] == 1000 - total_tokens - assert headers["x-ratelimit-remaining-requests"] == 99 - assert await router.get_model_group_usage("gpt-5-mini") == (total_tokens, 1) - - await asyncio.sleep(0.5) - assert await router.get_model_group_usage("gpt-5-mini") == (total_tokens, 1) - - - - -@pytest.mark.asyncio -@pytest.mark.usefixtures("router_minute_pinned") -async def test_acompletion_stream_counts_request_before_headers_and_tokens_once_on_completion(): - router = _rpm_tpm_router("lit-3058-stream") - - stream = await router.acompletion( - model="gpt-5-mini", - messages=[{"role": "user", "content": "hi"}], - mock_response="pong pong pong", - stream=True, - stream_options={"include_usage": True}, - ) - headers = _ratelimit_headers(stream) - assert headers["x-ratelimit-remaining-tokens"] == 1000 - assert headers["x-ratelimit-remaining-requests"] == 99 - assert await router.get_model_group_usage("gpt-5-mini") == (0, 1) - - chunks = [chunk async for chunk in stream] - total_tokens = chunks[-1].usage.total_tokens - assert total_tokens > 0 - - await asyncio.sleep(0.5) - assert await router.get_model_group_usage("gpt-5-mini") == (total_tokens, 1) - - - - -class _GatedIncrementCache(DualCache): - def __init__(self) -> None: - super().__init__(in_memory_cache=InMemoryCache()) - self.first_increment_started = asyncio.Event() - self.release_first_increment = asyncio.Event() - self.increment_calls = 0 - - async def async_increment_cache_pipeline( - self, - increment_list: list[RedisPipelineIncrementOperation], - local_only: bool = False, - parent_otel_span: object = None, - **kwargs: object, - ) -> list[float] | None: - self.increment_calls += 1 - if self.increment_calls == 1: - self.first_increment_started.set() - await self.release_first_increment.wait() - return await super().async_increment_cache_pipeline( - increment_list, local_only=local_only, parent_otel_span=parent_otel_span, **kwargs - ) - - -@pytest.mark.asyncio -async def test_success_callback_running_during_pre_header_increment_does_not_double_count(): - router = _rpm_tpm_router("lit-3058-race") - cache = _GatedIncrementCache() - router.cache = cache - - request = asyncio.ensure_future( - router.acompletion(model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], mock_response="pong") - ) - await asyncio.wait_for(cache.first_increment_started.wait(), timeout=5) - for _ in range(50): - if get_deployment_successes_for_current_minute(router, "lit-3058-race") == 1: - break - await asyncio.sleep(0.1) - assert get_deployment_successes_for_current_minute(router, "lit-3058-race") == 1 - assert cache.increment_calls == 1 - - cache.release_first_increment.set() - response = await request - - assert await router.get_model_group_usage("gpt-5-mini") == (response.usage.total_tokens, 1) - - -class _UnavailableIncrementCache(DualCache): - def __init__(self) -> None: - super().__init__(in_memory_cache=InMemoryCache()) - self.first_increment_started = asyncio.Event() - self.release_first_increment = asyncio.Event() - self.increment_calls = 0 - - async def async_increment_cache_pipeline( - self, - increment_list: list[RedisPipelineIncrementOperation], - local_only: bool = False, - parent_otel_span: object = None, - **kwargs: object, - ) -> list[float] | None: - self.increment_calls += 1 - if self.increment_calls == 1: - self.first_increment_started.set() - await self.release_first_increment.wait() - raise RuntimeError("cache unavailable") - - -@pytest.mark.asyncio -async def test_callback_observing_stamp_before_pre_header_increment_fails_leaves_no_stamp_behind(): - router = _rpm_tpm_router("lit-3058-fail") - cache = _UnavailableIncrementCache() - router.cache = cache - metadata: dict[str, object] = {} - - request = asyncio.ensure_future( - router.acompletion( - model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], mock_response="pong", metadata=metadata - ) - ) - await asyncio.wait_for(cache.first_increment_started.wait(), timeout=5) - assert metadata[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] == 30 - for _ in range(50): - if get_deployment_successes_for_current_minute(router, "lit-3058-fail") == 1: - break - await asyncio.sleep(0.1) - assert get_deployment_successes_for_current_minute(router, "lit-3058-fail") == 1 - assert cache.increment_calls == 1 - - cache.release_first_increment.set() - response = await request - - assert response.usage.total_tokens == 30 - assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in metadata - assert _ratelimit_headers(response)["x-ratelimit-remaining-requests"] == 100 - assert await router.get_model_group_usage("gpt-5-mini") == (None, None) - - -def test_track_deployment_metrics(model_list): - """Test if the 'track_deployment_metrics' function is working correctly""" - from litellm.types.utils import ModelResponse - - router = Router(model_list=model_list) - router._track_deployment_metrics( - deployment=router.get_deployment_by_model_group_name( - model_group_name="gpt-5-mini" - ), - response=ModelResponse( - model="gpt-5-mini", - usage={"total_tokens": 100}, - ), - parent_otel_span=None, - ) - - -def test_pass_through_assistants_endpoint_factory(model_list): - """Test if the 'pass_through_assistants_endpoint_factory' function is working correctly""" - router = Router(model_list=model_list) - router._pass_through_assistants_endpoint_factory( - original_function=litellm.acreate_assistants, - custom_llm_provider="openai", - client=None, - **{}, - ) - - -def test_factory_function(model_list): - """Test if the 'factory_function' function is working correctly""" - router = Router(model_list=model_list) - router.factory_function(litellm.acreate_assistants) - - - - - - - - - - - - -# def test_pattern_match_deployments(model_list): -# from litellm.router_utils.pattern_match_deployments import PatternMatchRouter -# import re - -# patter_router = PatternMatchRouter() - -# request = "fo::hi::static::hello" -# model_name = "fo::*:static::*" - -# model_name_regex = patter_router._pattern_to_regex(model_name) - -# # Match against the request -# match = re.match(model_name_regex, request) - -# print(f"match: {match}") -# print(f"match.end: {match.end()}") -# if match is None: -# raise ValueError("Match not found") -# updated_model = patter_router.set_deployment_model_name( -# matched_pattern=match, litellm_deployment_litellm_model="openai/*" -# ) -# assert updated_model == "openai/fo::hi:static::hello" - - - - -@pytest.mark.asyncio -async def test_pass_through_moderation_endpoint_factory(model_list): - router = Router(model_list=model_list) - response = await router._pass_through_moderation_endpoint_factory( - original_function=litellm.amoderation, - input="this is valid good text", - model=None, - ) - assert response is not None - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -def test_handle_clientside_credential_no_metadata(model_list): - """Test that _handle_clientside_credential handles cases where no metadata is provided""" - router = Router(model_list=model_list) - - # Mock deployment - deployment = { - "model_name": "gpt-4.1", - "litellm_params": {"model": "gpt-4.1", "api_key": "test_key"}, - "model_info": {"id": "original-id-789"}, - } - - # Mock kwargs with clientside credentials but NO metadata - kwargs = { - "api_key": "client_side_key", - "api_base": "https://api.openai.com/v1", - # No metadata key at all - } - - # This should fail because there's no model_group in metadata - # The function expects to find model_group in the metadata - try: - result_deployment = router._handle_clientside_credential( - deployment=deployment, kwargs=kwargs, function_name="acompletion" - ) - # If we get here, the function should have used deployment.model_name as fallback - assert result_deployment.model_name == "gpt-4.1" - print("✓ Success with no metadata - used deployment.model_name as fallback") - except Exception as e: - # This is expected behavior - the function needs model_group to generate model_id - print(f"✓ Correctly handled no metadata case: {e}") - - # Test with empty metadata - kwargs_with_empty_metadata = { - "api_key": "client_side_key", - "api_base": "https://api.openai.com/v1", - "metadata": {}, # Empty metadata - } - - try: - result_deployment = router._handle_clientside_credential( - deployment=deployment, - kwargs=kwargs_with_empty_metadata, - function_name="acompletion", - ) - # Should fail because empty metadata has no model_group - pytest.fail("Expected failure with empty metadata") - except Exception as e: - print(f"✓ Correctly handled empty metadata case: {e}") diff --git a/tests/unit/integrations/test_langfuse.py b/tests/unit/integrations/test_langfuse.py index bd55216e4bf..3c0238372a5 100644 --- a/tests/unit/integrations/test_langfuse.py +++ b/tests/unit/integrations/test_langfuse.py @@ -2197,6 +2197,58 @@ def test_log_event_returns_the_v2_dict_shape_for_the_alerting_trace_id_cache(): assert returned["generation_id"] +@pytest.mark.parametrize( + ("metadata", "expected_source"), + [ + ({}, None), + ({"trace_id": "my-unique-trace-id"}, "my-unique-trace-id"), + ({"existing_trace_id": "my-unique-existing-trace-id"}, "my-unique-existing-trace-id"), + ( + {"trace_id": "my-unique-trace-id", "existing_trace_id": "my-unique-existing-trace-id"}, + "my-unique-existing-trace-id", + ), + ], +) +def test_logging_get_trace_id_reports_the_langfuse_trace_that_won_precedence( + monkeypatch: pytest.MonkeyPatch, + metadata: dict[str, str], + expected_source: str | None, +) -> None: + from litellm.litellm_core_utils import litellm_logging + from litellm.litellm_core_utils.litellm_logging import Logging + + logger, exporter = _steering_logger() + monkeypatch.setattr(litellm_logging, "langFuseLogger", logger) + monkeypatch.setattr(litellm, "success_callback", ["langfuse"]) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "callbacks", []) + call_id: Final = f"trace-precedence-{len(metadata)}-{expected_source}" + logging_obj: Final = Logging( + model="gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + stream=False, + call_type="completion", + litellm_call_id=call_id, + start_time=datetime.datetime.now(), + function_id="trace-precedence", + ) + + litellm.completion( + model="gpt-5-mini", + messages=[{"role": "user", "content": "trace precedence"}], + mock_response="ok", + litellm_logging_obj=logging_obj, + metadata=dict(metadata), + ) + deadline: Final = time.monotonic() + 5 + while logging_obj.get_trace_id(service_name="langfuse") is None and time.monotonic() < deadline: + time.sleep(0.01) + + expected_trace_id: Final = resolve_trace_id(expected_source or logging_obj.litellm_trace_id) + assert logging_obj.get_trace_id(service_name="langfuse") == expected_trace_id + assert _span_trace_id(_exported_span(logger, exporter)) == expected_trace_id + + def test_parse_langfuse_debug_only_enables_on_true_strings(): """v4 treats any truthy value as debug=on, so the raw env string "false" would enable debug.""" assert langfuse_module.parse_langfuse_debug("true") is True diff --git a/tests/unit/litellm_core_utils/test_health_check_helpers.py b/tests/unit/litellm_core_utils/test_health_check_helpers.py index ffe50c137d2..c0298bc1c80 100644 --- a/tests/unit/litellm_core_utils/test_health_check_helpers.py +++ b/tests/unit/litellm_core_utils/test_health_check_helpers.py @@ -1,7 +1,6 @@ """Test health check helper functions""" import json -import os import socket import struct import zlib @@ -1235,3 +1234,193 @@ async def test_health_check_with_custom_llm_provider( assert "error" not in response, response assert upstream.called assert json.loads(upstream.calls[0].request.content)["model"] == "deepseek-r1-distill-qwen-1.5B-q4" + + +@pytest.mark.asyncio +async def test_azure_chat_health_check_surfaces_provider_rate_limit_headers( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post( + url__regex=r"https://resource\.example/openai/deployments/gpt-4\.1-mini/chat/completions.*" + ).respond( + json={ + "id": "chatcmpl-health", + "object": "chat.completion", + "created": 1, + "model": "gpt-4.1-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + }, + headers={"x-ratelimit-remaining-tokens": "42"}, + ) + + response: Final = await ahealth_check( + { + "model": "azure/gpt-4.1-mini", + "api_key": "fake-key", + "api_base": "https://resource.example", + "api_version": "2024-06-01", + }, + mode="chat", + ) + + assert response["x-ratelimit-remaining-tokens"] == "42" + assert upstream.called + + +@pytest.mark.asyncio +async def test_azure_embedding_health_check_surfaces_provider_rate_limit_headers( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post( + url__regex=r"https://resource\.example/openai/deployments/text-embedding-ada-002/embeddings.*" + ).respond( + json={ + "object": "list", + "data": [{"object": "embedding", "embedding": [0.1, 0.2], "index": 0}], + "model": "text-embedding-ada-002", + "usage": {"prompt_tokens": 1, "total_tokens": 1}, + }, + headers={"x-ratelimit-remaining-tokens": "84"}, + ) + + response: Final = await ahealth_check( + { + "model": "azure/text-embedding-ada-002", + "api_key": "fake-key", + "api_base": "https://resource.example", + "api_version": "2024-06-01", + }, + input=["health check"], + mode="embedding", + ) + + assert response["x-ratelimit-remaining-tokens"] == "84" + assert upstream.called + + +@pytest.mark.asyncio +async def test_image_generation_health_check_returns_a_successful_response( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/images/generations").respond( + json={"created": 1, "data": [{"b64_json": "AA=="}]} + ) + + response: Final = await ahealth_check( + {"model": "gpt-image-1", "api_key": "fake-key"}, + mode="image_generation", + prompt="health check", + ) + + assert "error" not in response + assert upstream.called + + +@pytest.mark.asyncio +async def test_groq_wildcard_health_check_uses_a_concrete_model( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + monkeypatch.setattr( + litellm, + "models_by_provider", + {"groq": ["groq/openai/gpt-oss-20b"]}, + ) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.groq.com/openai/v1/chat/completions").respond( + json={ + "id": "chatcmpl-health", + "object": "chat.completion", + "created": 1, + "model": "groq/openai/gpt-oss-20b", + "service_tier": "on_demand", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "2"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + ) + + response: Final = await ahealth_check( + { + "model": "groq/*", + "api_key": "fake-key", + "messages": [{"role": "user", "content": "What is 1 + 1?"}], + } + ) + + assert upstream.called + assert json.loads(upstream.calls.last.request.content)["model"] == "openai/gpt-oss-20b" + assert response == {} + + +@pytest.mark.asyncio +async def test_cohere_rerank_health_check_returns_a_successful_response( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.cohere.com/v2/rerank").respond( + json={ + "id": "rerank-health", + "results": [{"index": 0, "relevance_score": 0.7}], + "meta": {"billed_units": {"search_units": 1}}, + } + ) + + response: Final = await ahealth_check( + {"model": "cohere/rerank-english-v3.0", "api_key": "fake-key"}, + mode="rerank", + prompt="health check", + ) + + assert "error" not in response + assert upstream.called + assert json.loads(upstream.calls.last.request.content)["query"] == "health check" + + +@pytest.mark.asyncio +async def test_audio_speech_health_check_returns_audio( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/audio/speech").respond( + content=b"audio", + headers={"content-type": "audio/mpeg"}, + ) + + response: Final = await ahealth_check( + {"model": "openai/tts-1", "api_key": "fake-key"}, + mode="audio_speech", + prompt="health check", + ) + + assert "error" not in response + assert upstream.called + assert json.loads(upstream.calls.last.request.content)["input"] == "health check" + + +@pytest.mark.asyncio +async def test_audio_transcription_health_check_returns_transcribed_text( + monkeypatch: pytest.MonkeyPatch, respx_mock: respx.MockRouter +) -> None: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + upstream: Final = respx_mock.post("https://api.openai.com/v1/audio/transcriptions").respond( + json={"text": "health check audio"} + ) + + response: Final = await ahealth_check( + {"model": "openai/whisper-1", "api_key": "fake-key"}, + mode="audio_transcription", + ) + + assert "error" not in response + assert upstream.called + assert b'name="file"' in upstream.calls.last.request.content diff --git a/tests/unit/llms/anthropic/test_count_tokens_oauth.py b/tests/unit/llms/anthropic/test_count_tokens_oauth.py index 96d909a4b3f..7f717091754 100644 --- a/tests/unit/llms/anthropic/test_count_tokens_oauth.py +++ b/tests/unit/llms/anthropic/test_count_tokens_oauth.py @@ -11,6 +11,7 @@ Regression tests for https://github.com/BerriAI/litellm/issues/22040 and for the import os import sys +from typing import Final import httpx import pytest @@ -21,10 +22,15 @@ sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../. import litellm from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION from litellm.llms.anthropic.common_utils import AnthropicModelInfo +from litellm.llms.anthropic.count_tokens.token_counter import AnthropicTokenCounter from litellm.llms.anthropic.count_tokens.transformation import ( AnthropicCountTokensConfig, ) +from litellm.llms.azure_ai.anthropic.count_tokens.token_counter import ( + AzureAIAnthropicTokenCounter, +) from litellm.types.llms.anthropic import ANTHROPIC_OAUTH_BETA_HEADER +from litellm.types.utils import TokenCountResponse # Fake tokens for testing (not real secrets) FAKE_OAUTH_TOKEN = "sk-ant-oat01-fake-token-for-testing-123456789abcdef" @@ -269,3 +275,80 @@ class TestCountTokensUsesWorkloadIdentity: assert result is not None assert result.total_tokens == 7 assert seen["auth_header"] == {"x-api-key": vault_key} + + +@pytest.mark.parametrize( + ("counter_type", "api_base", "endpoint", "tokenizer_type"), + ( + ( + AnthropicTokenCounter, + "https://gateway.example", + "https://gateway.example/v1/messages/count_tokens", + "anthropic_api", + ), + ( + AzureAIAnthropicTokenCounter, + "https://resource.example", + "https://resource.example/anthropic/v1/messages/count_tokens", + "azure_ai_anthropic_api", + ), + ), +) +@pytest.mark.parametrize(("status_code", "expected_error"), ((200, False), (401, True))) +@pytest.mark.asyncio +async def test_count_token_counters_return_typed_success_and_error_responses( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + api_base: str, + endpoint: str, + tokenizer_type: str, + status_code: int, + expected_error: bool, + httpx_transport_clients: None, +) -> None: + response_body: Final = {"input_tokens": 17} if status_code == 200 else {"error": {"message": "invalid key"}} + router: Final = respx.mock + + with router: + count_route: Final = router.post(endpoint).mock(return_value=httpx.Response(status_code, json=response_body)) + result: Final = await counter_type().count_tokens( + model_to_use="claude-test", + messages=[{"role": "user", "content": "hi"}], + contents=None, + deployment={"litellm_params": {"api_key": "sk-ant-api03-test-key", "api_base": api_base}}, + request_model="claude-test", + ) + requests: Final = tuple(router.calls) + + assert count_route.called + assert len(requests) == 1 + assert requests[0].request.url == httpx.URL(endpoint) + assert isinstance(result, TokenCountResponse) + assert result.request_model == "claude-test" + assert result.model_used == "claude-test" + assert result.tokenizer_type == tokenizer_type + + if expected_error: + assert result.error is True + assert result.status_code == status_code + assert result.total_tokens == 0 + return + + assert result.error is not True + assert result.total_tokens == 17 + + +@pytest.mark.parametrize( + ("counter_type", "provider"), + ( + (AnthropicTokenCounter, "anthropic"), + (AzureAIAnthropicTokenCounter, "azure_ai"), + ), +) +def test_count_token_counters_select_their_own_provider( + counter_type: type[AnthropicTokenCounter] | type[AzureAIAnthropicTokenCounter], + provider: str, +) -> None: + counter: Final = counter_type() + + assert counter.should_use_token_counting_api(custom_llm_provider=provider) is True + assert counter.should_use_token_counting_api(custom_llm_provider="unknown") is False diff --git a/tests/unit/llms/openai/responses/test_openai_responses_http.py b/tests/unit/llms/openai/responses/test_openai_responses_http.py new file mode 100644 index 00000000000..29af2f2c55a --- /dev/null +++ b/tests/unit/llms/openai/responses/test_openai_responses_http.py @@ -0,0 +1,531 @@ +import asyncio +import json +import threading +from collections.abc import Iterable +from types import MappingProxyType +from typing import Final, Literal, TypeAlias + +import httpx +import pytest +import respx +from pydantic import BaseModel, TypeAdapter + +import litellm +from litellm.integrations.custom_logger import CustomLogger +from litellm.types.llms.openai import ( + IncompleteDetails, + ResponseAPIUsage, + ResponseCompletedEvent, + ResponsesAPIResponse, +) +from litellm.types.utils import StandardLoggingPayload, Usage +from tests.unit.proxy.conftest import httpx_transport + +pytestmark: Final = pytest.mark.usefixtures(httpx_transport.__name__) +_OPENAI_URL: Final = "https://api.openai.com/v1/responses" +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) +_STANDARD_LOGGING_PAYLOAD: Final = TypeAdapter(StandardLoggingPayload) +_OUTPUT_TEXT: Final = "Hello from the mocked response" +_CREATED_AT: Final = 1750000000 +_AsyncLoggingMode: TypeAlias = Literal["non_stream", "stream"] +_RESPONSE_FIELD_TYPES: Final = MappingProxyType( + { + "error": (dict, type(None)), + "incomplete_details": (IncompleteDetails, type(None)), + "instructions": (str, type(None)), + "metadata": (dict,), + "model": (str,), + "object": (str,), + "parallel_tool_calls": (bool, type(None)), + "temperature": (int, float, type(None)), + "tool_choice": (dict, str, type(None)), + "tools": (list, type(None)), + "top_p": (int, float, type(None)), + "max_output_tokens": (int, type(None)), + "previous_response_id": (str, type(None)), + "reasoning": (dict, type(None)), + "status": (str,), + "text": (dict,), + "truncation": (str, type(None)), + "user": (str, type(None)), + "store": (bool, type(None)), + } +) +_STREAM_EVENT_FIELDS: Final = MappingProxyType( + { + "response.created": ("response",), + "response.in_progress": ("response",), + "response.output_item.added": ("output_index", "item"), + "response.content_part.added": ("item_id", "output_index", "content_index", "part"), + "response.output_text.delta": ("item_id", "output_index", "content_index", "delta"), + "response.output_text.done": ("item_id", "output_index", "content_index", "text"), + "response.content_part.done": ("item_id", "output_index", "content_index", "part"), + "response.output_item.done": ("output_index", "item"), + "response.completed": ("response",), + } +) + + +class _ResponsesLoggingCapture(CustomLogger): + def __init__(self) -> None: + super().__init__() + self.completed: Final = threading.Event() + self.payload: StandardLoggingPayload | None = None + self.usage: object = None + + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: + assert isinstance(response_obj, ResponsesAPIResponse) + self.payload = _STANDARD_LOGGING_PAYLOAD.validate_python(kwargs["standard_logging_object"]) + self.usage = response_obj.usage + self.completed.set() + + +def _response_body(response_id: str, store: bool = False, created_at: float = _CREATED_AT) -> dict[str, object]: + return { + "id": response_id, + "object": "response", + "created_at": created_at, + "status": "completed", + "error": None, + "incomplete_details": None, + "instructions": None, + "max_output_tokens": None, + "model": "gpt-4o", + "output": [ + { + "id": f"msg_{response_id}", + "type": "message", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": _OUTPUT_TEXT, "annotations": []}], + } + ], + "parallel_tool_calls": True, + "previous_response_id": None, + "reasoning": {"effort": None, "summary": None}, + "store": store, + "temperature": 1.0, + "text": {"format": {"type": "text"}}, + "tool_choice": "auto", + "tools": [], + "top_p": 1.0, + "truncation": "disabled", + "usage": { + "input_tokens": 2, + "input_tokens_details": {"cached_tokens": 0}, + "output_tokens": 3, + "output_tokens_details": {"reasoning_tokens": 0}, + "total_tokens": 5, + }, + "user": None, + "metadata": {}, + } + + +def _sse(events: Iterable[dict[str, object]]) -> str: + return "".join(f"data: {json.dumps(event)}\n\n" for event in events) + + +def _response_events(response_id: str) -> tuple[dict[str, object], ...]: + completed_response: Final = _response_body(response_id) + in_progress_response: Final = {**completed_response, "status": "in_progress", "output": [], "usage": None} + item_id: Final = f"msg_{response_id}" + text_part: Final = {"type": "output_text", "text": _OUTPUT_TEXT, "annotations": []} + position: Final = {"item_id": item_id, "output_index": 0, "content_index": 0} + return ( + {"type": "response.created", "sequence_number": 0, "response": in_progress_response}, + {"type": "response.in_progress", "sequence_number": 1, "response": in_progress_response}, + { + "type": "response.output_item.added", + "sequence_number": 2, + "output_index": 0, + "item": {"id": item_id, "type": "message", "status": "in_progress", "role": "assistant", "content": []}, + }, + { + "type": "response.content_part.added", + "sequence_number": 3, + **position, + "part": {"type": "output_text", "text": "", "annotations": []}, + }, + {"type": "response.output_text.delta", "sequence_number": 4, **position, "delta": _OUTPUT_TEXT}, + {"type": "response.output_text.done", "sequence_number": 5, **position, "text": _OUTPUT_TEXT}, + {"type": "response.content_part.done", "sequence_number": 6, **position, "part": text_part}, + { + "type": "response.output_item.done", + "sequence_number": 7, + "output_index": 0, + "item": completed_response["output"][0], + }, + {"type": "response.completed", "sequence_number": 8, "response": completed_response}, + ) + + +def _response_sse(response_id: str) -> str: + return _sse(_response_events(response_id)) + + +def _sse_reply(body: str) -> httpx.Response: + return httpx.Response(status_code=200, content=body, headers={"content-type": "text/event-stream"}) + + +def _assert_valid_response(response: object, final_chunk: bool) -> None: + assert isinstance(response, ResponsesAPIResponse) + assert isinstance(response.id, str) + assert isinstance(response.created_at, int) + assert isinstance(response.usage, ResponseAPIUsage if final_chunk else type(None)) + mistyped: Final = { + field: type(response[field]).__name__ + for field, expected in _RESPONSE_FIELD_TYPES.items() + if not isinstance(response[field], expected) + } + assert mistyped == {} + if final_chunk and response.status == "completed": + assert len(response.output) > 0 + + +def _json_object(value: object) -> dict[str, object]: + return _JSON_OBJECT.validate_python(value.model_dump(mode="json") if isinstance(value, BaseModel) else value) + + +def _assert_valid_stream(events: tuple[object, ...], item_id: str) -> None: + event_types: Final = tuple(getattr(event, "type", None) for event in events) + assert event_types == tuple(_STREAM_EVENT_FIELDS) + missing_fields: Final = { + event_type: tuple(name for name in fields if getattr(event, name, None) is None) + for (event_type, fields), event in zip(_STREAM_EVENT_FIELDS.items(), events) + } + assert all(fields == () for fields in missing_fields.values()), missing_fields + created: Final = getattr(events[0], "response", None) + _assert_valid_response(created, final_chunk=False) + _assert_valid_response(getattr(events[1], "response", None), final_chunk=False) + completed: Final = events[-1] + assert isinstance(completed, ResponseCompletedEvent) + _assert_valid_response(completed.response, final_chunk=True) + assert completed.response.id == getattr(created, "id", None) + assert completed.response.output_text == _OUTPUT_TEXT + assert tuple(getattr(event, "sequence_number", None) for event in events) == tuple(range(len(events))) + item_ids: Final = ( + getattr(getattr(events[2], "item", None), "id", None), + *(getattr(event, "item_id", None) for event in events[3:7]), + getattr(getattr(events[7], "item", None), "id", None), + ) + assert item_ids == (item_id,) * 6 + assert tuple(getattr(event, "output_index", None) for event in events[2:8]) == (0,) * 6 + assert tuple(getattr(event, "content_index", None) for event in events[3:7]) == (0,) * 4 + assert _json_object(getattr(events[3], "part", None)) == { + "type": "output_text", + "text": "", + "annotations": [], + } + assert getattr(events[4], "delta", None) == _OUTPUT_TEXT + assert getattr(events[5], "text", None) == _OUTPUT_TEXT + assert _json_object(getattr(events[6], "part", None)) == { + "type": "output_text", + "text": _OUTPUT_TEXT, + "annotations": [], + } + assert getattr(getattr(events[7], "item", None), "type", None) == "message" + assert tuple(item.id for item in completed.response.output) == (item_id,) + + +def _completed_response(events: tuple[object, ...]) -> ResponsesAPIResponse: + completed_events: Final = tuple(event for event in events if isinstance(event, ResponseCompletedEvent)) + assert len(completed_events) == 1 + return completed_events[0].response + + +def _install_capture(monkeypatch: pytest.MonkeyPatch) -> _ResponsesLoggingCapture: + capture: Final = _ResponsesLoggingCapture() + monkeypatch.setattr(litellm, "callbacks", [capture]) + monkeypatch.setattr(litellm, "success_callback", [capture]) + monkeypatch.setattr(litellm, "_async_success_callback", [capture]) + return capture + + +def _assert_logged_payload_matches(capture: _ResponsesLoggingCapture, response: ResponsesAPIResponse) -> None: + payload: Final = capture.payload + usage: Final = capture.usage + assert payload is not None + assert response.usage is not None + assert payload["prompt_tokens"] == response.usage.input_tokens + assert payload["completion_tokens"] == response.usage.output_tokens + assert payload["total_tokens"] == response.usage.input_tokens + response.usage.output_tokens + assert payload["response_cost"] > 0 + assert payload["id"] == response.id + assert payload["model"] == "gpt-4o-mini" + assert payload["messages"] == [{"content": "hi", "role": "user"}] + callback_usage: Final = usage.model_dump() if isinstance(usage, Usage) else _JSON_OBJECT.validate_python(usage) + assert callback_usage["prompt_tokens"] == response.usage.input_tokens + assert callback_usage["completion_tokens"] == response.usage.output_tokens + logged_response: Final = _JSON_OBJECT.validate_python(payload["response"]) + final_response: Final = response.model_dump(mode="json") + assert {key: value for key, value in logged_response.items() if key != "usage"} == { + key: value for key, value in final_response.items() if key != "usage" + } + logged_usage: Final = _JSON_OBJECT.validate_python(logged_response["usage"]) + assert logged_usage["prompt_tokens"] == response.usage.input_tokens + assert logged_usage["completion_tokens"] == response.usage.output_tokens + assert logged_usage["total_tokens"] == response.usage.total_tokens + + +async def _async_logged_openai_response(mode: _AsyncLoggingMode) -> ResponsesAPIResponse: + match mode: + case "stream": + response_stream: Final = await litellm.aresponses( + model="openai/gpt-4o-mini", api_key="sk-test", input="hi", stream=True + ) + return _completed_response(tuple([event async for event in response_stream])) + case "non_stream": + response: Final = await litellm.aresponses(model="openai/gpt-4o-mini", api_key="sk-test", input="hi") + assert isinstance(response, ResponsesAPIResponse) + return response + + +@pytest.mark.parametrize("sync_mode", (True, False)) +@pytest.mark.asyncio +async def test_responses_exposes_provider_rate_limit_headers(sync_mode: bool) -> None: + response_headers: Final = { + "x-ratelimit-limit-requests": "500", + "x-ratelimit-remaining-requests": "499", + "x-ratelimit-reset-requests": "1s", + } + + with respx.mock() as mock_router: + mock_router.post(_OPENAI_URL).mock( + return_value=httpx.Response( + status_code=200, + json=_response_body("resp_rate_limit"), + headers=response_headers, + ) + ) + response: Final = ( + litellm.responses(model="openai/gpt-4o", api_key="sk-test", input="hi") + if sync_mode + else await litellm.aresponses(model="openai/gpt-4o", api_key="sk-test", input="hi") + ) + requests: Final = tuple(mock_router.calls) + + assert isinstance(response, ResponsesAPIResponse) + assert len(requests) == 1 + additional_headers: Final = _JSON_OBJECT.validate_python(response.hidden_params["additional_headers"]) + raw_headers: Final = _JSON_OBJECT.validate_python(response.hidden_params["headers"]) + assert {name: additional_headers[f"llm_provider-{name}"] for name in response_headers} == response_headers + assert {name: raw_headers[name] for name in response_headers} == response_headers + + +@pytest.mark.asyncio +async def test_responses_converts_created_at_to_int_and_forwards_store() -> None: + response_body: Final = _response_body("resp_store", store=True, created_at=_CREATED_AT + 0.75) + + with respx.mock() as mock_router: + mock_router.post(_OPENAI_URL).mock(return_value=httpx.Response(status_code=200, json=response_body)) + response: Final = await litellm.aresponses( + model="openai/gpt-4o", + api_key="sk-test", + input="hi", + store=True, + ) + requests: Final = tuple(mock_router.calls) + + assert isinstance(response, ResponsesAPIResponse) + assert type(response.created_at) is int + assert response.created_at == _CREATED_AT + assert response.store is True + assert len(requests) == 1 + request_body: Final = _JSON_OBJECT.validate_json(requests[0].request.content) + assert request_body["store"] is True + + +@pytest.mark.asyncio +async def test_responses_mcp_followup_forwards_approval_and_previous_id() -> None: + mcp_tools: Final = [ + { + "type": "mcp", + "server_label": "weather", + "server_url": "https://mcp.example.test", + "headers": {"Authorization": "Bearer mcp-test-token"}, + } + ] + approval: Final = [{"type": "mcp_approval_response", "approve": True, "approval_request_id": "approval_123"}] + + with respx.mock() as mock_router: + route: Final = mock_router.post(_OPENAI_URL).mock( + return_value=httpx.Response(status_code=200, json=_response_body("resp_mcp")) + ) + first_response: Final = await litellm.aresponses( + model="openai/gpt-4o", + api_key="sk-test", + input="Search for recent weather", + tools=mcp_tools, + ) + assert isinstance(first_response, ResponsesAPIResponse) + second_response: Final = await litellm.aresponses( + model="openai/gpt-4o", + api_key="sk-test", + input=approval, + tools=mcp_tools, + previous_response_id=first_response.id, + ) + requests: Final = tuple(route.calls) + + assert isinstance(second_response, ResponsesAPIResponse) + assert len(requests) == 2 + first_request: Final = _JSON_OBJECT.validate_json(requests[0].request.content) + second_request: Final = _JSON_OBJECT.validate_json(requests[1].request.content) + assert first_request["tools"] == mcp_tools + assert second_request["tools"] == mcp_tools + assert second_request["input"] == approval + assert second_request["previous_response_id"] == "resp_mcp" + + +@pytest.mark.parametrize("mode", ("non_stream", "stream")) +@pytest.mark.asyncio +async def test_responses_standard_logging_matches_final_response( + mode: _AsyncLoggingMode, monkeypatch: pytest.MonkeyPatch +) -> None: + capture: Final = _install_capture(monkeypatch) + + with respx.mock() as mock_router: + mock_router.post(_OPENAI_URL).mock( + return_value=( + _sse_reply(_response_sse("resp_logging")) + if mode == "stream" + else httpx.Response(status_code=200, json=_response_body("resp_logging")) + ) + ) + response: Final = await _async_logged_openai_response(mode) + requests: Final = tuple(mock_router.calls) + logged: Final = await asyncio.to_thread(capture.completed.wait, 10) + + assert logged + assert len(requests) == 1 + _assert_logged_payload_matches(capture, response) + + +def test_sync_stream_standard_logging_matches_final_response(monkeypatch: pytest.MonkeyPatch) -> None: + capture: Final = _install_capture(monkeypatch) + + with respx.mock() as mock_router: + mock_router.post(_OPENAI_URL).mock(return_value=_sse_reply(_response_sse("resp_sync_logging"))) + stream: Final = litellm.responses(model="openai/gpt-4o-mini", api_key="sk-test", input="hi", stream=True) + response: Final = _completed_response(tuple(stream)) + requests: Final = tuple(mock_router.calls) + logged: Final = capture.completed.wait(10) + + assert logged + assert len(requests) == 1 + _assert_logged_payload_matches(capture, response) + + +@pytest.mark.parametrize("sync_mode", (True, False)) +@pytest.mark.asyncio +async def test_responses_stream_emits_valid_events(sync_mode: bool) -> None: + with respx.mock() as mock_router: + route: Final = mock_router.post(_OPENAI_URL).mock(return_value=_sse_reply(_response_sse("resp_stream"))) + events: Final = ( + tuple(litellm.responses(model="openai/gpt-4o", api_key="sk-test", input="hi", stream=True)) + if sync_mode + else tuple( + [ + event + async for event in await litellm.aresponses( + model="openai/gpt-4o", api_key="sk-test", input="hi", stream=True + ) + ] + ) + ) + requests: Final = tuple(route.calls) + + assert len(requests) == 1 + _assert_valid_stream(events, "msg_resp_stream") + + +@pytest.mark.asyncio +async def test_responses_stream_error_event_raises_bad_request() -> None: + message: Final = "Your input exceeds the context window of this model." + created: Final = _response_events("resp_too_long")[0] + error_event: Final = { + "type": "error", + "sequence_number": 1, + "error": { + "type": "invalid_request_error", + "code": "context_length_exceeded", + "message": message, + "param": "input", + }, + } + + with respx.mock() as mock_router: + route: Final = mock_router.post(_OPENAI_URL).mock(return_value=_sse_reply(_sse((created, error_event)))) + stream: Final = await litellm.aresponses( + model="openai/gpt-5-mini", api_key="sk-test", input="oversized prompt", stream=True + ) + with pytest.raises(litellm.BadRequestError) as exc_info: + _ = [event async for event in stream] + requests: Final = tuple(route.calls) + + assert len(requests) == 1 + assert exc_info.value.status_code == 400 + assert message in str(exc_info.value) + + +def _alias_router() -> litellm.Router: + return litellm.Router( + model_list=[ + { + "model_name": "openai-offline-alias", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-test"}, + } + ] + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", (True, False)) +async def test_router_responses_alias_uses_underlying_model(sync_mode: bool) -> None: + router: Final = _alias_router() + with respx.mock() as mock_router: + route: Final = mock_router.post(_OPENAI_URL).mock( + return_value=httpx.Response(status_code=200, json=_response_body("resp_router_alias")) + ) + response: Final = ( + router.responses(model="openai-offline-alias", input="hi") + if sync_mode + else await router.aresponses(model="openai-offline-alias", input="hi") + ) + requests: Final = tuple(route.calls) + + _assert_valid_response(response, final_chunk=True) + assert len(requests) == 1 + request_body: Final = _JSON_OBJECT.validate_json(requests[0].request.content) + assert request_body["model"] == "gpt-4o" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", (True, False)) +async def test_router_responses_alias_stream_uses_underlying_model(sync_mode: bool) -> None: + router: Final = _alias_router() + with respx.mock() as mock_router: + route: Final = mock_router.post(_OPENAI_URL).mock(return_value=_sse_reply(_response_sse("resp_router_stream"))) + events: Final = ( + tuple(router.responses(model="openai-offline-alias", input="hi", stream=True)) + if sync_mode + else tuple( + [ + event + async for event in await router.aresponses(model="openai-offline-alias", input="hi", stream=True) + ] + ) + ) + requests: Final = tuple(route.calls) + + _assert_valid_stream(events, "msg_resp_router_stream") + assert len(requests) == 1 + request_body: Final = _JSON_OBJECT.validate_json(requests[0].request.content) + assert request_body["model"] == "gpt-4o" diff --git a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py index a8f0d23e560..73332728964 100644 --- a/tests/unit/llms/openai/responses/test_openai_responses_transformation.py +++ b/tests/unit/llms/openai/responses/test_openai_responses_transformation.py @@ -6,6 +6,8 @@ from unittest.mock import AsyncMock, MagicMock, Mock, patch import httpx import pytest +import respx +from pydantic import JsonValue, TypeAdapter import litellm from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj @@ -24,6 +26,7 @@ from litellm.types.llms.openai import ( ) from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import Choices, Message, ModelResponse +from tests.unit.proxy.conftest import httpx_transport import time _ARTIFACT_FIELD_PATTERN: Final = r'^(?!__.*__$)[^\p{Cc}\p{Cf}\p{Zl}\p{Zp}"\\./[\]]{1,200}$' @@ -3445,3 +3448,127 @@ async def test_extra_body_merges_with_request_data(extra_body_mock_response_data assert "temperature" in request_body assert "custom_field" in request_body assert request_body["custom_field"] == "custom_value" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures(httpx_transport.__name__) +async def test_aresponses_forwards_previous_response_id_to_openai() -> None: + first_input: Final = "remember the first turn" + second_input: Final = "continue the conversation" + first_id: Final = "resp_previous_turn" + second_id: Final = "resp_follow_up" + response_payloads: Final = ( + { + "id": first_id, + "object": "response", + "created_at": 1734366691, + "status": "completed", + "model": "gpt-4o", + "output": [ + { + "type": "message", + "id": "msg_first", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "first answer", "annotations": []}], + } + ], + "usage": { + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + { + "id": second_id, + "object": "response", + "created_at": 1734366692, + "status": "completed", + "model": "gpt-4o", + "output": [ + { + "type": "message", + "id": "msg_second", + "status": "completed", + "role": "assistant", + "content": [{"type": "output_text", "text": "second answer", "annotations": []}], + } + ], + "usage": { + "input_tokens": 1, + "output_tokens": 1, + "total_tokens": 2, + "output_tokens_details": {"reasoning_tokens": 0}, + }, + }, + ) + provider_url: Final = "https://api.openai.com/v1/responses" + + with respx.mock() as router: + route: Final = router.post(provider_url).mock( + side_effect=[ + httpx.Response(status_code=200, json=response_payloads[0]), + httpx.Response(status_code=200, json=response_payloads[1]), + ] + ) + first_response: Final = await litellm.aresponses( + model="openai/gpt-4o", + api_key="sk-test", + input=first_input, + ) + assert isinstance(first_response, ResponsesAPIResponse) + second_response: Final = await litellm.aresponses( + model="openai/gpt-4o", + api_key="sk-test", + input=second_input, + previous_response_id=first_response.id, + ) + calls: Final = route.calls + + assert isinstance(second_response, ResponsesAPIResponse) + assert first_response.output[0].content[0].text == "first answer" + assert second_response.output[0].content[0].text == "second answer" + assert len(calls) == 2 + request_adapter: Final = TypeAdapter(dict[str, JsonValue]) + request_bodies: Final = tuple(request_adapter.validate_json(call.request.content) for call in calls) + assert request_bodies[0]["input"] == first_input + assert request_bodies[1]["input"] == second_input + assert request_bodies[1]["previous_response_id"] == first_id + + +def test_dict_responses_input_filters_unset_reasoning_fields() -> None: + test_input: Final = [ + {"role": "user", "content": "test"}, + { + "id": "rs_123", + "summary": [{"text": "test", "type": "summary_text"}], + "type": "reasoning", + "content": None, + "encrypted_content": None, + "status": None, + }, + { + "arguments": "{}", + "call_id": "call_123", + "name": "get_today", + "type": "function_call", + "id": "fc_123", + "status": "completed", + }, + ] + + validated_input: Final = OpenAIResponsesAPIConfig()._validate_input_param(test_input) + + assert len(validated_input) == 3 + reasoning_item: Final = validated_input[1] + assert reasoning_item["type"] == "reasoning" + assert "status" not in reasoning_item + assert "content" not in reasoning_item + assert "encrypted_content" not in reasoning_item + assert reasoning_item["id"] == "rs_123" + assert reasoning_item["summary"] == [{"text": "test", "type": "summary_text"}] + + function_call_item: Final = validated_input[2] + assert function_call_item["type"] == "function_call" + assert function_call_item["status"] == "completed" diff --git a/tests/unit/proxy/test_health_check_max_tokens.py b/tests/unit/proxy/test_health_check_max_tokens.py index 091de1e24b3..cfdecdba69d 100644 --- a/tests/unit/proxy/test_health_check_max_tokens.py +++ b/tests/unit/proxy/test_health_check_max_tokens.py @@ -1,7 +1,10 @@ +import asyncio import json import logging +from typing import Final from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest import respx @@ -1131,3 +1134,166 @@ def test_transitive_probe_expansion_terminates_on_a_router_cycle(): probes = hc_module._dependency_deployments_to_probe(a_only, router.model_list, router) assert {d["model_info"]["id"] for d in probes} == {"b-1"} + + +@pytest.mark.asyncio +async def test_background_audio_speech_health_check_uses_model_info_voice( + httpx_transport: None, respx_mock: respx.MockRouter +) -> None: + upstream: Final = respx_mock.post("https://speech.example/v1/audio/speech").respond( + content=b"audio", + headers={"content-type": "audio/mpeg"}, + ) + + healthy, unhealthy, _ = await hc_module.perform_health_check( + [ + { + "litellm_params": { + "model": "openai/tts-1", + "api_key": "fake-key", + "api_base": "https://speech.example/v1", + }, + "model_info": {"id": "speech", "mode": "audio_speech", "health_check_voice": "nova"}, + } + ], + max_concurrency=1, + ) + + assert len(healthy) == 1 + assert unhealthy == [] + assert upstream.called + assert json.loads(upstream.calls.last.request.content)["voice"] == "nova" + + +@pytest.mark.asyncio +async def test_background_health_check_observes_the_concurrency_limit_and_queue( + httpx_transport: None, respx_mock: respx.MockRouter +) -> None: + request_started: Final = asyncio.Queue[None]() + release: Final = asyncio.Event() + + async def complete_request(_: httpx.Request) -> httpx.Response: + request_started.put_nowait(None) + await release.wait() + return httpx.Response( + 200, + content=b"audio", + headers={"content-type": "audio/mpeg"}, + ) + + upstream: Final = respx_mock.post("https://health.example/v1/audio/speech").mock(side_effect=complete_request) + model_list: Final = [ + { + "litellm_params": { + "model": "openai/tts-1", + "api_key": "fake-key", + "api_base": "https://health.example/v1", + }, + "model_info": {"id": f"audio-{index}", "mode": "audio_speech"}, + } + for index in range(10) + ] + tasks_before: Final = len(asyncio.all_tasks()) + perform_task: Final = asyncio.create_task(hc_module.perform_health_check(model_list, max_concurrency=2)) + + try: + await asyncio.wait_for(request_started.get(), timeout=1) + await asyncio.wait_for(request_started.get(), timeout=1) + for _ in range(20): + await asyncio.sleep(0) + extra_requests_started: Final = request_started.qsize() + tasks_while_blocked: Final = len(asyncio.all_tasks()) - tasks_before + finally: + release.set() + healthy, unhealthy, _ = await perform_task + + assert extra_requests_started == 0 + assert tasks_while_blocked <= 5 + assert upstream.call_count == 10 + assert len(healthy) == 10 + assert unhealthy == [] + + +@pytest.mark.asyncio +async def test_background_health_check_timeout_marks_a_blocked_provider_unhealthy( + httpx_transport: None, respx_mock: respx.MockRouter +) -> None: + never_release: Final = asyncio.Event() + request_started: Final = asyncio.Event() + + async def blocked_response(_: httpx.Request) -> httpx.Response: + request_started.set() + await never_release.wait() + return httpx.Response(200, content=b"audio", headers={"content-type": "audio/mpeg"}) + + respx_mock.post("https://health.example/v1/audio/speech").mock(side_effect=blocked_response) + model_list: Final = [ + { + "litellm_params": { + "model": "openai/tts-1", + "api_key": "fake-key", + "api_base": "https://health.example/v1", + }, + "model_info": {"id": "blocked", "mode": "audio_speech", "health_check_timeout": 2}, + } + ] + + healthy, unhealthy, _ = await asyncio.wait_for( + hc_module.perform_health_check(model_list), + timeout=4, + ) + + assert request_started.is_set() + assert unhealthy[0]["error"] == "Timeout exceeded" + assert healthy == [] + assert len(unhealthy) == 1 + assert unhealthy[0]["model"] == "openai/tts-1" + + +@pytest.mark.asyncio +async def test_background_health_check_timeout_does_not_cancel_a_sibling( + httpx_transport: None, respx_mock: respx.MockRouter +) -> None: + never_release: Final = asyncio.Event() + slow_request_started: Final = asyncio.Event() + + async def blocked_response(_: httpx.Request) -> httpx.Response: + slow_request_started.set() + await never_release.wait() + return httpx.Response(200, content=b"audio", headers={"content-type": "audio/mpeg"}) + + respx_mock.post("https://slow.example/v1/audio/speech").mock(side_effect=blocked_response) + fast_upstream: Final = respx_mock.post("https://fast.example/v1/audio/speech").respond( + content=b"audio", + headers={"content-type": "audio/mpeg"}, + ) + model_list: Final = [ + { + "litellm_params": { + "model": "openai/tts-1", + "api_key": "fake-key", + "api_base": "https://slow.example/v1", + }, + "model_info": {"id": "slow", "mode": "audio_speech", "health_check_timeout": 1}, + }, + { + "litellm_params": { + "model": "openai/tts-1", + "api_key": "fake-key", + "api_base": "https://fast.example/v1", + }, + "model_info": {"id": "fast", "mode": "audio_speech", "health_check_timeout": 2}, + }, + ] + + healthy, unhealthy, _ = await asyncio.wait_for( + hc_module.perform_health_check(model_list, max_concurrency=1), + timeout=4, + ) + healthy_model_ids: Final = {endpoint["model_id"] for endpoint in healthy} + unhealthy_model_ids: Final = {endpoint["model_id"] for endpoint in unhealthy} + + assert slow_request_started.is_set() + assert fast_upstream.called + assert healthy_model_ids == {"fast"} + assert unhealthy_model_ids == {"slow"} diff --git a/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_wire.py b/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_wire.py new file mode 100644 index 00000000000..2b71b70c6bf --- /dev/null +++ b/tests/unit/responses/litellm_completion_transformation/test_google_ai_studio_responses_wire.py @@ -0,0 +1,158 @@ +import json +from typing import Final + +import httpx +import pytest +import respx +from pydantic import TypeAdapter + +import litellm +from litellm.types.llms.openai import ResponseCompletedEvent, ResponsesAPIResponse +from tests.unit.proxy.conftest import httpx_transport + +pytestmark: Final = pytest.mark.usefixtures(httpx_transport.__name__) +_GEMINI_URL: Final = ( + "https://generativelanguage.googleapis.com/v1beta/models/" + "gemini-2.5-flash:(?:generateContent|streamGenerateContent).*" +) +_JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +def _function_call_response(signature: str) -> dict[str, object]: + return { + "candidates": [ + { + "content": { + "role": "model", + "parts": [ + { + "functionCall": { + "name": "get_weather", + "args": {"location": "San Francisco"}, + }, + "thoughtSignature": signature, + } + ], + }, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 4, + "candidatesTokenCount": 2, + "totalTokenCount": 6, + }, + } + + +@pytest.mark.asyncio +async def test_web_search_preview_is_sent_as_google_search() -> None: + response_body: Final = { + "candidates": [ + { + "content": {"role": "model", "parts": [{"text": "Search completed"}]}, + "finishReason": "STOP", + "index": 0, + } + ], + "usageMetadata": { + "promptTokenCount": 2, + "candidatesTokenCount": 2, + "totalTokenCount": 4, + }, + } + + with respx.mock() as mock_router: + route: Final = mock_router.post(url__regex=_GEMINI_URL).mock( + return_value=httpx.Response(status_code=200, json=response_body) + ) + response: Final = await litellm.aresponses( + model="gemini/gemini-2.5-flash", + api_key="test-key", + input="Find current weather", + tools=[{"type": "web_search_preview", "search_context_size": "low"}], + ) + requests: Final = tuple(route.calls) + + assert isinstance(response, ResponsesAPIResponse) + assert response.output_text == "Search completed" + assert len(requests) == 1 + request_body: Final = _JSON_OBJECT.validate_json(requests[0].request.content) + assert request_body["tools"] == [{"googleSearch": {}}] + + +@pytest.mark.asyncio +async def test_gemini_function_call_preserves_thought_signature() -> None: + signature: Final = "gemini-thought-signature" + response_body: Final = _function_call_response(signature) + + with respx.mock() as mock_router: + route: Final = mock_router.post(url__regex=_GEMINI_URL).mock( + return_value=httpx.Response(status_code=200, json=response_body) + ) + response: Final = await litellm.aresponses( + model="gemini/gemini-2.5-flash", + api_key="test-key", + input="What is the weather in San Francisco?", + tools=[ + { + "type": "function", + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + } + ], + ) + requests: Final = tuple(route.calls) + + assert isinstance(response, ResponsesAPIResponse) + function_calls: Final = tuple(item for item in response.output if item.type == "function_call") + assert len(function_calls) == 1 + assert function_calls[0].name == "get_weather" + assert function_calls[0].provider_specific_fields == {"thought_signature": signature} + assert len(requests) == 1 + + +@pytest.mark.asyncio +async def test_gemini_streaming_function_call_preserves_thought_signature() -> None: + signature: Final = "gemini-stream-thought-signature" + response_body: Final = _function_call_response(signature) + event_stream: Final = f"data: {json.dumps(response_body)}\n\n" + + with respx.mock() as mock_router: + route: Final = mock_router.post(url__regex=_GEMINI_URL).mock( + return_value=httpx.Response( + status_code=200, + content=event_stream, + headers={"content-type": "text/event-stream"}, + ) + ) + stream: Final = await litellm.aresponses( + model="gemini/gemini-2.5-flash", + api_key="test-key", + input="What is the weather in San Francisco?", + stream=True, + tools=[ + { + "type": "function", + "name": "get_weather", + "parameters": { + "type": "object", + "properties": {"location": {"type": "string"}}, + }, + } + ], + ) + events: Final = tuple([event async for event in stream]) + requests: Final = tuple(route.calls) + + completed_events: Final = tuple(event for event in events if isinstance(event, ResponseCompletedEvent)) + assert len(completed_events) == 1 + function_calls: Final = tuple(item for item in completed_events[0].response.output if item.type == "function_call") + assert len(function_calls) == 1 + assert function_calls[0].name == "get_weather" + assert function_calls[0].provider_specific_fields == {"thought_signature": signature} + assert len(requests) == 1 diff --git a/tests/unit/responses/test_responses_api_lifecycle.py b/tests/unit/responses/test_responses_api_lifecycle.py new file mode 100644 index 00000000000..8805bc97d10 --- /dev/null +++ b/tests/unit/responses/test_responses_api_lifecycle.py @@ -0,0 +1,109 @@ +from typing import Final, Literal, TypeAlias + +import httpx +import openai +import pytest +import respx + +import litellm +from tests.unit.proxy.conftest import httpx_transport + +pytestmark: Final = pytest.mark.usefixtures(httpx_transport.__name__) +Provider: TypeAlias = Literal["anthropic", "gemini"] +Operation: TypeAlias = Literal["delete", "get", "cancel"] +CancelProvider: TypeAlias = Literal["openai", "azure"] + + +def _invoke_unsupported_response_operation(provider: Provider, operation: Operation) -> object: + match operation: + case "delete": + return litellm.delete_responses( + response_id="resp_unsupported", + custom_llm_provider=provider, + api_key="sk-test", + ) + case "get": + return litellm.get_responses( + response_id="resp_unsupported", + custom_llm_provider=provider, + api_key="sk-test", + ) + case "cancel": + return litellm.cancel_responses( + response_id="resp_unsupported", + custom_llm_provider=provider, + api_key="sk-test", + ) + + +async def _invoke_cancel_response(sync_mode: bool, call_kwargs: dict[str, object]) -> object: + if sync_mode: + return litellm.cancel_responses(**call_kwargs) + return await litellm.acancel_responses(**call_kwargs) + + +@pytest.mark.parametrize( + ("provider", "operation"), + ( + ("anthropic", "delete"), + ("anthropic", "get"), + ("anthropic", "cancel"), + ("gemini", "delete"), + ("gemini", "get"), + ("gemini", "cancel"), + ), +) +def test_unsupported_response_lifecycle_operation_fails_before_http(provider: Provider, operation: Operation) -> None: + with respx.mock() as mock_router: + with pytest.raises(litellm.APIConnectionError) as exc_info: + _invoke_unsupported_response_operation(provider, operation) + calls: Final = tuple(mock_router.calls) + + assert exc_info.value.status_code == 500 + assert f"not supported for {provider}" in str(exc_info.value) + assert calls == () + + +@pytest.mark.parametrize( + ("provider", "sync_mode"), + (("openai", True), ("openai", False), ("azure", True), ("azure", False)), +) +@pytest.mark.asyncio +async def test_cancel_responses_404_surfaces_openai_api_error_with_exact_url( + provider: CancelProvider, sync_mode: bool +) -> None: + response_id: Final = "resp_missing" + error_body: Final = { + "error": { + "message": "Response was not found", + "type": "invalid_request_error", + "code": "not_found", + } + } + api_base: Final = ( + "https://api.openai.com/v1" if provider == "openai" else "https://example-resource.openai.azure.com" + ) + expected_url: Final = ( + f"{api_base}/responses/{response_id}/cancel" + if provider == "openai" + else f"{api_base}/openai/responses/{response_id}/cancel?api-version=2025-03-01-preview" + ) + + with respx.mock() as mock_router: + route: Final = mock_router.post(expected_url).mock( + return_value=httpx.Response(status_code=404, json=error_body) + ) + call_kwargs: Final = { + "custom_llm_provider": provider, + "api_key": "sk-test", + "api_base": api_base, + "api_version": "2025-03-01-preview", + "response_id": response_id, + } + with pytest.raises(openai.APIError) as exc_info: + await _invoke_cancel_response(sync_mode, call_kwargs) + requests: Final = tuple(route.calls) + + assert exc_info.value.status_code == 404 + assert len(requests) == 1 + assert str(requests[0].request.url) == expected_url diff --git a/tests/unit/secret_managers/test_google_secret_manager.py b/tests/unit/secret_managers/test_google_secret_manager.py new file mode 100644 index 00000000000..ed93aa2861d --- /dev/null +++ b/tests/unit/secret_managers/test_google_secret_manager.py @@ -0,0 +1,83 @@ +import base64 +from dataclasses import dataclass +from typing import Final + +import litellm +import pytest +import respx + +from litellm.secret_managers.google_secret_manager import GoogleSecretManager + + +@dataclass(frozen=True, slots=True) +class _CachedVertexCredentials: + token: str + quota_project_id: str | None + expired: bool = False + + def refresh(self, request: object) -> None: + return None + + +def _google_secret_manager( + monkeypatch: pytest.MonkeyPatch, + project_id: str, +) -> GoogleSecretManager: + monkeypatch.setenv("GOOGLE_SECRET_MANAGER_PROJECT_ID", project_id) + monkeypatch.setattr("litellm.proxy.proxy_server.premium_user", True) + secret_manager: Final = GoogleSecretManager() + credentials: Final = _CachedVertexCredentials( + token="test-gsm-token", + quota_project_id=project_id, + ) + vertex_chat_completion: Final = litellm.vertex_chat_completion + monkeypatch.setitem( + vertex_chat_completion._credentials_project_mapping, + (None, project_id), + (credentials, project_id), + ) + return secret_manager + + +def test_google_secret_manager_decodes_secret_and_requests_latest_version( + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_id: Final = "test-secret-project" + secret_manager: Final = _google_secret_manager(monkeypatch, project_id) + secret_url: Final = ( + f"https://secretmanager.googleapis.com/v1/projects/{project_id}/secrets/OPENAI_API_KEY/versions/latest:access" + ) + encoded_secret: Final = base64.b64encode(b"anything").decode("ascii") + upstream: Final[respx.MockRouter] + + with respx.mock(assert_all_called=True) as upstream: + secret_route: Final = upstream.get(secret_url).respond( + 200, + json={"payload": {"data": encoded_secret}}, + ) + + result: Final = secret_manager.get_secret_from_google_secret_manager("OPENAI_API_KEY") + + assert result == "anything" + assert secret_route.called + assert len(upstream.calls) == 1 + assert str(upstream.calls.last.request.url) == secret_url + assert upstream.calls.last.request.headers["Authorization"] == "Bearer test-gsm-token" + + +def test_google_secret_manager_returns_cached_values_without_http( + monkeypatch: pytest.MonkeyPatch, +) -> None: + project_id: Final = "test-secret-project" + secret_manager: Final = _google_secret_manager(monkeypatch, project_id) + secret_manager.cache.set_cache("cached-none", None) + secret_manager.cache.set_cache("cached-string", "lite-llm") + upstream: Final[respx.MockRouter] + + with respx.mock() as upstream: + missing_value: Final = secret_manager.get_secret_from_google_secret_manager("cached-none") + cached_value: Final = secret_manager.get_secret_from_google_secret_manager("cached-string") + + assert missing_value is None + assert cached_value == "lite-llm" + assert upstream.calls.call_count == 0 diff --git a/tests/unit/secret_managers/test_secret_managers_main.py b/tests/unit/secret_managers/test_secret_managers_main.py index 32040251795..d32555c3669 100644 --- a/tests/unit/secret_managers/test_secret_managers_main.py +++ b/tests/unit/secret_managers/test_secret_managers_main.py @@ -3,6 +3,7 @@ import json import logging import os import time +from typing import Final from unittest.mock import Mock, patch import pytest @@ -230,6 +231,13 @@ def test_oidc_circleci_success(monkeypatch): assert result == "circleci_token" +def test_oidc_circleci_v2_returns_the_environment_token(monkeypatch: pytest.MonkeyPatch) -> None: + token: Final = "circleci-v2-token" + monkeypatch.setenv("CIRCLE_OIDC_TOKEN_V2", token) + + assert get_secret("oidc/circleci_v2/test-audience") == token + + def test_oidc_circleci_failure(monkeypatch): monkeypatch.delenv("CIRCLE_OIDC_TOKEN", raising=False) secret_name = "oidc/circleci/test-audience" diff --git a/tests/unit/test_router/test_router_provider_endpoints.py b/tests/unit/test_router/test_router_provider_endpoints.py new file mode 100644 index 00000000000..708322f4fec --- /dev/null +++ b/tests/unit/test_router/test_router_provider_endpoints.py @@ -0,0 +1,732 @@ +import asyncio +import contextlib +import io +import json +import uuid +from collections.abc import Callable, Iterator +from datetime import datetime, timezone +from typing import Final + +import httpx +import litellm +import pytest +import respx +from litellm import CustomLogger, Router +from litellm.caching.dual_cache import DualCache +from litellm.caching.in_memory_cache import InMemoryCache +from litellm.constants import ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY +from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper +from litellm.router_utils.router_callbacks.track_deployment_metrics import ( + get_deployment_successes_for_current_minute, +) +from litellm.types.caching import RedisPipelineIncrementOperation +from litellm.types.llms.openai import HttpxBinaryResponseContent +from litellm.types.router import RoutingStrategy +from litellm.types.utils import ImageResponse, ModelResponse, RerankResponse +from openai import AsyncAzureOpenAI, AsyncOpenAI + +EVENT_TIMEOUT_SECONDS: Final = 5 +ROUTING_SELECTIONS: Final = 40 +ROUTING_MESSAGES: Final = ({"role": "user", "content": "route this request"},) +EXPENSIVE_COSTS: Final = {"input_cost_per_token": 1.0, "output_cost_per_token": 1.0} +CHEAP_COSTS: Final = {"input_cost_per_token": 1e-9, "output_cost_per_token": 1e-9} +CHAT_RESPONSE: Final = { + "id": "chatcmpl-1", + "object": "chat.completion", + "created": 1700000000, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}], + "usage": {"prompt_tokens": 5, "completion_tokens": 1, "total_tokens": 6}, +} + + +@pytest.fixture(autouse=True) +def httpx_transport(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + monkeypatch.setattr(litellm, "disable_aiohttp_transport", True) + litellm.in_memory_llm_clients_cache.flush_cache() + yield + litellm.in_memory_llm_clients_cache.flush_cache() + + +@pytest.fixture +def router_minute_pinned(monkeypatch: pytest.MonkeyPatch) -> None: + pinned: Final = datetime(2026, 1, 1, 12, 0, 30, tzinfo=timezone.utc) + monkeypatch.setattr("litellm.router.get_utc_datetime", lambda: pinned) + + +class _RouterLoggingCapture(CustomLogger): + def __init__(self, model_id: str) -> None: + super().__init__() + self.model_id: Final = model_id + self.success_events: asyncio.Queue[tuple[object | None, object | None]] = asyncio.Queue() + + async def async_log_success_event( + self, + kwargs: dict[str, object], + response_obj: object, + start_time: object, + end_time: object, + ) -> None: + standard_logging_object: Final = kwargs.get("standard_logging_object") + if not isinstance(standard_logging_object, dict) or standard_logging_object.get("model_id") != self.model_id: + return + self.success_events.put_nowait((kwargs.get("client"), standard_logging_object)) + + async def next_event(self) -> tuple[object | None, object | None]: + return await asyncio.wait_for(self.success_events.get(), timeout=EVENT_TIMEOUT_SECONDS) + + +def _rpm_tpm_router(model_id: str) -> Router: + return Router( + model_list=[ + { + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-fake", "tpm": 1000, "rpm": 100}, + "model_info": {"id": model_id}, + } + ] + ) + + +def _ratelimit_headers(response: ModelResponse | CustomStreamWrapper) -> dict[str, int]: + return {k: v for k, v in response._hidden_params["additional_headers"].items() if k.startswith("x-ratelimit-")} + + +def _openai_router_client() -> AsyncOpenAI: + return AsyncOpenAI(api_key="sk-fake") + + +def _azure_router_client() -> AsyncAzureOpenAI: + return AsyncAzureOpenAI(api_key="sk-fake", azure_endpoint="https://azure.test", api_version="2025-02-01-preview") + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("litellm_params", "build_client", "route_url"), + [ + ( + {"model": "whisper-1", "api_key": "sk-fake"}, + _openai_router_client, + "https://api.openai.com/v1/audio/transcriptions", + ), + ( + { + "model": "azure/whisper", + "api_base": "https://azure.test", + "api_key": "sk-fake", + "api_version": "2025-02-01-preview", + }, + _azure_router_client, + "https://azure.test/openai/deployments/whisper/audio/transcriptions?api-version=2025-02-01-preview", + ), + ], + ids=["openai", "azure"], +) +async def test_router_transcription_reuses_router_level_client_for_each_deployment( + litellm_params: dict[str, str], + build_client: Callable[[], AsyncOpenAI], + route_url: str, + monkeypatch: pytest.MonkeyPatch, + respx_mock: respx.MockRouter, +) -> None: + model_id: Final = f"whisper-{uuid.uuid4().hex}" + capture: Final = _RouterLoggingCapture(model_id) + monkeypatch.setattr(litellm, "callbacks", [capture]) + router: Final = Router( + model_list=[{"model_name": "whisper", "litellm_params": dict(litellm_params), "model_info": {"id": model_id}}] + ) + router_level_client: Final = build_client() + router.cache.set_cache(key=f"{model_id}_async_client", value=router_level_client, local_only=True) + route: Final = respx_mock.post(route_url).respond(200, json={"text": "hello"}) + + response: Final = await router.atranscription( + model="whisper", file=("speech.wav", io.BytesIO(b"offline audio"), "audio/wav") + ) + public_client, public_logging = await capture.next_event() + internal_response: Final = await router._atranscription( + model="whisper", file=("speech.wav", io.BytesIO(b"offline audio"), "audio/wav") + ) + internal_client, _ = await capture.next_event() + upstream_requests: Final = tuple(call.request for call in route.calls) + + assert public_client == str(router_level_client) + assert internal_client == str(router_level_client) + assert tuple(str(request.url) for request in upstream_requests) == (route_url, route_url) + assert all( + request.headers.get("content-type", "").startswith("multipart/form-data") + and b'filename="speech.wav"' in request.content + and b"offline audio" in request.content + for request in upstream_requests + ) + assert isinstance(public_logging, dict) + assert public_logging.get("model_group") == "whisper" + assert response.text == "hello" + assert internal_response.text == "hello" + + +@pytest.mark.asyncio +async def test_router_speech_returns_binary_content_and_logs_model_group( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + model_id: Final = f"tts-{uuid.uuid4().hex}" + capture: Final = _RouterLoggingCapture(model_id) + monkeypatch.setattr(litellm, "callbacks", [capture]) + router: Final = Router( + model_list=[ + { + "model_name": "tts", + "litellm_params": {"model": "openai/tts-1", "api_key": "sk-fake"}, + "model_info": {"id": model_id}, + } + ] + ) + route: Final = respx_mock.post("https://api.openai.com/v1/audio/speech").respond(200, content=b"audio") + + response: Final = await router.aspeech(model="tts", input="hello", voice="alloy") + _, standard_logging_object = await capture.next_event() + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == {"model": "tts-1", "input": "hello", "voice": "alloy"} + assert isinstance(response, HttpxBinaryResponseContent) + assert response.content == b"audio" + assert isinstance(standard_logging_object, dict) + assert standard_logging_object["model_group"] == "tts" + + +@pytest.mark.asyncio +async def test_router_rerank_returns_valid_response_from_public_and_underlying_calls( + respx_mock: respx.MockRouter, +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "cohere-rerank", + "litellm_params": {"model": "cohere/rerank-english-v3.0", "api_key": "sk-fake"}, + } + ] + ) + route: Final = respx_mock.post("https://api.cohere.com/v2/rerank").respond( + 200, + json={ + "id": "rerank-1", + "results": [{"index": 0, "relevance_score": 0.9}], + "meta": {"api_version": {"version": "2"}}, + }, + ) + + public_response: Final = await router.arerank( + model="cohere-rerank", + query="hello", + documents=["hello", "world"], + top_n=1, + ) + underlying_response: Final = await router._arerank( + model="cohere-rerank", + query="hello", + documents=["hello", "world"], + top_n=1, + ) + + assert route.call_count == 2 + request_bodies: Final = tuple(json.loads(call.request.content) for call in route.calls) + assert all( + body["model"] == "rerank-english-v3.0" + and body["query"] == "hello" + and body["documents"] == ["hello", "world"] + and body["top_n"] == 1 + for body in request_bodies + ) + public_validated: Final = RerankResponse.model_validate(public_response) + assert public_validated.id == "rerank-1" + assert public_validated.results[0]["relevance_score"] == 0.9 + underlying_validated: Final = RerankResponse.model_validate(underlying_response) + assert underlying_validated.id == "rerank-1" + assert underlying_validated.results[0]["relevance_score"] == 0.9 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("model", "expected_model", "expected_api_key"), + [ + ("omni-moderation-latest", "omni-moderation-latest", "sk-catch-all"), + ("openai/omni-moderation-latest", "omni-moderation-latest", "sk-openai-wildcard"), + (None, None, "sk-env"), + ], +) +async def test_router_moderation_routes_through_wildcard_deployments( + model: str | None, + expected_model: str | None, + expected_api_key: str, + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-env") + router: Final = Router( + model_list=[ + { + "model_name": "openai/*", + "litellm_params": {"model": "openai/*", "api_key": "sk-openai-wildcard"}, + }, + { + "model_name": "*", + "litellm_params": {"model": "openai/*", "api_key": "sk-catch-all"}, + }, + ] + ) + route: Final = respx_mock.post("https://api.openai.com/v1/moderations").respond( + 200, + json={ + "id": "modr-1", + "model": "omni-moderation-latest", + "results": [{"flagged": False, "categories": {}, "category_scores": {}}], + }, + ) + + response: Final = await router.amoderation(model=model, input="hello") + + assert route.call_count == 1 + upstream_request: Final = route.calls[0].request + expected_body: Final = {"input": "hello"} if expected_model is None else {"input": "hello", "model": expected_model} + assert json.loads(upstream_request.content) == expected_body + assert upstream_request.headers["authorization"] == f"Bearer {expected_api_key}" + assert response.id == "modr-1" + assert response.model == "omni-moderation-latest" + assert response.results[0].flagged is False + + +@pytest.mark.asyncio +@pytest.mark.parametrize("sync_mode", [True, False]) +async def test_router_image_generation_returns_valid_image_response( + sync_mode: bool, + respx_mock: respx.MockRouter, +) -> None: + router: Final = Router( + model_list=[ + { + "model_name": "gpt-image-1", + "litellm_params": {"model": "openai/gpt-image-1", "api_key": "sk-fake"}, + } + ] + ) + route: Final = respx_mock.post("https://api.openai.com/v1/images/generations").respond( + 200, + json={"created": 1700000000, "data": [{"url": "https://images.test/result.png"}]}, + ) + + response: Final = ( + router._image_generation(model="gpt-image-1", prompt="a cat") + if sync_mode + else await router._aimage_generation(model="gpt-image-1", prompt="a cat") + ) + + assert route.call_count == 1 + request_body: Final = json.loads(route.calls[0].request.content) + assert request_body["model"] == "gpt-image-1" + assert request_body["prompt"] == "a cat" + validated: Final = ImageResponse.model_validate(response) + assert validated.data[0].url == "https://images.test/result.png" + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_router_acompletion_headers_read_post_increment_counter_and_count_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + capture: Final = _RouterLoggingCapture("lit-3058-async") + monkeypatch.setattr(litellm, "callbacks", [capture]) + router: Final = _rpm_tpm_router("lit-3058-async") + + response: Final = await router.acompletion( + model="gpt-5-mini", messages=[{"role": "user", "content": "hi"}], mock_response="pong" + ) + total_tokens: Final = response.usage.total_tokens + headers: Final = _ratelimit_headers(response) + + assert total_tokens > 0 + assert headers["x-ratelimit-remaining-tokens"] == 1000 - total_tokens + assert headers["x-ratelimit-remaining-requests"] == 99 + assert await router.get_model_group_usage("gpt-5-mini") == (total_tokens, 1) + + await capture.next_event() + + assert await router.get_model_group_usage("gpt-5-mini") == (total_tokens, 1) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_router_stream_counts_request_before_headers_and_tokens_once_on_completion( + monkeypatch: pytest.MonkeyPatch, +) -> None: + capture: Final = _RouterLoggingCapture("lit-3058-stream") + monkeypatch.setattr(litellm, "callbacks", [capture]) + router: Final = _rpm_tpm_router("lit-3058-stream") + stream: Final = await router.acompletion( + model="gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong pong pong", + stream=True, + stream_options={"include_usage": True}, + ) + headers: Final = _ratelimit_headers(stream) + + assert headers["x-ratelimit-remaining-tokens"] == 1000 + assert headers["x-ratelimit-remaining-requests"] == 99 + assert await router.get_model_group_usage("gpt-5-mini") == (0, 1) + + chunks: Final = [chunk async for chunk in stream] + total_tokens: Final = chunks[-1].usage.total_tokens + await capture.next_event() + + assert total_tokens > 0 + assert await router.get_model_group_usage("gpt-5-mini") == (total_tokens, 1) + + +def test_router_validate_fallbacks_accepts_well_formed_and_rejects_malformed_entries() -> None: + router: Final = Router(model_list=[]) + + assert router.validate_fallbacks([{"gpt-5.5": ["gpt-5-mini"]}, {"gpt-5-mini": ["gpt-5.5"]}]) is None + with pytest.raises(ValueError, match="must have exactly one key"): + router.validate_fallbacks([{"primary": "fallback", "other": "fallback"}]) + with pytest.raises(ValueError, match="is not a dictionary"): + router.validate_fallbacks(["primary"]) + + +def _routing_deployment(deployment_id: str, extra_params: dict[str, float]) -> dict[str, object]: + return { + "model_name": "gpt", + "litellm_params": { + "model": "openai/gpt-4o-mini", + "api_key": "sk-fake", + "api_base": f"https://{deployment_id}.test/v1", + **extra_params, + }, + "model_info": {"id": deployment_id}, + } + + +def _routing_router( + strategy: RoutingStrategy | str, + a_params: dict[str, float] | None = None, + b_params: dict[str, float] | None = None, +) -> Router: + return Router( + model_list=[_routing_deployment("a", a_params or {}), _routing_deployment("b", b_params or {})], + routing_strategy=strategy, + disable_cooldowns=True, + num_retries=0, + ) + + +async def _selected_deployment_ids(router: Router) -> frozenset[str]: + deployments: Final = [ + await router.async_get_available_deployment(model="gpt", messages=list(ROUTING_MESSAGES), request_kwargs={}) + for _ in range(ROUTING_SELECTIONS) + ] + return frozenset(deployment["model_info"]["id"] for deployment in deployments) + + +def _enum_and_string(strategy: RoutingStrategy) -> list[RoutingStrategy | str]: + return [strategy, strategy.value] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("strategy", _enum_and_string(RoutingStrategy.COST_BASED)) +async def test_router_cost_based_routing_selects_cheapest_deployment(strategy: RoutingStrategy | str) -> None: + router: Final = _routing_router(strategy, EXPENSIVE_COSTS, CHEAP_COSTS) + + assert await _selected_deployment_ids(router) == frozenset({"b"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "strategy", + [*_enum_and_string(RoutingStrategy.USAGE_BASED_ROUTING), *_enum_and_string(RoutingStrategy.USAGE_BASED_ROUTING_V2)], +) +async def test_router_usage_based_routing_skips_deployment_over_tpm_limit(strategy: RoutingStrategy | str) -> None: + router: Final = _routing_router(strategy, {"tpm": 1}, {"tpm": 1_000_000}) + + assert await _selected_deployment_ids(router) == frozenset({"b"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("strategy", _enum_and_string(RoutingStrategy.LATENCY_BASED)) +async def test_router_latency_based_routing_avoids_deployment_that_timed_out( + strategy: RoutingStrategy | str, + respx_mock: respx.MockRouter, +) -> None: + router: Final = _routing_router(strategy) + timed_out: Final = respx_mock.post("https://a.test/v1/chat/completions").mock( + side_effect=httpx.ReadTimeout("upstream timed out") + ) + respx_mock.post("https://b.test/v1/chat/completions").respond(200, json=CHAT_RESPONSE) + + for _ in range(5): + if timed_out.called: + break + with contextlib.suppress(litellm.Timeout): + await router.acompletion(model="gpt", messages=list(ROUTING_MESSAGES), max_retries=0) + + assert timed_out.called + assert await _selected_deployment_ids(router) == frozenset({"b"}) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("strategy", _enum_and_string(RoutingStrategy.LEAST_BUSY)) +async def test_router_least_busy_routing_avoids_deployment_with_request_in_flight( + strategy: RoutingStrategy | str, + respx_mock: respx.MockRouter, +) -> None: + router: Final = _routing_router(strategy) + upstream_hosts: Final = asyncio.Queue[str]() + release_upstream: Final = asyncio.Event() + + async def hold_request(request: httpx.Request) -> httpx.Response: + upstream_hosts.put_nowait(request.url.host.split(".")[0]) + await release_upstream.wait() + return httpx.Response(200, json=CHAT_RESPONSE) + + respx_mock.post(url__regex=r"https://[ab]\.test/v1/chat/completions").mock(side_effect=hold_request) + in_flight: Final = asyncio.create_task(router.acompletion(model="gpt", messages=list(ROUTING_MESSAGES))) + try: + busy_id: Final = await asyncio.wait_for(upstream_hosts.get(), timeout=EVENT_TIMEOUT_SECONDS) + selected: Final = await _selected_deployment_ids(router) + finally: + release_upstream.set() + await in_flight + + assert selected == frozenset({"a", "b"} - {busy_id}) + + +@pytest.mark.asyncio +async def test_router_simple_shuffle_ignores_cost_and_spreads_across_deployments() -> None: + router: Final = _routing_router("simple-shuffle", EXPENSIVE_COSTS, CHEAP_COSTS) + + assert await _selected_deployment_ids(router) == frozenset({"a", "b"}) + + +@pytest.mark.parametrize("strategy", _enum_and_string(RoutingStrategy.PROVIDER_BUDGET_LIMITING)) +def test_router_routing_strategy_init_accepts_provider_budget_strategy(strategy: RoutingStrategy | str) -> None: + router: Final = _routing_router(strategy) + + router.routing_strategy_init(routing_strategy=strategy, routing_strategy_args={}) + + assert router.get_settings()["routing_strategy"] == "provider-budget-routing" + + +def test_router_track_deployment_metrics_updates_observable_usage() -> None: + router: Final = Router( + model_list=[ + { + "model_name": "gpt-5-mini", + "litellm_params": {"model": "gpt-5-mini", "api_key": "sk-fake"}, + "model_info": {"id": "metrics-deployment"}, + } + ] + ) + deployment: Final = router.model_list[0] + + router._track_deployment_metrics(deployment=deployment, parent_otel_span=None) + router._track_deployment_metrics(deployment=deployment, parent_otel_span=None) + + assert router.cache.get_cache(key="metrics-deployment", local_only=True) == 2 + + +class _GatedIncrementCache(DualCache): + def __init__(self) -> None: + super().__init__(in_memory_cache=InMemoryCache()) + self.first_increment_started = asyncio.Event() + self.release_first_increment = asyncio.Event() + self.deployment_success_incremented = asyncio.Event() + self.increment_calls = 0 + + def increment_cache(self, key: str, value: int, local_only: bool = False, **kwargs: object) -> int: + result: Final = super().increment_cache(key=key, value=value, local_only=local_only, **kwargs) + if key.endswith(":successes"): + self.deployment_success_incremented.set() + return result + + async def async_increment_cache_pipeline( + self, + increment_list: list[RedisPipelineIncrementOperation], + local_only: bool = False, + parent_otel_span: object = None, + **kwargs: object, + ) -> list[float] | None: + self.increment_calls += 1 + if self.increment_calls == 1: + self.first_increment_started.set() + await self.release_first_increment.wait() + return await super().async_increment_cache_pipeline( + increment_list=increment_list, + local_only=local_only, + parent_otel_span=parent_otel_span, + **kwargs, + ) + + +class _UnavailableIncrementCache(DualCache): + def __init__(self) -> None: + super().__init__(in_memory_cache=InMemoryCache()) + self.first_increment_started = asyncio.Event() + self.release_first_increment = asyncio.Event() + self.deployment_success_incremented = asyncio.Event() + self.increment_calls = 0 + + def increment_cache(self, key: str, value: int, local_only: bool = False, **kwargs: object) -> int: + result: Final = super().increment_cache(key=key, value=value, local_only=local_only, **kwargs) + if key.endswith(":successes"): + self.deployment_success_incremented.set() + return result + + async def async_increment_cache_pipeline( + self, + increment_list: list[RedisPipelineIncrementOperation], + local_only: bool = False, + parent_otel_span: object = None, + **kwargs: object, + ) -> list[float] | None: + self.increment_calls += 1 + if self.increment_calls == 1: + self.first_increment_started.set() + await self.release_first_increment.wait() + raise RuntimeError("cache unavailable") + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_router_success_callback_during_pre_header_increment_does_not_double_count() -> None: + router: Final = _rpm_tpm_router("lit-3058-race") + cache: Final = _GatedIncrementCache() + router.cache = cache + request: Final = asyncio.create_task( + router.acompletion( + model="gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + ) + ) + + await asyncio.wait_for(cache.first_increment_started.wait(), timeout=EVENT_TIMEOUT_SECONDS) + await asyncio.wait_for(cache.deployment_success_incremented.wait(), timeout=EVENT_TIMEOUT_SECONDS) + assert get_deployment_successes_for_current_minute(router, "lit-3058-race") == 1 + assert cache.increment_calls == 1 + cache.release_first_increment.set() + response: Final = await request + + assert await router.get_model_group_usage("gpt-5-mini") == (response.usage.total_tokens, 1) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("router_minute_pinned") +async def test_router_failed_pre_header_increment_clears_counted_tokens_stamp() -> None: + router: Final = _rpm_tpm_router("lit-3058-fail") + cache: Final = _UnavailableIncrementCache() + router.cache = cache + metadata: Final[dict[str, object]] = {} + request: Final = asyncio.create_task( + router.acompletion( + model="gpt-5-mini", + messages=[{"role": "user", "content": "hi"}], + mock_response="pong", + metadata=metadata, + ) + ) + + await asyncio.wait_for(cache.first_increment_started.wait(), timeout=EVENT_TIMEOUT_SECONDS) + await asyncio.wait_for(cache.deployment_success_incremented.wait(), timeout=EVENT_TIMEOUT_SECONDS) + assert metadata[ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY] == 30 + assert get_deployment_successes_for_current_minute(router, "lit-3058-fail") == 1 + assert cache.increment_calls == 1 + cache.release_first_increment.set() + response: Final = await request + + assert response.usage.total_tokens == 30 + assert ROUTER_USAGE_COUNTED_TOKENS_METADATA_KEY not in metadata + assert _ratelimit_headers(response)["x-ratelimit-remaining-requests"] == 100 + assert await router.get_model_group_usage("gpt-5-mini") == (None, None) + + +ASSISTANT_RESPONSE: Final = { + "object": "assistant", + "created_at": 1700000000, + "name": "offline", + "description": None, + "model": "gpt-4o-mini", + "instructions": "hello", + "tools": [], + "metadata": {}, +} + + +@pytest.mark.asyncio +async def test_router_assistants_endpoint_factory_invokes_provider( + respx_mock: respx.MockRouter, +) -> None: + router: Final = Router(model_list=[]) + route: Final = respx_mock.post("https://api.openai.com/v1/assistants").respond( + 200, json={**ASSISTANT_RESPONSE, "id": "asst-1"} + ) + + response: Final = await router._pass_through_assistants_endpoint_factory( + original_function=litellm.acreate_assistants, + custom_llm_provider="openai", + model="gpt-4o-mini", + api_key="sk-fake", + name="offline", + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == {"model": "gpt-4o-mini", "name": "offline"} + assert response.id == "asst-1" + + +@pytest.mark.asyncio +async def test_router_factory_function_returns_invokable_assistants_wrapper( + respx_mock: respx.MockRouter, +) -> None: + router: Final = Router(model_list=[]) + route: Final = respx_mock.post("https://api.openai.com/v1/assistants").respond( + 200, json={**ASSISTANT_RESPONSE, "id": "asst-2"} + ) + wrapper: Final = router.factory_function(litellm.acreate_assistants, call_type="assistants") + + response: Final = await wrapper( + custom_llm_provider="openai", + model="gpt-4o-mini", + api_key="sk-fake", + name="offline", + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == {"model": "gpt-4o-mini", "name": "offline"} + assert response.id == "asst-2" + + +@pytest.mark.asyncio +async def test_router_moderation_endpoint_factory_invokes_default_model( + respx_mock: respx.MockRouter, + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OPENAI_API_KEY", "sk-fake") + router: Final = Router(model_list=[]) + route: Final = respx_mock.post("https://api.openai.com/v1/moderations").respond( + 200, + json={ + "id": "modr-2", + "model": "omni-moderation-latest", + "results": [{"flagged": False, "categories": {}, "category_scores": {}}], + }, + ) + + response: Final = await router._pass_through_moderation_endpoint_factory( + original_function=litellm.amoderation, + custom_llm_provider="openai", + input="hello", + model=None, + api_key="sk-fake", + ) + + assert route.call_count == 1 + assert json.loads(route.calls[0].request.content) == {"input": "hello"} + assert response.id == "modr-2" diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index cf3e749df95..8a7c9825f0a 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -37,6 +37,10 @@ from litellm.caching.in_memory_cache import InMemoryCache from litellm.constants import DEFAULT_MOCK_RESPONSE_COMPLETION_TOKEN_COUNT from litellm.integrations.custom_guardrail import CustomGuardrail from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.humanloop import HumanloopLogger +from litellm.llms.base_llm.audio_transcription.transformation import BaseAudioTranscriptionConfig +from litellm.integrations.langfuse.langfuse_prompt_management import LangfusePromptManagement +from litellm.litellm_core_utils import litellm_logging from litellm.litellm_core_utils.duration_parser import ( _extract_from_regex, duration_in_seconds, @@ -50,7 +54,7 @@ from litellm.proxy.utils import is_valid_api_key from litellm.types.caching import CachingSupportedCallTypes from litellm.types.integrations.custom_logger import HEADROOM_CONVERTED_STREAM_KEY from litellm.types.llms.openai import ResponsesAPIResponse -from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams +from litellm.types.router import CredentialLiteLLMParams, GenericLiteLLMParams, LiteLLM_Params from litellm.types.utils import ( ADDRESSED_RESPONSE_ID_FIELD, CallTypes, @@ -74,6 +78,7 @@ from litellm.types.utils import ( from litellm.types.videos.main import VideoObject from litellm.utils import ( _invalidate_model_cost_lowercase_map, + add_custom_logger_callback_to_specific_event, check_valid_key, CustomStreamWrapper, filter_out_litellm_params, @@ -6802,6 +6807,7 @@ def setup_and_teardown(): MODEL: Final = "anthropic/claude-haiku-4-5" + @pytest.mark.usefixtures("_vcr_outcome_gate", "setup_and_teardown") def test_validate_tool_choice_none(): """Test that None is returned as-is.""" @@ -6946,6 +6952,7 @@ _SCALAR_DEFAULTS = { "api_key": getattr(litellm, "api_key", None), } + @pytest.fixture(scope="module") def setup_and_teardown_local_testing(): """ @@ -8786,6 +8793,407 @@ def test_get_valid_models_from_provider(): assert "gpt-5-mini" in valid_models +@pytest.mark.parametrize( + ("provider", "api_base", "api_key", "model_id"), + [ + ("anthropic", "https://anthropic.models.test", "anthropic-test-key", "claude-test-model"), + ("xai", "https://xai.models.test", "xai-test-key", "grok-test-model"), + ], +) +def test_get_valid_models_discovers_provider_models_from_http( + provider: str, + api_base: str, + api_key: str, + model_id: str, +) -> None: + response_body: Final = { + "data": [ + { + "id": model_id, + "type": "model", + "display_name": "Test model", + "created_at": "2024-01-01T00:00:00Z", + } + ], + "has_more": False, + "first_id": model_id, + "last_id": model_id, + } + models_url: Final = f"{api_base}/v1/models" + upstream: Final[respx.MockRouter] + + with respx.mock(assert_all_called=True) as upstream: + model_list_route: Final = upstream.get(models_url).respond(200, json=response_body) + + discovered_models: Final = get_valid_models( + check_provider_endpoint=True, + custom_llm_provider=provider, + api_key=api_key, + api_base=api_base, + ) + + assert discovered_models == [f"{provider}/{model_id}"] + assert model_list_route.called + assert len(upstream.calls) == 1 + + +def test_check_valid_key_returns_false_for_http_unauthorized() -> None: + response_body: Final = { + "error": { + "message": "Invalid API key", + "type": "invalid_request_error", + "param": None, + "code": "invalid_api_key", + } + } + upstream: Final[respx.MockRouter] + + with respx.mock(assert_all_called=True) as upstream: + invalid_key_route: Final = upstream.post("https://api.openai.com/v1/chat/completions").respond( + 401, + json=response_body, + ) + + valid_key: Final = check_valid_key(model="gpt-5-mini", api_key="invalid-test-key") + + assert valid_key is False + assert invalid_key_route.called + assert upstream.calls.last.request.headers["Authorization"] == "Bearer invalid-test-key" + + +def test_check_valid_key_returns_true_for_successful_completion() -> None: + response_body: Final = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 0, + "model": "gpt-5-mini", + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "ok"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2}, + } + upstream: Final[respx.MockRouter] + + with respx.mock(assert_all_called=True) as upstream: + valid_key_route: Final = upstream.post("https://api.openai.com/v1/chat/completions").respond( + 200, + json=response_body, + ) + + valid_key: Final = check_valid_key(model="gpt-5-mini", api_key="valid-test-key") + + assert valid_key is True + assert valid_key_route.called + assert upstream.calls.last.request.headers["Authorization"] == "Bearer valid-test-key" + + +def test_function_to_dict_parses_numpy_docstring_schema() -> None: + pytest.importorskip("numpydoc") + + def get_current_weather(location: str, unit: str) -> str: + """Get the current weather in a given location + + Parameters + ---------- + location : str + The city and state, e.g. San Francisco, CA + unit : {'celsius', 'fahrenheit'} + Temperature unit + + Returns + ------- + str + A sentence indicating the weather + """ + return f"Weather for {location} in {unit}" + + schema: Final = litellm.utils.function_to_dict(get_current_weather) + + assert schema["name"] == "get_current_weather" + assert schema["description"] == "Get the current weather in a given location" + assert schema["parameters"]["type"] == "object" + assert schema["parameters"]["properties"]["location"] == { + "type": "string", + "description": "The city and state, e.g. San Francisco, CA", + } + assert schema["parameters"]["properties"]["unit"]["type"] == "string" + assert schema["parameters"]["properties"]["unit"]["description"] == "Temperature unit" + assert schema["parameters"]["required"] == ["location", "unit"] + + +def test_duration_in_seconds_one_month_uses_the_fixed_calendar_interval( + monkeypatch: pytest.MonkeyPatch, +) -> None: + fixed_start: Final = datetime(2025, 2, 15, 12, 0, 0, 123456) + fixed_timestamp: Final = fixed_start.timestamp() + monkeypatch.setattr( + "litellm.litellm_core_utils.duration_parser.time_module.time", + lambda: fixed_timestamp, + ) + + assert duration_in_seconds("1mo") == 28 * 24 * 60 * 60 + + +def test_prompt_caching_image_check_uses_default_image_dimensions() -> None: + image_bytes: Final = base64.b64decode( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8/x+AAwMCAO+ip1sAAAAASUVORK5CYII=" + ) + image_url: Final = "https://93.184.216.34/test.png" + messages: Final = [ + { + "role": "user", + "content": [ + {"type": "text", "text": "What is in this image?"}, + { + "type": "image_url", + "image_url": {"url": image_url, "detail": "high"}, + }, + ], + } + ] + + with respx.mock(assert_all_called=False) as upstream: + image_route: Final = upstream.get(image_url).respond(200, content=image_bytes) + cacheable: Final = is_prompt_caching_valid_prompt( + model="gpt-4o-mini", + messages=messages, + custom_llm_provider="openai", + min_token_count=100_000, + ) + + assert cacheable is False + assert image_route.called is False + assert len(upstream.calls) == 0 + + +def test_get_valid_models_discovers_fireworks_models_from_http( + monkeypatch: pytest.MonkeyPatch, +) -> None: + api_base: Final = "https://fireworks.models.test/v1" + api_key: Final = "fireworks-test-key" + account_id: Final = "fireworks-test-account" + model_name: Final = "accounts/fireworks/models/llama-test-model" + models_url: Final = f"https://fireworks.models.test/v1/accounts/{account_id}/models" + monkeypatch.setenv("FIREWORKS_ACCOUNT_ID", account_id) + upstream: Final[respx.MockRouter] + + with respx.mock(assert_all_called=True) as upstream: + model_list_route: Final = upstream.get(models_url).respond( + 200, + json={"models": [{"name": model_name}]}, + ) + + discovered_models: Final = get_valid_models( + check_provider_endpoint=True, + custom_llm_provider="fireworks_ai", + api_key=api_key, + api_base=api_base, + ) + + assert discovered_models == [f"fireworks_ai/{model_name}"] + assert model_list_route.called + assert upstream.calls.last.request.headers["Authorization"] == f"Bearer {api_key}" + + +def test_get_valid_models_returns_static_fireworks_models_without_endpoint_check( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "check_provider_endpoint", False) + monkeypatch.setenv("FIREWORKS_AI_API_KEY", "fireworks-test-key") + expected_models: Final = litellm.models_by_provider["fireworks_ai"] + + actual_models: Final = get_valid_models( + check_provider_endpoint=False, + custom_llm_provider="fireworks_ai", + ) + env_inferred_models: Final = get_valid_models() + + assert set(actual_models) == expected_models + assert actual_models + assert expected_models <= set(env_inferred_models) + + +def test_get_valid_models_uses_the_litellm_params_anthropic_api_key() -> None: + model_id: Final = "claude-test-model" + models_url: Final = "https://api.anthropic.com/v1/models" + response_body: Final = { + "data": [ + { + "id": model_id, + "type": "model", + "display_name": "Test Claude", + "created_at": "2024-01-01T00:00:00Z", + } + ], + "has_more": False, + "first_id": model_id, + "last_id": model_id, + } + upstream: Final[respx.MockRouter] + + def response_for_api_key(request: httpx.Request) -> httpx.Response: + if request.headers["x-api-key"] == "bad-test-key": + return httpx.Response(401, json={"error": {"message": "invalid key"}}, request=request) + return httpx.Response(200, json=response_body, request=request) + + with respx.mock(assert_all_called=True) as upstream: + model_list_route: Final = upstream.get(models_url).mock(side_effect=response_for_api_key) + + bad_key_models: Final = get_valid_models( + check_provider_endpoint=True, + custom_llm_provider="anthropic", + litellm_params=LiteLLM_Params(model="anthropic/*", api_key="bad-test-key"), + ) + good_key_models: Final = get_valid_models( + check_provider_endpoint=True, + custom_llm_provider="anthropic", + litellm_params=LiteLLM_Params(model="anthropic/*", api_key="good-test-key"), + ) + + assert bad_key_models == [] + assert good_key_models == [f"anthropic/{model_id}"] + assert model_list_route.called + assert [call.request.headers["x-api-key"] for call in upstream.calls] == [ + "bad-test-key", + "good-test-key", + ] + + +def test_add_custom_logger_to_success_callback_registers_once( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", []) + + add_custom_logger_callback_to_specific_event("langfuse", "success") + + assert len(litellm.success_callback) == 1 + assert isinstance(litellm.success_callback[0], LangfusePromptManagement) + assert len(litellm._async_success_callback) == 1 + assert isinstance(litellm._async_success_callback[0], LangfusePromptManagement) + assert litellm.failure_callback == [] + assert litellm._async_failure_callback == [] + + +@pytest.mark.parametrize( + "registered_lists", + [ + ("success_callback", "_async_success_callback"), + ("success_callback",), + ], +) +def test_add_custom_logger_callback_does_not_duplicate_existing_success_logger( + monkeypatch: pytest.MonkeyPatch, + registered_lists: tuple[str, ...], +) -> None: + logger: Final = HumanloopLogger() + async_success_callbacks: Final = [logger] if "_async_success_callback" in registered_lists else [] + success_callbacks: Final = [logger] if "success_callback" in registered_lists else [] + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "_async_success_callback", async_success_callbacks) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + monkeypatch.setattr(litellm, "success_callback", success_callbacks) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", []) + + add_custom_logger_callback_to_specific_event("humanloop", "success") + + assert sum(type(callback) is HumanloopLogger for callback in litellm.success_callback) == int( + "success_callback" in registered_lists + ) + assert sum(type(callback) is HumanloopLogger for callback in litellm._async_success_callback) == int( + "_async_success_callback" in registered_lists + ) + assert litellm.failure_callback == [] + assert litellm._async_failure_callback == [] + + +@pytest.mark.parametrize( + ("registered_lists", "expected_async_success_callback_count"), + [ + (("success_callback", "_async_success_callback"), 1), + (("success_callback",), 0), + ], +) +@pytest.mark.asyncio +async def test_acompletion_does_not_duplicate_a_logger_already_in_success_callbacks( + monkeypatch: pytest.MonkeyPatch, + registered_lists: tuple[str, ...], + expected_async_success_callback_count: int, +) -> None: + logger: Final = HumanloopLogger() + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "input_callback", []) + monkeypatch.setattr(litellm, "success_callback", [logger]) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_input_callback", []) + monkeypatch.setattr( + litellm, "_async_success_callback", [logger] if "_async_success_callback" in registered_lists else [] + ) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + + await litellm.acompletion( + model="gpt-5-mini", + messages=[{"role": "user", "content": "callback registration"}], + mock_response="ok", + ) + + assert litellm.success_callback == [logger] + assert litellm._async_success_callback == [logger] * expected_async_success_callback_count + + +@pytest.mark.asyncio +async def test_custom_logger_in_global_callbacks_registers_once_across_completion_calls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + logger: Final = HumanloopLogger() + monkeypatch.setattr(litellm, "callbacks", [logger]) + monkeypatch.setattr(litellm, "input_callback", []) + monkeypatch.setattr(litellm, "success_callback", []) + monkeypatch.setattr(litellm, "failure_callback", []) + monkeypatch.setattr(litellm, "_async_input_callback", []) + monkeypatch.setattr(litellm, "_async_success_callback", []) + monkeypatch.setattr(litellm, "_async_failure_callback", []) + + for _ in range(11): + await litellm.acompletion( + model="gpt-5-mini", + messages=[{"role": "user", "content": "callback registration"}], + mock_response="ok", + ) + + assert litellm.callbacks == [logger] + assert litellm.input_callback == [logger] + assert litellm.success_callback == [logger] + assert litellm.failure_callback == [logger] + assert litellm._async_input_callback == [] + assert litellm._async_success_callback == [logger] + assert litellm._async_failure_callback == [logger] + + +def test_get_provider_audio_transcription_config_resolves_for_every_provider() -> None: + configs: Final = { + provider: ProviderConfigManager.get_provider_audio_transcription_config(model="whisper-1", provider=provider) + for provider in LlmProviders + } + unexpected: Final = { + provider: config + for provider, config in configs.items() + if config is not None and not isinstance(config, BaseAudioTranscriptionConfig) + } + + assert unexpected == {} + assert isinstance(configs[LlmProviders.OPENAI], litellm.OpenAIWhisperAudioTranscriptionConfig) + + def test_get_valid_models_from_provider_cache_invalidation(monkeypatch): """ Test that get_valid_models returns the correct models for a given provider From 0e26edfdb95c8288a610cba4746af2bc1f2fe312 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 8 Oct 2026 02:38:12 -0700 Subject: [PATCH 30/65] test: move 81 legacy live tests in pass-through, spend, batches, audio, search, guardrails, image and ocr dirs offline (#45288) * test: make legacy live tests in spend, batches, openai endpoints and audio dirs offline (partial) * test: migrate wave-1b live tests offline (guardrails, images, ocr, search, openai endpoints) * test: fix wave-1b review items, add responses/ocr integration tests and firecrawl unit test * test: anthropic messages router/bedrock/openai-bridge unit tests for wave-1b nodes * test: anthropic messages logging, prompt-caching and tool-search unit tests; drop migrated base nodes * test: finish pass_through_unit_tests nodes, logging drain fix and mutations * test: migrate anthropic passthrough tests to integration wire tests * test: fix passthrough migration wire spend row lookup and wildcard config * test: migrate hosted vllm and openai file passthrough tests offline * test: move assemblyai and vertex passthrough nodes to in-process unit tests * test: restore unlisted router node and fix logging worker drain in passthrough unit tests * test: drop spend-row BUG skip and sharpen non-streaming skip reason for anthropic messages * test: use public presidio alias after merge Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test: restore the batch and file unit tests the migration rewrote away * test: restore full legacy intent in anthropic messages router unit tests Drop the false BUG skip on non-streaming aanthropic_messages logging (success callbacks do fire; the skipped body filtered on the wrong model), assert the logged model_group, messages, cost and usage, cover streaming logging for both Anthropic and Bedrock invoke, assert dict content blocks for Anthropic, Bedrock invoke and the OpenAI bridge, add the Bedrock invoke leg of the router test, fall back from a real 401, test system-prompt caching and streaming message_start cache fields on converse and invoke, send the legacy tool-search tools and beta header, remove the type: ignore and bare dict helper, and add in-process native /anthropic passthrough spend logging tests * test: fix passthrough migration wire tests and drop false native spend BUG skip The native spend row test read rows by the shared master-key digest, so it matched other tests' rows; read each row by its own message id instead and assert tokens, total, spend, tags, provider, api_base and end_user on both the non-streaming and streaming native routes. Merge the streaming test that never checked spend, assert exact tags from litellm_metadata, stop rebinding a Final in a loop, require cost > 0, and add the chat-completions bridge cost case the legacy test covered * test: harden the spend, OCR, image, search and guardrail migration replacements The OCR spend tests now build fresh kwargs per case instead of mutating a shared fixture, and every payload case asserts the exact logged spend. The OCR wire test reads its spend row by request id and checks exact page pricing. Image edit, Nova Canvas, DuckDuckGo, Firecrawl, Bedrock guardrail and Presidio replacements now fake only the provider HTTP boundary (respx or an in-process aiohttp server) and assert the outbound request, so the DuckDuckGo limit, Azure base_model pricing and guardrail masking are proven rather than assumed * test: cover Exa and Perplexity search structure and max_results offline and retire the two base search methods * test: drive batch and file replacements through the provider HTTP boundary and real logging callback Replace monkeypatched litellm.afile_content and AsyncHTTPHandler doubles with respx routes, read batch logging metadata from a registered success callback instead of get_logging_payload, use the real managed-files hook for the GEN-2166 regression, assert outbound request bodies, pin poller ownership explicitly in the migrated DB-sync tests, and require the scripted upstream to be hit in the responses error-status wire tests * test: assert file content download headers pass through the proxy * test: point spend coverage references at the tests that replaced the retired spend job * test: assert passthrough identity, spend and route dispatch from the code under test The AssemblyAI non-admin test asserted metadata it wrote itself and leaked a background poll to the real AssemblyAI host. It now drives assemblyai_proxy_route with a real Request and waits for the success callback for its own transcript id. The Vertex spend test matches its log by call id instead of taking the first event. The OpenAI files wire test hit the native /{provider}/v1/files route; it now calls /openai/files so the passthrough is what forwards the upload and delete. * test: mock only the HTTP boundary in the migrated audio tests Vertex TTS tests no longer replace _ensure_access_token or AsyncHTTPHandler.post; the token comes from a mocked Google OAuth endpoint and the synthesize call from respx. Speech tests assert the outbound body, the transcription cache test polls for the cache write instead of relying on test ordering, and the model pass-through test checks the multipart model field per model. * test: wait on a logger event instead of polling the clock in anthropic messages unit tests Recorders keep payloads in a rebound tuple and set an asyncio.Event; tests await it with asyncio.wait_for instead of a sleep-and-deadline poll loop * test: freeze module-level batch and file response fixtures as Final MappingProxyType * test: fake the presidio analyzer with an in-memory aiohttp connector The blocked-entity tests started an aiohttp TestServer, which binds a local socket. They now hand the guardrail a ClientSession whose connector answers /analyze and /anonymize in process, so no socket is opened and the outbound analyze text and entities are still asserted * test: type anthropic messages router test helpers with LiteLLM's Anthropic TypedDicts Messages, cached system blocks and tool-search tools now use AnthropicMessagesUserMessageParam, AnthropicMessagesTextParam, AnthropicToolSearchToolRegex and AnthropicMessagesTool instead of bare dict shapes; tools are converted to plain dicts only at the acreate call, whose tools parameter is list[dict] * test: type batch limiter helpers with TypedDicts and wait on the logging callback event instead of polling * test: give the migrated OCR, image and presidio helpers precise types OCR spend helpers take ReadOnly TypedDicts for kwargs and responses and use LiteLLM's OCRResponse/OCRUsageInfo instead of local pydantic stand-ins; spend metadata is validated with a TypeAdapter. The presidio fake uses LiteLLM's PresidioAnalyzeRequest/ResponseItem types, and the image-edit logger validates the logged payload instead of storing an untyped dict * test: signal callback and cache events instead of polling Recorders keep tuples and set an asyncio.Event, thread-safely, when the payload for this test's transcript id or upstream URL arrives. The transcription cache test waits on a Cache subclass that signals after async_add_cache. No clock polling or sleeps remain in these tests. * test: assert the batch limiter hook updates the caller's request in place * test: tolerate model-list probes and read native passthrough rows by owned key The router's OpenAI-compatible model-info refresh (litellm/router.py:10710) sends GET /v1/models to configured openai api_bases, so the wire answers it with an empty list and excludes it from the provider-call assertions. Native /anthropic spend rows are now read by a per-request virtual key digest and call_type, then the row's request_id is checked against the message id * test: expect the OCR alias in the proxy response model The proxy restamps every OpenAI-compatible response model to the name the client requested (_override_openai_response_model), so /v1/ocr returns the scenario alias. The upstream model is now checked on the drained request body instead of inside the peer, where a failed assert never reached the test --------- Co-authored-by: yuneng Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .circleci/config.yml | 95 -- tests/audio_tests/test_audio_speech.py | 236 ----- tests/audio_tests/test_whisper.py | 109 --- tests/batches_tests/test_batch_rate_limits.py | 896 ------------------ .../test_openai_batches_and_files.py | 580 ------------ .../SPEND_TRACKING_COVERAGE_MATRIX.md | 2 +- .../test_bedrock_guardrails.py | 100 -- tests/guardrails_tests/test_presidio_pii.py | 211 ----- .../test_bedrock_image_gen_unit_tests.py | 112 --- tests/image_gen_tests/test_image_edits.py | 179 ---- .../test_passthrough_migration_wire.py | 584 ++++++++++++ .../providers/test_ocr_router_wire.py | 69 ++ .../test_openai_passthrough_files_wire.py | 49 + .../test_responses_error_status_wire.py | 71 ++ tests/ocr_tests/test_ocr_matrix.py | 15 - .../test_e2e_openai_responses_api.py | 115 --- .../test_openai_batches_endpoint.py | 463 --------- .../test_openai_files_endpoints.py | 113 --- .../base_anthropic_messages_test.py | 90 -- .../test_anthropic_passthrough.py | 472 --------- .../test_anthropic_passthrough_basic.py | 8 - tests/pass_through_tests/test_assembly_ai.py | 102 -- .../test_hosted_vllm_passthrough.py | 71 -- .../test_openai_assistants_passthrough.py | 23 - tests/pass_through_tests/test_vertex_ai.py | 87 -- ..._anthropic_messages_prompt_caching_test.py | 135 --- ...ase_anthropic_messages_tool_search_test.py | 78 -- .../base_anthropic_unified_messages_test.py | 231 ----- .../test_anthropic_messages_passthrough.py | 287 ------ .../test_bedrock_anthropic_messages_test.py | 48 - tests/search_tests/base_search_unit_tests.py | 59 -- tests/search_tests/test_duckduckgo_search.py | 138 --- tests/search_tests/test_firecrawl_search.py | 42 - .../test_ocr_spend_tracking.py | 296 ------ tests/test_spend_logs.py | 2 +- tests/unit/batches/test_main.py | 218 +++++ tests/unit/files/test_main.py | 66 ++ tests/unit/images/test_image_edit.py | 152 +++ .../test_anthropic_messages_router.py | 619 ++++++++++++ ...hropic_native_passthrough_spend_logging.py | 181 ++++ .../test_amazon_nova_canvas_transformation.py | 34 + .../test_duckduckgo_search_transformation.py | 76 +- .../llms/exa_ai/search/test_transformation.py | 57 ++ tests/unit/llms/firecrawl/__init__.py | 0 tests/unit/llms/firecrawl/search/__init__.py | 0 .../firecrawl/search/test_transformation.py | 32 + tests/unit/llms/perplexity/search/__init__.py | 0 .../test_perplexity_search_transformation.py | 57 ++ .../test_vertex_ai_batch_transformation.py | 136 +++ .../text_to_speech/test_transformation.py | 98 ++ tests/unit/passthrough/test_main.py | 53 ++ .../test_bedrock_guardrails.py | 141 +++ .../guardrail_hooks/test_presidio.py | 224 +++++ .../proxy/hooks/test_batch_rate_limiter.py | 249 ++++- .../test_files_common_utils.py | 140 ++- .../test_files_endpoint.py | 48 + .../test_assembly_passthrough_route.py | 172 ++++ .../test_vertex_ai_live_passthrough.py | 107 ++- .../test_spend_tracking_utils.py | 238 ++++- tests/unit/test_main.py | 149 ++- 60 files changed, 4014 insertions(+), 5401 deletions(-) delete mode 100644 tests/batches_tests/test_batch_rate_limits.py delete mode 100644 tests/batches_tests/test_openai_batches_and_files.py delete mode 100644 tests/guardrails_tests/test_bedrock_guardrails.py delete mode 100644 tests/guardrails_tests/test_presidio_pii.py delete mode 100644 tests/image_gen_tests/test_bedrock_image_gen_unit_tests.py create mode 100644 tests/integration/messages_endpoint/providers/anthropic/test_passthrough_migration_wire.py create mode 100644 tests/integration/providers/test_ocr_router_wire.py create mode 100644 tests/integration/providers/test_openai_passthrough_files_wire.py create mode 100644 tests/integration/providers/test_responses_error_status_wire.py delete mode 100644 tests/openai_endpoints_tests/test_e2e_openai_responses_api.py delete mode 100644 tests/openai_endpoints_tests/test_openai_batches_endpoint.py delete mode 100644 tests/openai_endpoints_tests/test_openai_files_endpoints.py delete mode 100644 tests/pass_through_tests/test_anthropic_passthrough.py delete mode 100644 tests/pass_through_tests/test_assembly_ai.py delete mode 100644 tests/pass_through_tests/test_hosted_vllm_passthrough.py delete mode 100644 tests/pass_through_tests/test_openai_assistants_passthrough.py delete mode 100644 tests/search_tests/test_duckduckgo_search.py delete mode 100644 tests/search_tests/test_firecrawl_search.py delete mode 100644 tests/spend_tracking_tests/test_ocr_spend_tracking.py create mode 100644 tests/unit/images/test_image_edit.py create mode 100644 tests/unit/llms/anthropic/pass_through/messages/test_anthropic_messages_router.py create mode 100644 tests/unit/llms/anthropic/pass_through/test_anthropic_native_passthrough_spend_logging.py create mode 100644 tests/unit/llms/firecrawl/__init__.py create mode 100644 tests/unit/llms/firecrawl/search/__init__.py create mode 100644 tests/unit/llms/firecrawl/search/test_transformation.py create mode 100644 tests/unit/llms/perplexity/search/__init__.py create mode 100644 tests/unit/llms/perplexity/search/test_perplexity_search_transformation.py create mode 100644 tests/unit/passthrough/test_main.py create mode 100644 tests/unit/proxy/pass_through_endpoints/llm_provider_handlers/test_assembly_passthrough_route.py diff --git a/.circleci/config.yml b/.circleci/config.yml index d8dc40433dc..58860e3ed95 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2074,98 +2074,6 @@ jobs: # Store test results - store_test_results: path: test-results - proxy_spend_accuracy_tests: - machine: - image: ubuntu-2204:2024.04.1 - resource_class: large - working_directory: ~/project - steps: - - checkout - - run: - name: Generate LiteLLM master key - command: | - key="$(openssl rand -hex 16)" - printf 'export LITELLM_MASTER_KEY=sk-%s\n' "$key" >> "$BASH_ENV" - - skip_if_unrelated_changes - - setup_google_dns - - install_uv - - install_rust - - run: - name: Install Dependencies - command: | - uv sync --frozen --all-groups --all-extras --python 3.12 - - start_postgres - - start_redis - - start_fake_openai_endpoint - - attach_workspace: - at: ~/project - - run: - name: Load Docker Database Image - command: | - zstd -d litellm-docker-database.tar.zst --stdout | docker load - docker images | grep litellm-docker-database - - run: - name: Run Docker container - # Point the proxy at the job-local Redis (start_redis) instead of the - # shared remote Redis. The Redis transaction buffer uses a single - # global pod-lock key (cronjob_lock:db_spend_update_job) and a single - # global buffer list (litellm_spend_update_buffer); sharing those - # across concurrent CI pipelines causes spend flushes to stall or - # land in the wrong DB, which is what makes this test flaky. - command: | - docker run -d \ - -p 4000:4000 \ - -e DATABASE_URL=postgresql://postgres:postgres@host.docker.internal:5432/circle_test \ - -e REDIS_HOST=host.docker.internal \ - -e REDIS_PORT=6379 \ - -e LITELLM_MASTER_KEY="$LITELLM_MASTER_KEY" \ - -e OPENAI_API_KEY=$OPENAI_API_KEY \ - -e FAKE_OPENAI_API_BASE=http://host.docker.internal:8190 \ - -e LITELLM_LICENSE=$LITELLM_LICENSE \ - -e AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID \ - -e AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY \ - -e USE_DDTRACE=True \ - -e DD_API_KEY=$DD_API_KEY \ - -e DD_SITE=$DD_SITE \ - -e AWS_REGION_NAME=$AWS_REGION_NAME \ - -e PROXY_BATCH_WRITE_AT=2 \ - -e LITELLM_LOG=ERROR \ - --add-host host.docker.internal:host-gateway \ - --name my-app \ - -v $(pwd)/litellm/proxy/example_config_yaml/spend_tracking_config.yaml:/app/config.yaml \ - litellm-docker-database:ci \ - --config /app/config.yaml \ - --port 4000 - - run: - name: Start outputting logs - command: docker logs -f my-app - background: true - - wait_for_service: - url: http://localhost:4000 - timeout: "300" - - run: - name: Run tests - command: | - mkdir -p test-results - TEST_FILES=$(circleci tests glob "tests/spend_tracking_tests/**/test_*.py") - echo "$TEST_FILES" | circleci tests run \ - --verbose \ - --command="tr ' ' '\\n' | awk '/\\.py/ {print; next} {sub(/\\.[A-Z][^.]*$/, \"\"); gsub(/\\./, \"/\"); print \$0 \".py\"}' | xargs uv run --no-sync python -m pytest --tb=short \ - -vv \ - --junitxml=test-results/junit.xml \ - --durations=5" - no_output_timeout: 15m - - store_test_results: - path: test-results - - run: - name: Stop and remove first container - when: always - command: | - docker stop my-app - docker rm my-app - docker stop redis-cache - docker rm redis-cache - proxy_multi_instance_tests: machine: image: ubuntu-2204:2024.04.1 @@ -3591,9 +3499,6 @@ workflows: - proxy_logging_guardrails_model_info_tests: requires: - build_docker_database_image - - proxy_spend_accuracy_tests: - requires: - - build_docker_database_image - proxy_multi_instance_tests: requires: - build_docker_database_image diff --git a/tests/audio_tests/test_audio_speech.py b/tests/audio_tests/test_audio_speech.py index 998de5ecc3b..781dff654f7 100644 --- a/tests/audio_tests/test_audio_speech.py +++ b/tests/audio_tests/test_audio_speech.py @@ -8,248 +8,12 @@ from dotenv import load_dotenv load_dotenv() from pathlib import Path -from unittest.mock import AsyncMock, MagicMock, patch import pytest import litellm -async def _run_audio_speech_litellm(sync_mode, model, api_base, api_key): - litellm.turn_on_debug() - speech_file_path = Path(__file__).parent / "speech.mp3" - - if sync_mode: - response = litellm.speech( - model=model, - voice="alloy", - input="the quick brown fox jumped over the lazy dogs", - api_base=api_base, - api_key=api_key, - organization=None, - project=None, - max_retries=1, - timeout=600, - client=None, - optional_params={}, - ) - - from litellm.types.llms.openai import HttpxBinaryResponseContent - - assert isinstance(response, HttpxBinaryResponseContent) - else: - response = await litellm.aspeech( - model=model, - voice="alloy", - input="the quick brown fox jumped over the lazy dogs", - api_base=api_base, - api_key=api_key, - organization=None, - project=None, - max_retries=1, - timeout=600, - client=None, - optional_params={}, - ) - - from litellm.llms.openai.openai import HttpxBinaryResponseContent - - assert isinstance(response, HttpxBinaryResponseContent) - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_audio_speech_litellm_azure(sync_mode): - await _run_audio_speech_litellm( - sync_mode=sync_mode, - model="azure/tts", - api_base=os.getenv("AZURE_TTS_API_BASE"), - api_key=os.getenv("AZURE_TTS_API_KEY"), - ) - - -@pytest.mark.parametrize("sync_mode", [True, False]) -@pytest.mark.asyncio -@pytest.mark.flaky(retries=3, delay=1) -async def test_audio_speech_litellm_openai(sync_mode): - await _run_audio_speech_litellm( - sync_mode=sync_mode, - model="openai/tts-1", - api_base=None, - api_key=os.getenv("OPENAI_API_KEY"), - ) - - - - -@pytest.mark.flaky(retries=6, delay=2) -@pytest.mark.asyncio -async def test_speech_litellm_vertex_async(): - # Mock the response - mock_response = AsyncMock() - - def return_val(): - return { - "audioContent": "dGVzdCByZXNwb25zZQ==", - } - - mock_response.json = return_val - mock_response.status_code = 200 - - # Set up the mock for asynchronous calls - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_async_post: - mock_async_post.return_value = mock_response - model = "vertex_ai/test" - - try: - response = await litellm.aspeech( - model=model, - input="async hello what llm guardrail do you have", - ) - except litellm.APIConnectionError as e: - if "Your default credentials were not found" in str(e): - pytest.skip("skipping test, credentials not found") - - # Assert asynchronous call - mock_async_post.assert_called_once() - _, kwargs = mock_async_post.call_args - print("call args", kwargs) - - assert kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" - - assert "x-goog-user-project" in kwargs["headers"] - assert kwargs["headers"]["Authorization"] is not None - - assert kwargs["json"] == { - "input": {"text": "async hello what llm guardrail do you have"}, - "voice": {"languageCode": "en-US", "name": "en-US-Studio-O"}, - "audioConfig": {"audioEncoding": "LINEAR16", "speakingRate": "1"}, - } - - -@pytest.mark.asyncio -async def test_speech_litellm_vertex_async_with_voice(): - # Mock the response - mock_response = AsyncMock() - - def return_val(): - return { - "audioContent": "dGVzdCByZXNwb25zZQ==", - } - - mock_response.json = return_val - mock_response.status_code = 200 - - # Set up the mock for asynchronous calls - with patch( - "litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler.post", - new_callable=AsyncMock, - ) as mock_async_post: - mock_async_post.return_value = mock_response - model = "vertex_ai/test" - - try: - response = await litellm.aspeech( - model=model, - input="async hello what llm guardrail do you have", - voice={ - "languageCode": "en-UK", - "name": "en-UK-Studio-O", - }, - audioConfig={ - "audioEncoding": "LINEAR22", - "speakingRate": "10", - }, - ) - except litellm.APIConnectionError as e: - if "Your default credentials were not found" in str(e): - pytest.skip("skipping test, credentials not found") - - # Assert asynchronous call - mock_async_post.assert_called_once() - _, kwargs = mock_async_post.call_args - print("call args", kwargs) - - assert kwargs["url"] == "https://texttospeech.googleapis.com/v1/text:synthesize" - - assert "x-goog-user-project" in kwargs["headers"] - assert kwargs["headers"]["Authorization"] is not None - - assert kwargs["json"] == { - "input": {"text": "async hello what llm guardrail do you have"}, - "voice": {"languageCode": "en-UK", "name": "en-UK-Studio-O"}, - "audioConfig": {"audioEncoding": "LINEAR22", "speakingRate": "10"}, - } - - -@pytest.mark.asyncio -async def test_speech_litellm_vertex_async_with_voice_ssml(): - # Mock the response - mock_response = AsyncMock() - - def return_val(): - return { - "audioContent": "dGVzdCByZXNwb25zZQ==", - } - - mock_response.json = return_val - mock_response.status_code = 200 - - ssml = """ - -

    MNZ$s*_K`fAS;CczDX>4J{s4B%DM{ju;)$C%kXyiy*tLgt>;) ziC=82)t~wI`y#wPcf7ln>!-7_s}P))&Av1xS#6^Kpp(H2p2+VL&WT&4hVn}I6kU{O z%0HzrsghU&xadFR6}rnE;5yofwnM+?5}f1wR#Ozzn#oo1Q zHk|~0;xeJDG{bbz(#-a!{ii)|(4?TE=v-Js`-Bb)Z5&!Zv_R;ykjRi5K`(5w<)*Zh z?`%}peabJ-E9brJ{F&9@ArY1~Hmzp*fQ-gjzMP@%=89bpht_Vga7cP;x(=V`9NP_S zuiVz2mbK<2Q+av5m<1kjQ>dh!sItdFb+;dtkqdb518t=0hHphluhrv+W2470z|q+u zJM+3;xa5k^#uH){7+3MsjRyZy?B@vl!6L`ctfj7oh z`K)wRJSt4%E09z~$`tAYf>~Lcq+Wmn;!9L|8-0(wqPMPRJoZ{F(1*Uxf{2ncoTr^0 z=U7)199Y(QPQw3OR`=__j7agZd05bju+NdpVlL(S8lRbGNxr)I)AA+dZIOFPu5Qr- z!_Nk3ribK!R?vIOQ7mgnddJkHlv}@^{Q8ozBy~#q(#)~h%N!Q>S1LDxHP8?F2Dvt7AIO-K zRw%XQ?+U+zQq$6&WVUb=_qddfdM6{5loVyTgDJ}Vr@5}Vvbia`fcwnIe12{~ZoW3ZdLRQC|U(OevJIY<{ zBoT`B)3V9(#k}71R!R|gz9KZC zWA!QO-^w{}4`jQw-Rs?rJTcz7zFW#ewLQI#x@5KRR6HRKm!IMefs1kn`Cm9*4-;R3 zQ}By#flM?Ryop)BQ?J3BATN-X2l^ZBhk6eVNVR+~J!|2Y^}{*A*%Ypsot*QW=bhQk zPOdwyVD}Wa4_-@+y?1?u)LGyeC(4ytp7}^~iHIU%vuMfk6c}=2LPza*d20 z5Si{; zu#ytOI;n=)ZT%eN2z?kaEjlaqYTW(!lkp|v>cqrGR0&QoPZXk<>~G+1oij3Xd-|KS zt7!@81u`~g9?h=klsw;jC)F-=oZ&~ry~>BerPwCC;t%kr_?E(2v4h;-JiuDczCLJd z@RQ)R!Eb_^+68ME`cxIU(|QAcs5i>hI;TR`fsD85De3=Zw8^TOGt-&oUgoo*+gJ(e z;t{CVN`WsioqQsR{2JlC*crL$QOjQ2*r2Py$3x17dO`|>ybj8X+f*8w8i~1}U!0-7 z_U-n_ZrQcn$(%o2^E~%`_0((nPj(0%zqN!*!Vh>UWI{jD1f1^k(7O}{SN##m&o6=B z@@b(qJSWaz4(|wW_=jXVcZa=2uc4TKt8avNl_!tqj(fKIFZTp@qWdInCU}b*H*&iV z!y$7II$)ihADwphC~u^?gnbaMTBe08i2ND*0Mx<41sfF#E;OjX_`KKRK1Ux4j|`5m z{1zs&*8Vj2DaVrR$gJ9#`7@VhF31YY5uDdu-`x}7P#x@9<~F;(x$3zaqJQ^LS)zSs zkNFy=M~+S?w!==J?pbpyH@U)0|~R_AH;^$v6gtI9EA5huz&%ziBg{Pr$IpE&Rlsti|r%5;Bic&)tE zpD`<0DOly9<^$FX_EEtrLt2F%39T6F49*ku+}hiGRf-nIaxQv6d*{EaBq$w~U&=~< zO<+`aw6ppu++MT@Or9m+p~N7w9}WEu)q_xZU!bGdTtgsR(Ty4{HI`q>I!?A&S(FY6 zGT)6mk8Grv-Wsm7KjG;3S@}mP2bZEDaO%qAZQ>c=-sQ@`ZtLgxFK1@X#hi3UV|R*g zn*NBKGA#@`7G6F^kGq?vH?l^5UQ3>0ajxk15mFcleq&u>`Y9CS?&x7S8%*Bq@Knl# zC)#W0U#z?RzVP zcZ}EPH7Q&D`?Xwb7uQg@D19|KEIZ-*{U9h$uq}98&_a7h+a-&|93y86x1i(yN=NIt zwbJSc{}*Kukoon#LcVw2MP4&}tvM)HvN_vuIUNnJpE zBmB#OL4H=2_!p>;wLEkYiv&xcI?nd(Qf+yG+z_bHL2;rG%8y6Y*OV&g&d*fMaG#ab zSoJepl)ftA@FB|M<&iOkU}VQTZ#i$fPI`tYhqSxKB=M>FpuJ1zrihHF#F(eCh2YD7 zKKejp@9;ArC9t|$ndeJu_*`5`wi$P3999K2FT9@0DY0);LcEOC{DM_iwr z7CDt2NzTpg1m7a{50*nNOY<#v?A1f;VU@#^!&`>W4{I7aFxYLIV##fKA@1jMksRY+ zwimd-LpXKq1>0n>o({aJo|@nfSAKe5c?x^>BRaU<>FC;2@=wz|bQE`q|0W)jL(L}3 zE{of8)l$u}zyyH zdF`1BH^IirIR9O>lODuUp{8^4<>8tBR)TM?oGAB~bEF>9OWe4SEVLGKAsXia+b9{g zqSe7YQOn?guvNIh?;`oR3GA94t}XHxSHgj0bo1`>&h3ev1{lf z4A4{Hn^zsTBptz>T_gQ9l@#w>PdoQ+*I`#b_ZiPf-(deuZ4fI*yuwm>l{wq8#+qh* zZCzrmVExZh-qI4?(++Zd=}(}It-1Uw;Db&1 zv1BY))%Zd;>m#+vxHTk?{}|jvn<)L1WTm!0)jtn>miO8ky*zzR=d%i6v!3N9k~n@3 z&kJSYX;lk2V}j5NIApfaN<1$nNXMj?(iLfj^c?qjCG)+ZUF?Cm>Voyv*Z)+R2hPHB zh4;Vk-&9|MTM&v4w+Z*+H6xqJda&1iqmPh+JL)o6Z)A!`=uV)Ot*8_86{}B2RUNGI zeyh^W$NOa87Gwiu)d@NW6<}Vux@D2g7L*V?IQUSo8oVi(5BAve+H2V2tb5RXZ)Nt% z1*JT~RqisYL2H7&b5k9ymQ)}3|3l@GTjkYV{sR7?%3j(6T`Ys^GyVVuy9knq$__ov1 zP{hB5H``&M7iYKkS}FbfFVwtrA8ybXC6BYbvT62PK}kVng31ON@WdAFeQca< zf_0Ads`U@6+2WGdi|0v*kqJNb+R!aF_qV~XwSMYvgw9Tie~7=Avcwnb3-vwmW#jIt zk6JD|1*)&|@R=zkg#wdUW2s;*5BJ3`mVsusJVd$ySDFm4>nDTn{Dz#y%}b~F7i2Vi zJ>b8i?NQhHt$rD|Y@JjKqgNZyvi$+fDGH8tQOrxTX&M!P&}~K5Sq-}HZNyJXg7?!K z_XGO*&bawzgAgw?4d8$1PC<`2_OO zSi#x@vsj0&OO6_@tpeKo67F!bptS!4MfL@{7%aO(^c}F`%jntNg8r&Ec)T%CGaUeD z-w9sY888hqv@c-Hd6gat^WDYA6JHHwg_6(z*?$5RT^yUo?GiT1eJsUn(`^fF&#bGg zpR9{*E$zqbpX{&go$Pt+>uoEna5k62g>dc(?Vz6p=W{vy!&`wX+Yg=%UK!)bEa8lFΞBV<~GHWu9u9fV*7M#DQWi(FaE3d2x#L zO*#u-#W3L$mk8&M7GSp=MOW{=wg8!54s8HziD!S(zWM-k50lh#|HslK0r&^J@_9}Je>$Q;JnuI3c9ri0Q+-|1m-fRrdzMc3ql>`%odmr_&-P&|sN zT$VeqixcQLZYoAIPdhIi-W#YAtg;GOSIvoLEwco?f$rvU=1-2ZoM}Dcz1f*hSPP|0BwLpWL+E0^yVfb9~{71`cG<_GrR-5^}RK`4ZP*O`Mr;mMki5k zVUFXdL?y}T-Kw{=BE;;D-0sJIm;Ar@&GBS{B~#Sh+0)1Gm!Imt)h`^}$8uG74p3W4 z-Rv*MS$#TO(;{e9Cg^^;!<#8-9kI`)q{HAbE5OAw!4|KAZjzr^f^~XDjJ4BI#OZKl zXU9^*r}Ne0d&nu|1?WbU5c|`CQPjQ(|Hh4SUl8^nPCpHc6&{k#Y!5xZanzW>{`A6T z>iCmb7&n)Mc2xUkNiX5pHAg4WA+L}k#U%R0o>||`W>hi08t)A+JDZc~nwfdmXd@I> zsoOZFPhleAQq=oqp`Mw_indO|j$O*`bb&b*qfF-XlupL}GHPj18jVs@sJWF4^w*;Y z20ppk`eZgWbDIWy02|GoLSXL0sif>N51MoNT4+8rJ6dn7<>(MJ@F}-Q&0)R_k-tcl z=uSUQAHsG!8&)|7ih1q%T-RVbSBLw0Kp&!KC9+t8m#Is9krC805?#$;7$ISJgELGw zN{JHb+@$qPQT*#IXM8f5q%GZ6tGKRsj`;oX#JTD_HD`C%7f-DJ>VPHwZ#>!Dk70L~ z((KPbH%syx^RM zY`o<@wb#qTKg`YQyw@)nSIifdZD(hSri0n4+mw3hOLe*9ucM5XL@c;OeaiH$Ld>fi zk3yhRO2Hh;{B}=j&Qr}N?Dq=h@N7Wk@;p4^%SJ=GwU42Z){NWPQal28y%HS-{fM7> zOFgBT)I!%t&82hb2c#Bi6Qjn!`MPdYpfYwHhE;l_8Ruyj{NQ}3AD^afof&rdLGMZL zcW)q!+4Fj8qay4OHyOeWSlQiR&=%rZ_QEJBO^^s6m}d5% zr=%U6@6^N>RZuv&NgwTg7&k3YTFZ$h%~Dh-tBAQc%cJ3@wia61kGZW+&D3UT;{aU` zNB9~ECwm~Lb*T5WH-;IX`FPUN#$>aPm6L9->e$pZs0Yqt|D#ZR+bOHqzwzwWc=}U@ z)1}Z%Nr_eHjJp0aG+qYKzu|9J$NsH1SEBhUqgOZEdSL|+)VQ^hH@Q&?>4~N3rnXk8FR<09u z=Da9VzR(-NHrNB_V7+m`*v_n)=5Wn}^#r1^4BjhfH?KY87VK6F2jqjj`F4~4@UKDo%zXV z4d=F-6^-BBgjU~Dd$*8Z{6qKCR(j}4py;xe>A;_@g2Z{F(Vv`1%(M!17k{xM-0Z{5 zN&aPZrAI9k?fW^X6ATx-67z2nwh=v)g|9J!TRNK_wMq0!y%GMSx2pp);BuQC;9l?K zzL&w@u4WqRLhoB{>{|UZ=W-l3BZjD8n)#M}i?OD{pIgP0^kU)$^mkfI!%;?^jpAVl zx>9?oOC~X8ZYELZOQMX`(grb#4xQt4XFZ~q%MDYkofv|)!W4s3!~d7NIT zqC90PdUf{02f9MvNEP~q1}1kaK8NXL(~o(WE>Yin>D{QPMPplMqbz#Y3}vNi!=F12 z$Dp;5*?5Y!_H2C%EG?H&&+Q_p8tppoO6|_=PIT>ZHFu%O%`}^vOz+gRKWZv< zg;EorJzl}&KFE+hu2g#CVo;^rf(5Zu{j#P|m1_P%w88{66{&mAnra}-0zD#1(- zH#373OJ7b^^8qq4xfT1Y9*LT$E23np8oW$1b|CirDM1wozK-d)1R1l zH%y!jZ!J(f$_}mPZ%sPh{t}05gCE=#5BCk#g+{0*b+Dpf#)#$>V+TBpqDBPqMptmh zS9%^iZ5^Yk(U`ec*I`=xLzXa%9*-2n18M1uc~56#6x@$)^j~JPDiNnHvd*Fi<)W*x zG%M@R>AGgAs30Gt+tk;QKa5*;56fN;eD?q>5fjv7zVZg*Tu!ra<6mRY}L(RxhhLnDIKwv4;+pKrOwN8E*a{kbdE#~{r7ZQhtjvC4pbVLeP9-xyIW1R|w@5+K zQO@3Ew0d6&Q?LygxXo+nMy|-I$w?2~ANny*(*X)1Nf&Ay3IjjrD9(m@!eF{oHEhW` zGJ|)pxhkQO^oE|t^=QMDCn{FW0KT4cQimBeuqh>pMC)@#`*9mr!v`vDPQmuRLG!Q~ zTDq~s%I{EwegV6BDt(k!P`KNh%q6N%&YO!ToCISqfR5c+Afz$GRsZ3eOQI=rlaAS8 zsD9iPn$vG{hV%MP&ZX2+I)X9%P>QK5)GO*yb-CI{?W6{)vzWY)ph}K&>JT!!E{c~v zCzt#d>vV~IY{i7YLfEUysOLt~bDEX@)#Ip7)Ieq6Bbm{aLM)J^7d8ID_FMriNk^~K zJF^8{&>DT9jnTPjLe4b?*7$3b8D^lv5lB4J1Ak^QJ-)d#1pnBb4xc}C;?5I0p%7FG zRUnUep4U#LOaBHsYJ}op5?y_eh$Wjbl z#8)ku6BA^=pa*XzEcuF79%g(VrZcMv*~~RgeIGKqT}CSYe>hX3j+tBNaOlM99kVjf z6FQsT6)%Z<~$rNje&If29t;8p~p|-yCkc=3g-DW6b1ezT3JZm zZ=$rFi}{Kn(n4t|(ePcg)T8OMJ})gHS2&2q`Ut6_6o;M3D_)_~zcl&id^)Pr*{{%~ z9ziTq&Ps2kqMA^j4(qb?ZdanCqyrZJA3j|_*zViuy*|n3yT#|TtioVsg}LG1tn+kw zS0hU*3?K6bndVrSXw}TdaQCvp%FId*+Rq%x+8iZEH?r1MgWHmhBTcWpHC?|}iZ~iaQ`@auYEF>p+;9X+qnlKho12yhF(b+|kI^l8fri6DkgzuFKviy9H~L1)W4(4$8|y?2 zkq=AbXT3JJ^TZj=c;bay$;`zt^17#VS)?OEoJTKII`H5lSStw|uoaDwF2uoZdaADy z@B9=$il4;<@f$kap+um!!BA(@1x?qUSVjyYj$OiM%Z9o|ZqxzNz_|O3H93P;K{K=j z27n2U;%hQ%(iPo;A>{NvP5gqWZUpkX7-Fw*&eQ~~^GEs@V%X1(XzN!+5Ai2Vz0Kx2 zu<;Z4hlkv;v;2P?aZyo}_lKe(pAH>^1$d_us0bBD?w z2GmCbq9Xg*&VEBzo(J8K4OUBHrC<=BvP38)ta{wpUBtO7@%kH>k}#Sc*pf`7m}v!J z9~Tf=|HTilMM)wYWrJC$FfBxpG9@}hCK^rUu`pZg3s#Wb9ro=g{)9;u)@ov%muN(c zK!>6b>Jol@TGWYrIW7M^2Cu3R5lb%dG|CBE#c7b>yFx=dG$rx;T6}02 z_pKp4QtgD&sH-est=bV=+$7!~!1R=BC{b#H#f{1(?hzhYjbYwZ_ZLyUP zJ*gbz&O6NkJfCcDLtjl4C)qE}```vPoc3t56L#PgR`-z|AgV$|YpcCSd`NagKEwMp zob}$i?PSItgEQkH!@qI16o>WgZf=oFo&8#H)|66WYH-&z+MN+63d;N zE98{Uv%*7ck>38DRv0>!4Wxl$3-ginKx#+r+hQ#X(>tFN9hAN3)Sb51nq|b6=v8L2 z+S#SV^g>G8YmO1lh$rkC;2t<|!3NcwF2qX*?1DT|C$N;u#NnUmtzvQjOj4P=Zxfc% z3+CJqcEe$7t9jN~YVQ@R8Ik6AbbeObi_KxwA~duOKl8LP%;&nv36X@$baFIAtFox{ zkJN(9W4+K^ZYexBE84B)&tiKcrQj#+64TiG%?SGragHzUYk`JnX}hgepI(q(=<3JV zy@>3JqqlZmyod_$c226lR32N>6?^HAimL}C?Y;HgelNa|DshiakoEsYrF5&1S(5E1 zX0RPEWf#_)ORX55Zj+tE>?wGp7*t%ETIV=vwaHrIt=sk()Zk5{E=tcYto-6hv#prh zs)N>3XZyG?)BH`}W(WI-*#W)Jfp(}l+paF=6^@yEt@Gj&Nw!uQPAR?e$T~oG-6f?v z(^^BVy37{XD2y}u3I2|8Vh;RDIi;l%L3}s@joW8-7~R&r#7J?7E%{WJrTW5hE0=Ie zN&`wi5jI#yDJ6=Im+eLJDQlh?fzQY$l(SOWvslHm=1uD!-J!#YvtKa*v9aCE9BO%_ zP~iwzh6JYa&MaXpL{H!u8Gbrpko4LPKw&)F)4w2c;yi zv1i64x^n7LlQ^k2qBikexJ*8kgZj?}JI3rH6o8+!4#l43=yyFNGu|hRHfyqm)vS?1 z8F7tu2|eA5LS||OV}uOMM(ZdH5(N=uHIbUPlu?w-ykx@TMbuEt3NX8tnPZKCV0Rsa z+j=>%0$5XdE4!IOc}?Xw720PVrK?6>agiK>6MX0`>q*-a`Y;Ci>A^q7=~q! zX4S@v{p|hLWHc{73wf;8b{lkJZVIP}#}A6@6+bZ5Tw)gGq&Uty#GXdo{;i!#ABSr4 zOY4~tVtLV%$OyM$HMeIUs)y;w`8&z6+zIeJrt3WM4n}?VjriT_W9n$nEko0yzuA}> zNluik0>v-pOS=i0wGs4uPLNiCbMlGoVPb3IptDwQyN2`|mGFGlSF@eEQv9wLF>6R? zmE~shq^4qfWt26;aKV>fz|FmCUa>xlD7t|KmXQXd$XnbVFH{xJ86P=|pTso+J-BEn zZWHU+r@ZZ@K5~6J)(Y5_m03btJy7T@eGxq7C%dzhh?aVkl~qb9%`sz8T3^II6t`wz z`?`_u*A(N0bEZ!Zc8-|GzF@wQ3krYDJ*W{pV#ZH?xrdpVc@MMfI8NtRV~7wXr?ba_ zY)7lBbvKhrPZ|ZO-JX_KT8GTD;t4G95NjX) zW`tcx%!J~AjmQ2)FVA>Xp<9SO?Lv}ewE|Cm3|f1}u57**Qp(Fv7jIx5egg?tOCT!F8j0LGO{@`ixJ*SH(i zN=N_oXdz0{?YveQVy2Sd&iPGMULd&bN5V=whcpm9uW%u!RbQwgeX^&Rt?Z^!Vdh2e zvuj#c$Qx|t>ed(22`9|M!gREBA6a>^`qNQQ+JNT7#pDSXQQ|CmcW|p!glct@>zg5QoWsj1Jag6k9i={5=wV(bK4sUjf4y zV-*qum>ijl(`ECVBTyYWZ#K6tp+%!IH$WDv6R)I{Hkx(p9Qe-dRz|CVZ&4F4u(vl+FIpt^B#Lt~o1mz;TG&Wrd5Uh>tmq}C5qk@1 z=)ruBCgVVI;V648GeC4RNJuOHvZ|8fZDN*sI{G}bQm+Xo`d?{1w)enT>&=uBjZ7-F zl!_YhX`-$Ts5V{`#)v7&Xicy=*@$aXVA;`ev%0Z*H>Ct~xwS?5 zDNG@QJwzOKA5G(tC=zeM3wO323frXfaHIa^omY~X)D-fg*7nPe;r&%Rn{<|VVsy=>+=mb4?VLrDkNuGwi`I#aDrKe`J2c0Ms~UqWqb0l$n`WATQ|hcLRE8+T^S#ofk?I$(?doH9`KUg ztk@YeSg+7G8ZL~t8rx|kP25RlF`g<<4p4_3Aas+6HjjgSW=GfWvi%c9sCL3AklwVY z8sY=#^3ts*c&Tq@2DA)p;^pbQ{-M1})U9}8zjJ6(Hf1M$dUJvB=pIuUx=GGHfOEQz z)6rg-Ccb87h8vrhPS|X<6?QQ-x-N>#6Nxk52@S3Hb`5aFw^n*)%%uixIz|@pi8KG6 ziAbxNH#rl;hm%EqehU5A%1jmbmYkt4f#tM^82uWt=wu>v2O4nK>6T1FYq33<%y@FQ ziDV`RiGx2eV+1XHZgN}Ve{>109(*z%uG11P_75`z{LxpdOD?yVye%Wy!9jEnYmrm_ zOSCN(s_sAT+$1oAnZY~9Vfg_E-f^9mR0m}_{l z%;*@m=F`>YU!!`6V&7t-JB4VtJ-6U2^WZfo}OG`0<+9hJUq9 zxXpf6P9DwX6k+xXKPAWp@=5$mD*bC>)6kV<)O^t%6mpr3?W_1x-8{=d#j#G8x&*E>gD za3aj_y4Zn@RAi#8uU1Bof!@r)d z)xtT*(O#)6-4;69y{S}wB-5N^{%0sgPQ8owchV#nLTi!|>9^{V)E*YvRI2(_%nTsQ z^H7#s3F{3_4(8{qky9znL3-~i9nlS`s9s}g=}@JPTu6!)TM)zD2e+Aoy12*cNj)f+ zISt%>uKpbSHV(doSKnj2q?5}Z6zm;nR#{g6Aq?tAFc6~9vFr)TSP`UoFdTs^@+P^q ze1l9VjvDGrG~fRPsVra@ur|?ucFhLZn_FPMHH< zKvnM!n4l@ViSz+nO8lL;f;qw6lERY)dw(PyPMYCutbfw?87aw$|5biCO1LU`O8GDJ zkN58sI3UH|l(~XV1r<(}C+J~{wt=32)qb2`;r|Rws+#l@R^;-8 zKJnS&x5e#=%Md>({&~XB#Qffy-dFk~Qzud`DrHlOpf}LPan-S0OYQ99Oa+Uvgln|( zzV_9zOP#08kRM2?m?}}me#-p7k#q`NNy?wJACCOzgc%7J6SgJnpf_<4eGK!wqx2K> z9=|g`S<8j?Vhzcpvv8SmL-8uh)FzIjj0q>BJmvu^zqzQ`97QRqJNi1a&_r5ICD}toBp2~< zLF*EA#&=ZX*1+mJ$(?9KrKA{YA?IPFkqv5QA}J7W3dyvG7FW(3$Vhnz$s)h5twKz>Q$ZTLFx*k`<#2RfIHC} zlyn>R-wSH#Z;!WB_-mb*-829;b_dY66I6%(p@#e$E!#^}I)+$(n59&XYTF1VQ$Mic zn4D9b2=WHZxW2Gt%b=`!i`v;AB8G}+Ll(C05sUXWW2rzr)?4VCy^%@Iq}7Q%6RW~6 zU6xoLt&gvX!;*@6KkGg09deAe&7;EEp67qSPYm!&`90N}G)>cXNc}rS8vmW{3$7`y zKF+6V7}-=s@5uPwF+Kj8e~$h&W0uDbkF6Es{OkDZ7gIZ~S7J(iud&MvVcOky!Imzm z(ay!5w*H;~C+~dPZ@k|}zjXe2{j>R{aWB)}DaGlKyJrnIdl&}&QKJ(k#Z8at_*eOR z`|tUfd~wa=izIYQtdlg(TiQ5knZgZek@8l(fPTSG&34vv2Y6P(3QBNKa%s+Mj$-Of z^px6)*@SD>FGJD8=zN%&kTciT86vIo0gKRCJ{t|DQRvu!n!PNh65gm-VnskG(7PclaC|lD`*-&KE6&uH9DgHcn_Q7#bJ)q?u!1p{{_EIp1YnH z|Dh@WVU~E|G_gS?Q(W+G?)S@m$=T2Gn8>@28Id$CzHsc1zqMlijs3(t_4_DIEQqNV zqs5+&`&dE**#-mg8ON*`dnHg z^rDC0iayaBnHZc98#gevbxfL=88O#ltHq~G+)6LOcm1RB%bahAh%Wg8lW>}9F+KVgnK>TcO34G_wcWu-8)8?W(&V|drqY2XzrY9^*@FYf{MEf!+HFFaK#DZ!iS0BG@ zs53Bl&Bn~6TJhOpYsNf^*%BKZ z*FOFWr!_wQX?#-rvxLuyTfAe8I@T@X%TiLbd_x_r6>;5m3w{^<&iSqL%kK9B-LD_c zyV@8>R`r40k(pkbnYYx{tY^IR)=7Gr@NfL&xQ(%uW5>jnj2jT2FR`>YtufF%VMT&q z-W7XGy7W{&sEnlhr?=y|BbQcL%b^*L<&N8^z>SpCOGAXuSpSA-)x^VYzkv$MSvZ%P zo=dN(uh+lp!PL0#gO1&Scaw*xQ$(?PIFDUj;%HgMJA;(4(RYGX&JrY#j8lLz?J;)|oa_<`DbbC@lYh>v=}lh_H1 zZ#VH-0F0)Zut9DSGnGWQya?6B|qr zpGg{$XW9=_`oIN zsi$P(%cl*z}vJwpAtGU2CK=Zt4QJ#t&7mtWS6MRRQPhsSp)5qao zV_{V4-YV?h2s0}+rlV3k+@(A){Z(bKQi^%zx9H)V?r7|2=BV!I=Q!<%cf5klRNFCD zZLNHhF7cC&kk?6g-=5^L3#}J$848&e)$NR&?$K0^KcNyd&niNu9K^JUaXfEl`k*$D zU!R~>)*Ot=pE`5|xPVGFzL1k%2A@8fuU2&DZ{@XF;gyu7Yw8~7T(`<$J+h(v9Lw1c zfhBax>}dwWhMGZ^!PHt5r3%BG@T02zlt~Q(_!$%6n%d-EX}CWz%+VN57F!97=^=UM zG8k*;;C6}Z-(|W>(phc6>n>Ymu~(;wGyIuoF^BB=7E?S66YaehE2BZZh{@|iBtK~h zDB1$5x3N^Zt5H+kX^kO=s=z$0Mpg~qR4rHlMNu98YOM!tcn^1b9`!(7%z*NE4AyrY zY}GO7Az$UhYE%KXaW#ExdEQ20I&RuZsr zlHr#HLm5dukr`QJ&Ux*Pu;voXKW1?z0jz}E69Tic5|!z%c#Up6Sy8hJdR!yP70)ph zeH?QZ1|(O0kARzvL;*h~U*F-EtijXEWaK8;+7#xf>*S3?>2WLzAMw7~9lO^VR%enu zhMvGS@N;va-*y1ra$&i(ynx=7AZE=@R2nOxa#?wtw3W~Pjay!p>T+KybZMDNK7&ZT z(f>YO6KfEWds=!{L@F9LnecFkPAGq=DlDm?s33c!(_&fJmIsJ13sAXr;1Nor1KG;D z%jqixk1_=;-pN?<@u=*}+=32#+KuGFuehb7c?OGF1Ve}>4})ctgfSILJ+?4hhVo)0 zyopAhEob}9{oC4RPJ!h#haq2HqfLT);4$cCs{4OHYHdN)aaHe|D z2hc$3B(0KcG?*62W6`zCMCV%uc5b+2il6B3C?WOdN%BgYI3cQ-kNF0Dv37Z>P(1># zI0bKEBq&xD{P-iLBwS3sot?Rnd+Eb}LVac{D8_cG*tI~>ih~;e;g+R_k$4QgS{^Km zm;8P%@msNEPV8|y7xrQGp2PV)Mu*)*JXIUe!gg>Yj=^w<2g~n;I^SCuF4xGPF2iow z1EZ`reDIoN%J;$5_hVH8LA88yVp74Hvp~57I29+cay|L%Autn{py$4oQ+y82$O+gn zYv2uagK__xtnCt?^$du{OIR&~nFg^O7T^VHPIpmD+E>^=+n4c@B+3v(+zN{HoOCpn<%)4VnEiSX*u3N;Kqdeit`#hD&nq z_RA@h7M$P}%5-G`thN2hHL}}A81K}W zyLk>S`z-RHo#1t!uo2l%%L`&2Q)+y9efZv`v3yjv>xS0Ns4c_c3nCeq#9ke4nO|DjS$= z1rV_G@UaSEDbv8!Ph@e%N6$U$PG`U@SFcAf&Kp~zr z2u4MWc^%HxBbaF~xl0Lr8|0KTh(Sxivn-2ssRG|L2i)2u;~AZ~h4I}DVOFj&&sct( zIssN^5%`>KrFbbpR#ijY??|sz&>Crlv?p}WS8ynf7wGzgtF6$y+lDPYB;A66n3^u1 zdD1(o!abQemY=Q&-%ONPEMIRyg$y%`>LoYkC+<}MM) z7*^VkNURF@eM_daG{OfohLtn{n_^Qf@#o}s6myFXQR6hc1Ydo{ExOO$-Un(}4fJ$8 zxo=lE6%FuLMK}k#^#eBdSFp4&Sn+t(XMrj9 z5|+^i@W8{U^_|08zU0iDLWf;JD%q;yj>eM@?!cS8Zl$zB=}Vk!%3e+I^zUPZ3T7r^hBFM`VH?9G%^Abj*(hz=S$~RG=eLli{DlIVWn(1^q2*kK2x$mR*Dr0KdjqDtm6Ldg89&1~o3p`B&f~A$=1w{SzS4hFR=??;Opj&`Z)Vi~b9l?p z1=toH;krh4>#p!#-r&gTEbZ*CtwQMKfb+Y%ssELL^8pY1SNm=8v~uTmDb8Jv(+mHk}B4t(#VVtjI+oz$h%*GES++U75TYtgVPIG-;IH2yb`s4+Mol9vU0-4JW3FK8 zh2gclY4yJ7s-D;P=o{!ZDnSq8NjtrCTd4%5H`94g)3hquLuY@_8~@gUg#)ko7w|LP z4cz~^GP<%lFFHD_QN?#0qNYMsTTP%)Uu7-ilJJ$3jfk7 z*^2d}29gE)-xUuL2@Zae-jGCIdw_aQetcL)EYVo%B9-9t#G(K@jlRubV=O(E`$3(G zgOZ1W?H8q^?vqgs6m|o?AvHZ8E3t^bc#3%9g@@djFIE~(1Kb-Tgv)TCbE35JiJajv znZr>~-0r3mws&{>Lif=Vd%kZ=1htSF&9&PuO@X-@-nO8fg?fP zp-S2oS4qE*{;&LV`DJ(4aCLUAab0lrL3KPEYG}WyQM^`KDTS3dc|H9xGsP<4uVbnI z+@cmY!u~_H_1kF9r*2DTHyT!XK2*uSQ}^gY$4)~!_H$c9Q0b2|J!G(Fv1%{LW>=9l zq$A>7iNCA>b9pB>stg(FF6>lK`Vo6^^2^}?!i9V=jkUMu2&W18a(^@K2WFdbW|$%D6QGqfA@ z@a$BZs0GxVYJzeIE#CL^7j}|H;Co*SUJ&X_tZHYNnfb|7z2g~ zB>(Z^e><9Gh~I7FJN7Azezx)S?ro*l;w(Mt70ub^O?rv0)2EP|XSjxCO+lYVL2lPK zyDE886PV?%iTWam6(_*bEk@KJz$U5Us%#U;n$}r^Dqh<8QI>1!2(J0LJVe))h6!Meg%gTDXt2TnJZ3Uq??8u}| z)nsR|GrP05ww+tkPpygG&ItC)p*B&L($(IN45*frUwRI{qlqQq-a3d|(__05ux7LI z9MkxW>xkH+=`;I}*ti(k;dJsFUkAoJod$lRt!!2cF_*cOl1m$10^o+u{w~h*!{Hj%1}u6ETg(hfJVrZ4I3L>)iC* zR7VezgV7fqOqxJIiUPT*6%-f9oDedvv!(Gs=BL}T@whqa%Ml#U*1YUbRuVb0r5S?CX$bM@sv5xlZY}kc+&q;XTMGs`i)PWNIsRHeeuQ6 zKA-m!-?pFXRYRh^@5F|iKvp_~GkNhnz1XEM=;bUkdeL2EGeajMnB#c5ihmf3P)f0^ zZ}ws48|w667MBI)>}^*YIy5To5n7U#!9O8CMSE7h{fFo68VPH(}8of4~yTCd-aef-$f>NitOeN z_h_oslV>bOuG}0SFcYM0H`#{=?~t7cZ6G`QgBYzX_vESIAWy7?YSL)pvm)Xb`sS)J zUHBZa>3+^_1kskSQzJkBHjX>##>V95T{7`$QV=N@V_&)?N3n;AlgGfhu7s`aLCi3K z=&BW;_!N=!YSw!iyVR2%D?zufAJL7QuAL^>wrT8Ecf8e1cK0oL&1ZZ|0+EiJeecVC zI){DU&etvW{4-FkAQN&Y2cmhN5m)KBxB>f|2P(&)`eZgUq90Bx=*3wpKUo@BV zidI|O<*?OhX!v(kr>eu%H0o02g3~j$#1Nl zKiN<c}GGW^<{b)E=c>jUfD zfAmH7}&@QpdW$~-NIC>E_q%DZRWNu;vx$b`A&l}ite{S$^D#@^3Q<;@BgjoT$e%3gOnnfO-^%D~;Ym3vQ zOY%LX37w!l)vaoMqPzJh^>5T>Gi@(e^Jt-t)T@mNsC~QG<-)sm_DO9pO=#FN)%SVHubraAbl5!^BlrSY}*{Z7jlal!r$8gUeX!M zxrFn2mffCDMX51fXC-(09p5YY`@Y=Qy6jCVFqm+5Wj740A*^~N2<%BJ^1;b-pSPi) z@rys(N~BQ=th6fL(KiwKBw14`hj54{tn}lbM-)NZrHYJLu(L5d@+dv&5Q&uU)~Z2(|NxwdD@#EJs~O7xj>mT3zJOoS&VKojIM& z=+$1W)pxdZCep3FnW+e~i1GfS*&L@VPF8;#P4~z#^d`m=?JgFUz@PwEzE`ZQ#6uQHV+-%iPzbySdhTGL=7Lo9ZJ|&`MI;$n>C!#YoG}y_`kvI+H~&PLE`XXoT2JO-Zk)m z^}r@F_aVf5 zVI|trIVqhvBL{_wa-8}^dx0W*4x)(*jz`Wm?&a=rXlxGE?rVpftx&}HsV#+7R196f zEa(GfQa3WA^b*?WhrzmwgBkBfBVne!i^%sGtpABbsd=mcC;)sjd$E=?s98kQL6?s8 zyh|iCg>_5fKDEc^IEl{=aP!}Q0u9G=Z6Y(d&TVT0`k7?qg^B!&dRI6$>N!5YKdiZn z$zDB-JDbCL2nIO_ET$-`N*RfHyW+olQ!|v&{&``jU^jD!iSL;%{^c%g(*sLo-_nBN zMRKzWiOsHJ!>{D*H7-x^_|1tO;J~6r+gy z^O23ECkkDH4T#{g9)=+pg6;LGa->_qQu!Q@Q_i+YXKQF!2?Z8IUnSRuMzw+r+ zaHWi91eyv?<^;Y);lj-%hZOw2hpy^TXh7X$Le5owJ;ltT7v3mzW0SnS(Vg93y|aD! zZ(NUEmtBus?_HBz9@k*vlE>O6Eu+@gv4{Oj zPy*F+%3L|66iEc#37hj2t+Vy^QS88Bx|Djdq7~?;?#X&qKshHCd}RenY8p@R6;??o z_j4mwt3TeO7XJALx2rVq#zeFYbJ9mJkEnhu$fT28RuX=I7#-!tEdh&p2U@!XENMSA zq7K$CX3DtCtEhl{WWHW&_Ocz=aS-dH;orN`_57Wft1s3$1)lIKm4wfnl|p2LgT=pK z$PVg%zvyfY1jC<8|3q(ex3bHT(q|MF`k`sEk@`(T>NhG#f0A?sg{SW5VmMK?+DV-N z>>D=PYuFiHBAm{|!g+`@RARz#d_)MyzQsi11hA(+M440QSgHuFosaJ8ugod(X+iBI z*4SsvBnoL`v@;qIi{wMcsuTRQHbxdI`Mwx`2@@#VGI^|F@_*atiOKsJ>Fvl=M#GqC zZLud%T?`Zsfjkcu(<_J37@y{BqcwKCQ3p6aYO`D+%qA`6wq2)OwO#w2rJRek`dSSw z(b3W|K<%X@$%W8WIDr~k5g6{XiKspkM@<7KcuPKh7e#|~-~(&1*x$@lFb^`IZQ8?h zVl^$icP-+KPw=Db5P$fA-{z)*-v|#^oBXRjxo|kPGCjQ>;lzgDK)0jt{TIM=BJdL3 zs8KH@Q`*N@KQf!^$r?Qk@$whY&nS!69|Ak?G?96I?DtOMgictyj&z=e!SK(HMT|hv ztqiwsBsvco8UYjKRq`No;u^_~nOU7A_f{?_2b5yUG}Q85P}d)Up1?U?_Y&^lFYIkg z>Tdx|8S75wah+Hi<}>zR;gsFMUN6SiuEA~=LjCU>nRY0hlEcaJ263{BgT<|3l9GZB z+aLT{A)^JW6b#QW9-nsvy|3!zX}(%@rJ{}qnc8eyiTr|A2j%du>|Fe%7!Dk zR@nm`f=A3i3o$RlhOvaMViWUC^JmjxmLS}dq#UU zxg$_(%jEcsrdds`hjzztTwSk-N>TYMCo>WMdK4c%i_VV5;QG%&T~=d3mY|;V0#=J< z&cF^?bH1MXP3osAuhrRq-T;7AO%TOld{tFtk0u}YcXdpGG9-J2? zjEQK)ERNwx^bKdLDxdK+_ir8N z?HgamjK18!jIh+obLy9)Gjz%LV59^CpNjQbg)+n=X68KLE^h-3GtIPg=S!$9odrAj zK?UnNYugv~?O<~R>K`R|_AO}HYy%^xVUD)4pcuFw_D?9CC2!z6Et97@&N|~<0q)tZ zF-#Y|2Mco%Xx2bib5D0ax8D$VqP9}4p(x4>r5bv-X&m3w8tM=9pwkKqm_$FqYE2DR zhON2CIzpAK6)JQ0&`~@KR`BTUiyizO7fcb}*WN!&;u8XH6nDV|LE zA}4(qu|iLxx<15pDXnWD7}LQx#)6&S=gG2=kzE0$dk;e@$*2lWo?!Mv)um`M2K$=$ z=bhCPmPaJr*2BaB;yBQQSTR3}E7hd2Xr1MP%`ghZhi>vnw3E7{=bM&^?rt=cCbPHA zso`M7x zAj8Uv64x8^61ZfnS(}*gD3Rm>kg`DQ3%K%9Dk0NzhFv<;>95^U=co;^&IjCe{c8JF^|Wy1)pDql z;b6)rW_4#u!Y;L;x=b!4-mrdSN1lLL?=-q$b=%-qGnn`Ewcba}tZRWb>1%yF)}lTP z=8|B9k!Y}0!Ip+o$!^A8_9x4G57%b`UUobAw`#8=pDITleUMCvo|`kCe6Cds2G>n)>_?vXAYB8!h(Rh+Md-|p z3PWMzof!mTb(h$^mes}F0WvU~JoAHo#<*;5waSg z*>lyexc^c=x5sc^a*R-KtF^SUuBq;fXUE67^h2-ACK#;tL^yy}ko)bm9px@4E z+TMamaS3M=wkJ06W~O4W7zBI++H;!zk5}{N^oDw4_1@+ps}#{~*JK@`TA(X7m0kff z?zvqXZ+M9MTQ~}bgYXz*Y=g*XB6oQK7|9e+G@pjSU@9(uIqUz>5AwvENUujKUOkH1 zS}bw14^cabE!ic^6$cW^beqdhGOkm?L}0FvlhT?gjjUQ^W~h>^o#Gp~?AHLp)PT z>`)@u&K`23o@6EgZ~=aCD^H@3HIjWCjdoFMzQ+>J-C~8xVA-})m6$;7ZZ?ScM-*K8 zbNlzzG`( zRL^ZJFt^wRrDe)ab)(ur`6Cs@8->ZeRfm@3%4*tGSZmCE`t+pV39S+qB}`6SnH26dbO|<6 z4eW|wnrNV{(i|$34G`TxK}nh!(2Q)${_zsln1XxD%OB zl4^-h@}e@_3e~d|@_wSJ7ij(z5xqhVy4-!5Kefm^PvTKuVeL1=C0kD9w}gnj4l!OH zcr4$*3w^4>ZK+rnh5{7qPOnTa|F!+S+}nfQ1MQA$a6HIUb@0lPXt<#{M^e<%F-e>{0%BAY1c9iyl^ zn&eHI?Nepz15#EUyFHMcKNmk`pYedHt!Gdf?W?CXhudSBGf>sh!_k0_hZ#ip*_7{! zzvF;*-j&8L%6~?{ZvWm)XwB=Gu1tWrwMZVPB&bug&CbcNyJh7ka{-RR6K|_mB?AA< zl$eirzYk_*V~Mvb^PPVu_D#y?EvYXxQn52fLC+UhTZ#87VVz+Fleu0M7Bcm;5`8PB z#X;nOCNo?<5zqSAU@rI~XJJ?sx0hjuZV+|tA^y`FI765Q%5~3OW=H^I{TTi$PaE!jT&V!s4KcG8A4|D*V7_EJ``D|3&0zzRE>T z&O!$CPwK+W8ct2|1^4tQ)_5N^rATh_WqipB;)lXa{j3Uyp&jvlcPx2x7<*0dKY>L1 zYso2E!wE|Z3#AdK6O=PAKI|#WFrzRap9G4iMxHqZ0 zx0in0$YSLrQw_3HS_MH3)|&gRt*{g5>u*RU`en-LOEwQ`MLjK`+Xd@ zqOf?1fknQ6i&-6B+&z%HRc1~z0cE*v=1i)RP3TM+V_5nFy$ZHgzq@=X?wIi&dTnmo^Ze2eqB9PU8S_c>OFLVzspBp@fA=WD^1mR${@KoS~#DK zBTQ5|<{gFp@HTIIJ&&=GNU^z|&l^G)o#g%FeWxEVT7g=GS|zBa#9>W*HKNZbnB;~n zSsH!F{7mYdhlWp67;tatig(gQ>XJ}x5#a!XKwhN2B9J?l#^TAKRkyKAd;S-fM3VGWeIw!@UjPJFwb>^40;iXX%Z+})GtmA^yB@EJ7-2~24ajM@-luCK&b*-#%C z3+kB#_TCd%P4|Q|)I3M>X#(l$_32b@0Kc3}OfV8e zdr{Q4GI9oFqK^txCH`O+j&iyl@s1CyD#RCc!H62eQ5a7hWCTp|zo5v8#9JRY6)iXy z|AL+Wz#fEQ`9~XT&B^u)vA*JR$XX^COb*9BHB=3P*_Vrc#R0Bc?$e&Pp7frfOh%=! zOvhqJr8aE&`%E#oAWwsB`!8BWMXYGBl+xTK9d=F+J%jG`y7V93V`z>y1r=MNF9yrF z2{U08dyxukV;~Bv^R1R38Xl%oH-u#v0Y7OBY=9^#{G$bx6+eLr+C6bG5zsPXnM%YK zVcd}*Dl8KGxy$$^Uri$vJhndjKM>D#0-tw{+Es3t1oTtmhpUkzT?D^~hsnMdpS+3A zgAVkq4v}A@qpK>1$(RIrGZ^qGcxhdTvRd)qF)(%8QIGoqf|vzXX(znv6DkqKsD7US zA^Vq|ZUxp^m+bB>e6?oql{*tnFM}I&pUkfye%Qy^e+in>f+)eqyL2VT3$pc!oOUHy z{Y0|eF(4e3_-X^%F*y16Rk7Ed`M-_y!Y;-d9D^ZIhblu0tnvrW?oRwiHgfJ~LOGa^ z%$gRL!^T=p&GrU+JB+nB#66GUQ_P1)6~dgVnXrF~VdG}N=q$o5Ey!s*C%RTLtNt{-sox#h9NVe0c|iY{D@9N(zYjlYi71M*$P4G%XUJSn z8yEGj-cH_xBt5B&cc-@yeLP;~gQwuW*VQA510H)b5p#LypIKwJ~q}&mm>xp1Y-R_NXi9f(_SfCzf~?-Qerlg+6%D=P*P5vhU~NDSft5 zQ=^{4`dp`S_KiBiAK056h{n2-;rpkCXsPIvaAv?+=oyd zjIjc+SSjgmC`R;gm>8o7+~J#8r}boTW8qTxo_`YT^nqlP%UHRqWDspYBwmpPgyKKu zQs>QqeZNE8wmvzVI|;gUg!3?i8<7b=1lJw~s#WOBzhm}DNu?>=>XvFM=Ghn1`fDvU zo!QmH=y@sWFw_H74mK!RlufdY>ewMVYkdCjDa>R8%e)ag)Rd@fBvHV3?|JVz?=9~m z?`vLc_vQv1+xyZ@p3yCTt_GPLSq=;4Tx@LiL1$X{)$n#^5d?+o;Z9OgMAgzx-!HC44bcgs)smL1IdmmkT(s--&dw*=sHMQ zKi;<>b;Mib#mkBJhjDYOf@GCrr;3xqILMf05*Ke^UFRh4LgckGlB=#OlNt2a z*^8gNrYqGS-C7CmZ-D3Jw#Q?k`hX?107=<_ee!jr<>92QAR^fY9w5N@mY6@W91n7y z3g%ehrw}fVlYWpRpi0iIPZU~6J>(to5-C_XkEJgJLo-rLBRv;d+K2dTKdgrKAR^$Y z*x7+#n`2=Qq(x!nFqZbDwVhMd6dMvo)R9UoCiRtjGMVKOs;-ly1nHCGSRv<#Xgpxu#;@c~5@mrCbhS60N2*Obr4cP!HZ#Brz^GF@UJ^bBk^u}tUz1#@A z{|kF|oqB3N_NNXE_?~p4o!}n)hvmpY)RYqb@;73=M`Y&v_-e*UNiF^dk8dmXpcbEb zE_K-dSozO(K5`_K!-#Vet&yCSXfo$|_D3>=<5UxTHL!w2((T#PBh-ct!Gd@N@4|aystOj_9W}U&Xv?;y!($Zo zHAbw1aLFq)VB_oaFJ^(VJVp&MjhKK{O0=A0|83|vYb2DgJMsUE_}mBW{z3=)oq5rk zY-c4iSPJ5mjz}mESWS8~BZpz5?|{*|S<8#u;%n&1l!W^ki&vQkySa?mO=>0Pq?2F` z5%V}Pkxw^{&s~*!R8v?)eYd_ana;hd!a&N0qa8@}2(&PHDJbUK44d}h59 ztbz0^z7(oiSB**54n9e!SQdS{+V*>4ym*THaE}->){dY?xSiK|Sb<>r+?I{cZ<4qcgL!`x~^SocXxMpCw77X3Sy&z z*h!ZKGg(d`Vp>b4v(yL zx+=bgy8Qi&T`&uq@-k7pM#$A*w8K>A0Bt4Nyprr|SiqU@@GLg{IjpR1hKJaV+lhl8 zgvv`8x+s0&kVSZCmB~iCD@2X7pV~!uWC+&fI$&0gI=kukQoE@2E=HE+I5l<0kP$OhR^iK8A}7eau7%R8;OmSB%RAVY5kUW}TU42A&LQZfsRmbe z*@bB*oOzW9yqjg%_9Ll<_2Rz(=UOs+qp<-7pf93{+ZJV4-7_?w#Cql$_OmJ?BXE9_P}0cWmZV&z|U!zEpG^qrZ^K{)0`NmGfRZ;@ob?RHgxgI&l3?=VwD# zb*f<+(TBx`De64uG%XlPstG^LLw_A5X49DNg&mY=swAD-JVOzsHu1b~L9nChY9F z?C`E>m`=W^7P?49Kdgqg>p;cdpu0YJsoqpyrQz4u>63N<%*AM{ocWa21_jD4gkIUp z-|ck!xnbywy(N1kJL93}z>c^>oyRCBzc!xPRALOBuniWo)1;mu2wiId-Z8|geSzf+ zc(XbZWeHL*kvfzRq+4}Xc9B}3N5JI)a_MhsT=JpU1}R0bHG31IOCzeh23iYYMW?}i zVS|mi(nH&6cf>ev1I;1|6i!WSVZ&)6X77o31{<=d*~z6$G1PRrYbWV-mrG4F40TpO zKTLEE=cK@s&S1wzElxS0CKyIx;d$T(T<3Xq_W1f^+gRv*V>ZkoGTe&1X&@G+4J)b( zoL5*m;oPV-2bMLFl3VG8>2z$Qd*)KZ0q0>d9rq2p)%;2$XN04nvpIh3E9YJKvlkuz zCe!P3{J;CSb_`r^Z}X5-oyf`8ag=ZX~)pr|2SJ~e{k~6K(&&xkeugx zN1}5E9YJe3YiUQFb`m&V-hksK$gsbi7tp!gu%-Ss zTq5gtRGUCowD(BvbJ!6_IC-!M>o4nEM%Tao=m=zylk=+S!BGtf^(VH=14CCT4v!i% zauyCOt!n6=?wr#S&C^;U84{fvkdc{mjhGAt3`CzdQO;o9AEEO3tuuuz%UI~^hVu_) zfwGPKaWuUclh{|#z%Y~Q#dTy-099UttP^A6t5N`sARhLTDK#%$;K!%V1xgvJCg);n^uRWxXCs#81)?TVNXG+IQd~3) zR(7Bn9L%pcROp1B%h1`Z0oLPe?9cwl>zqi>LE3sMvle1&PUO1#L?d2nFPQstXFaNo zE^C#X_kq+tYOWe-e>r`W{_sLKqBeEt7+el7Wi>I&>GXIwLtOuiW}^e>Hl&*ebRJBH zE9y$An(s}1AQ~*(a&Fe_hP_HhDAL49VBL`w7kK9aZ%sjuwc*MTVEP`*Gz_S(fg3#0 zQnA>!`E(0*Je572=vVm~i`*5x^@Wu$;Do=*bW>Pms7HO{ZT3E{!5_MYgtxH9G0r~t zO^1-IPIT{iErxtj9X=ULT#5ad$m8w!0&TDyh7y^3L?6D9=!w(B;;XQydOW)Qq4tPg zel@YdJ*ZK+XvoX{fbH}e)8L2NL`?q$uGN@(dGPAVy%E@6Ex7j_@sn8QaD%Zo;{%UC zHs{4sD~DB;ml<_u%wnv)3f9_HcIr&iXIY*31rbx7i_H~+)br%7Q@}f%d0r#(`4g@_ z#>&Id?H7SV4rJmZeD34O^AB1!j{4m~oDX#bz^Rt2I`mD&Ba6fM|S*=+cQ zHt?n|K#(DjjPGbV4iv}Yeu`e34n)h*O;L8448%Lnz^WgL{rw&8KE=9cQe!m>Uts~( za$mXvdLnDi8e*|LRxt}dEO%dEaGOfIo!Cud$`tuX8soK58>=yId!ZUUenR!Z{VjDi+ZlHP*+=ha&Kk=e9;@mrlhNOY} zN5H%#+POW}b1m$ZQs~st?4|okoM0c<`4EHJgkSZMNV|i0T5YsWB4c!Ax5Qm=a|xc$ z!a~f8ynKy@*kHJcjWrElyboSWbGk@;B7#vKZ)PBQyW8yJABR0%fe2<7J-d{Gylck} zBq5%|v{Kie7aAP(zv8uVKli5g%S2zH7Y1~H>X*aH?ks~Ko=U!n>+T}nD3{vN5w|x3OUne0j~^g5!}V8 z0dU50!xtsU*qR;w15HP$;vZzJ%HFfj=DIG&h({`xaF@I;52zQPK_uSQ6hc>vqEz}z z9sr96snwW#XIW-Gm*`v%D!(?eU+W0D?EH>UDuGO7vP-Z-q#gO;1nnAfdlnw$ZKB6T zv5dz+Cmry-2h#y(5NCzm()S7ejb6D)oT@#3X*7}k2zXR#?PMS3C!jbU3Hc9|_noOu zE(*7o#A4o$9$GPUWl#&9RKyxUEm$otdG+D*?8LR_A;qh}Jq?I(Po%1M zCo=H~H1dL2RyZ{(pO8rkvbhx_4d*ow&DRHteug9*iM(*3lB5b(+{DXDBFfhYdEK2g zRKW);ghiK1e5{yW1C>GcHV1T7lP7J&7nf5es>{wxY_Ak}s|^{?rQ~7)i8)?UW5~qp zCMVE@`lPn>pXp9z#5#3{x`-|`K6E^4L#F%^F&|aQBHl2AnUp{R?*bN8$p$%z0NjT% zC10=^%-lpPf7YXVE2*;UK{kH}Hg5n~@)G#rL*TE<#9nK|g}u>`$Kl6=VD17EavlEn zC@}d7{g)FsR0D&ZkQLpCN*}~$-HI$;gw1#gi1#C!IE%RLJ0e+O^j#T1Bys@quQz(i zuJ2|#5B=Qa6-u<@G#W{AyG@8u?MH{r=BfAD6`l&r-ym(>sN3ntPYtBUBxvR`(UJM+ z`RnW|A1aYuDKwd#kv0yZ1I^#xhFBTJW{R`5}1=^B+9-COfHl*|4{JsOFt!0h7 zk)`**_ZnVODL3W(p+is& z09wb)Yzh#gFO7AShbt7WYYg4|h3q`Q>^GvVh7*ykgxypCh*TlY(GGdr8$He`7JSkk z?rY1pd02BEwCO44bP+n4jQw33{FFeSM3GN904C3Xzf17l8Zh|@zEhp=v3v`H$JXq_ zS_cJZqsnv;TpY$b~2{`x#5+F0~=I&?RPU+t*0wXkN0z z@dVfn6iF_Cbq!)puFJCS+?Gx;p`yUdxk4GxPbpJ3$uY0rKh= z^t>48u3@%Q`DQB;@-id5087ze;5OVn4yZ2!_OZk*1DH*IY_8JqbUS9yo1ee2BO{Ug z=lJ~-$&^WdV>ZLUXwvVvIq|_@r4$`UyvRnsp$_vcwrUjg@(db(h8~fgQf`NM~+;|b#tYfyH_<J`auIDpxWMO-9bo%cE~S3=(#AcE`k>Bh2K?; zn4CKnyBi*91!%kzcPwGGH;^S4809(qABZN&!4qDv%16lXXk?Z^E=q^oS+KeXUH=LA ziCq7TjpRwZ>`yS!k=c!9#*@%eQ^+!{!&~czrYOmXCGm`MA~&MBDl>6IR^*1S%Of>3qStM8N)q4(D$=l;hE_X~e!#hNIh-aOnuJ#(5uQsa~!&=rt zW2K!|y!|Nl+E#UU=@?f_CevgK1`c6E(2NGxn>lvrhPcPuGgK$J30o8E*bX zgsD21?2gx(A4zo@yXiPEdWlSmMhj_Bl+-yMW=?;@>r24ZL@;&(IAu72nGfNmG`md0{tagS!2Zk=gJ9I01+GJV4jg zW|pJjf^E!L&ggsz&c#1Gip+Qd@4STS*Moi0;D_M7Rs4iOEt7!KYxdEULNi>0mgXR3 z<^Z*k@W^IpS;6lzGJ|O9yc8%hJ5=fdB!l3E8t6Z7`0E)X+<*qRFyas9YJ+~fn9qOE zNNr|a8M2D5-BkJCqvPE&vaR0Adm*6b<03nLPVqM!Tk|rNyF0O<_LFfBC+;nF zXJdLCbVAmzVBN3D>bWqlAh0+9nK6?nLZrcg)C`4IcM;d#gO_!h$Vw=7^=T}axoG^x zz$ggrtiZiVK;6jSpGcDx;QuAOeg~LN#wHz1eMK9j-ycwOMLG)wA^ie*{z%{+gB)Ff zmaLC;$*B>@#$f2V7I#!7*VYqxS`lx>0}HP*(z71lmPDc!;?uhPUxc+*gc~M9Q5T_` z?NHT9sQwnRZ5vc>f@c0eQssb(lc2U}q~}s*wHK_Xqtm`adn>{3bnw2EnfB$`t%2oX zsQM)II2YIoMGPfRy$nirl98OG(^D6$haJSot?b5IM14yn5wn-{s(ZmX7Fl#j44`w} zBl-d?qtds7(wKZ`TcUndv3BpkUkl*KwMe{0$cNQ%Vhy-29?4mk&p%*y-Ur50fNu)8 zIgd;es>_G`i({2Di1*5Qen zUI^mzo%lC@vZkeA_dZg)CZ5n>sBAd$F<7~&hMCT|%%JCcNAoD-94gGZoBUlSS~|E+ zb8X~0!IIadgXuB((f&%b(wmC2O!Bjxor9?FJWh332-VY<=&?APohManr`TQi%T|uM z`h8?6K5@qJBr35hXffJDa+U|xgT^>xBV#GGp^}JSwT?cBBRN0YN|t}Jx`}b$qv8HT zCXE1obJ&}CmrfcTs4*)_m2eyO(~PC#MwtB~b!H>z(X^SI^*JC^3+t^9QMO&!I)jMw zccKdNrW(y&grQLFS1Mr-b9QNWs#Z@Up#td)zYnbwjHfL4sH}a!Ge6Bvrk$L_?7Akmt+L~chS4U};n2!Z5@eMt;kbQy{<8ad~^F?}^gqz$r`Kq$9I(s~qP$N9rcuU<* z=JJbnoSj_*crBu~X(W5TQf;pGJnXJrM}6F0YQ9(EH*dvL@P)ejlZ_fdws0vt6^9IZ ziJl(?zc)tLCS%7ffzA)&wMJ6GHH}I=S0qad{trZcG{eqqjgQ>}KXC!v(gUgtMQR10 zRod%Oo$OdV%fWUZIDQdQ+!wtVf?lhHjgx@(xQEoR@ZJtBBtrW!+`kkm6zjs3Xnqd* zM-8P`ceTSr)whZVJ(;I^ka6unzUU7kEE}Eo4GYwGQ*O&**Ojh+S=N}Vo8pZpOwZ}1 zw8FB-62dOS_vRj^I%;=AHS!dL>A!e~zE9n4bknxhw;i+HvF&HSoYhv4E?W0&eQnEa zd+gh_t%g+g@lPa+G@6>$CG15Q1y--ob-SK9hU$>h>U*;Fn~_1uSOn?lymrq0S{ifx zWPirKwma+txx?PwmQc_|`quPDdz3~>F2ZguNR}!NZBSHcfcMoL|7sf5R+ovF`ICh_ zN)})_R!&>=;u4@7jc1+;-lnnp;2bn^9BR4Fjw+|440(tNM3kbnTtq8YqB(mb1Fw>& zbE8ryr}ByDe25aJ+*aQi7nmN=l_<#6m6}{D5xRy%>$(yx2vGJT$^D!GT7AbpdnZoQ zE(OIjrBW$3^PfzAxl>dk{>!W`K^<|(d(rkjNU-I|yP`z9gTaQYHibP+h2WIx>^>__ z9dd@q9##W2|V71tX){*+Oxl|UrI&x~?oUfD-#@f`2edi46FQz3X zZ_@?pgHBTa+1uD(bx}CcmCCmoP-+0(rnUl^<@PXJxHX+~HgZ@kwyE|wM}~HW4BiB+ z+az|fbV0+Fr`~po@iS*nUt|YqH!5!ZjX&9s_yPZF7@76^P}z9s>K}Tn=3|dgNo3(% z{`aC5wmMXj4?A{?vl<%aBU$c+Xw6q>(>>HOIkitH>Wz;Dn@8RJ}If z43fQc5-LW`d@!`?POPde8i7-2v8ops51IOyXS4h7t!Waqp>wF(0gsTaT35fcD-++-e@9T)8F{Odx-C3Vc+E;);v#{Lf6n_{Lz>6 zu~-bZR3`hd0UO7keIRmHLrHWX34J71ZepP-pNF71>tgGeiGSas!bM`Zwb4=1$8rw! ziifEXTf%;UUygd@_Y%;Jji9eGhTckkqpNw5OB+iu%Sk$ST%r%yJ5GE5Vg6)ZV)i!I zGi7q(T?X0ChjjWa=h$R_X{%@ZVf}6mp-;#!+kAG$l_83~4lVc@yFD*{@eSpHy3yEz z^T&Ibhnn-5$D2Zt;{}Xw&`Dd+MRTc(o6Y{s1ym0&;FRcedoG8cBZZwtclj@a{xJFQ zV`h_|$}ntGa?qdr1t-+cGoC^dw4^?961yv^DeLf3H=&2cceYS-yMeB2o2ij60HqA4 zpH?0!|4U;{T)?NOMhs^sGmEBk+j~Pi+WI{9>1nLvo#?H`P|bXF)@1fZ9i_Id8dw^E z&U($BiAqSQL)7&3#eNwAel~EXUp=)y`(dxCA5=e1*1l|PWxB~8m&>NgrgrRHn?bJV zITq_xszA%p4W|he-Djzu%x=$*?AghV*Hz#kNjpkCSQ#qar>U;ShR~cld%p^*e=EoE z6M7NcJ3w`@Cly9l*b!ZWUCtIqS*rCnP<7e?9^QpybVn;iFsnj%0=0n37^pUZh-xf4 z`v-PSLpl?+p+0^IBdkZRzGIiF2c1M*v@H5o`%>T77@O&{HX56(6OcGZ#pDxUBXP{( zz-0=Ou(%%E4M1*5rQ8&^z+M>Qv&e0EJ8P@edEatM#UfK@!i+jM4OOQD()O~iK z8$^91TVMR>?nJXDP_JB#jL&fNN;W*IcSMMOAoER}t9=lgX9E`1WvHYU@EnNjUW|u3 zf#}Z&A{*}bJY}G-PIv%qiKZBc4(uYbDc#)*@Cje^E7n3!VwW0KVLg?C z?3iyrO;Jzyv^dq~6R_F7(4`}qUDD6lC!OV}Nu}i){Iy7O5Kek%Rm0kULFQ{3(zZH& zL>DAdJkg#0L>gLy$ys1{2l1&!_`!CfN2i&|Loojo${{`m`n@~*8%EhJ_G`8)?8u$R9%3JKeq+Pm zoab{vJA>z85w({AYP4RcsZJR~1B zx>RZ`#$D!$mMF_P_N>%$nFBpdM9MTZ#(`6#+E-~~NYajBhg7xqwqK_^L@av{ zb|ZW3_9pCdEUWpDQ=dV7Y(=DY1$tH2V6Xf<_E`UAtVjoo)v8HtOLd%sSj;@^`jXTo z&R{jI@QZ@%kJ;0(-B#CL77IdrzAx0v+%t5gr+ctES3Rg+Bv<9FPE?-Y9TX(av5u90 zfScP>Pt_2-<%ZrTvd3c}aF2sBn;_r6YAt~#Cv_7isR%v&M-_Z9l@hD)yUbKh#*^ul zYU7{q6{m4?#}MRsf2y$e0ijsBWLcp~PL;;Ls7{>cE%hRi>@habLDkKaoldH6j0=p4 zaXNb;Tj8PklXZ@Oe)`crs;T|0Z7%1rUFXz=&Fs?g#UI;99PF8)4_wkxUBimjqvvv| zN7y-W8L1JEt>WRFg+3X9yi6tUnvY%VW1;C%Xd^GY;&>_*(0ycB>*G80;F+bV2D37+ z{m32J$IzJ?=R;`hOg#5vShh3SPx?3BU<*1YmBe3ci%h$W?Xnv9CJ}Qf1D*86HXjKj zuR%dKk?H-AOesVsZV;C{N$h_ZF`W*?dEOHf*hkFjD_PWH_;J(mV4pII6|c>J9}tP8 z9ZJ=#V%%!7nZsS4l5y_p;_33;e3Jf|;U?8o#8^qKrr6j48RTeae{J)&&!lU`5xN`v zi>5lusZDk8Q08jv$DvXv9L{(~Zpj0E70MpL2=zR^RC#rga)Qd4zEZOWRHPSher*$7 zE}B6}W8we7nj0|}PvY6-h)Y&vJ?9KnsohFas^gFCVqc>~dCOBJ^O^|H5b`}s;IC!$ zx!TJj6|<@ z#d2>&OzRXh?+Qg%Wo_Pkvj8dYioLUhex|Z7Yb-G!S7<|Oe*OZEw~%IGRC#OyBKbHQ zV=WQ>Ke3;>UtY^A)!p+B(O(##%)Q~24%iNJ(mb?nB9yTbLq&@z&l8%0cR6rbcF z*E|ymB>fgr@P=ZLp+>69Id2!r{Fl7aer)|a^k1k9?Ywk^Grv!)Ih$_DZbKeQAJ`#` zwI2IMa^V5wq7PuLRe*--B8|QyMW!>4w?uVUvC`34GymdKHzxZ!izv={V$q4v!S4)o zCp;~a^Etgf{D}3|&~2o4%-IEdZmVu9heA(Xkw>BU>weBOG=l-Hai5x)bKs#Cv7uG0 z^$8ImJJGawpmdNZ?PIcJWuT-HVBiuMxWam$8!SZ8O2hZ1$vfTz!qdraRUwkZiS77x zEs0sSho`P6b?B27#>p9XjZNs4@y7JTv>$8xJ~ZZ~dMJeq&FC*7Uc|iSGF6_n!N@I08R>ueU?>yA9iyZt` z^two&dRPk8v6Xvc&lEzk8PWaoh?%|y-fvjfEu_e0G@3v5<8tbURjQ*JA_eAf2GC(> zcoMQ80(8KY28@oeOz?pUx;r#9X!`8Ml!I2}ZlgNx4_?deT^3 z4S4f3*l5Zw)eOAI&B)s+%v7ocCC6QaoM#rcwSkOh9_)c4gl~dohk$WzO-n>mM zetBeE6Jl!DIHAi-q^E*uuj!M?%n@1dOgBuUIMJ{PzU>5M547%u*Yq4Kpay5)G)J?I zApYY{jYegn=+WdovLTmippPW08H&u6z0*=(mx*7p6;Z z16rwpRrv&)U^M(H{P-G;@<_|l^5I1_LE4YQSBW5Imk(^cV_(irc1;*mP07uC>eK5i zJ2u=8WMz9~`x3I*U7)OG=qh)5_N=BqN*vv9nnM$@NSZTfi{VuHM&Plzp{<`m5#w3Q zXDDS0aS1E@F&KJU24te4rfk5v8y5d6=CBFrya?Gd0D0Afe3cKLR&!>3n7ol0{uoU? z!abs$oaoI;$AQtDLa!G3e{tei&;l~tx5(*ag* zL1n9%?|EwQYLk~diw_eHrBR*BDDRMT3clbwJ~@SDFA+z{v$>PUS%9y&6Q~|Um-iy( z9Y9>oNsjh4GdzxEm5g5PiVhgW40eFSvBz`{JABMPgUy?*9D01j=lV zk#>o26(l;*tAKSLY>2KVl=DJ6i1c%M4|_AqHqJ#H2TL*F=aE&H$GLX5l1yB z1K54y>RiIky-3GKG-ns$Gz*}-u}J6^L~rFBGk@&m+Q`iy^k-T4AP?ERFX+46*g@-v zvz;eb8HV;8#~P)YDM7EN*nu4E17DUvzjPuiAr+Fpcl=3SIsu*F$!LX%FJ@x}^^nag z(4VWRnb=G|e>PeDc5qvEsy*_+S=o^+27H#XdNyn>S02D_m8!zMXpxW5h4jj7i{)jZ zx0`eh>qu_ttJV{(lz`NeI+AJNwih)jsqlJZa%34)nI}R=*N_}xS{l_)E#Tg@(ArBR zl98F^rp7iA%q~Dyh&@~qo+?5VuqOG5S>zrfbSQ+7joro0qwD;jErQHn15H1temXarwkbRQH!^NRjx~^>HNo0A z2wfe7|DG|M4?r*-X!=1F4ziIyfTa&utV9it)VVZ63N!%!CBSMb6%Ltr(+7d>5=Q?A zyS6nFEf`C!A~~zxQ15!=_bW!LszbFKyxEjYb^`SD63<6!X@6<4#XF>m#d(L-$|8 zs#^rz4d-VDqwIlRuJDpN{KHV*Pizdact&z>SJvbMwH9FxQZ*OBGd6Pf5b(B#xq9=o z%HW_R>mP`fdx5;_cIe8Be7EBQWZLE!#xFANkcH!ocC3b60YcgkGzf}=@064bhE>Kp6OZ z$#`d>hy8H=4RCv%(ZBPuL3>i0_JG$dqK`LNMLhri;`=DxB?DPHGcntGtfM})Mue%z zjwnF;E7#uTzHok@XZ=aY^Gd9&JMizy-`4mqrNBo6sJW%EG{m4PMR%rt4zS4M$>))bqlkLA}9u8AdLP1Z=?g zmErb={GXR+SkOlbPxj^6Ihc#o;#Ff_^`R`O8_EQhN6`^i@Ia&4z}86SCGf*Bc*Yyr=z~NZgH<(!xrVUf za=M-_!wie)qkRQ_QU!Gn$@LQ~Ci3?^xcq@kmD*QVonNGu>KAlw!;dQiE~Mr(4=^ap ze*vteIn(-zTRx*n!qZDBDZNs?_f0hx3cVpK&_f+sGFD0S&i6+y4a? zq&Bk?6f;K(#gTe5wB3gD1KfnXwG0NcGiTWJMQnS{I4k ziBYBhqX%oXVPQcSSbHAOT1nQOgSogfdR1tkCp~$yG(Ed1LDdVv`wo^0oi6>?9JfY&8%r-l}UBNDn0bEg(CuPCE z@j%9tX9|~eV6_ccYXdMutl*yeNW@F%n0P2(fud;}1Z55l>!&myl*NqNCkMX)3l`4V&d4xKz8R&bd4 zA3*Y+hN^_Jt}@i1k<93C-pZ#qyiA%!x5LK;`yfYQQPhe!q$oO$yE zsgmCbg&zj`*H~Q`EBeXRU&tW&LL+&QqHUmphCn_y^Yejnt$d%(I8yzc1#G^dwV&~u zLsZ~n8$NAO;3ijQa^>%*{p6cuRw9}vQupp3!AZ}-P9fxMcXX1-wMO8;#`86#QUEYj znSm=130 z!3%wO*B;()3iZ{49?LR$=Ms%z|K`LD;EAsC@KalNOkfnU>*wYBKRzvpC4c(AF#;>G;=b4>EbF<#l$CK z$q3}dZmS1}6oF&p$p$1vDzx^STHgYluh7{O?n(nrPR9Jmj}@IS*2x$C`vzWq@ofh5 zX$20RI@fqGS{@+c2UOj89{{wgzkhIlp6%;hcj{Lraae7guR^^vihP~lHr@z{RS zztD>@ z0Udrepy|SYMZr&be#)>ecPQ`^e103aNN%;=b2f+#}loU1}B#TFV%re4KP#(Y32i+#PFOffEjEk|%2Roo z>`tr;Z?xlTp_-=HX?=)aEJvdMLp4Z0e2nG1CSW}bgaZac_f@#2A3C}(7EB2D)!|hJ zNEQG}fj}l3FnOWV^aiBx6)58cRDFX_qR>sU@8<~bUL#Qw`FoM?MOT02e>;*zYzE<; zXV8XNmJvv)8^~YrgUvOzql41{4aBA!HOFIMNjOsNXC8+_NA_# ze!f6bWOPAREB$=jpui%`yB3%z1Rf-QFWlLPF z;&nc_j?VwVc(O~#4X7r9gE%O#5LYYkQ%V(KCWaYwnB-r%jE?)l3gZS`R)=j`U=>-3?`&Q+DpEB z1#V@JRefmBo%-STKtXJvKwkOb4#fzfXsRe<$TQlamAawB8$*ZbV4k8I_*M46xq>e@)-1a` zG-PK@__P_1>e`77dkIUlBh(zs80{IWCmbqu(rvKMY61zd&wrnOC%b^I zW8L0j4if#Zz$#q9NC{qEJV`7bdHsN5wuqm~x-?-)nuo2l?kkM2=lmC2>IHDZ{PVvyo6X@;|Gcs~rZf5wAxjlkA zuVVXb#j=*l>I>LYm*LrOP}ohh&kg1t!Ae(Q<%wsynQLyd{%Gh^wO zS%plVt=rpg;kU0){SW5;2v|IWzP-_;Swt(ucClj5h;>j4z6xMIuE-RT5!Aa8R~rx2 zcEvvGz*?)oV;1Piz;zPY@@8ehNaR*X=f*&zBzP*o?~35cjXkwbnVncI=UB~kcq^2Z z+~N5jb$KG3FOtWOSEORuT_duwot54KW3ga9k#7WM5*zFUt#o33+29s>U?3&@;l=XE zyHZHm>^e`ihQo&A6*PjHM5DxlFY#IGK)-?9>B~6fnYjnj#h}|A@o;-Ga1hVTpZWKO z7OTKN#lUAxDAf)=Jy=mLsLl=f^O2eT_RYUB3yDHSfCD*0@dK2RfJBh}t2@EOHNH;( z?w|O37fEo7x%dJvE33bZtPro~7rY{$WrF#)j4YK+B3lc>KTTMd?8Z@oytpM)q(9>tG%0fQH>e4{E z>iNRYvL9BgfoIGm9iGe1)5X5@*11ZkP`rge=putxGIN($*$=G8X3&S^L|eh}-I%E# z^AetN=E+=r!HVnt`cwN z3XuJTOcME97~E6_`@xJ~k|z(x2B^h+)4+j5>|D6QpLu41k!)P!#yv%WQx5Q_fdjFh zf|;%OdOkod7K}QuN%AoB40!V%ymVcsFdyzN5AMGsU7jO10^w=#(F9uGS;Ku^vRCah zP?tT$VzFj|okXq==ev7Q`x$V5i0IB1yxlHDhQ{CtZv~GB`0NIEyYmc_&Tr|gTcX6W z>rgB>y1w$fVAficS2^gQ0hBEitC)!6scc~ zxrml3!X3r9Pb^&r70}JF>Q?G8{ozDfW|BP^O%!CdZr^o6P8CG|+L2$yflUBZqJrV# zz@;2CBWu=Jk61(^Qyk1yY=EhH#CsjmWiiqAwb0Z(aP-@^i-&hEam`$=+Qe%gvg#~r zzN%B*6FB?{)cFMZ%Rtk4LeW8RWGmq3#R@D)2eG*PSV19v8^E16YpTzD+JYzXy!_!) z6Fecixcs;$C;z*0jY!K{NY<7>s58>1Aovj*U%btSIt85o?t9>x^UU}<5WfmN?SY=Q zGtZ4kniIT#0>#*Xzi_P=YZe|)4>)?JJ$vvq4Kd3 z*}?vUyThTzYm6bDv`9=1C`FGb5!rN{am1&7&icHVO=+-Ogdg!cY67!b%&Q&L-U_IOpdA}A({{Q) zBl(RXysyoQg7~x>&`V`zlJ&~L%)hYSocdjD!NZ?;YzxRowZ?Dh35;ta_e8VxMUvHn z8YG5Y5_tY(9gl&V?5n>F&0l7QyP?G?MBo1+S~&%(l$c>8YkLFdUWOhX^ZqV)qWc)K z`zW+}0lJmwax6HuBJ)eKmRiiO68M#TF5%A=X;kM+ z3|y>0$ul*FN2@aLmQY!4UCKOVepi6u3TC?w9(lmpW!+zZsC0KQfE)4ec0voX+wcrO ztC65-5PlYVbAi|MFn5veF~BC36_$e1^1`bs>y>?1 za%Sx|V0RK6Y-e6Gcr9mU?~xRe-|^u}cU_A3>b#j9suxeeiq}#FI0nL_xsVP)XfKU> zWZ$3#4f-7(p&c9$`V5R7u%a8x)WK@R zl=r|S0{BS8FoGFp!VeOet<0U38BKI{b*O4EG}j%@3gLf|3KDH91^y-W>4U5-uhV*Y zK9hLIAJ9ihWR3^3wKFd>JZfUzUywW|__2-dsRZf#AbE0kpzjAIc>>iOj3D{*LX27h zyfk98nm`~Yqe*T;q8U=3YUbGrGB%wV2OyJr0LxY73KY)aXlSfwcmmhqh+ELy zL7OJU^`}BoBs>)$_*B!7DwWTbm-(=6<}GNXQaY!@4=#U zla{Il1My#3q1c28j4e?)v8l7OUeWNyfXMHRKrHgppvO$jtTz^nMnLZ$Sz`pV+YhI# zMMFzAVi|N82JDUC!@wvxp*xB9=3u=wkV(~m-(TS5w;v+bszmW#G1f=sVS$bs=@!UP z)>MVNh4#hDw=(ayKw~=^{~!>S7?|{7k^S!ONS{dfPI4PZ;8n>$Z`3hh!`?7KkrEx2 z(`&0ShqgSuCmOyRu=CfoP%QLb4EVS+S0fnkfkp~LJF1TiKM{D z$rk4Kh!y3;y6y+OYk?QxCyA$t1d)7eE##Bf36kOZfP{X*Q=T%TmCR!w94k5ON6bNi zZa)CgZOrflPx!>lCC@KD`bFmb8W<|UNY2~7s-{M&&A_= zrc0X}%;za%KSxts;R)jZrRY!#;yUR*AYEbNbjp@GAur_D53Z8T%mrO9NdFwM0uz{x z#FxLY&eup}iQATjFI}PT0A9J7S$QN#7kEKp4aIcHQWt(Iz;~`tn#f#n)Dz_dSQdw;^SAz!{;?w-IWUN?(y4D&seYvj?DMYv}fDX}GdEu`h|>HAk|P({;lq z>^X_x6l0x@;PD`6Iy)=thp#e;YsFh@z@3etm!gd6%@q}}wR7qGDwIsOT_{pKtthbn z4fshs#uvCtN0(3hWI~H#<%t&^#wx?O%gF!5kTO!Y;LgbYj45?6U7=cmt4L+>lZ!zU zqLFekN3qKVpArR50Xq@}kyw_<>EBtjTwvfmu)7XqCD!zSclUvu$W$wKgGgXE=3WAM zmV*$ zil^~j?2?{X6-~fp5Z1t8J?dARSyy5n5*<>3xp4dr?4|YCsnVZg7XKZEV`Fp)ngD#o zig^yEWuN;O{uiEp%YUB0R0GF9fwkBc{?LeIbp&o^bYKQ|UW8+JAjicAzXOi$!hwfa zjmQ?Us(#C051=jC{}Qa+kJ+_AQ}u*4hT>mTfVR?jY7}>g2f2aY`{57i#T5frhXIFh z#@xdb4!|FwJmoUa4TsO9PDrfN@7yn0E6LG`wVw#zN)1Cn=q-?$OHUSw{G9*_^L0tR z4;V-%7_oDALt!70zXiaIL<1z|>xl;##af;+l63T(h-cUn`1a?gvkq}H>;I@j>kRYS z$*f-T42cAWA)8WwnyT*rdcllE5}XIBKadr6W-PCCpy~}=3P5$$pg^%iB`+(Pp(;?a zWPv5FEF->zYO{gO1mOA%Nq!&roP{1F&MkRO>Eq^!bdy&d=yf)^o%LwwapZRg!Ta6e zMd<}N0NvdUN-4|O;z5*xcf{}h@ITD$1r`UO=2+lr0Q&*Dj;sPyB);$``hPu9^BsCU zgI$ zQy8fc0Jg;TkQ{a@W4Q2tN$&f9+fXW_#KX_VXURP25g1$rK6gV8kD&mmM*7a|r5Y(! zkE;nB9egHQsSq?+1~`dTl7&7g!n0iAk9@$p4DZC|kT{OSx{`Pw5B~S*KF3m?agA{m zq^j(+Xs%ruaht?&_bqm@NUq*T$nSop(pdvIWdEG)_D;Lm_3f7!_UKS{%GRx0UfgLJ| zU_3dKA{QJbu#>7cfu;Bs9;~uDIIIgMs704B3M46w& zI=aH&5_K6)UUWQM-$2JhcClOVniEW2cnWC{*Lz}HbF%w;3H#7BuFyCueefTckke+oX}BK z=1`OCIwBXOdUq*Nkg3Sxk?1BlCF2iX4fv6&t_HwH{Lw%lAiIl-ac>EpeV4gP?0+9N z<_Id8IYAW~k+{xXe!t_1ubA^=aQlE6NYq90G(w9KrQVH(Th9nP$p?vNwGiq%tJ9ct zJ`zZ#a*xbC6bQcuf2DK^s16=QqnOCdh$pXM%V%R$;S7!IQ=nYQHe~Vp8*6?9c4SY> zLpW?7V@Z}(EYx^l`9S}4J=$gy^dc2X_qkT$s*y+%(IENZWRcCqb&m7r87eCmIc;JV zlGAty1xO53>P?Ol#r+Fdtm0iL5D;x}2?*U~7Q#;wACcIrL^`tQxzrEH_rzZtMl87p z^dTN$5h%1FIBW!rrL&m$K?d%S8Z4=f5^v!%<6UO1iBMeEK>0UI(^I0pKCE z9+}KR>emvPaVqP+&k7@%QePL!I#E}^^ZJ_}{Pz-&Gn$=D)0`T_Y0+#wz0vgr{K3nQd}t0zFx z0gS|d^Z*A1fo2@5&D5hpnhtUCpB_SaQZXW3(3`;3jhRPTXg3{hmiT5gbY^7~@vk1i z4PnSi*}D+B?2iHg!yRFzr z=lMjcGJ;rp9psvv@+1}7Vn<80KsvPfnrk0$_iZpDJx!yTMF!EeLcmCB38ey1BJ@U9 zDVgIEj3@p;DkF#|EgjQTT}OybF1bBBwu=D{3;=SOU|-^yXW{avKqC&xBvzhOM#XY} z0#AI!d>*i_6==V|sYI73>3yK{6&(|e6cEWNR#!RJkpoHfj+d-46$+8Om{@TBP>Cz^ ztIfO%>2WHl68j1L_;Qz6cD`I255&bX`i1T+tV@FLj4iZT6W^{qIC94de$Pv4`^?NU zC#y9<>5*{0R8dJy#P5uUKjTTnO>m<^v(iafs_!MwD%n%9b3{751CPI@#cS3gRc0b{ z#TO0*D$>7HD9Kl!pZNDV!BQ~zl4!36CS;#W4c66OuNdlqjwpb}H?yi1Kt$p$lA)4% ztSlYwIf0SHNzF)tK(6@*epp4eVg%W!vB(zbez*w!kj|M&Kv-%deYr=4uB94k6Q1Qt zxbh>baxzcB+c&hrKD>_&@M#QsGMOhyC7whDt1z>E=)vhke0n2)CF(B~MiRf1=x7PB zBfg{5Rrey_ISuX_gXSE7H0#CRK1dIV(w9U(=ldU@hoIS|_Nf-QG=WW5z75b+-FGe_Va3*v`rN5#Md^($68jPO7JvI7d@~)N=_2@$dKIZyzoo-p z0pyK=0C*QY;1GGO^x#h*SQqnUGT&q|cmG)aJs@ z6HTguo%-0&t#zt$M>|N(o7nqek6z^|_n5^!=>I)qN`=fbp1K_??+mk*y6Z=9<~1NM zF<0qGY+%JcV9Jkien%>uNM+fbkq$3EKyn-fKYN+|UN~Rk?$T{iYW2Usy(w_|OqDk-AYtU5=Y=Cx9heQqQgZ)lCSE}#?q87MCdUtk&2AVR`Gv+VV+72{I z9jj<`|-KdD5{Jr^@d4sn!xfuWXfEq zb`L+tk(yG2a0S>LVs;W`4bttsLX7f;86SZs%BjIj%IefA5p}_29bny@JDNhbje&eeDC>~g! z#_scsZ5oQ$8V7*5>J%lM_RJuU4igFT-}NH z5;>RLmP8+l^R6|YNllLhT`7IP6X1D~8rP9G8=!?n=>HkSTK4Mjwln8QR`VR$nv;>* z@a!>I!i||*0PvPQ*F{;8)X#k8jw`@06*(r}!&_ai1tKvmz;Xx~-+5>~q2=nxy()|$ z9WVQ^wq`ob^Z>qdu^-1`J&Fz3gOz(Rwpb2Q`yBQ+9JIAtIZ?E{1xv5K0&c?5Bl;YgK1tgI#Y3IcQDgGp~~SDxU3 zmbSuCvTM}BDn#Rm&X-KP1zH=d^LHEgydP_xz)v?gPwWrLGGvD;y`fF17Lbgg>>$1d zot;P1Yz1Gtbcrf4iZ5JOS=TBJbgNXJoDZx^eSQWQaRP;dNaLlfNtpP-Vw%g#=;3i^)4VD`F};rqeV#r}(af62KhUBO!@ zr}yQh4`h3K>l~)vZmRN=bD@XRT`PgTGOdxtkMZ^Uu+uXyBbH{b>>xbKdr(VhDEkmy z2Hoi}(2^b`3pqXRKTgnW#p!hy@b2rv8*jjgRIaRJmW%P;hSBxqKlJSHO!Gb@*h(NF zyB}Rxb7g2;yrFh*gVYYmX{INUb;H=h=}!NoXIccO8o0A_v@*W?OknX6nNkirXo+D9 zQM2vXXQQ!{{)N`Xx9Se3Nv5Fyu=EBOawdfAICudj#Y>VC6$Y|9`6#>GuCV+5E2kIb z=T(HgstN4LzrdKwiSadoq9vmwov0rH^9#f{Co=PONNw3&l8iL^i0*#^7YR2-qmjf5 z746m<9+4WZqDYSz_-qf*oQYR{5KfrKF06@qJVokxEYP4t1SS6_vAN%ouo`fYRNhKF zJRL12`~4)pRuIfcFBXYKOY})%Vp8{(42?=1(*8&Xythqzi>FPnN2uM1K77KYjqyqv&4oS01pYSe};y2zY}biQ-)1Gl@vaj;DB_ zDNzH7+lt*>NY_$1pcp?SbUZi_n19A2mi+-L5=r7k#kj5{enj_C9A=^eH zjeqYYZh!^l!>gb^Z^?xFvYIl;mRG=IFLbyIs+8{BTcE_Pz<39?h2)K8r{6g!=0826 ze+ejmVy>rk`5Ml;G-Byhd18Hj2QwyTfU>Gc*6+?NWiRj}C@>9gK+Z~#T{BXlCE8K& z(**fB4!JpltoJBg77v1NMX!iYG?f+fQiLL zOEqR2guauy_Dax3T{K^5R`!PI#6NJ=Y-n&Pa1ySU>Q>nWCDDp1K%^l*d61kN;qg9t z1?YGrZ+mu)S7x_xGrAA-W>4^+XuFxfcMR`Fq4y3jP6G6P4_og(kh~8(?=k*K@UfOF z{zQ*X=b5X)%zf4&a&;x|Ut=@L-Zt6CBH8{U;7%&RN70v{HxaWLU{(B^+gxLU!lV+R zDH8IyVW84ZZEWKpVK6Ig#4qGL<@MsQ*-GTxJaq2URBG|!+M!j%UmvJwhXu2rZnP(XN zCKhQL9YH(Ng|ak#bxzR}cp6>zhXAbsbQ$QusmV|2$o3HZ)t6qvr{G{8qTpBYV7j66 ztaR%OrOQbsU7teOt=*4T5BmDFp`S=k-j70}tN{-_z)uS_oEtqi{OC>gl2cy~(3#{Z zy(B83yB=T@ccMz;2=T59$cGisX**=uZMpzFWk$8=Iey-;*6|;&wRB#Jp*Nzx)`x!E zv-!5O_LL5;Bk3`n+qTl$%-Yi$VSQx_bqq7CGM2U6azCH#lUGxpZ$58)Ci_(NY3toD z`v{N8mW}FQ$NtQ(sd{Gmk zdR@(aFk55KMeh4tm%3Cotx{67h4w4fo>_}CTV+0Ht20(-e#mNU zi?ol_+zsE9XybeHUCZBYN$zf*8$3sPuJb72UdJ`oT-V4s+}cUo5m6q8r)12v>*+0`OtGe~H?N3KfXB(x7v8*}PC6DVP*8n#^w@t3gEfrl> zn|7%444bqtdnsFVmX^6K^Gs$)R#9sM+d+F9&BO3a>1*6&+F-8jGR>uzOPKkNX`QjQ zdcjc08Rl4FUu3&wt!xdjcCm(Ad)ung&)VUzARVsI0q!oRaFn#|`IJx1apb0h`Z@ba`xCn_JtSitE$DC59}i$Nof;3&-L|zlkkh!D(<^Bj9Zq|2 zO652D#71$gKfTMo+7sx|o0Hxvd*~%RfDVMSkl`)Z4|A73OQq-qUyqY=+?C^K$DN$g zu$;brnGQ1@-^N4L-H=N+3~!VI#)hVD=E^STT>>oCEoRF~ml*Ri(=p=)y5V0%B9`T> zUprlVedtQx%JIlClumjlv2%lwpS$TLJ%t{wpA?JQnA7B2s7=+vs!HeEhn(`XknUri z$^!OO^rX9Z2Q8G2izOUM_Ve~h_CfX)_Q&>4j@nutB+OD^^^eG@2a$VkN#-q zk&4^tZ!}O#p=ak7`Xy&@j@@PY%Kn3V7Mt)A`_@}m{?>NKnvpdyt5DW}toW=m)@L?PM=OmE#>k>=Y7OHY<7~QJhO3@xMP;O659juJ zXhR%|?Pbh)d$w4shPjhzIXVx^6RXxZ@z|q zTl8ab;>y&8S>Lq=rb=!*vkmoW^$lM=}J3G*FQ>^?SEYrCxY%vYIlna?s~GBYzvW(8)g&w6aF+^Z|*yJ*5{d-yGE`uIjj54_3i8R#C^SKwqtPmfW(X67Jt?fygwEG{PIiV?-PIh zosuK-yFE(@b;<7WGkZPXWWPS(cdg%3pAOj{yWg`kGM6+~P!hEs_L*7H>AO=~r6eYI zNy(p@oVq&ACB1g~==2xqyEA@e-m%tl6fg`jUUu2*w%TL0=PS?io&!8@d%SZ$;r7|m zgAEW(v|-j)>7P@)2);3y|rP6$?6i*Fqf~E2-ik#b=-cr zZg!pI+Q!x7`oeP5Qq|JarIz`q@uYIn8RckV&tofNU7GbGGhgPOj6E6kGRtP|vo5r6 z&|Vpyt1~%?BHjGT+`>G;)ZX}6X=Qk*b%jHhS^H$k119VtQeA zvD~)UEek9oUGA6;s9{WRWuo7X6FxRv+ zbWL;hb6f7Z($d;xp6QVK(NNL3%u&(a*7n*u(b~&8+B(GA+uG53(E7z%+;+>h%09u- zOX~+MRi&HVVD*&hZk%YO(>lHX+Nu>4@_E{4y47ac4%tjtBki1z)O?m!9$sEX-!nPV za#YCC$hW>vgjZs=JswqE?-*}uceCoJO-Z`-bL@{&Kd$^p{OOmJkn%CzVjb#;GE_C~ zvc$XlWMAYRl|9bG-L10A3}b@fPwlepduHqOjFcwHb&_Hd4M}U0MkjAc z$&nVC-XZf#)*9Qtjy+C)^^mEKWwu+0$9T^lo;y7+c?@??bp1b$t^zuWY>Bq__{1GZ zaCdiicXxO95Q4kAJ8W=wcRudAKmrNGGuGby@9fE8VRu8O`@MQq^{Q^&Tab6MsnlrV znHHvu5q0k`cW2kAJZoN$yjHIB?wj6H;!(MkmSrrVLf9Gn4|AC9i@liB>ns*<(3#+P zX47|mhT3-FCr4Y^8bW8ET z+hOCQywTmFH$)ByZyq`|I4r33m0eGP6M(jxd(z;1gbOEOoKsY;zCN*n97V0D|Uh)spd-c}yTRmv6U z+Ta@R?(Z4qz3p2q-jkxlRC2xM)~?8O552QCgA9x%y~Z52&E_7=6lSfuWiGJWx$ zey%&YKXX!Y?&r>Tee;YLen_n9)y^30kvCIJWp*_8l#en`x7@bI+veI>TOVr=%O6t> zz9-w4u1iK3FV)KOTA{qRkGrueFK?Txgr~nRP|8xEfJ#O)UAdyBdFEc0^Ogk56>|?$ zTkZ$VlA2aVc`Ftdf_=-pK5v9DPYjU%Dxb93Mn#gRuQNKE%}wT8@pZXb%v6daW@tl{ zEa|j(Md&Rg2p@!DVs~kw>{PF6qYNvU_ZQPOnBj~C5#k5W0yKr3X^7Olzfx-b`=Hoj@+KLV<&YCKM@JcwKC(7%IXW ze8JhXy`+7?`el6`J9;iE{V?LidF>gf9hw9T8k-O*~?7!SV(|+@7%PH$0>vpSXiMB+VInyKVBWq!^=}XjD@|7`C z3sP)SMd6HB_k?%{dfWS63v;Ac<+R#QcM{V{57m?z$U3;e+$7G~g$O-#Mo|<@CEU~5&Y$&}Stitb+M^JKksjqkxk?eyIDSi?iax1Xl z7t<7NG!$f_^(Wwt4$!>Ncj}~%GOiIZR25``%NZM6m9??YnNCbR-J2>$f_VpwtRJ-F zP-$3#EPs`@0-6{Pp`24rDK8h2nur^PC?Q%{DRdU6NO^LG>e8E&cj;$r7k;iO!2HJa zf-lUSp!X1MwL4Or@2u-)&cduJnWZw1XO+&o@5|7tavL4?u&dF-3kb0{VuutclrJu7 zN!Y}oRQpHMTRKzMB+ffLcVgz4zbR?8|D^pbop}(Esj0iFx00|P5!7r9qY7{_7Q;R- zuv17(*pIM-VQWJl2G0(B??|^kF(t80scrfyd8n_QYkGFEj9Y)^rAnywOs25NnaC_kVz$*duoJH)1EVWH}>`VF*xy4A(Rw=useZmUw zR`>6`1-ZL(`sbX_*_+$jb-^=IXe`&$Q0JqgxaX#U)+hFR&VqrD16Kuh59sPRW}Ri? z*;V8X{e;>?=`JsmK8mssE6n%(@-Fg@_8#^I`Cj>g#a&XQGC=L9H3Xw$5D`tbrKZp$ zm>TR|_A5J{{fw%`cj$T%^uN?Zu=qbl&G;F#QwpjJ!JaOA@Aw&7^vA zW2L_OTkVNFy($#BkAqwGjy?m-;n%hKVEdnrf6fG7t3%tY{sEu;EhS2G8_Be8YVHt& zW`#Y7NQ!J6>4@kXmK|InP_iGj1o8vvYsLd*zcA08mQyNgcSb_Sn#|4F#q$UcCD`S6 zYGor3dPf;_ZT1G&&6H%Ot)HxoY}IXhtUD}S%qst!JItPf{>x>kKFl-oWJLd+MoDPPN!X_{-cS*BR7TK=&Vu`DoOGri^ya9!94 z#sh}d!ej_>OwUrE%cI1qzCj+3tBq@i>#TdbR}$9AVOnK_CYMql=#uO(ZY96UbkIE4 z^4ns!ezUZ*yfHU3k2baCJ8|!r=hSs#roKUm7ms*j+&Q^hbL!=6&56&YU60(iyhaj4fZ~7(Mbr_|(wB!BqoaIhxv*oBzip)1wGl&GhZaYnatO{qvv2e_H%K zlKw99S&32mEk+w^g)E;s0SKQa@2ULd9JF zsBV@ep_BKVt7NX6wJEb+X6LN^Ih;G)_d_0}Zy?7pB+r`LS*loXTPxdE*)G^x*veYB zncwqPZXHvPrl>c_TMTHV{Lp@?la#v>E$;T6_D=Ht>%HWwE;f{3s=JMnR3+vSnD!^~ z3g6W;E@{6NMh+I8zEID2SCzcYxk%y!HRtYxAM!%Xr+KMNht^|gKj!dNzj<+ z#TMo(n%7vTI0^;64BiS0|x@vL3a=o#4lQp*KP&=o4{}EDCKviONscM~!qK)cU}BYb?{}Y30;!@)_x; zxK&&#W{X>;=D5wQRVt{XQLPxO7FMlLM(VDcOY5K4t;!CEF4A^om+fbpiCL2Fd&6 z6x^rBsTZ|iqd%dOZRtKtSN1nca!L4pl^#cB66K9UxCa(fCd)N&*Lp2)Q1+?Iw4?eZ zV+m0Y{lfZCV%bM2R3&JYO{e$Jr|3b@3OPX8sk&rYqMtEazoOMdJ?%MEB9g&5z7t%& z3vom1NOU2GQM2eFjGJk|R$;#|m7xtagqlV|$rqij+F%+?NvhEs&rHeDn*p>%2TBV>~#|KIR=19c0bhOK54bIb82TbSS_OFSIeqJ)uHMl zb%GkHo>4|B#gq&=7Rnwqpnuaz9|LunH0YD{K;Q2UYI)h{F%^XlQ@mafjMn|MS86r& zwlYUquDnvpL;bM=Gz2KH>37x#q3V?neTAJw3R#Hm&&*<{at-)D{7leb95FYv?6O2y z`&nC9KUtg>pJ_bLbBCC|bOov(IT$R>UTBo$fSWm6?T7s}BX`)Q)`jNGYJCzEpE*9Ko=|sbw3vwNan+S4AO5RPy8P=Ns>@!90x_FgQ#s6hN4G9+>;8Ra$N_V z2^M{O2?{}{q3^zp+z5q(8esEn1l^=V;IIFMI^YD-KzFkon9?qyhj0niJ2y0iCIJUJ z2fY_RBEAi}Tj!zFc8u%@EiZw_5oI4ZT$=f&A*dHWJ+NKTzqs0p9PDc$WiM(RkE>UTbai zo8U|Bg>KhlRB9!3x@w~1=I_tm1mE#L=;#$f)p8Vi+k2tQ@djFP6QB<{7W#LGP~EK# z%^)j!ha;c{p_5tE2D%st@UF;Kf30s&w&-7$`^ho+Cw6i{t^~vMt(tJQyZKf7e-KZ!k2m7)Rdtreh-V*fe zT4A+61C8!Rf7=B;rgg}47DC&t0n`S@z&3gE3s`PjVy|YSPcadFy5Hy*-a-dA9z5a2 zp;_|)N>0b1cfz2jSUgM-bk_nl>iB5XVr}yN3SGaC$z}>Bd4gWj;NdHq)n| zNz|I$i;D6IaEj;P4MyN+DcZkKO7TK#DF8dV57k-ZAkbbu4BOZS)WRp|@A$|C(36=; zuZOoON$&+y{sf>7cA?`u*0=?AlWI_mtN?{261_4%1A>1(eJ8p>lhEh%Ph_>QuXNN^dUP0kt9}5ReGEm1?&#m0 zMhDbCmsS&)ffr;Vbs8sR7&Q%w3#;)YiRkh+K=t_y{Qn4Oo;8G)%L8Zt)I?_49_loI zk!40}={Pgrpe|GoHd6x{dgIYAY=B-uX>=k60;_QjD5Ueob>yd|@#jZIJT$P9V1rRq z9Vpknpvuv=sE=et(vOSliv3W4lZey1YEjy1=;smOCY}qd$0uVX&?zs8HaPWbU~#3$ zmt+oEk@8W4>DKfdN`Zffg+@bhSXxWy;*5u$T|=;=_XRiiXsr4HeG%BX8{&DAfRXa6 z8n43BIdBIm0Iu9|=s51jiiTpw!3Jw?iYn|BbjD7gmv#wi3}3T{wsl>IKFhjR`+?aSa~zK&jTV<^tGGM?(S(b<~}Tx1bsVhymS;f93g zxu^#j)ARy*Dcs=7K=I?3-U@%O7y7M!-Uu0e(@bdnoX~4vkLRNAHwPMp|00IBN6)uA z?8&d5nN3_H6=;jMhj$$XjRpeVfF)l+^XM>n7yn!bG%RCO*RMgl=Cig7_3gR1ZREmF zo3Jb6aK3LsU#1Z`j+_S783|qR{#gBk@bG1zt56PoQ3k!w$LQ8J1EZFi2sb=9*%#3r zx&W<)3fKh_D{>zCh*ltN#-ii19=~r!4}UtmW^1S&)F%s)A>;%w-mZpED?s|-MdQ&G ztxHtLI(&eiX@PF&abqUpL;_F=myNxMevk1kFLA=+@Y7ZR*U%qmC;==+wc!c&!%L+C zh4&ksWD9ny1bY5E(Zvpdwj7DijQ~F41it?n&Q><~-v{9|`WageqZ3{c+3R{li*>Lk zkG=souQC3)Kz%);$Y3LpXn`Iz5C0cS42L(~1;#rO{rWpbJA95G%XJNww#|qFVyqdu z-YXFw(%}Ds!7-hTuIxyB?sRzVc(OZ?N5#Mm9*@sd5z`t21Jw+gber&0-+-RDj*j^f zpv9`-&3~fb{sbMX{!mOPhtqWcovN+mPBI?vu@hLi576k{1csSYdK?)3|7WZLrlmW& zzKijHE)%iP%L#>gV7d_jn_Le}%6mLnA!Kyn=;_56pY*xtuMs$z-;DCWC?p`N`QWwd z1It?#c!)tj22I3Q928hf029y^cc4eG3Fu-HbMa)cz#a@n_46s#?f|rzZW5cw=R|w* z2y&xiz)QG^9q>P!fgd^wi(HExe2H~A0?WFEb?Zh91@d7r^wlwg2uqlPSalxJ#7Pby z!^s=?^Dkr>ClM<=(3F|3Cn2Y~tQAG4`VL}7L3F~yfm)c3oZt#3Vs{y{5TzC)6L3Pc zbt14*!=U960_BT8u#=-WzrV3ge{qJ!;`i4;(e}Wb&P8W@J#6V4v5=gPwQG*9Wohh~ z&ln5q9)*b4nOKEx`!T4hEWnDtgkEF^;_T$$u#I6VWJe=GT|KWF{p~oW|Z_oh{fK==Q4Lv_^&N+1M;}L0&0R@tT zlYJ1lr!%nYbFiO@z)!3Nw&E7*&7;9KR0VjDX!QE)g9X10*7^dlHMh`LEr>nKj~pZv zXW}vZ%1G4jEW|P(EZoRi{Q6f*jj4G5qv#0UG3o(3TN-`A4Zv*_Mckf%)%gx9{tk@L z6l6E)_z89K|J^vTCC~}3OPKIBt6=xB#5>rm8#dh@*pztU7VKdp*2#gN`W4*d+Y$F$ zLAT;0YIjUKpY7I-eW(Up4Go;RU$tgCPVN-ggoYbee_)<2 zVxDL_R?^Sm+7DQzuE5-V$MdhkPPKt`Rf2uZf=b2{eA-7~H=E-9{J76m*z=12eWzdD zygf7q{Mv^p=otgyCm^U52k%Sk(rcj=jiQ z<6$`;(T&}S&#H<2ss&qkjwh)>%mL1)BtD}NxdnZ3FEBh^(HXvnJ#Pi9Q%~4KBAzw@ zS=y^@%!-1`S4^8tZp*a%0FugtSWI9t2)eI+d__e z8OW_Oz?IxYw5g1_qdeny*Z(7#H7hYG;CjzV1M zhKLYCe!zbEaaaxEqe>w*)F3)z{Yw}-kW1Y_zkVT*Od;?$5#axy0Drd~SkTMJ{$Jy) zpTv6Z#y?xb5{6-CdqNGM1HP{@;!zNhh#N*RJkb`+@;v4xla01FtGz zO~D!fe2;&6D+xJm8{neeAewGQG?)#H-6^b1XXw;r6AIqB1HLN&$YQ@%%Xi>JKEU?B z!&9t>H+=yc=!>UXja9CQ-70`gs}gi2(~NnDt_g@a!-+oF@pAaPwUGN|V#Pvn%llw# z!rmW%)po&`4Ytt*w&+LWufcb>fpv3G9Gwc@fZ~X82XSKdMi#5q|1ZAf9H!-aF!Ko&|O;hZs%fKvI9NtytfL5Tu@2d4HmXn~U|YX%no0pL=fLw$L&WM0|IrY+ z#}lGAISXFAyz%EhJKl+Gu`R4>88VJ-@J+iAQ{SOBG!T*Q9-e4A-oCYwkNioLB>fXP zyRi-*kX(P z@{roN33SAp`}GE%5M_ZuUylrHA-?A<&T4J=A_How^WkawVm108H+qK^_Ul5OhtKAr zZGIC`U@hWyENr<3Vj5~!u+1XKB5M;ZjVyg5{?007TYgQvr^v9j;E8CUR2w6*)PZM8 z!-`hJo799A)Wff>u;2Dr$Ki-<3f@Zs%Jw<-@Gd@M1kUcw|15qEvg&@QJJiKle~Hr& z5C7_qUC~g0Z;zO?6keh}(ENdjOP%0vM#1MF!D`k=ZtLfGU4WRW2K{5}Hju?lwYXJ!k6WrpAm*$dvL zBWz$V_Fxh$HWX{x1n;p8aW@3k@C-RjB68Jtc%v7Hw=Y8Iat(Jh z|3shv=Vig~_)+=Skw@Kv#T>;upMw`DjrH~Oy64A<{DCv>#eVp?3iiNG*20ebv$N0f z^Y6gp`1N+C!=sJDEJG*6WEvTEWgu|X;!&#c7H}&dFg;q82>FeE5kw$Pbz!R$hnwy+o9rjAyEZU48{y@#4PH8kK}V z{R^<2N73hNuO&b&vZodYH0KEXmmj!8F4WCfTNyWmAgE+lgz~+YY5|47{7eaEIdhkJ z%uEKRc0aS2naA{CWV$01-z!t!$bRGltV;+gKUd(vcH-AUtn6dFcMD81oJ2m{8+wo@ z$pET8aD0B-Z8s4=KqSILTi2KOeSLnU}(`2uT%0~~>e3qdG^$%*HC!mBD z3_rF5xbM=qy(9qTb_yqbF?`K$ta5Rv_qlKv{Hm{les?sqZhvVn@J<%gv-;_?P*M5~ zZLhz$vr)!LsGR=;t*<9gSUZL4r-Hs`H_f8$REw#yVpY#6r=XR*6<^Dhv&u?ko-$6U zuB6G^#xM4GS?spN98}Y#3D|YGkA} zd}DB1XFH3Bd1}Nd`Gvji)4BHSPZ`bAqto+eG|ak_d(D$Ame#__rA#7!#@y4o#kK-8 zOsnlvY$GfMO^4Y%bP>u%b|S99)8*pbn&Qp=KpFl%s)c{mg6hBWOX-2=60QoAm>~9+ zHpzdKhT32K1(8NoW&7}z%>yj2EXOS2mRe@J={nbl{fG9EcA~5PP+g+T#^3fx1Lf~> zM`aCgY)jM-D9-OibP9wAs0BRpBT^vCK}DTH2V)%2X{V^_)D`GG>&WoR5*JV#xd^1` zD9x$m(~3aLu?L=Oq}CKQp$sSmhiX-|wonpnq1D#v!s5uyRB0d-Z^QeX*LwoBV#pzKpwwL41eEVe-wfX^Uu7XlSSR+7u1SUDB66N|Sh|8I z&o9Txo8%n%tg=bHsM{z9H_}qaIXyTo>}t3ra$VGm=mRl{Fn9NGVkuC4Lw33pIVky$w81-Qk{O&luk3Pma0TPH>&EJ)wgo6v`skUtLv-dTkCTQe}yQim;70Y(+Z&8G!M7q56EBAq3(QE zU9BX^=Yhb96JG%_a7I`ozL72{)AbfqaqgEn!(KP&QfT#vkC7jvrbHEwJR4piY(_BU z+-<%{Khm;=M7N$(DQj8!sXzaw&Hhs;oya?D4)d6u=5opsn9UF?RnuEhy_MI7A&SiV?gwb;cs&ojiG?W*Yh?RI<0`Q8af zr5(yy?XWSEY)ao?3Uaah4|J(!n(vxPO9{(6bFewuRF@CnPBH!H*QApust;8k$e~h- z5GdU8?e+Zw)MF{(nQ%`0AXQYhshU;=kvW%`fvVp#RMzcOIK1LrC}@lSDX8y6T|z5tSZ$_t&)`R3&Ro&QY%BeqN2&jN>{jga%UIZR`1tWez5HEYP9jFk7^D}U3zqQj~!#NJBtUm2s1^TjF=u# zC%kEBLXg|B)3TMVL+qBzdynOg%W9QTBK_6ha_L1fGBQu(#JSBvj8e<^Onqk~fK<3- z>1e%Z{cL@OZemx=BJ>a>f+9_CMKMkLHN*tEuy_pm#V|nBWe@$O9$g#ah|LJ zbbT6GlB`15jECBIV4jM}t)*MyaWPu5%M+Eh+9Klyd5mtv4(5vR6ZsLmmG8h+hsVD_ z_AxqWE0xnym{?I*;OpiOWli#DA|sl{ zOv)c#;BJ9YaeE6*EPSir`+ViX9Jbz6Te+(zE~iZTxnKW$um2_BQ}0jLzm`vK^jFNi zE!v3<+!kBypqAm=qtE2G#WpYSBBn%S`;cnR_Ld6VW6Cf#X~mVL;v`=gZ>%@UHwHMM zQeucu)?38=Jg-Y$@4VZ216+gM>7JrOXL*Z$knUm5=UgB1E8=H#F^nq?iGCb0EwoXf zX5G!s(u0Mt+#cz5Q!`S^q#RFi{O+7qBK=xcLsxI%n7Wtz!nZDnJRQUDjnU4k>py`13g-PwVnJw@u868>)^ZW4f7uKqHwyQqdFlY; zG5MGti7AXyrhMkz=Go?@CX%OS?Z>cm8Zrq{$7TtMBUT2M_$ z6}dqX?WwX$UMoEkJBdZbKH_?5m~uhuMR4>y#MASpb7r&Uw)wkhJ->`irqT_MDoGuM zmfpgi1a~oa4fp@BLruKotR=VHE>oM#as?w5|F})YP#osPGaVpzv-#3QnVkFzxVo4{8y{=eR-U8 z(WuHMm|1(9fQ!M&VbPI~B2yzqgqebiIHy@_nRc*a>4Rhx(M7MOevwWJ9emNg`o7A( z;ogt#UwOZC(y}M#gyt=D_w|jC)~dISc=`rE&sHHYJLE!mwa7-1-+@Gl4hnNjHperg zjn;B$Z=Kv58I%8PPwkePn6~z>BlDl^MS15v&Ba#gaH1GvUucybTPoxP#U2ijYv}G|l*6srTRX2)*REyv82O!stM9hdx1FCT|nde9OIQ zo`;@T?>p~jUj%URA#!nLmD(2_SbxpCET+CD>B-tVl~&&%J6fQ4l>_jzdo-86fJ|Vc ztp@^Xg}^KJ0t=5@|q@i*Xir(`90d8#Kv z%sVLzQkxL@nFFTN_6tGL;VF^3qv9ibg-;DB75LNs0!Tp*Baj>P_i}Avl4rS#bZv3X zb*H)qy1Tl{ihV}F=fKiI=K{|KG;#K^>z4AS zB<2KBL%k?WcAv{3v+iYF&X|xnJ!^LMnVbcA13X2=+G|nj2YsmIUhv zYm#NI*~>?9Lm7%bO|~T5`cbVaX0N77Tg6G@F0p_VB-c>tsZqEsY%_|I_ox+2U9LAj zlz+joTz=Nf{Gwg}neqiF7fF65^~BdNNs?QupS2doX`(2#hJMeK`0LyNHjQpZ z?ZdfTp*2_Q0&$%S%&jOblZPoY)s2`1%0P`^EBZ9kk^hcG#jpWp+O9ymy&ftGyU=}) z#0_Y-{#C06ROxS}v+_SVR-P=Sh{MH0LVw|t&;>IOJ*3je-2)U-od~?gbW~)5s58uc zQ?A_1ENU>dqwaZ!Lf9H3GT?CIor{Yyp=YH*5th z$GCaaCoNXm>)DF_!x_1Z%l^4bZ`ZD*~6ae&GfVGu%gLRQr zMJ`ghY~MO6{xEQ@$!rZL#h%s-yqU(=OV{TDb?@ea_2{Wtvk^Ox3%n zUO7~5Dh(262!DJ5!vBQYQcI=2_FJzE%s~<+rV7ws==w}yW;J4NL-MZCTaVRjYCYs4 zDN+q-vlu8o6MhOEL@RDrNpig6P&=sy)%@CNZ7^)02WIv+=?65EHccI@c2Rq)F_?gw zh(oN<=xIs%KRr|e8$I&8<~}|`{uTDzxVOV5Mv;{mHTFD4!qNP>t8EpyK7Y} zE6wZqrff7llx&U;L=L90R;aJ!k`gJ#3%`XU;!3HXTuT|K(mFaB)LOQUsU$L-5L-Um z9P1IwOY?9O!R1ll!&R?IZH3QX%Dccb#*^g<@x2lj;#=c+_os8?W{H`XMbw>!@x}j>_AC zBFyn^^K}-&#Z>9EI+pm%OtLf!hzX64kRsnkos3S*mz_Vlz}A@3;U^rM*_B!o@u25P z-m2`$>EC_}KNEf|Oum?A&YtP{D%H`KkS4aZxve8KcxPDmi2dPJLj!{vJAc?HYbWy< z{tc5tKGh{9Nop(7VskM|^a6K&#wU30yLY>{dz$*bV&3eO6bg)wQ)^GuV>0;tmX@|p zwrw`eI?VdQvK{-ema&o5wW(5J-*orGJXh|++`zn3z$_N>n0*7q`brC(Cqpq8lFEp{ zS+8NMap$?STrnv}qfcN1Mn5?VU8#_s3ntRX*=>?$zAf+|hZfTx&c|fs!j}Uc*dvp{Fr@ z*-+p!WlrbDb9LFHbU|t|;nl}rax514DkF2&1=yi+Np*2C0WZ}q(T z6(_VO(8V1w^g*k2q7mjEUwJB;uRJrKmG57KX=i3_@5S-r< zOV*WNc^|pB+^LyA)4n9n{odpIhvb`oF60FHsw<4qm@32dw3Z4S7$!#6jSh*b6<#!C zW8faAWdCSQF;!zrlEXENd`1}S%k-Y~*7DZ(EN~^|4#_E)Qz5swtEZ^8e>bgwzTfIN)uqrg}?= z&4nwzKfq_lNQH1ePS*G0Z1!P9_82c=3UQfvzd6=yF$vrYwh5C!nMo5SLAz;Vv4Sg< zwF;qLfJdti)RAFCkRo{vsFpkU>VeyM8}d7l9+b90sSYd1FP!&&_Vo8m_muXw_MH(r zN|lu=S~XPmx8QCY%mbTNJ_wUB`>9n#Ph^5-*&>eijqwVeAn!}>UEf9FhuB<#Yu!ZNvx_K~R0ZMNME zN(^rlwKb|tWJGx5&=0|_As@EUmd1~!ZmG#aUr%#a|Gdb&_qjcDKj$pT$;;`Rm*=h` z#L3ZW5lrZ&X(#pe#09DrTY&$>zv4&p4fv;ge)9}Vkad&AW$MRuqNfu1^l0^~)JE9r zd6ic+r+?Pb%;H(+v&p<8?ghSO(q{FW(Ua!+F_upDe$KW5C_%ePcuj))J`Ka7Xn!=2$|_60M8E>5KaFHsAa_zOzB zyhe(U%u*q#m9zuI8%1S@TpTlmE9I+ls!S^7lr zGkZ?t*5FitL+JIG%!Su6{!F53Zn zSG&i$(maKWroo=1s#2gJyMN?rS>H4A(g$bc&pMrR&2`=TLj08RyV(*`V*$=BVMyacP;DCX5g!3HOC1;tc7k zd`|6Z%%YaEiI}RpWSeGBv)`~cw1?RG!(aT%^de*Q&dOIYNbq|5c)NQqcu)AE#s5iP z6}R@z7)Va0SlULh0!)Iu-nEoRSI;wP}732Gcz1I~!|g=fA4KH1k=I4|53 z%gJMvYq&>NB_@%>ffpaigmBYL6|H&pbI!T}cb$u!Wdg1Qo(b*|S}$~EaJhi-wrwVj zHRx;Pa=nur;eC{IGsFG&*x&2v3o^T9SI8~o`tDgIcG0#{Kl!${QqHvjX9DU3+;GM^ z2RmBW7h72~&&ANkkV|U%T4NGHV0QntF;1_c%~MrnkTOg@feDR~?$lgI&XDYV**kJL z#Ky&*;ER}69?2Vme14aa_aelV% zvF0!H-W0tofA3}|@L~$~Zaca-4~Y&y)-6YM`yy3=N+6E{Y4I9Jy<5Pnw~?z#`Nfu) zT!;}a39H4~(jobw@?9;c_XR$r0+5lF!FI3`82c~uZ>9tn&cEjinEILm%{wjqYypmc z9kQbyV%a_GYs)k9DU*W_Wmi+bjWt@ba!YO`SCd;|EzikamC9-}ZMhD1WXu~m$Yns_ zywUTtLfS0cmZl(E8?5}rv!Abwmbe*E2N_@T3QT0ZTw+@#BDc12Itw z$O6CqGK2o=MKH^?0B)}qY^*l$klQggmO-_pkI)`^0CScxvo%u*s7P)U*ZY(YA3aec%vJd8`Cf^^v=MmG(bPM6;K`i z&s+?|p@M#sO`osj*Ep?$HXC)3AwaBE0*0b6x^EIu2RQOZq!pf^2ReV9jXZRwyXj{A zw?^yDVF5AdYW@a>?kq5zf5=+YaB2e9ECB2qM}Rn8hu*fGJch0>0Yt<$bcu_hkMJ7I zXqV9mybmO17u@(4po{89Q`N!B%mu2)L&N}Est}pz!L9{b!Oz}Y0BFv`=t9&1D^Yh; z3RZvz>>7{|UUWUe@x`F)?(ZA7#wVD7X$%4;uoP+u_t5<~0(4p{avphHpQ4{kMm$hp87d;W^mG$V39wZA=TY-c>O?9F2fOvfh{LW&a zP44MEu>udYsoDU{0;Fg?(Piw29=8LX+;ykw3}rX3 zDhAc?aB>ka82eHEIu5k(4|Ezof!Cq~I;bCXJ9dKw2Ye9rJqbOFCR8GIlKMi)l!`v_ zZ|Z;4C2AYB6sJrjmy#7wdsvU2@OmJ%aZiC^Hmra$|NA(JeU}nr!R)@ z*@s$C5O#PkR&3>e^9DWe6UzWOc@*dl3s8wIfrefT6v||Df3M=ZO8|wE4usuw;4GdH zG$sN5qN|(*q)<8IBYtXA+`2D<@u`AV8o$b*k9kzf)M}#Yl%co4Q@P;XF2l07fMqb5 zjG!8Td7~>;22+dofTm6b*5D0rCd+Zg4`QY^NBw~t(;YP$9rr_sKb!S1APj)0AxeX5 z@E_P-G8sp;hBxU>4Zuv*1Zp(ZpBjmNxt;n>PRH6D!^xawq+kv&8-A%2a`X%86V;3v z&p+Bg^u5Oc|8|v#ASaU-NsSDpWH2D!z)nSzAAu8_2u!qJP1_7lX~l|6M84kun52`y z4iq$Ap|>Q#|0QYX(Elw79DoNHej6B~>XX+`>AP|B7X(+{#@*9bM)_f=zq02I)w<)6caKFfs*whH{$XBPqF8l;fD$XX*>)2 z@B(Oya>%F7Al9@XD+BkPAAaQ)_H`Oo>|bE-2II^p;|)6_>L1r1!3N#HuY1rfTLJ7# zIItUKaXM$iZa$+UmxeVQf=Qt?SomdfA9$7WumX>OU#I}ipm5?6aHDqMXp%4!kpw^f z5cbdrIIRSDAivhHpU3kbtm`?T&}!p}bKwC}^){GF`3uzHV%S;=A!GieHQ4}X`325M zePH#TnuLtn~^^ z6g&VL;xuC37X14_c#>5(L9Y=FM}o0yIp$7|=z9^nym~)$_D2J?;>Vb@1Tx7Atmk%c zizVx~u{zt~@$Unr)edMB^*=;YX-rxiL_YG0+zyO?BQhVE18m7ptWplJ_+8*PbIHN* zAr0u2m_2zyKcr{T6=@fygvwChN&*52cwmDv9%HTcB0lT`;^GAUxl7*)^yX20n?4>n zaEQ+9B)HMch#QGO2s8l_`3U@XTVQt-_~*GeA6cY}d`_-{7p#K)%>)9|kGJ(}{Ph7w zW)QHc1MueUfa-jWn2`u}&)Ps6-NUCQVLb_~XGQpckwB1K05ZkTs`?9f>cjZL>^G29 zeqPXM{0#$h33|mLnsWrL{vG`KxYK=c#8g5gDxhPl*D9Q%(H` z(T2XqjAt8QR`C-uuwAB1OMTSxT3Mf(4}sNYig~wXFM1-COu6iCbe9R>5jz8gw*#ob z5!yIpa(k73m8$AAl~jMqiBcRY5gmlu;!f$dJP7!e%Zf+2qK?pq5?SPDa0T5$K6-{s zA@||btzd?+H`py~JvNrD$WCEjvG>`L>;|R{oda)N3W&msIFUt-)_O%PL~V*`%M971 zxaHND8%>mIOH-wq@@(aS`du3fEbbTJ@ID<172jt!!y_Nm~UD~y3 zsQO;Hq$Fb6awPKKb-?))08>~aO+{{V2$|mk*yja33i;L%uwTxBf8K;s^$-#B1hS1p ztjHjM~EXGktg2QI%qG|N|^U9uB?&&C)uQ#Vx}+?A`}(#xzs5tKbXI|8}rbO zyo7nxxxjYpk;26SU}H&f2Y|sPL`WA0pxgBwXo6na9`q6}lOlDIK8%ytk3LN=Wlphy zd>@m|+|--d$g(HGX=qH+_ znbK3K2C90amA>k5ZJf>_ADan8+(F=$4}uZv9TkA7#M>;*8LY@kY#91KC)oftgbAb@ zP&?6ku)ak4VR)8&VFQ{jR)NDWTp}JzzH@X6iqCJNB3FNiSIvWp^z zFMtt z4WlC1Wi+}bcxjHB&RKuiZ#ot@F4}k4osM9q;mmP5Yf7D}SOkmT*=-Q}I)>nrpYTS>#@&B|MKE3&Zem{H`B)6O9OP&_z` zO0(-&g{{c7#QNM}3$pP*?%b#Lk%y7-`dMzzVy2>oF{+G=bC|9|f1r8+@H+#Q^4AA619wjN;}1>Jd@YTUFB>kUW$@_ zW0qY9=f*i{45sUUssYA3@(vThH#YSHzvFXMu4JK-JdWxXy>e{Ub(DIrxHBTbf; zN=~_@lA{jA4RRl5qqYMhTNJaWy>X8o!1m#SfEFpmm*nokTg;|- zndoG!MBZcq=0t{vD}_1cA>fY;2lL!Jd8SevJ8@V22J}NcOjl~!I#e$P5T{9!uEUgM z53osW57x`thQ16Z}#Trv4EJF}>J=m_qKQcG9!pF&cq`hG8?AN6b!UDAS#34$jp$ z<{jMvJX+^5a|qQD!=y)Pwbac@sB%S~3VSVvnf)H}Zd5e`l{m$a|HxLQDKKuGRF^td z`-vOu7Vw`G!@Y^aEvo?87u=A`Fhf@e`PEw7@zRL$WPjX)<1zJH1K8s4$XZL{lMVnq zoQ&A;0E|WTfLT<)u~$?5t_)RN@;y|3hsy17Qnty*fT_Equ-bBC6BWq)&t#b2nERS$ z@C*3ArlMBa*4(k%ao+gH)J?dHEuI_H&>FE>2P^Ab?PdoLpx=%Z(SH>-*1aAIoh*n_Oie{(cmY^)wmn%Cnkh}kEb`$BTyfiOg+bx>2&zTWke(J9E7PhWfq=%i<~a!!?`)B*zvQ+ zt5wuz%1*3XSG5h;<>sJ{*NiNN`QtD2YV@rl*$Ql3whmjAHRCK+V?NO95J@T`qOJuG zWj%5r_(T?f<>8nbrKTz(&U9t9gSrlVpbEHi4%S*~I=FF5;@(>cRhVO#)(ytZCXU)k zrBWV{-ACbPwFbjY4X}8mBEpRXBgSrIl>30vc3_@v8PNMTk*7b>#%OkJBTzG|Fopj= z)Uy-i>DY}Pay?k`K{-@GtOXCk4WKb5X}PfA3drNn5|hbrY7(`cnuS|R1eJoE^&zs; z7~HA)pe|CIVyN55y5|5h@(I(~-%+D^3C^PtTCO@(4N-S0C6!0u-M z-g%_X(*F{#(PzJAS_72dW&RGghMUa?m}gm5Sb5tMTb4c9QQaBp*l+D&s?2tTKm0<4 z(YvXGWIQHNMjBQlU3;z!mO2TweWEwO$NA=auX#p$Xm1yvAheMx%X?%|KB4?lqmU6F z0ZO|DY9E`4msBQGjN67!Tgsi|UT}-Jo9t$0Iq-Y~=x}B$DnBtyH+mU%z7*Ml;Eg?6 zd!QgYDyNl7YA?01>QlZbv54*`QD57H-{<4*`BpoH=@}I@MxOjl-ll4S=eR8`V9na& zws{iIJ)6z~+rT--!{h?#y^G#NZ6zOKUT{4oj;G@$#_9oj17y6TfX}{$?Dn&|S6!p- zfj_CGJq71O4@}ez$4%rL>MF%i=P8U^qk}F>R|e-oGAv;j?z2~LM~y+|w*l2AA2R(9 zdJA-&;(^TisV-Jasc)5}|FGASl>W+OB>@ZW+?qz? z#x??I@@%jq{=nxA!x?A>d~!T0OcLcT@^BD^~^$2U4b&ai? z-EA-LC~f~^Imqu~MpKDkZ+5}vLV(u3f*aIt{hpSmoREGCZr^is(3c~#N)Y-08Cz1! z6&C?Ru1eJqz5IUVht>|v11t0zm~XPdPu^o%u`>`Ihp;tSnw`%)K~(fnP3d%WY(F!b z7z4OYJA5!yy^K2OAzr}Q-he$Qf;z=dHBoJb(>@M6a~L&;RJACgdq1pJH{{HtG3T8| zwg*1+Hkb^?P&KJ?xZR(jvM3u)V^@%T9DKha$SLZAx8;S>5%2X{tq8u5bwJ$bLuD-xQ`=K=qc8BBj)^CVXkEb=0pB51l;taag%NVhK5+w?arWM`W=;= zwm@C<#0*6oDx&$}XDecc($62CZB)b@%u^x@J~|k=ZY$(M;iQC9Yev*;L=M8;z7Wpn zPRv^P6-p`8D6Zlw4b!GB%r!HpBhi@3^Q(Xr$8+>WHK7ZhW(Sb{UvWRXj9XJ9%pqRD zoKtIPcZ4FEIWb}Hp8)ykw7OaB3 z{nWE`#>kI~Mm(O}iCg|j%t?lm5#%R4UtiQuTB0Jd5byH@|2>W@jSA@k%#Phf{A`V? zd^DzPQc%Bqgd6G+^eDRPQK&5Wz|2AGPWZHP-9w*TJmP_<9tFz_!^#bJQ(Hc=;6S^h4)^iIwLmx0n z_a4(nV=(j72frtwKIm6&@T(aW$4UjDZX&>n{pvdeyn~8rs9#HSG4}f#s%XRh>lPfx zjGh%$%~R+zw1i*JM!oVb`bgvH<8&}{m}$g5VBPE|b|Cu=vGO!?k9mpwcsR2cF|HHX zp__v%emEx17Qkvh!B=iTeW-!%K$M$=s!*DG4z-%sQ*J?DGQ zQIph0=(Jf>RXIal=f23-7-Ct=kf~+K6@QHHKc2kGuh4cL?lN--X`KgNB3Bm1zcl(J-6jX@4riP}zw8*JlLoo6ZYE#gW4AWcJq)BE`)@ z89czPd44-$>zAc${Q(YI!x=7MFfwP8!gzCa3a_a9>_d_vB!bCn_h#1c^A|5~4`P@*NQRU+a zJeG&54buZJkSEszf)Hf-4}4Y-R9%p6D+iDtQ>nyMiu1b&WhkuS1yQhrthy-GMTSB9 zbX_NMGy!d&oB7m(tGgouKeC^%^{mjv-$dOzb2jV16YRr&9m=Oy@biCxO~{xL#BMCe z)DSdiESc_G0|R-_ApW`vdi)KFvx;xCnq~6y zMQQ{r0nH)`AIWup4}a#x!x@Hkc$2kFL7Mo;cfTk+F|7=ArD{ev_r^oiB0NnrGY570 z+6tY?3O%J=){{)dMAjGBfd2j)EMj#bG0>2%9%lpVs1&pw?Dj5JR2dtge_$V(7)#N) z?XVuc5LsVFMS>yJXerBC8g}3f_WGP27PzbX0}shW?i=`%&l+c1n0NSDv495e$$oZ& zpQ;ht4JVpaQaDDO&B#pu0~(Og^5>C-z0m)0aCw5BoBZw`=<+>efyWYWyA}xLMp@R> zJ#bP>)cYah>e20@wO#@$Im@Z%HPsD#*Rzos_ePj%T0xxR9G>P5!esDFO`-3-c=}%w zX?uW$*_d6emUmda_bUNUVgVqHdWu5Z;k z2ZHe8&7mUG9OCgE$&QyyL%>BCeMU=D9cNMCrJe?F?}LtS(XFHhw&xksLL}2VWb<*} z<3?&%Kn|=heHV^$zVTEsT8IW*!k&HztRc=^3-7j%m~a~?Y75%+88*VGz)g@bS3wUr zsnR1-b!~{4LIsLcEufdBt?fzCZmOV{*Z-t4szc}}G{ze+ib^=!ko>Wn>QppfTT^9W z4jQcoboPq+5vxJAw?d{(5K7~xZ_LUU2gZOvX$u`k2I_Kp)lBWVW71cJSWqw$Aq|au zRCgkCUs27$#@~trM(A(!w}Aq}NaB>s_&Eq&^wiXeJ7zwl&J#4=eqs!FsS|mYm_{16 zq$Hh=+nJGc9h;2SgA8slt z9tv~>_0v(`g!g7I5qVXeuXh6N69TfXZXk;OP}LbZo9GgbqTk9u3k9TV!hO7`#>2s!N^- z@rN#YVbdDU(G|$2|EhaTiNZl~ys4;OiRz&AW~1gL@(nLkYrUJP1fIU0%&#yOPk(Br zeL=nsg06jmBf=&zSU4J}!8(5ECrtnmuNO=ni{#X0H_6fu3LZfQ9KeT3yqb!bpD z4FYk*E$0gK83udu3HxFPR{S2SKYS!6p9i~g2v$-(xN$3Ykxs~#)Xa$gA6Q+>&^v>$ zk87h%yJ1@&Lwfy(T>J;C=m%Dd2dz93Tc~khId*A9?x@$m*la|*@5E1CoJt%`@tQp2 zh8_d{Di>I%Y(y>6&|JB(Y6nuqq68?R5G>GS?%zf|2@@3C4X)Y?vS0@lGF(t}2jj*8 z@?jR97=d}0Mjp*Vzg|O|7&c@YRIvhS>V}HS=fO>84 z@(%2xyVT8S4QF0q{mI-qf8qD1;gA6LwK-U`g^0hr$EWZREj0(-u$o)`b#~9drpyA@ z_Xqd9N!;&t@Yy@qTEoHUTtp*mMK(O5J6{A6;3;bv!*9P~?HGOBJ=BPKg=JcvIvRD6 z?c2aT3`YJ8L93p@T8;rXHktJpHgytK%r-2-%2>450|WR;0ME4*n|Uaf;xr_CEh^=_ zXTCC)LMJc{Ij~xs+_Nr#hjOy>3231XX#I;k@ir@-#~O^zzNgviPQ2!G`sdlhci4fa zm~m(35zc-{%+BbUUI+2qZ*9EQ-?LdqhSsdiK~&E zj1KC#nT-S7#1Z!ND}NKQVcsKc-*6ME#;6szJ9lNC1Ci$jt-i+gs*mNA&UwDir0!ps zLjpWg3XYixJ}V0Q@jG;y2m7lSws%(kZuFR!pFGPe=AjfLc^))>XyUi(`*Yb2>Sxzp|UkQ{!31 z4)lpp|7+6J}>r$oWL=E2DdHPoC&R zez=heMhES3P>7vVNaPu(m~S*A868k_L4g5gZFIhtsGhoj)BT8yc#6#Gina6!T&4rA ze~zp(sIw;W&BpHS$M2vb;X?3bk493SVDuvRGWRh$S{&*s&+27v*@dC!D(pxaqb49l z8$rockPoShZQ^~o*spZT9ezCvvw8 zD=5dlRl`%+j{8z6R`Qs-QLi{nA3vSJT8o2T4)fyv%cu|dt?`w{$ls8>8kN+HuBu+1 zdOcI`2ym3NosM9MX_Uj!e!j$ulY<$J!!UdLU(Tvd1Pn1lFV)Rt?!bK_2Pa=Gq#9uzbpSc)8V7x+{Voy9l zy?I6@o>r3II{BU-zA&U>F|>i<&CvMVRkX|%_U{Gvy~*5ZjmFKV~on^5Tj4I(G7h7 z=Va97>y3PD4?jnIi-awxj+E7I*y<}vgi}s&$f5!LZPZgP^nS*?lWsiheL9(wp6IukRia$_(UG&KbRA$3>9U8ciU%r zkBv^bRp7}MXwbI&l$+HS&eTvp9Q)4op&2`kh3ywRd!(@Ts4k+-%M6mjWJ5HFQwVj&+M8}^|1vj zHuHa@uf5R|J(XFz_+K&Tp&+sn4(|4fLPn?H|XFP-Q9T^=UDh{R(3QOnk_Mt zW}jyAw$bt0&);R9c^3(7Vx@+*t7zwyNTjOSz~#aXW#9yD{}`Qa@W<5p((n_z?1 zWu6t_Aw%l?3Qcz5w{_Y5ROa-M*Bzer5*zsxynT&%7~QiS*bM(=T84%kIe_kagof4m z%`4XO9m!pkFB`Y?%NgG^jV5W^u(G}snT%3W%6RqrJ4!-`+((%HBB{TV}J?jc*%^~odQA6F# zdoG?cR(#3)~vx}CV1#APx}m)1{gh!lfKRj z5_pHPvf}V{fEh%?UkZC$4}a7Y?0y&7SH>-EFqZNWEboVkuOCS^|uG zhqFGJ$wQamrC6}l9k8RD#ras!ihH1mCs4xw z%`uQUV>_piiybM$_b}uU!A_)%C-Xg**~MdU>>j8ri4%)q*NuHn`2Q=E;7Tha3#?0n zlD*KkVU-)z827^s=lIt^OCE&_ZonHC;D>+l4%fvhHSGQ;{QWHz>wfa79o+2qWJ>ry z@uKu+9mg5LsJ=v-|%2M)<}MyYjjN*3;v=RJXMHy73N*l`0PkL;KzwNB@!FE!2N3xcdi|H z@F$}w8Z%aTzH0Kc()^T_sP=B=I+nZbS@t%7W-vp=@7SHatksB_pJDf2;IZz2r{H%+ zpGb6N2Q`C_;^DiGSN$hf4i8b4?#K|G-F%JCT*U2bJKrax4S!0NiMG~8SrE7S* zL#Z6Qj|lZleI4=dc>Or>I5gLv zoM;(j+!Z{(dxajNCN8II=oWKPOMj57qb-9#HU2cOp}xtVRC)L!?xafMXS|F$@!Q;3 zh6kblXtHc}67QZ&j$B!?f~t`61!QERc`>T z)tD}S*@?<#=ii24x2Eeeh@>7MPX8Qqg_*p$8BpR5^z?a7^#3fAFRb|?^u2~0tO|Jc zjog%8%$pc(SZEfdx2C{5b?aWmnE} zB&%A&ZEZE2-x06YH_l@e<6R-Mu&!1P^kq8zjlZd>^zFW@-UL%NSDj4X;~Q!?830eU z4f^dsccCcR^KY$3?9-eHu1`Uaf(L}m327d(DL6IA74*Wn(=o&zW{b15H-8c52z5{;uswJ8E^jePaJ)LaOnVr($kIc2~s--45J5LGk0n{UyphF_>G@}UCyCJ&Zj5PB^Kx-$W} zRwwX<+2kO9vH`X627nm-MvlrE>Z$Hkj*ycwUzw+@1|!@~ElD4x?}7Z{ALjM8iq7dl zGeTa5ehOO|?uZDB*bp8Zu2W6tbI?<#VqZuvwjqLBk5zy3-}2q{HuAQnV|YPoJywz* z$i1j~p!%bfa!N&N7OkZ2!7g>0)>bdYea244r6Tq-f2MEcTKil_1!rGpN#{gI7rVvQ z!_r?GA&d?j)XJ+irM!O*6&23-2Kb8k%J_QtMEMByQ41=UmC^L5YR4|8X~CdXs)1o? z&poHSFj!nFtuptqe6SR-HnFy}R;4adFEVK2!PB)P-*_nz-mUZwOJrxSDr=OJN5k84ZvbOgqIs3tL~C37lG`W3(wXG{7ZbGB3Y3)=@mMh zyu&@bt28ooHb@;qvi(LrR5HCpYiZZjuHaA4a@uzFf*Pj{Cwg>7YG8e7U+mI@{h`&u zt3`;BVUf2ZazsRgPY5kTAKR6-9_GQqO}&SjCTrebl~LPs!`nuF>+hsK)7le#af!v*_1@;y=8NX>ZsX-lc-vKSZRG zgj!(Ur(2d-hthrTzU`K6j_thlu0^nfng12X3-zcyIgHNm8r3{o%K_488p@04;TMdo z%B%gVUnN?V2W0<6sjzvtd9*pm+*SHb>?E`@RRU`;TRTg~wb6=9M#O4x%ag#VfA@EV z!osOJcnZ1sJCZqAuYuOPLchIB=mVyQ;P~bGW{nuu>{*=oi~GGLUV^ViO3r% zMOBZw99b~3Ld1-)Gr>!p$<~KraA1cr*LTKUEB#37juiJ#6Sb3GrnXHV>{h*T{`1;C zlPW$kH?$76Rkr_O|6;#oUvBSgPqEdr9jE&3D9a{ulC(f7Dn&{gKy|;TKY^e(QRn-w z`@+13+*30Srwi$a(XdEW>)e$* z-#uk~CjWJ1x;7$ER)|9uy|-ksJ+YOwyX<3ao2=z6o25jdnrVq{)%Gc!{UxYI+Qs|A zbHtPES>&DWTP$zzucOLwEIBlJu@;+BjlC#&=f}XEE*3fy{r*6%%VoR*9kCMcqP05` z)lDHjw-RmKm&oS_Z5=txKh^VO))fR@>xVawg0F6`G^a0SQF_~ja@L>KBic{>tSLcU zV>xK=>dF)13F{ttIciMwtLQz^LUgsLqLIbIg^+OPVaqd9xT;g7uu)1f2mW1(-tq)ROScm4c;1(BQztVaL9eK-WEHqQ-5`nG(~t8xJ=LJ zIanhy7C2e9RuU^me zfVzy%4eJQqoHEIm<3_x6-B5{&*ytl}PfqQ?&Q`cH)?YzyK$bPhJX4 zvzuJ*kwR@D50Sx5_(Te!w>M!)E#>|+B-5sh13jt2u`a5YPXj;QlOB-{x@<<0hZRG< z#~>`Iri?v_2xfvVV&7S**%)q7Y=1gG2OXv#vNODA_%9JnBNHRPL`FpB48ItB*Ky8r zSU95Hk>`8Xre~zM=qN5FA4rNy-ut6c>VFxf^rur=ZSA&NPn}6+mu~)ientLAJ}tMFzxX=)wtJ6zcDqwDZeIRh4XvW`z<1hH*qtq7 zYx=76JQ)?-_dQ>HzbMVM?tvCUUg^5Ily#u3o&B7BpM8Q|vJbFbuqIlHP^s{`G!sqw z1nF(mMf`yswwgQa4(dFJdP!}TdS3}w7L#F8g1YXf{Y|Kn7^^PVPU`|aHD`-6rM6H^ z9N9PdtufYPmIU)v>9jZz%u8*3t(sH0P1n=zzS~g6aw-ay@?G)ml~>S{^jCGg_CI|j z5r{jcd|;}V3YUeqpir&|^XbE=B2|ijap(?$GKnnlc<_X+>DTE+S06lIYcGW{>mhBBt9tR)LUwQtZ@g3tL#!W>)(!?!J9&dhp&$K7yX(us$KMq z=si*EB5H(2y6#&yi&OMY{zaaw>BUk~lKUl1PgK6wPJEO!^5>xRg1#VafKb7*)jq=| zgxJEChKEO74OhdKg$@fo;%sG0mex_zdb2uS+3EMo?a718<@@1{q5^unudaMb{vv1zz`cM4~E^chOIe*Yav>sZwWBQ{b(k>{2K0Cx^frJ_Padi|`MK zy3b@Kw^E;X^ni*eU{NBnpvNZB=L*BnIUj^P*RPSm$iCOXhfKKpfWvCXiCBlScouT;@? zfc;$nCVV(K0jcsfd4hbF>Rpw|X?)|qt1MENXk7!cu*7oH5fidHd}d_6sQOWdqr$W7 z%(^<}UDhVi{;;1ezqN_@NZTQ=bYD;H{X2i-~Trk+zC zi7eMf-hAFJ-c)Z1`L+L+`c1!O+AFp+zp^y9ZL`00+;O&db#q;Ec5!sGC74T#mjhF^ znaVC%@y_t{b#Kn-k+C|XwR^Xxo3DYtvKmR<2NyM9+F@T!vaGO}J`3?7=jSfGA8&#PhACVhqdld1;X$RO^2mRd`@#o* z3nh*^nrUha?IN{3=HeA_fHz%>cjyg1h3UxHs(7v%Kv^UC?H1w_2k@=L;q#clZ~p>e ze^m2fg>_bLgX~`|*O$xC8K5{h$kF~4{``uW9QG;N61}ELqR!_dXX((z5rw0wM(L5i zL>-Je5S>3ui|C#a#X?FrV$4kg7yL^+`O}A_%=*zf`C?MDr0Aq`NxObzO&jA`piDF^ zHJ`OjaoU1&g^mmB6CTH_T6oT|N+Eq+`Rtd>&oZ;cBh`=o{c>&Jch3s0#m%4~){`2YsJyd8Zb+M#VVC9hgqy2*2Vy|l(VjW{?U_K^3ptJHp)u~*Q z6XEU{pO<{4jlNI5V)8_4|H}US>{TDV53zU)`8QWZi@Bp^vUR;}p1r7}vg18Dv0H45 ztg9@4n3qa9#7Ur|I%?tS6lE~IxBpVQQh)2HR^Py(>22}g?KF)B6WxqFgUdosBw`t% z8dgmQ`ge99+n^TK-b%b%4t#4y^i(I3o-J?)ou^Z)q?Z<>U8Vx%Ph~xKJ_~kmUS$j& zdfSoN^@d!i?OG{nO@wn+9l&&a29IS1@4b_|!vH*j>%fH06U+$u*>7fR5i>1Iw&*ia zxuc3jc!C?*_X!1+9`5@ohm%_+)lI6MeDp_`lqIRN)5MH@o)P{>`YGv+?W|J>S{yVb zD9P1^ipzdiM$p;d=#WamrCojP(=2(Y`bsVhHg1f%L*DB7Eq!Q8!5^07#mTvUu1al? z@t7_V6_mSLlyJ)2-@e6_CFIx8y`h&vZUmKfHnD9tHx!4NuInB(Qi+gTdqX|z-4eYA zLVOkFJpQ%*1!y=`elHisGf@@&_ExV8`l_<@(|p@H-d@CcjcnFA!L@>4xjr~{+X`9k zi`PtV^s-tUeY3mEJACte={~_fQ88&7^$cz&eW;x@8o4TO^yQ90)!FW%?Cpif8Z%*ddNUfv9`18pde5-wDeF;9N z+*aNw8yyl7S@(9oO$`eSkUH6a2~G%`6>&1+MnttpHL_lGgDk1h`yxw*op2qto)tp1 zM)C+xmkdwZy3{f$%YSr89-S=z$dabItNU~6O@xr#Wdgj5Z+g}x8I z=xX7pW;<`WZcb;VokWHHbd|Kf{Ox?*JT=_6GVW!(%J`UZKVxS`N`})D={xG*s2wyN zkZM?MRA*S{c;k>A!OnEYILEK{Bi7%|H-uV&Xsv>BU;c}wj4K?BrEBNxLlYBwrx0-cdNiH zjmR>&hu>}y9b3jx+i4c{V1B~M1CW;e@Nf3OpFGy|oa#`MuspAlop4Mzi?^zd_=otG zN@~-{-Mml!M@*m_9*BNeCp*Ys38VXnK;GsGwHejS4(p*{diUtxsetuB9mn5mA;Wu7 zae4%PhPL=1jQ(A<@g8jqtkm0b`wjRrnvsH1dcO=_4*cw`gT5?$GST0+VEGI0<=7rKCAzR?A zs{51V-f~NM0AF@l;NOn^XUZwgLp4eO=j_nxR~0m3#5O=JmNs%FTB{7 zGPTrp(3NHBG`w36B2qAvted#_k=l*&4)C+Ty?afXDq1K1Hk6 zM@_|hnN6`Ph4E)BB=^#!mQd@e^|7ReQC~lby3Ot7rt%H>m_J>ar^V@4s4Q4WbehLl zr#QlbW(IqLTL){dInJ++O^yPNwOTZIkaPCI~EodNX>r_R@4 z@x1s%{8KC^#*%rG9R#KjSWcea9XLy*R!u|toKL`;G=ZErD_;0--1@(Pj;$*GPmB{U zQjIISxQ;B>Mc|nN`fJUh&8PN4K76?EsFqPuDS$RhQkHN-FzXR|75o4OHhFQN8=b!Y z6K078rCU-4pt68vA7wug>Jv#$mnm<*%)YhAku3c7u$cN36l<_kA5KT@f;T zhJlVx$A3CNF9<%l1M!+`^f9wjH^7F~8cE!!8j@imbhrepY#;pgL-Fp!v2WSAzxE^! zHj2K1Pfac%hIm0)>b*sAue`zNjhWjcVpj$>wIQ#HWD5b&C7WB_e_< z-bIz2jmeBZm**ap2dS2b6kA2sgPqKxWHP>PknwUcGaEEfFAL^+EOgzA6Usqs<1uoq zHQ9q%i6Vt_uEmLUOrqw^2|YjOb^$y>e|9Yj%VICpz_M`jGgC>losi7>n-FCi%YCa5 zy|t_28#$F3{jCjJbS2dCle>2rp%hhqoMho>^i_I8Rm$ghLG6N@{Ft5SzxMc7W^mGT zh?X`cZtn+~V8n)v8i7WC>)(*P)4@n~=4&_+@o_}Lhl7tYdK3nM(l|*Jvq?Z9?p2I_ z6tU1?W#U$`$fNw+>VN39@Y+a3a|%&OZV$Fm0<{8;lK0aGEO;n#`Xq7n_C#?bL9rSL zs}02D8X@Jp`cwP@H`$$UaBmr+&NGR&W)STkk(u8&j9Af8ynqj=_P32#;Ye~owq@?e zVtDKcdpeIO=N@t_)5ybG$Eu>BXam7v^w+cz*Ei6M9*_W__R$rMxW#V6m*|3)nh|mA z1P2$zTRexqKLdqboZO?O#6O-Qr3zwGl*Mj4hlJ@!{jASq^>jr>BoVzlP9{)M;+I3o zSKSW{eHD_pueB7;fyFpT?99iWWaxgl`zsm4iL5Oz{JDv9D-2H>INcuLDONI$-_### z#7rHJ5lGE~;@ixu$r@0kk#SLxh;|k}agZ#)6m~toHc*?uZF39Roi1T2a~=Zy9VCNh z4ECvA+(3S1VX)4z#5=D;#m`yORCug9l=FjG*vZ{008LhbFBPf;AEDBLgPGmLX1R&% zs{(qz8kqf_RE>I1XNz5ECL>S6M?}5>+;t9}ycoan66nD|@D&Zj6LTI*Exs*8<*I?H z_?I~ALZrDJPyAc30XxC0za<+f9qrdf*g@^SclumV5hi4jfla6s$VmilF7?qmp^3&| zH?<-jJDAT~u{ZYf+`&|zjYkSwgudK&!}(he;<*bLBM;uC!qDYWJtsBX9_eL3$=oG| zUzRvtB)K^E$@5$bXZ2z($DuzD(Y*&q+kMC>CwV+|K%@T+Hq}G)b_a8@Qva+Ka@^>A zn~e0nz8ICjvLe}@dg7guyx)i7Y zN^o~35}^lws|aP}7dFF(g^~Xg$m4rR|J^Y11)mbN_zS)}V9HGv)gYu+8$5Syk&&Ch zW4zT{AU%xk-8ZP`RT?Q?13fa2=Wb+QeNma3AXndzEsI(CR4BDQ zXd5HH+{@Yw#L!;Q5=ZeCTtbhn0ejq;Jcb3#(7=bJfdeUqG+T@0xs5J3gML^`-d{Yj zJOeyP6xf3>a_f@d`MPL`1w8#7JNue5NFr-L7koONRcfG!E_2dHkmsY37b|#Q4WjwA z$v#}cbB{njx4APL`3L*(Y3Sg3sBn+h(PzpFPkS@7uO7o;5;Eu{7#;(ey$t=(ldsKu z?mTBclF@pC&UnV$_aY~pyvx93FC^NKK;HBZtddl+#qOdpOZY29|@PMn~$FpQiHevr_pswSbl#TOvf&LngJle`0 zjYi_NCByPJboF=e82do}97IbO4_w2W@B}17VRC8?GR_My=(9nITqdS^5uIV=U*Dl> zE^Sp*+e;a^V56IDHi>93*HfU*6*CmbMm+!^Z7T7G6AXI zo_W?m8zynONqp)$`(7M6k&w?d=})_vm7RrN_JHLQq1WD^n~eOoP3+kN#@i0X`%z zD^RMDSJE5ncoo&RZbKoF5F0wvUiB9^iXkI-!fr5pJ6 zfna@%3;_dg7{z|igx}^s38%mkJx4x0WL4Xd49(C0UpND!a&jXyOJ!!IkmuGFsXl{~ zy$l}kGj{6&aBE+%zHUOVKC-Q%u$WY&+zBK_bE0OcaL5yM{$AEr7JvM6>VO7k%79vU z`&PlX|HH2M0zRf7r_!C3lt=R$|BnLs5DfxCW4Dc3I2kWrnXkv8m40`xTu zI<1N{tVGq;a`YXiioxttUu?MF_~dj@Z@XF1Rz_P1r_^Vy2DbJJl4lM(6N`1Q z9H}*xFQdN5J^1AVJYq)<24N+}pbh)6o0E|XGZ}9(_w!#E!NlGQd>K?>LaG^PQ3HkA zhuKXgRO!x>{CoKhVsGNYOALS27=%A1J?dmenX1?e`E&u(MPi;Q}J+r)O>^LHk< zS7p!YfG{nJC0iIytIv9-LnGbb;Od-=8Pv&M-hYbEZD&^uWceF(Trg+$3$#{|6^&yq zi&?`Q_Mko75zME^Ku3OLVV7j^DSjx;42`?N_*UUF!$6NVf|3hy2b1tYoMb+KGV9Bn z&uzZnL^pfc!EZ=NBO}MiYR22!92en?}4ryx%6Vc{I(lkxmN8H#j}tM-i(HPGun(U76cz67$lIj=^{G&>yD z6V1^NjopUbX$TFs=X-tjtvTO|^Hdj4NQP=opywtd{U;#PM)1>2cJ2~SHPBIcS!;Rr z_ZiQB!t8>Ot!+5;O5&AI$a2_jzU3O5$qOZMOoMxT$asEze8gEQKl$q7%Pl~?>V5y~@j2GQuC2za(6r#%lTGZ#s* z6i)Ak{V|5~H*zy{MvRB6b})l2%;_EbP>wa`;j0s;8!S{|H~c2LCYX-!_r6s4pxS0& z8+$q?P(;X0t%ggiV~wK*%b!MJl@jCR(v_$_zdZ)6AYwXrps95EU-t*n4zr7W$LG27A{~Gvv5Y(_3 zyY&{FOzE%-Do^9S-yDW$Vi2LN@DXBV2>&zDem#i3!KIu z$k=*hVpZd}?XkMXVJXH_FUZIq-wfS<#QSv#55-1ia}gSS!{^?!_G`>@Ef$I!-hI!$ z4S|2t95ODI3YIyz&z297_NnOModh~o)P z=)Vp-lRJ~^?dWKc=hsEb1yLjBDyy!G^jV73|AX(7nfVz`;WAR@3>-2O>97?0?;zA? zN226pC*E@R?E_`Ta(nCp4S(kShCF)=)#X7?h+sJ#=%M<|sz3D77#?Z!yW5}#T0xnUm|Guy{?4xb z!zvAnb|&`PP^`G3+-H8p{%U~#WfUX+i)>7SCUx{nJwDZy(>88&M!w=Bo@CgWZJ^H* z?9VfJH7PTjvH<5`^f~C7d0%VH-o!xBosf2;_-r#MvK*)T7D_zI-dy8#fHfsB_Y#>F zVKKC<87rg+9)>SSDhKkcCc9}^XfirXLV83pLSgtb7(P#d>;K`TjOqbK=9rnx%-f9j zn0+_Ao4MieCcM{-wmlCutC_ie#yxx+bZTh(c;tYg<5l+BovCvl^1UkZa~k~A6|24_ z92N_u8a387XIin#kk*}fjpFaU$rkAcwY6b~XTxUM^Iy$b)Y!T+P7vT)?@Ur5tONR;EO=}V?Y)R9RinfU_bZ6~}1jkza{ z$39%fYXOvc6kBs7GJF*jaFKPNMar#%iWeiJjePp){B>~VT{Ac6h=r_pD>8Z)yM7+| zJsTQWj5HmAAMGQE@q_eAFnqYbQ9ah^z444*LoYxT`?;Sv@k}+wzV8oh9AkvpP>G?7 zk7MzVVrB)o%i05RcpH~8#(cb(Q>h-fi+qJqaO6t-i&Kd?R}=ph+eoe+O=dpdZey|?0cC|jYL|A&52S`TjGL_S-6&OfQN0* zlP;*o)o5)Zy-J$Vabz;o(U~(iompkB88M&bRB;_ zpCjBYBk`Ap3$Hnyg`A^N;ouYN8^sMh8@{St^fswaMaI@-Qw$^*q$xj7C5ETy_28Y# z+|Qz^>ok_gYobs{tWB(F0$p{^&}nEbb&Ru%kE!?PhIU%xbM6}04~G9INU@TdpBkz6 zs4RGc6`TX{^q7kGjfuf7&@NL|u>d&jyF{XvAbF~A|N0AWz*ppY5;tL+@D+d7bgYDY z%+{z4G=iEj4Z)B8#Phs^nKq!;MOCs$O7l}4ekwuT%(!UZ;_<;ncV~Pc}+b&f_wf3PZG}bUSDXqQ8<>MEiwYfX>C4bC+hR%#-=)D?6M7n_{QnR!HmdhPetWZV#Auf_k=4IxRmf4oa7RB<- zB3g4=gRQ456)msL?aa@lLDDU;uDC!rixz1_r=xoM3;3`e5fN3rigf8nJbDQ{*^&Gx zS*t@Vxj5Q8FIwO#9_cnvp_^=vvGDa}@h_?v4--Fvp(!oAgA%{uG2KHA%^UD<3h2+0 z>T~5ec|`ZgLs`j;*DJ^Pe@^mhPLTgIMB4#+?X?!F_b1w%K!!w7D8`6y?7}a(2&uji zz8^~rBPSKRw&*LUu-XnhVh=Lm8eso5Lk3(CW{88Ospj352i7;X5B58boz4la5OFA!Hti_J~h%^KEj)_U|eK4Te7ivCRYvWgT-L|(tQLwUYV}yi<1N3cV}((P3h+e;YkAY}C2$ zPetHT;08kA(Qn!?{UMoLd${cv!x|WdS7ie<-VP044h!Z2I#UX)*K_GhwF-2U{1uc< zB-LcIAf0@4Q#q(_W#x^9g5vLz$6Up_-Iij{>FnlO6f`;58WI`uJh)GAQ1JYqC$0o% zSH~RN2uo|Jkx(&EMQfnUl7oC7Jq5wv&&bH1Q6ghYMk%-7?ee;OE#-&)8EPqgS0J0v zR$MDxG@r6ew_0s==@;3+7GhgzeL=PIa^Q|`(f6mhFwk^}`2A?`)#ItU9YY8J19ViKFUQ||eHr@O{e`M7*uP6k&u(Eo_8JX`4Pw3FPn zr`k|5t`6X38i4;hjJV-sVLTZd2SJ0h04*^ApXwHDO=%be;7qF1-1j(FEc3=m68kpG) zWN`FHll`IP(x!l;EJcU+Q2#*riEp28y>F-QtFJd0%0>RlN_lm-_E>LbS})ubqs{#+ zyRFAd?*L zCRuHxIlZw&$-1$_ZY=CM?D2E-aBd;LVB?jAdX+&!fZwE&16UlpsteKVc0^F#;2Ft7 zO`uw23{NFfZk!gc=AmP6R{F~6Vd{7L;~A^-*fyw36wqGfukud$Y?={S(v}eJOP&kBNEAT`jqga~7MEZP;j) zEd?zj%_k+DJ{VVs$%i86PGkS9^lbv z{D`NuCjdDmyszK@J!UeUv?aiM=(*f$Jkh>xn1=l zdTHDY4-tt=!IG#5jVvd7sw2{=xANNG%l|~KE!LJ<0tP71u}Hz3+SG*VJ->^1^BHk~D)(OhJ|cWFb31cD6A;mj+4-D6nec3o5$4 z!XkShPUa4@2Ak{y)yOBBztD~PZ(%AOq|0h_fAP2S-SVt(N)X}#f=@evG00y?<_nsXqw%4cN~)%=6F@6p4Rs*ek#So1_n zDeFP&V{4rCqIJKuB%OwLkx!E#o))H28@Z=G5=37a^#PhPxANV8pVt%rH@}k(Yc-YO z$_e^IM3Lu|Ppzp=1xuJ0RO>S^=>qZ>Wo_%=5vj z27?P;?!4?A;@skxZGUBrHxCyd27wvUsn;C1j%uzFFMM!sbgB@1>? z5v4x4Oiz>+~KHzH1=(mQ~U*0D>tWWpx0k%-?e;HKy5)ihX?xHz(lef)`%48>X_d^`g)lwMAMRW4K9OEEEk|l>%I=ifsU^}Tr5AHQ z_dNE!@gG+w>4|}A^nP|pN9m?mm>w{zrCO3rDoBR-BXFE58hohLVQXxwY`bGEU~Oj^ zW9}t+g;}Pm0i8ZwTb14ZZ*ns*&0+FyiG74;HRg2HO%KTA^k(!H5g|VQPUf_8M)>^A^*scxmrOY71zqe{5 zGb0;vUg3El(KgWC#mfyK zM%_iHxF`OeewY8PoFMO!NBJKq+4X2)oVkR(gKJ92$nb}e@zE!K6Zz@w`Y`aSNFX39+2}Yl~of@P{?^dLkF39>Qg0N=?Tql z)$dWWJVyG@e8D={{><^j*~Ycsb;woM72#}P|86NSH8OS8*7{$0N4qDaw@)jOHU~Q_ zkRI)x>k)l_%0-pt*rCNJ9e6Wv!n9f_PB*6aVmWE0^qP)rzmtKko4b&Cm|}U!{r9EW zZ1#bGyKMqFqxaT!gQ0Fqmgg?)lXNVU#q|2Q4*mJH7^rU%mhB9@TAThu$&UuZ*xR@LRsA=}QmumZy;zGIlZvhAIvp1HdiZknLo@elSr_9*U4?hwykp0Q*w znB_bZIqk*lr|B)W-BQuK zgpP4<^_E&;GH2^4qmhoqz=0bXFpudcZlK*B&^O^0-3eOY6TZ&PB8K?VGCEci5Zg+- z%x5eotaWUE*yh>(A@_BYxeFO}AeIBU^#HlSjg>wALjITXE4dyu6-vTePxXHTr%in` zyI5xxFW_CWFzQ-7`%M`4DQ9jVG9FOBWVt??6WfHRWDj(ra#Q_?mm`6Gs*9A}%2#g3 zPq-bo0%zz1U8sPe4hI$Gqc`8*U_-0oQ#PtK8I_rCQlF+Te&usWfz?C^?16Puk7-Wd zq<+eFzr!CZhsqvb6z7{2`#X!ePU{-DF3dImw3T$F1zW=Mh7St=7FH`PHgs;t?BG&C zY0j07X7)U`?UrZKVxe`Qqh?d9C@=l<{n7rpNWF6MaCtTq^OSt;8p=+!iT)~3430Q2 z4nvymGmA*L&UDw4Ez7M5aK$Y$zjH_jh4ysViq@mGFW4dV$pbwj50t&WrM}_5-JI|q zxeg~hidrLA)HV14D?+coo7z$#XrR=S`Vr+UA!G_KHpiG_=?s%Y?1i_eF|nAx&{!t2 zUly~Ec6MS9QG?5PYer-uW~zWbs)aXR2J@Z;gzy*PSMjI#K$>iJS=v}S;ScL>sX~Xo zqtafwL7c^tcv(M-pQtf8%&i! zkw{{bz)AYjc(mu*MVL`~WD+b?PAOsP81=O}20QMKK7hDW zNuuDL!51VjXD67~C3xtUQ;}W$e^g;>A~NE!pIUfx$Zf)Ilnt@+FRIeS(cjfNTmfa z5T}+^Lj8N>I5~~`l-1wSUq`73Z-q0nbyRO|M7Qpy8?$AEg`U|(e#_9rfGZikn(Wf>CSrk!7`L*hIwF@5YocPPe zvBGPhl{>7{5{ilAsoQgyE^|xR+47>D`V{BE%+CTZuW=(>PMk|JUFCMVk~oLa zEoUk}C$W1g@c|AXXK250OZY5AF|Uq%6%cy-QIW?7vT*H z=XUc$t&NTG8`!AtfsNGvzmHcWk$Ax)LglK(BG z%T`q_yY>ZZs4hOEE?B)U)Kd6^v*9moM7+m{jrHahwgB6{Kk_()+4sfQv`z0xu4{e0 z6c$}2s5~=BKIX#A4-H~23n^$$Zb)2XkKW|kFUR? zBh<0W{L8!46`2PLcc6>t9@?%^W2+k5b?76Cu~FS_RkUhQB#n!e)KSpNaM zh7+`rc*C>dHB46}YEI3>Rx75@MDxxgQkNeo8b+MA8QLcvkHH5d>>BPlU*XH&d2L~L zyu@dga7T;;Wq+IwOgeYbeIUVe;u$iY-Iq`2B<`I8g&3%vm()eCi;rg$XvM2|9L^HC zn@pTxC3w_ec(5`rD?LJ16W^RfHrEb1MeX2qhe$>wUgbr^?~WreQ$QIA#7NhJk_z$}Aca#qE!b`bMT;A3om!q?l2G z)yT;$hyAvnb3VharGcp|KxFKza97+QRW|?6Jl8zV{Qp=w3#cmkr;l&ld%2_?*w}&H z-Cd8Z*xg_&b_aH1cPkj!E#}`=?Cw?+;l|ecSWoT9ns^IK+7JW-YP$PqN#}V7z~jEj$6mh2UF0 zq4Ss@8JYRyMcz{*_X`O+j+|6Eyn_-%rc`ddAhf!WQ)C(edU7TAusXShI_z;Q zPd>>riXvIcrCP+D$eeBk|0Qrs^SG0FAXlP^XtJD{g@|&Kue=|)_&d< z$NwXUCe;LMV;A;8J%f>U7Iu9DmZ8kPM)Pxi)O9apo>~%8`YExCfyC#|aoR&c1Ewcl z+#T%h1+vCRs7$zkgqg{dp{smcg8!2ZiHvXu?8}`X61*UKx4}nQ;iE$2!s^o}#YIlM z6c`f^NRwY!8K1B&rh#%Bj@;-1Mx!NYlWm-CbMl)N(SBRNA`~Qlww7)>CKI~aQU4T% z>~0|rB}!PB=k0+aOGAx4u(HBZxh5yboAw|fth{?pFn=$=<4i_lr@?l1!)-a?&>;NP zx~V4+i)69-#8&1v#`9SY&e2)lRc^fkI_L~Lv~t@k$q${UKFSwPYzB5P0=md(YG|7? zoWMqM=U0#k1<4^#2cHl~?6NVEWHlHas|sN~Iip78`(JYRTfke5=l3HyC(Eul$eyQh z2Npu-EBJ*_B&3DT&j}8%B35=i?7Y2j$yT}>z9LR4icT>Jyw4dB3N5)!owK$O;q zGePC%fUjR+6`o^XR)tLh+4kHZ%NK(oH~?ND3Exl!mr);znuJ`K1oh12SzS_5ASK{j z8!Jn~->|x#W#YC?@FVBpXh?9~*_Z;9f&p>hE{M`^;+=jCq&DmO&IVbs=)%`LO z>DdFGU4_)&#mx>z2TP!!i{QPCMy~G1Q~Jc|?c!&xnusoVJRwj@P5N%-=0x8UgB%ZX zpfl1lo}P}Ysc?%yu0*0Ae9#b^v9^j);dT_u>j0;?mN(P_O(0RJR0f*p4qBoL5+f9< z8v#ONJ=}2)v`rdLK%WWHCf%Q6HRNLhSeW_BGfbtd08Cf zWmb?kAK?F-AcUTAN5P=s@_Qi>N|IO04j)+v+z`-l9`5!v-slpLJx<~h*5rX)P{T|# zsD;@og4LVADI}sVTwtoKsVYUlSXpRkli6f>h-L(F^2_;t5H&lUkR9{LE;OKA_mvbK0Yk9|5e;j?^dsjSWL{giuX33q0f`eB<-P#}m2BmdK!nNRy#h zXstM#cI>$^D?7$%yrI+TZ%`=3=%cZbu0?M0x(AWdTcPC!aQ#DYI*ZT{tKs5E@RpN! zdUNRLp3%;nL9K(+JOnpJl6g#r1nNV!{~J?`ienYc#X{N2DaRvo4)NK9CzgQ+qp0)g z%>O4qQPV*zwM8!dV4B+yDt|1jd<`h8FK3*c8!th9^HBKLnrJr<$#sy*3rQ65G+p5J zJfPODq93do9EI6gPdL97H`MB<_n@7VZ3Jr95n%L(F8gg#x1RbuVRRAyrnX137E+=$jLYCKo>G|-bq+MSA=k? z;T~~TJ=lYW)Z5C548W3#MY4OigZ=E}IJ9A-F6Irp6~u*9rJRAW*hN-9guQ>@ zcSDgtwaC4dOkb#_bJ7hFB9_!m;0$A$7X(??SAwEf(D>2h%T6zddG9o2MO@mF{m>N4*Ue=P2mX( z(Y-6+sCHyP1rUBikT?V3p_j-wq(>&T`*vI5(@#;|?Dpd5)HR27x@!!_BP4-dP0ajX`3ZfWofRP4N@w zWmR$XhHhu_-DPlkb8cTGQ*WW?Ch+`y+|_#SWfJE$g0%@)1^!6G;^^V>P`x!{N5LnL zV@=_#cmoX=r7Z{`g{kyMA6x2og6rAMDuC#@2fatz|%SY)39Opt|`6osrkl?RW|i)P`8!$BZXfqwkDh5OH<5 zz0*Xk9w0+L;)`BJ3tBzEck|nG(ZlVr5xVkcH}*W6d+*Qhyy8rnWs|(%q_VB8T!pU8)r&o9{2D*yso4T<+V-?~Pxi|1bZrfFS`j15WyX^V>>! zb9qOc?V|FD9wtM@&So!tv6{;}h@8MDcUI4IPm*T^xwe1ap=vhGt7YVL_kdX{1BccE z<5F8Lp){~%v_G3nEV#U!< zr7e+KkLRejp6XI}YyHWhhk-iEFa8z}N?qk?L}#MqFnP2*PF^gp;{Rdfbl*w?r61y4 zDx7tEtDH=#+=d6$NB^iD)nbSt%b?o2gCp_9*Yg?^(6rmB((FeKyn`OfWtjrsi;f}W zOQgeiJ@=6&c4`hosa1TBPj-YDDI zY)Jg4G$_{bWE>tDHY7_Lq--;MG?OQ-#h)8X9-=cp(Hbwb5Sgm$`a$qu;oM$95KQSn zMl=8)`3|2vopBEwQ#?KWE~!E4F0xR+JX1WoJVWRt*+QReE|jj=f}C&s_66(-oER7p z_%?8B&{X;~9#69*C^}%C-#7XeKck1~IyqjPX{Oi5dw;mzliRKQ@65ji$z4;PrDSt$ zbd`4JC-31=59>|MUBWmq2RY)E@;RlvU2|-9zVw~tx53ZX@2T^IPh-bR+c4#ZluK-9 z9@d@OZtq#9)ZcL5bDwfgaogSFUD;g@R~6R{*9rGaj}uJJFg?b|#SFlla)OeXEJ7}a z+rHJF%YM?<1T57CI`bWt1LS;CL9sX!16LDa%&OC0TkYmO>hbp+cOP{Z^ZfH%@orH2 zfY!RJUpB6rZ-|NSBpRDrxuLYTJ-2nVzqaS&xlwjMdrmr&)s;I+3&kx|(cI(wE)mH- zqP8c4D3M1jPItH`>QGJBZs~`N1<=-SAx11tua))EO(~=v_qB0$fg6N@}=)L1T>;2;mOzj%(>p{9zRcB_Z3$kS5Ma!*GYF7?;Gujkk>ZG`9HtX0Z06= zGooWiK$bKm(j86rChfhTF7(W7;Cs+{!N+6YqHGbn8R^tpE=O{Kq?3OKBt89CF{P8s zbj7>Zdy>7MG|fmB#z>)Z3p$49V=DZ7TPug|Q^z-xUlqT2Ux)7lpZ1PrwnOqK5d=M$ z+0EYB9@SlnsP`6cI!@`ex0g4Y_nqgrr#1b|T&h?5r)Mx*2%E)EQf4~AOsAWDW?NxJ zmo|!3nWUN4=&nB{yOfrmd0oi87g0;`XAZLRh13S>X0?`9N*`kEG;ay7#P`x;at}3Z z+iag~S?o6YFW0s`QhF*k>4~+Q=;%*6#2iB|C!kyFkm<7%*XajeRfWIyBE#o^Pxo@V zc{$rFLP=zEEAmw>Wfj!m7BY|H>BC%JIW8yB1Lz^OAV)zVZ3QFSLocZ(gEz>c?WN<* zRqqTCHlg0aAb1uK2Omfb(GOni$}DmN57R+RB`X@g1@_WyrX9xMCyvH9TnECw2fA#I zu|%IiKdFLbNqT@`dh0nyr1=bS|0sG?eWKU+K)7OpC!gBFxGi;X-1Z9#>>sozut&hy zfTX}uX`iIKoNiZ|kbpkUlk}+kuC!Om%T>hf#w_);dtOTQfjF}oaOn`2+#xZtSbs6t<=3bs|YqTtw%*c;nt*dmo_ayO}zIL92TPghTQ`nt!_ zJGE6x?UW}e?_DCt_Xd)z?hyZm4us(>nOq+c>BV{eTU-%8)**2)tl#C{Wu#t4wG zTWr6;-Zljjb{b^*OeW9n!+sfwh1C%M^D^=Dk7PmeBN;=8i#!y{5)W)l|EeFLF_w|X zXbWQYx0p*>C8d+w$z%DbD!-PhO0&cn0`{zNAH4E;BJf4DAF4(aGr#s!y{kR~O;=z2 z z_jT`I@*@3JNgF~otp~lzhSRmg4nnvUJ-q7>KaEYr2It~bj%#$F5mKZz_Nu<${i6dO z`~UI%?riUS%dcsG9*`~Ik8fU|f3{6Z0p*MwBu9X@n@pF(0_r<&QPr(x)Z3AZn8yB> zijTxUVm)k}400p6w_Km7XAoUSHTsctR_@Aoz+z0L9?(Nny}E9KxoY8A=^p03nZLX=w0po0&cE1nV3FWnBIddcm?9mH^u2v9{CvhxVZ8{js&AvL~1GS#{w-!{7^KX zp>d{zh|LLKgwhGMpq81BZkjKs?=MCU!AtE#vN%`DC9jrExslRU$);>Xs(hs{s*hYu z4wrXgiS3XpaX*XDq2YK4mT%bzJIR`m(F6amGm*76#JZm1Io83ZJ%pblh= zmMCOJBYc{SXzh(ed2fQ~D2Gi{mz>6HJulS-m5g_K7%}kg;9kq|uMMC|vrwrJ4PGEE zc4>b6=vGMRtC|fQUMA3*bI1u!@t*O1=2VxciP~p#n!L)<)_0X(5x*PGZ_b*2H3JF< zB?XlaDjSf~Z?p55PfMQ@j!pJTwrz5VG*U=w1{!7bJzz-^wW{=>9EnXb2voAFFEBom z_xK-DEEwDPA(@$XVryzQ9!Q6z3(^&72KkEmVo^abKVr)r2P0lb`$BGQI@Fy_{Y1t& z0Yv(1uT9MlmFCb2>#dDQqAG*P53P~D(SL7;)DphSCEX_TJB=CVZ8@D9SiX&^$Z3hs zs_RubiQi1J_^6*IS~vh-p^2ah8^w`QO?f4Bl3hLnzr7XziEX6YQgv=vrK9->WNSZq z+7*pZKi%eH}uhX+YQ0` zWj9~ZDW@#)?&{c73O>kZJl0agj-O*;55p^BJ~wf-GgxJhxMz6cAMrgBh`VLRYgt5QsXN}`9paw9 z$no?b24(e-uxfSJGm$-#JJ|{bE}3p@6{JqmJm_c?y0|FvvYVKQ$K90(U=~ieCZ276 zyu|<_B+c>pk0M#4@#UxToEW+_?Ba6||8I*Q`~t7AB=ca(!;iy|or6F*_jQ`6$jb z4o~wi_IO|Dx(TQ9o@&(^M5?Q>ugdUMO}uJrveX1WR$aLzF_nVg9u)%9l)~uqGd*Q*vTljLs!-%_N9T*IV!#Z zfu6*T1yHH!4=@fRB9$)*232zqHl5v_ud@mi@&NyObPsp$~LmIThy4rwN5TugG zh%(Y6tQsdMp-X&RAOm!m_~R%}FdbfN0v^sSDzc(Ezhc#29?ZLe(PLYR%v~#|nkiddx<| z!G=PkrK#4ms$OrAg|McZ?&sfWi8_7~iqnzLs>-`eETKEN{Y`SPGF5RYBW*uzEA6!$ zPaQRU+WC}WK0`alFZ)7!3Hxi?8ry%iFZ6J`FwY$It`y7JVPDs`u50REY$_ z?WQ&w999!F(I@J0#u7m>sJxp6Z|5dsJO|E7S)DAnn{KVc$Zh^6leL)G@JjLt z^{HMy3woy+7VmX_l)z zfq2-&q?<_~_ImS#`pDV@CcGTs_w01&`^DdPuqdv8khnq3Um!@l22?T(B?hz`dMJi` zJVkuan!LD&ct|)=s~J#Re?0D)#H6fR&yz^Bg4h*SMV70y2l_1rlB;P?9&QxY zd7v~-Ql(ztrB2It1U*lXK6o11^t!1A7?l%`i} zg1kYlD?g)m+(zmEdJ~V0C4%Emq<2KB{2j{e)<%Z41+|aElK_nW~^sO!GM{U8VACJXJ%hqOJXLs@MkN6ffzzx!Ub?|M=B4iBEO=z(a%Iu8p6w?;mo6CTb~)P(Dk?J&G(1+ z%3rAXy%B(=F$`I;gztRA)`l{Xz|UF|>+@T!KH5 z+ac()1ge>vp?xOuF-^RRtr#lRr^$4fOXisoI#kK69$xC66b%Jx3gQmQRe9Hol52IUhvfhwWnFj`O z8`;=IqcQs62AK`3Khq@oX^B_`wTUZ^VLu0{?AeX9egYP6f)ooj=%MsM@<=6^GtgNc zNWb8L#5jk`)#TsObLvw+N%y41@Nio?pg-bS6FaOZ84nXQ zN-mHJov{YWVlh~ggHx!ZNJ6LFKzdt!GW_7^SyW9IC67Lau9TBGlUKx``qTM#q%@t6 zImGw+NqwbR(jHKF9_l&pgQ-x-F6ZSwN5Shc@=WmY$)MC9Ns~< z(STOuzdV&C#ld_pN2}!Jp6?P9oQIUFrYrgt?sznI=R|EV$nyf)J8JA=@OW?FqZNUB zXJ}{Oi+^acQD`z94sM8)TS`pUhm(11T9Y?tkokHDDxv{$cq=spkLf)c%rvf&tYs7Q zx`vfKB7z+#mzRsmxxlpil|Irf`?3^Hb?0~~0(mVkDz3?!>`S;m@?=-RGo=bnC7PTF}EIsUD=nr zJqOyi5;5g+=-NSG-!C%-&X*i#L%6j*I;|Z~UcmjXAwu1Pyt^G=EIZ$^W-X4#ZyV0{ z2O*uWa%(ln?OFX8RI+H+>}YT5=a1A(X-6!wO4tVys3YqKx@ia>#q>2=Zf%R&Rc*^;rbM+ORDJd&3nIn7>(xO~?ng@< zrVgwk$Z(C`HC7+0PN}*4V%Tr9=on|^7Uz@u>knmLrG_X3UAr-Lry`l%FmlY}$WxWV z&bUM_WF9hnHfPfguc|MWk!j8buh)h8+%ec24d{;hR4hd`xeB#rl)c=yhiFS4dR3p1 zYU0tWWJym7iNvGtVRgCSvFGsc0ICvNV^dW~jXC^}+UjjsQ+udt&4=c>pq(QY+LhZr z2X`;z1V;1km%0P*rT|qde#~#o4~-u{9;fhmk*L`gJ|7zy$>KKRM!zDtr+_D!#d*X) z2h-5t*^E3`-%c=f|3G=JCf=D=55a0_4<3I!XWSJ1R~g$TD}OUZ2n0$`VIZ+UC%w_L z+S)7CmC4kR9QSehR`GQ?Pl0?+aE|v)@24|AFRwF^+OlX{ZkuBBB@6Hxc|1|tDik*B z7-xx^2I#po7ZEfc&l7i3&wHk)4E1IqBAMCq+#LdXQ}YI)n+AfyOTfE~F{TlT^pZVY zPt;^8afCk1V>3ZWw^M?ZEAn_M_-#}*Rg>Gue)0+_MJy|BLk3kR53OTg?W9_|B9btf zdZ1!N+podjFV)p#NiL}Y#4vAz)&Hb7qJzW}Vq_`^PMwKO-?1%vGGXnLyiBp%#@Mde z?qVOdx0RsEdjmE6#g$}v1W}PbQV@BBqT&!?H?fG$M4&rzW`B9=Ci>c!1}l4!xrT{! zpzcGJ%P#gM;upnpqNDJ7Wa5q%dZ`|C^_zN&`SAbMRGYjNu@Qls>`5}wHnX?!04beL ze+7Q8v=&dj>S8d*^E{Qvpq%9dzfq0k^4#>Kp>}wS_mQ_Qwuq{p)}H9~gn4qf-QTB> z&nQPF`#0M@#}?o6V(pD{NH@l1Hq zh)Ir7+A4#T@5+1IQKrT%bNuIsVS>y+DwviK#YmC+%e$mFqJbaD*yqO=$zxO^|1es8 z>OJgD^oCJg?8ctEL;Por>Q!ffm0Sw8{xnlH_vlpm8Nca&D}YY8jed`UP9{MoHPKhW z(tP4zL+PBf0@?5hi>s<$Q=6#1XRgLbZ&mL;&qKHFYT$}ZnND=Jo-52X-*w-W#=XUT z(-W;8F|LaDEl~<{{Jtjo;N%kojyQ+|Auxsp|FdE~YxU9#SMb z_QplMsws%;dE07cBvhr2=$JfJ8Np=CF}CZYHx91)em*kGpq-ne2;208Nwyw5DqK*O1jww)T#j?#9t=_PE+ z3`q2BM3L8;i;bOhV2H-jJ3!Rw6crs0J@4Ip-L2h`?j`O~?lSIqE>B9Z>yYb}dz$yP zW}0=SIZWE@BbSx8%j<0eeNOs%>0~j(ua)0fzccAIf!o@`8O`{OR^8AvpG zf;Tfh?sTn%o`L*n7dm*|6()jSFO3yzzVrN^ zlEwD-FX=zp|CYb$f6c$L|7yP`zGr<#JC@sn?bo5LNy-E1f$$4j-9fy2kUG)Z%~RB! z(-oI8H)VFpn3O#!FH+v5yh}+;spAT9_w~H<79vL43m>=}v4=szH8F#nQMscuwJo#V zw3WACvX66ocgQ|p9U~pT?bGd5@iGIrCspxPzDif9Cap$HsF(Que>evFvK4(C-srJLAMWfGmQ+bHCFRAF)Le!W2RgbL3KKC}EK5mZI zg?RZ7V=OwvW46YsNfz4DY5fav_-}Zh-SHW_V=u>Akwx$dS+ip0nQw==IdDxlTcdzdtnn3*X`{}3o&Ubd<{Oyha z$0$b~M@_qEizHU|9mJ?8u%p}_!__r^Z1TvpoLrLu7IPwW^;JYnYWEa5< z9i|g-DBS@1Q#s&e-Hq@TWe~JpA|@5gHO3eGrDse<4#O+dxlxaIvv-Vl5;>&wWI^+) z8neS5QJFLXJ+gx8lkIp9<-iGSrDJz_Eb5KKIf@YrD}$B{Bht1Stl0{BVgE695$&Cb zr;-ai`%!$e{}KJJ#AJ~au?Ps5W=PGk@@;vg^2PRtX|{&FvE!g)CtWU@_$>1Y_UY(Y zX5U0_gkefAWi!>@+mu|&5bA&)(2L3+6nzAp2eX)^(X9g3Ng$v9qlP7n+|P6BDL0Z! ztBp@|Q+rHoQXy{nh8p1cq-;=TAV>E+f)jLEF4K3+`^`t3|3 zQr3+aWE6JrM{rs}%#YnkJ~ahzU?VY=K6oL;saDx$oFyW77GGvPc)F^oc!?a;^4H^B z-hu0_K`f>v2pwzA$5dhp2SFumCYD|t+j|M;lpY`DBNlUZvP34-I#he8eWWV182;Qd zGAEU|o2=MMEy!iIBL3vibI@6W{H`$`k6;K=@*G)T-Pn&6U!Pb-2hhGtpp~YeP6EUc z^!J;{Zl`clWd$d?;|QK^H)cwFW|f-SwN?x64c-Dklt3-bQ}Cq)}3sO+Ie9UJEzl}59~0L1*e=3I$LbS3qJA!aB}Aj zswSk`=4*VlS>`aYlG#SzZVVBo%Aw|A?XyCG<|}NMoMd z*~p@PQ!AM>gokQQ?W(0m5vK?^R(tgFkbzH7g81O`=1mk3?x%> zo=yw(gd{1ya9#fd(?CC0b`@K z(e!7|>Rr!Q>5w^HZ)}9BVNymhgPKbZktPYZw3bGQ7;1jfZE_W1pq1t09F4VZVZ zW}&V3gjm{PYL$#OnoF5!>*Ot|+_PbX~A>x-|=9_k7C4!!6KiZ6^8LUH4|UW|;_Fmtck)?8+9qs}ni znvwEEp_0}_>_(3htWW);;i86lmR=R>r>Y){)g?-;jd!Xw9pQqIp58sXO$RcjfL4z_ z3hl*t;PP6Fx-`H@)NDk9{z>)8+FmmHDIU3*zEV3&-E?;0hhABbY)J5;l70o-!Li3tdOPVTdl6IL-)W70SX`=9po;W>~K(V*#h1&(G8x@f< z_Q^f*HYf5-_$K8y)=^2eT?!EsyovO1jMI)v-SujCyD!Xf*qk5bHlPywX*r0KJkb9m zZoQUHWKY$2<13X{vf19CF4gR>v?5OOjOy5bNE1b~xqkEjhW8V`Qh?BYshYg4R znmFkUZ!Y1SR7`xPZr7^XgNbdfXs5dd);x=#zxu^}vW8H#f@rb!spMexv zU>p=*8;7+ALVf9)xmhbO=9Xreue4~>p;QO;@j@LURIzO|8>?%?j-VNjsSV8}`6d;< zb&XTz2B);E}%u-*uiiV6>n0Bq$jdcqW=BR~exW1JR?34^gY z=;tUzh#$p5+Do&t@(2WHF0k@5rS!rZ{VS6C9(ju}Dv(+mPt9OyjWJ(aVMa-F&30NH zqn3DDEMd&hX9yv!2de4YxMtWTuO^Bny4B6Q8Upeer4=S~a#}G#yJ7yr@>7kuW+&Fw&uB^}zm0ytTmbJPb$(6(^W_P2$xkTDt4 zi-@M!%xG`+k)GgZP2npMd=wSS2;B{9M$%5RDVd>IahsUeNT;_IMoII8A^KrtBz-pZ z{#pgIyVP0=qPNpdaX9FsD#QR5N*=t7b9J zkid08DqZ96Xl&Q4!UAyxHdq_+BRT!*<|OLy+JGXA6i$NetWWob?({(!%UxgN{?nya zMYJG?QQf?be_Io(e!#PC6ZMV+vyfXnCA2dO7~vqLy7T|L#!;c3m>o}}6SiRwa*@ZW zBfe(j#OG^=9BU$u7FrPj?g%DgKc1)6(P|^n*8}7UT9LoHZ|+U?6=Wh*$2qrqbo4Ju zMqR+NvFhjsQ+?*8Duy`=;%V|>1~>l!D|rY{Y00yigIJ(Sh1^L2Vk-xz3bbaqj3Dxx zj{V$WR~@mtU(kO!H(BsqWWmdjX|j5HO)n-&~j(6*glYdd(E?{V&!=+iNcK*{=+Z6X66yyWPE(FU`LZJt1RT> z{lV0pMR6kgiAf|9Ayw$k*`9i}B}^uL&v!Rn8{3FxydloM1HYsjr&OFhE+ao9QVHe* zS5yEgb_$QIGe1`rtKG&|dg2FGqL15ouqhd->}w2W{~8hV+#KPZ->C!l_=t7BhxV`WlxUDDBdP6q$7wDQ?vgd$M6G{&qIqA*o-Y9#@|FA1 zjOI|uYSx+vj`A28r?g~S8{->Qfm;f*`>*&a^{|;=Q;)a+wANA9AH?b8p|WfE^^}vSGpfpc*x{T2ywEhfdkD4Ob-2k6c=$GYJh$QJ-}0n+>^YR*Nd$S48JU=u zchmwMUy+=cgL7;PMZG4Y=EDwrxFh^_?)V3|ykA)Uop^p;;T0Hw!koq>d~FBZk(

    + {mode.length > 0 && ( +
    + +
    + )} {/* Main Content */}
    diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/custom_code_catalog.ts b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/custom_code_catalog.ts new file mode 100644 index 00000000000..aad3df181a7 --- /dev/null +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/custom_code/custom_code_catalog.ts @@ -0,0 +1,136 @@ +export const CODE_TEMPLATES = { + empty: { + name: "Empty Template", + code: `async def apply_guardrail(inputs, request_data, input_type): + # inputs: {texts, images, tools, tool_calls, structured_messages, model} + # request_data: {model, user_id, team_id, end_user_id, metadata} + # input_type: "request" or "response" + return allow()`, + }, + blockSSN: { + name: "Block SSN", + code: `def apply_guardrail(inputs, request_data, input_type): + for text in inputs["texts"]: + if regex_match(text, r"\\d{3}-\\d{2}-\\d{4}"): + return block("SSN detected") + return allow()`, + }, + redactEmail: { + name: "Redact Emails", + code: `def apply_guardrail(inputs, request_data, input_type): + pattern = r"[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}" + modified = [] + for text in inputs["texts"]: + modified.append(regex_replace(text, pattern, "[EMAIL REDACTED]")) + return modify(texts=modified)`, + }, + blockSQL: { + name: "Block SQL Injection", + code: `def apply_guardrail(inputs, request_data, input_type): + if input_type != "request": + return allow() + for text in inputs["texts"]: + if contains_code_language(text, ["sql"]): + return block("SQL code not allowed") + return allow()`, + }, + validateJSON: { + name: "Validate JSON", + code: `def apply_guardrail(inputs, request_data, input_type): + if input_type != "response": + return allow() + + schema = {"type": "object", "required": ["name", "value"]} + + for text in inputs["texts"]: + obj = json_parse(text) + if obj is None: + return block("Invalid JSON response") + if not json_schema_valid(obj, schema): + return block("Response missing required fields") + return allow()`, + }, + externalAPI: { + name: "External API Check (async)", + code: `async def apply_guardrail(inputs, request_data, input_type): + # Call an external moderation API (async for non-blocking) + for text in inputs["texts"]: + response = await http_post( + "https://api.example.com/moderate", + body={"text": text, "user_id": request_data["user_id"]}, + headers={"Authorization": "Bearer YOUR_API_KEY"}, + timeout=10 + ) + + if not response["success"]: + # API call failed, allow by default or block + return allow() + + if response["body"].get("flagged"): + return block(response["body"].get("reason", "Content flagged")) + + return allow()`, + }, +}; + +export const PRIMITIVES = { + "Return Values": [ + { name: "allow()", desc: "Let request/response through" }, + { name: "block(reason)", desc: "Reject with message" }, + { name: "flag(reason, metadata={})", desc: "Let through, record a non-blocking violation" }, + { name: "modify(texts=[], images=[], tool_calls=[])", desc: "Transform content" }, + ], + "HTTP Requests (async)": [ + { name: "await http_request(url, method, headers, body)", desc: "Make async HTTP request" }, + { name: "await http_get(url, headers)", desc: "Async GET request" }, + { name: "await http_post(url, body, headers)", desc: "Async POST request" }, + ], + "Regex Functions": [ + { name: "regex_match(text, pattern)", desc: "Returns True if pattern found" }, + { name: "regex_replace(text, pattern, replacement)", desc: "Replace all matches" }, + { name: "regex_find_all(text, pattern)", desc: "Return list of matches" }, + ], + "JSON Functions": [ + { name: "json_parse(text)", desc: "Parse JSON string, returns None on error" }, + { name: "json_stringify(obj)", desc: "Convert to JSON string" }, + { name: "json_schema_valid(obj, schema)", desc: "Validate against JSON schema" }, + ], + "URL Functions": [ + { name: "extract_urls(text)", desc: "Extract all URLs from text" }, + { name: "is_valid_url(url)", desc: "Check if URL is valid" }, + { name: "all_urls_valid(text)", desc: "Check all URLs in text are valid" }, + ], + "Code Detection": [ + { name: "detect_code(text)", desc: "Returns True if code detected" }, + { name: "detect_code_languages(text)", desc: "Returns list of detected languages" }, + { name: 'contains_code_language(text, ["sql"])', desc: "Check for specific languages" }, + ], + "Text Utilities": [ + { name: "contains(text, substring)", desc: "Check if substring exists" }, + { name: "contains_any(text, [substr1, substr2])", desc: "Check if any substring exists" }, + { name: "word_count(text)", desc: "Count words" }, + { name: "char_count(text)", desc: "Count characters" }, + { name: "lower(text) / upper(text) / trim(text)", desc: "String transforms" }, + ], +}; + +export const MODE_OPTIONS = [ + { value: "pre_call", label: "pre_call (Request)" }, + { value: "post_call", label: "post_call (Response)" }, + { value: "during_call", label: "during_call (Parallel)" }, + { value: "logging_only", label: "logging_only" }, + { value: "pre_mcp_call", label: "pre_mcp_call (Before MCP Tool Call)" }, + { value: "post_mcp_call", label: "post_mcp_call (After MCP Tool Call)" }, + { value: "during_mcp_call", label: "during_mcp_call (During MCP Tool Call)" }, +]; + +export const TEMPLATE_ITEMS = Object.entries(CODE_TEMPLATES).map(([key, template]) => ({ + value: key, + label: template.name, +})); + +export type ModeOption = (typeof MODE_OPTIONS)[number]; + +export const MODE_OPTION_BY_VALUE: Record = Object.fromEntries( + MODE_OPTIONS.map((option) => [option.value, option]), +); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx index 73d9fb8123e..c67758c0d98 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info.tsx @@ -33,7 +33,8 @@ import { } from "./GuardrailFormField"; import ContentFilterManager, { formatContentFilterDataForAPI } from "./content_filter/ContentFilterManager"; import CustomCodeModal, { EditGuardrailData } from "./custom_code/CustomCodeModal"; -import { GuardrailModeCard, GuardrailModeRows } from "./GuardrailModeDisplay"; +import { GuardrailModeCard } from "./GuardrailModeDisplay"; +import { GuardrailReadOnlyDetails } from "./GuardrailReadOnlyDetails"; import { getLoggingOnlyScopeUpdate, getGuardrailLogoAndName, @@ -41,10 +42,15 @@ import { loggingOnlyScopeToChoice, skipSystemMessageToChoice, skipToolMessageToChoice, + streamScopeByModeFromConfig, + streamScopeForUpdate, supportsDirectionalLoggingOnlyScope, + toModeArray, type SkipSystemMessageChoice, type SkipToolMessageChoice, + type GuardrailStreamScope, } from "./guardrail_info_helpers"; +import { GuardrailStreamScopeCaption, StreamScopeFormField } from "./StreamScopeFields"; import GuardrailOptionalParams from "./guardrail_optional_params"; import GuardrailProviderFields from "./guardrail_provider_fields"; import PiiConfiguration from "./pii_configuration"; @@ -236,6 +242,13 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, "skip_tool_message_choice", skipToolMessageToChoice(guardrailData.litellm_params?.skip_tool_message_in_guardrail), ); + form.setValue( + "stream_scope_by_mode", + streamScopeByModeFromConfig( + guardrailData.litellm_params?.stream_scope, + toModeArray(guardrailData.litellm_params?.mode), + ), + ); form.setValue( "guardrail_info", guardrailData.guardrail_info ? JSON.stringify(guardrailData.guardrail_info, null, 2) : "", @@ -304,6 +317,16 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, updateData.litellm_params.default_on = values.default_on; } + const modes = toModeArray(guardrailData.litellm_params?.mode); + const nextStreamScope = streamScopeForUpdate( + modes, + (values.stream_scope_by_mode as Record | undefined) ?? {}, + guardrailData.litellm_params?.stream_scope, + ); + if (nextStreamScope !== undefined) { + updateData.litellm_params.stream_scope = nextStreamScope; + } + const prevSkipChoice = skipSystemMessageToChoice(guardrailData.litellm_params?.skip_system_message_in_guardrail); const nextSkipChoice = values.skip_system_message_choice as SkipSystemMessageChoice | undefined; if (nextSkipChoice !== undefined && nextSkipChoice !== prevSkipChoice) { @@ -566,6 +589,7 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, +

    Created At

    @@ -723,6 +747,11 @@ const GuardrailInfoView: React.FC = ({ guardrailId, onClose, )} + + = ({ guardrailId, onClose, ) : ( -
    -
    -

    Guardrail ID

    -
    {guardrailData.guardrail_id}
    -
    -
    -

    Guardrail Name

    -
    {guardrailData.guardrail_name || "Unnamed Guardrail"}
    -
    -
    -

    Provider

    -
    {displayName}
    -
    - -
    -

    Default On

    - - {guardrailData.litellm_params?.default_on ? "Yes" : "No"} - -
    - - {guardrailData.litellm_params?.pii_entities_config && - Object.keys(guardrailData.litellm_params.pii_entities_config).length > 0 && ( -
    -

    PII Protection

    -
    - - {Object.keys(guardrailData.litellm_params.pii_entities_config).length} PII entities - configured - -
    -
    - )} - -
    -

    Created At

    -
    {formatDate(guardrailData.created_at)}
    -
    -
    -

    Last Updated

    -
    {formatDate(guardrailData.updated_at)}
    -
    - - {guardrailData.litellm_params?.guardrail === "tool_permission" && ( - - )} -
    + )}     diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx index 96ade4b8c1b..d149ef98328 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.test.tsx @@ -15,6 +15,11 @@ import { skipToolMessageToChoice, choiceToSkipToolForCreate, formatGuardrailMode, + formatGuardrailStreamScope, + streamScopeByModeFromConfig, + streamScopeForMode, + streamScopeForUpdate, + streamScopePayload, loggingOnlyScopeToChoice, choiceToLoggingOnlyScope, getLoggingOnlyScopeUpdate, @@ -369,4 +374,56 @@ describe("guardrail_info_helpers", () => { expect(choiceToSkipToolForCreate("no")).toBe(false); }); }); + + describe("stream_scope helpers", () => { + it("treats omitted config as both for every mode", () => { + expect(streamScopeForMode(undefined, "pre_call")).toBe("both"); + expect(streamScopeByModeFromConfig(undefined, ["pre_call", "post_call"])).toEqual({ + pre_call: "both", + post_call: "both", + }); + }); + + it("applies a scalar to every selected mode and omits both-only payloads", () => { + expect(streamScopeForMode("streaming", "post_call")).toBe("streaming"); + expect(streamScopePayload(["pre_call", "post_call"], { pre_call: "both", post_call: "both" })).toBeUndefined(); + expect(streamScopePayload(["pre_call", "post_call"], { pre_call: "streaming", post_call: "streaming" })).toBe( + "streaming", + ); + }); + + it("keeps a mixed map instead of collapsing it to a scalar", () => { + expect(streamScopePayload(["pre_call", "post_call"], { pre_call: "both", post_call: "streaming" })).toEqual({ + post_call: "streaming", + }); + }); + + it("formats scalar and per-mode stream scopes for display", () => { + expect(formatGuardrailStreamScope(undefined)).toBe(""); + expect(formatGuardrailStreamScope("both")).toBe("Streaming and non-streaming"); + expect(formatGuardrailStreamScope("non_streaming")).toBe("Non-streaming only"); + expect(formatGuardrailStreamScope({ post_call: "streaming", pre_call: "both" })).toBe( + "post_call: Streaming only, pre_call: Streaming and non-streaming", + ); + }); + + it("emits both on update only when a prior restriction is cleared", () => { + expect(streamScopeForUpdate(["post_call"], { post_call: "streaming" }, undefined)).toBe("streaming"); + expect(streamScopeForUpdate(["post_call"], { post_call: "both" }, "streaming")).toBe("both"); + expect(streamScopeForUpdate(["post_call"], { post_call: "both" }, undefined)).toBeUndefined(); + }); + + it("keeps stored restrictions for modes outside the current selection", () => { + expect( + streamScopeForUpdate( + ["pre_call"], + { pre_call: "streaming" }, + { pre_call: "streaming", post_call: "non_streaming" }, + ), + ).toBeUndefined(); + expect( + streamScopeForUpdate(["pre_call"], { pre_call: "both" }, { pre_call: "streaming", post_call: "non_streaming" }), + ).toEqual({ post_call: "non_streaming" }); + }); + }); }); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx index f55148b7315..a898f3b9abd 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/guardrails/_components/guardrail_info_helpers.tsx @@ -343,3 +343,86 @@ export function choiceToSkipToolForCreate(choice: SkipToolMessageChoice | undefi if (choice === "no") return false; return undefined; } + +export const GUARDRAIL_STREAM_SCOPES = ["both", "streaming", "non_streaming"] as const; +export type GuardrailStreamScope = (typeof GUARDRAIL_STREAM_SCOPES)[number]; + +export const STREAM_SCOPE_OPTIONS: { value: GuardrailStreamScope; label: string }[] = [ + { value: "both", label: "Streaming and non-streaming" }, + { value: "streaming", label: "Streaming only" }, + { value: "non_streaming", label: "Non-streaming only" }, +]; + +export const isGuardrailStreamScope = (value: unknown): value is GuardrailStreamScope => + value === "both" || value === "streaming" || value === "non_streaming"; + +export const streamScopeForMode = (raw: unknown, mode: string): GuardrailStreamScope => { + if (isGuardrailStreamScope(raw)) return raw; + if (raw !== null && typeof raw === "object" && !Array.isArray(raw)) { + const value = (raw as Record)[mode]; + if (isGuardrailStreamScope(value)) return value; + } + return "both"; +}; + +export const streamScopeByModeFromConfig = (raw: unknown, modes: string[]): Record => + Object.fromEntries(modes.map((mode) => [mode, streamScopeForMode(raw, mode)])); + +export const streamScopePayload = ( + modes: string[], + scopes: Record, +): GuardrailStreamScope | Record | undefined => { + const perMode: Record = Object.fromEntries( + modes.map((mode) => [mode, scopes[mode] ?? "both"]), + ); + const values = Object.values(perMode); + if (values.length === 0 || values.every((scope) => scope === "both")) return undefined; + const unique = new Set(values); + if (unique.size === 1) return values[0]; + return Object.fromEntries(Object.entries(perMode).filter((entry) => entry[1] !== "both")); +}; + +export const formatGuardrailStreamScope = (raw: unknown): string => { + if (isGuardrailStreamScope(raw)) { + return STREAM_SCOPE_OPTIONS.find((option) => option.value === raw)?.label ?? raw; + } + if (raw !== null && typeof raw === "object" && !Array.isArray(raw)) { + const entries = Object.entries(raw as Record).filter( + (entry): entry is [string, GuardrailStreamScope] => isGuardrailStreamScope(entry[1]), + ); + if (entries.length === 0) return ""; + return entries.map(([mode, scope]) => `${mode}: ${formatGuardrailStreamScope(scope)}`).join(", "); + } + return ""; +}; + +export const streamScopeForUpdate = ( + modes: string[], + nextByMode: Record, + previousRaw: unknown, + previousModes: string[] = modes, +): GuardrailStreamScope | Record | undefined => { + const previousMap: Record = + previousRaw !== null && typeof previousRaw === "object" && !Array.isArray(previousRaw) + ? (previousRaw as Record) + : {}; + const preserved: Record = Object.fromEntries( + Object.entries(previousMap).filter( + (entry): entry is [string, GuardrailStreamScope] => !modes.includes(entry[0]) && isGuardrailStreamScope(entry[1]), + ), + ); + const nextModes = [...modes, ...Object.keys(preserved)]; + const nextStreamScope = streamScopePayload(nextModes, { + ...preserved, + ...Object.fromEntries(modes.map((mode) => [mode, nextByMode[mode] ?? "both"])), + }); + const previousCompareModes = Object.keys(previousMap).length > 0 ? Object.keys(previousMap) : previousModes; + const previousStreamScope = streamScopePayload( + previousCompareModes, + streamScopeByModeFromConfig(previousRaw, previousCompareModes), + ); + if (JSON.stringify(nextStreamScope ?? "both") === JSON.stringify(previousStreamScope ?? "both")) { + return undefined; + } + return nextStreamScope ?? "both"; +}; diff --git a/ui/litellm-dashboard/src/components/policies/types.ts b/ui/litellm-dashboard/src/components/policies/types.ts index 430864f93df..2830a22950c 100644 --- a/ui/litellm-dashboard/src/components/policies/types.ts +++ b/ui/litellm-dashboard/src/components/policies/types.ts @@ -102,7 +102,7 @@ export interface PolicyAttachmentListResponse { export interface PipelineStepResult { guardrail_name: string; - outcome: "pass" | "fail" | "error"; + outcome: "pass" | "fail" | "error" | "skip"; action_taken: string; modified_data: Record | null; error_detail: string | null; diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 2a2b5acae78..cd85908979e 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -27393,6 +27393,13 @@ export interface components { * @default true */ sticky_session_routing: boolean | null; + /** + * Stream Scope + * @description Whether this guardrail runs on streaming requests, non-streaming requests, or both. A string applies to every configured mode. A map overrides named modes (pre_call, during_call, post_call, ...); omitted keys default to both. Unset means both, matching historical behavior. + */ + stream_scope?: ("streaming" | "non_streaming" | "both") | { + [key: string]: "streaming" | "non_streaming" | "both"; + } | null; /** * Template Id * @description The ID of your Model Armor template @@ -37786,6 +37793,13 @@ export interface components { * @default true */ sticky_session_routing: boolean | null; + /** + * Stream Scope + * @description Whether this guardrail runs on streaming requests, non-streaming requests, or both. A string applies to every configured mode. A map overrides named modes (pre_call, during_call, post_call, ...); omitted keys default to both. Unset means both, matching historical behavior. + */ + stream_scope?: ("streaming" | "non_streaming" | "both") | { + [key: string]: "streaming" | "non_streaming" | "both"; + } | null; /** * Template Id * @description The ID of your Model Armor template From 0721cffab2ecbde51cdef9aeca0ce1c16b3e0aa9 Mon Sep 17 00:00:00 2001 From: yujonglee Date: Thu, 8 Oct 2026 13:48:28 -0700 Subject: [PATCH 65/65] refactor(python-bridge): take NativeCall directly and fold routes into per-route folders (#45413) * refactor(python-bridge): take NativeCall directly and fold routes into per-route folders Co-Authored-By: Claude Opus 5.5 * test(python-bridge): use rstest for updated tests and keep embedding's stub parameter name Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../callbacks-legacy-python/src/adapter.rs | 9 +- .../callbacks-legacy-python/src/call.rs | 44 ++- .../src/test_support.rs | 24 +- .../crates/host-python/src/argument.rs | 121 +++++---- litellm-rust/crates/host-python/src/lib.rs | 2 +- litellm-rust/crates/python-bridge/src/lib.rs | 13 +- .../crates/python-bridge/src/marshal.rs | 19 -- .../crates/python-bridge/src/routes/AGENTS.md | 4 + .../src/routes/audio_transcription.rs | 8 +- .../src/routes/chat_completions.rs | 150 ----------- .../src/routes/chat_completions/host.rs | 2 +- .../src/routes/chat_completions/mod.rs | 71 +++++ .../python-bridge/src/routes/embeddings.rs | 10 +- .../python-bridge/src/routes/inference.rs | 89 ++++-- .../python-bridge/src/routes/messages/host.rs | 18 +- .../python-bridge/src/routes/messages/mod.rs | 37 +-- .../crates/python-bridge/src/routes/mod.rs | 175 ++++-------- .../python-bridge/src/routes/ocr/host.rs | 6 +- .../python-bridge/src/routes/ocr/mod.rs | 37 +-- .../python-bridge/src/routes/ocr/project.rs | 254 +++++------------- .../src/routes/responses/host.rs | 2 +- .../python-bridge/src/routes/responses/mod.rs | 102 +++++++ .../{responses.rs => responses/websocket.rs} | 113 +------- litellm/rust_bridge/_native.pyi | 8 - .../chat_completions/route_host.py | 6 +- litellm/rust_bridge/messages/route_host.py | 6 +- litellm/rust_bridge/ocr/route_host.py | 8 +- litellm/rust_bridge/responses/route_host.py | 6 +- .../chat_completions/test_route_host.py | 8 +- .../rust_bridge/messages/test_route_host.py | 23 +- .../rust_bridge/native_route_wheel_test.py | 69 ++--- .../rust_bridge/responses/test_route_host.py | 23 +- 32 files changed, 559 insertions(+), 908 deletions(-) delete mode 100644 litellm-rust/crates/python-bridge/src/routes/chat_completions.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/responses/mod.rs rename litellm-rust/crates/python-bridge/src/routes/{responses.rs => responses/websocket.rs} (57%) diff --git a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs index 70b7ecb45fd..dd3da8991cc 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/adapter.rs @@ -936,9 +936,6 @@ mod payload_tests { /// The payload phases of `Logging` on top of `StubLogger`, with `pre_call` handing the /// payload to the case's `on_pre_call`. const PAYLOAD_LOGGER: &CStr = c" -class Request: - pass - class PayloadLogger(StubLogger): def update_from_kwargs(self, **update): self.update = update @@ -954,7 +951,7 @@ class PayloadLogger(StubLogger): self.record('post_call', None) self.post = (original_response, api_key, additional_args) -request = Request() +bound = {} kwargs = {} logger = PayloadLogger() on_pre_call = lambda additional_args: None @@ -1225,10 +1222,10 @@ on_pre_call = lambda args: observed.append( def check(): assert observed == [(True, True)], observed ")] - #[case::request_attribute_behind_an_omitted_keyword(c" + #[case::bound_value_behind_an_omitted_keyword(c" document = {'type': 'document_url', 'document_url': 'data:application/pdf;base64,YWJj'} pages = [0] -request.document = document +bound['document'] = document kwargs = {'pages': pages} observed = [] on_pre_call = lambda args: observed.append( diff --git a/litellm-rust/crates/callbacks-legacy-python/src/call.rs b/litellm-rust/crates/callbacks-legacy-python/src/call.rs index 8e7645cae7b..8ea67292ae7 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/call.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/call.rs @@ -13,21 +13,21 @@ use pyo3::{ pub struct PublicCall { args: Py, kwargs: Py, - request: Py, + bound: Py, } impl PublicCall { /// Copies the keyword arguments once, so the legacy path's rewrites never reach the /// caller's own dict while every value keeps its identity. pub fn capture( - request: &Bound<'_, PyAny>, + bound: &Bound<'_, PyDict>, args: &Bound<'_, PyTuple>, kwargs: &Bound<'_, PyDict>, ) -> PyResult { Ok(Self { args: args.clone().unbind(), kwargs: kwargs.copy()?.unbind(), - request: request.clone().unbind(), + bound: bound.clone().unbind(), }) } @@ -55,31 +55,30 @@ impl PublicCall { py: Python<'py>, name: &str, ) -> PyResult>> { - lookup(self.kwargs.bind(py), self.request.bind(py), name) + lookup(self.kwargs.bind(py), self.bound.bind(py), name) } pub(crate) fn traverse(&self, visit: &PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.args)?; visit.call(&self.kwargs)?; - visit.call(&self.request) + visit.call(&self.bound) } } #[cfg(test)] mod tests { use super::*; + use crate::test_support::{local, local_dict}; fn capture<'py>(py: Python<'py>, source: &std::ffi::CStr) -> (PublicCall, Bound<'py, PyDict>) { let locals = PyDict::new(py); py.run(source, Some(&locals), Some(&locals)).unwrap(); - let request = locals.get_item("request").unwrap().unwrap(); - let kwargs = locals - .get_item("kwargs") - .unwrap() - .unwrap() - .cast_into::() - .unwrap(); - let call = PublicCall::capture(&request, &PyTuple::empty(py), &kwargs).unwrap(); + let call = PublicCall::capture( + &local_dict(&locals, "bound"), + &PyTuple::empty(py), + &local_dict(&locals, "kwargs"), + ) + .unwrap(); (call, locals) } @@ -91,25 +90,22 @@ mod tests { py, c" pages = [0] -class Request: - pass -request = Request() +bound = {'pages': [1]} kwargs = {'pages': pages} ", ); - let caller = locals - .get_item("kwargs") - .unwrap() - .unwrap() - .cast_into::() - .unwrap(); + let caller = local_dict(&locals, "kwargs"); call.kwargs() .bind(py) .set_item("litellm_call_id", "call") .unwrap(); assert!(!caller.contains("litellm_call_id").unwrap()); - let pages = locals.get_item("pages").unwrap().unwrap(); - assert!(call.lookup(py, "pages").unwrap().unwrap().is(&pages)); + assert!( + call.lookup(py, "pages") + .unwrap() + .unwrap() + .is(local(&locals, "pages")) + ); }); } } diff --git a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs index f252a45b562..a7b28f1b0ca 100644 --- a/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs +++ b/litellm-rust/crates/callbacks-legacy-python/src/test_support.rs @@ -173,21 +173,23 @@ pub(crate) fn local<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py, locals.get_item(name).unwrap().unwrap() } -/// A legacy call over the namespace's `kwargs` (or none) and `request` (or `None`). +pub(crate) fn local_dict<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py, PyDict> { + local(locals, name).cast_into().unwrap() +} + +/// A legacy call over the namespace's `kwargs` and `bound` dicts, each empty when absent. pub(crate) fn legacy_call( py: Python<'_>, locals: &Bound<'_, PyDict>, asynchronous: bool, ) -> LegacyLogging { - let request = locals - .get_item("request") - .unwrap() - .unwrap_or_else(|| py.None().into_bound(py)); - let kwargs = locals - .get_item("kwargs") - .unwrap() - .map(|kwargs| kwargs.cast_into::().unwrap()) - .unwrap_or_else(|| PyDict::new(py)); - let call = PublicCall::capture(&request, &PyTuple::empty(py), &kwargs).unwrap(); + let dict = |name: &str| { + locals + .get_item(name) + .unwrap() + .map(|value| value.cast_into::().unwrap()) + .unwrap_or_else(|| PyDict::new(py)) + }; + let call = PublicCall::capture(&dict("bound"), &PyTuple::empty(py), &dict("kwargs")).unwrap(); LegacyLogging::new(py, crate::LoggingOperation::Ocr, call, asynchronous) } diff --git a/litellm-rust/crates/host-python/src/argument.rs b/litellm-rust/crates/host-python/src/argument.rs index 13214e0e9ed..0b89f94cd74 100644 --- a/litellm-rust/crates/host-python/src/argument.rs +++ b/litellm-rust/crates/host-python/src/argument.rs @@ -2,78 +2,97 @@ use pyo3::{prelude::*, types::PyDict}; pub fn lookup<'py>( kwargs: &Bound<'py, PyDict>, - request: &Bound<'py, PyAny>, + bound: &Bound<'py, PyDict>, name: &str, ) -> PyResult>> { - if let Some(value) = kwargs.get_item(name)? { - return Ok(Some(value)); + match kwargs.get_item(name)? { + Some(value) => Ok(Some(value)), + None => bound.get_item(name), } - if let Ok(bound) = request.cast::() { - return bound.get_item(name); - } - request.getattr_opt(name) +} + +pub fn present<'py>( + kwargs: &Bound<'py, PyDict>, + bound: &Bound<'py, PyDict>, + name: &str, +) -> PyResult>> { + Ok(lookup(kwargs, bound, name)?.filter(|value| !value.is_none())) } #[cfg(test)] mod tests { + use rstest::rstest; + use super::*; - #[test] - fn lookup_prefers_the_keyword_even_when_none_and_falls_back_to_the_request() { + fn dicts<'py>( + py: Python<'py>, + kwargs: &str, + bound: &str, + ) -> (Bound<'py, PyDict>, Bound<'py, PyDict>) { + let eval = |source: &str| { + py.eval(&std::ffi::CString::new(source).unwrap(), None, None) + .unwrap() + .cast_into::() + .unwrap() + }; + (eval(kwargs), eval(bound)) + } + + #[rstest] + #[case::keyword_wins( + "{'api_key': 'keyword'}", + "{'api_key': 'bound'}", + Some(Some("keyword")) + )] + #[case::explicit_none_wins("{'api_key': None}", "{'api_key': 'bound'}", Some(None))] + #[case::bound_fallback("{}", "{'api_key': 'bound'}", Some(Some("bound")))] + #[case::missing("{}", "{}", None)] + fn lookup_prefers_the_keyword_and_falls_back_to_bound( + #[case] kwargs: &str, + #[case] bound: &str, + #[case] expected: Option>, + ) { crate::initialize_python(); Python::attach(|py| { - let locals = PyDict::new(py); - py.run( - c" -key = object() -document = {'type': 'document_url'} -class Request: - api_key = 'from-request' - api_base = 'from-request' - document = document -request = Request() -kwargs = {'api_key': key, 'api_base': None} -", - Some(&locals), - Some(&locals), - ) - .unwrap(); - let item = |name: &str| locals.get_item(name).unwrap().unwrap(); - let kwargs = item("kwargs").cast_into::().unwrap(); - let request = item("request"); - let find = |name: &str| lookup(&kwargs, &request, name).unwrap(); - assert!(find("api_key").unwrap().is(item("key"))); - assert!(find("api_base").unwrap().is_none()); - assert!(find("document").unwrap().is(item("document"))); - assert!(find("model").is_none()); + let (kwargs, bound) = dicts(py, kwargs, bound); + let value = lookup(&kwargs, &bound, "api_key") + .unwrap() + .map(|value| value.extract::>().unwrap()); + assert_eq!(value, expected.map(|value| value.map(str::to_owned))); }); } - #[rstest::rstest] - #[case::prepared_value("{'api_key': 'replacement'}", Some("replacement"))] - #[case::explicit_none("{'api_key': None}", None)] - #[case::bound_fallback("{}", Some("original"))] - fn prepared_mapping_overrides_bound_values( - #[case] source: &str, + #[rstest] + #[case::explicit_none_hides_bound("{'api_key': None}", "{'api_key': 'bound'}", None)] + #[case::bound_none("{}", "{'api_key': None}", None)] + #[case::bound_value("{}", "{'api_key': 'bound'}", Some("bound"))] + fn present_treats_none_as_unset( + #[case] kwargs: &str, + #[case] bound: &str, #[case] expected: Option<&str>, ) { crate::initialize_python(); Python::attach(|py| { + let (kwargs, bound) = dicts(py, kwargs, bound); + let value = present(&kwargs, &bound, "api_key") + .unwrap() + .map(|value| value.extract::().unwrap()); + assert_eq!(value.as_deref(), expected); + }); + } + + #[rstest] + fn lookup_returns_the_callers_object() { + crate::initialize_python(); + Python::attach(|py| { + let document = PyDict::new(py); let bound = PyDict::new(py); - bound.set_item("api_key", "original").unwrap(); - let source = std::ffi::CString::new(source).unwrap(); - let prepared = py - .eval(&source, None, None) - .unwrap() - .cast_into::() - .unwrap(); - let value = lookup(&prepared, bound.as_any(), "api_key") + bound.set_item("document", &document).unwrap(); + let found = lookup(&PyDict::new(py), &bound, "document") .unwrap() .unwrap(); - assert_eq!( - value.extract::>().unwrap().as_deref(), - expected - ); + assert!(found.is(&document)); }); } } diff --git a/litellm-rust/crates/host-python/src/lib.rs b/litellm-rust/crates/host-python/src/lib.rs index 00543f64085..323f5b7a79b 100644 --- a/litellm-rust/crates/host-python/src/lib.rs +++ b/litellm-rust/crates/host-python/src/lib.rs @@ -19,7 +19,7 @@ mod owned; mod runtime; mod services; -pub use argument::lookup; +pub use argument::{lookup, present}; pub use binding::PythonBinding; pub use conversion_cache::{FromPythonCache, ToPythonCache}; pub use driver::{CallOptions, run_call}; diff --git a/litellm-rust/crates/python-bridge/src/lib.rs b/litellm-rust/crates/python-bridge/src/lib.rs index cf1080a3881..92acf796e6b 100644 --- a/litellm-rust/crates/python-bridge/src/lib.rs +++ b/litellm-rust/crates/python-bridge/src/lib.rs @@ -17,6 +17,10 @@ mod tokenizer; #[pymodule(gil_used = true)] mod _native { + #[pymodule_export] + use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; + use pyo3::{prelude::*, types::PyModule}; + #[cfg(feature = "panic-test")] #[pymodule_export] use crate::diagnostics::_panic_for_test; @@ -29,9 +33,7 @@ mod _native { #[pymodule_export] use crate::routes::audio_transcription::{atranscription, transcription}; #[pymodule_export] - use crate::routes::chat_completions::{ - achat_completions, acompletion, chat_completions, completion, - }; + use crate::routes::chat_completions::{acompletion, completion}; #[pymodule_export] use crate::routes::embeddings::{aembedding, embedding}; #[pymodule_export] @@ -51,9 +53,6 @@ mod _native { use crate::tokenizer::HuggingFaceEncoding; #[pymodule_export] use crate::tokenizer::Tokenizer; - #[pymodule_export] - use litellm_host_python::{ForkedAfterNativeRuntimeStarted, ProcessReservedForForking}; - use pyo3::{prelude::*, types::PyModule}; #[pymodule_init] fn init(module: &Bound<'_, PyModule>) -> PyResult<()> { @@ -101,8 +100,6 @@ mod tests { "atranscription", "messages", "amessages", - "chat_completions", - "achat_completions", "completion", "acompletion", "responses", diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index df15a9591a1..4a7efeffdd2 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -19,13 +19,6 @@ pub(crate) struct RouteOptions { pub(crate) timeout: Option, } -pub(crate) fn messages_argument(value: &Bound<'_, PyAny>) -> PyResult> { - match from_py_argument(value)? { - Value::Array(values) => Ok(values), - _ => Err(PyValueError::new_err("messages must be a list")), - } -} - fn required_object(name: &'static str, value: Value) -> PyResult> { match value { Value::Object(values) => Ok(values), @@ -449,18 +442,6 @@ module.__getattr__ = fail fn argument_converters_keep_nested_values_and_accept_explicit_none() { Python::initialize(); Python::attach(|py| { - let messages = py - .eval( - c"[{'role': 'user', 'content': [{'type': 'text', 'text': 'hi'}]}]", - None, - None, - ) - .unwrap(); - assert_eq!( - Value::Array(messages_argument(&messages).unwrap()), - json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]) - ); - let params = py.eval(c"{'temperature': 0.2}", None, None).unwrap(); assert_eq!( optional_object("optional_params", ¶ms).unwrap(), diff --git a/litellm-rust/crates/python-bridge/src/routes/AGENTS.md b/litellm-rust/crates/python-bridge/src/routes/AGENTS.md index c2afed49b45..948c67eb647 100644 --- a/litellm-rust/crates/python-bridge/src/routes/AGENTS.md +++ b/litellm-rust/crates/python-bridge/src/routes/AGENTS.md @@ -11,3 +11,7 @@ The host driver owns sequencing and terminal events; the bridge supplies fallibl Use the shared `run_public_call` boundary with hooks supplied by bridge composition. `callbacks-legacy-python` owns legacy argument sharing and `Logging` dispatch behind `PublicCall` and `LegacyLogging`. Route bindings supply `callbacks-legacy-python::LoggingOperation` when composing legacy logging and may retain the request needed for projection, but must not duplicate the legacy callback contract Regression tests must observe that an unstarted call does no setup, hook and preflight rewrites affect resource configuration, setup failures reach the selected failure handler once, and provider work is not replayed. Retain existing read-point and object-identity guarantees while changing setup timing + +## Layout + +`messages/` is the reference shape. Each lifecycle route is a folder: `mod.rs` holds `run_` and the thin sync and async `#[pyfunction]` entrypoints that take `NativeCall` and call it, `host.rs` holds the `PythonBinding` host, and anything else route-specific (projection, error mapping, extra pyclasses) gets its own sibling file. Scaffolded routes without a lifecycle (embeddings, audio transcription) stay a single `.rs` diff --git a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs index fc7bfb08b75..4c0c2760875 100644 --- a/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs +++ b/litellm-rust/crates/python-bridge/src/routes/audio_transcription.rs @@ -5,6 +5,8 @@ use litellm_inference_transcription::{ use pyo3::prelude::*; use serde_json::{Map, Value}; +use super::NativeCall; + use crate::{ errors::route_error_to_pyerr, marshal::{RouteOptions, optional_object_field, required_field, value_route_options}, @@ -40,8 +42,7 @@ async fn execute( } #[pyfunction] -pub(crate) fn transcription(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; +pub(crate) fn transcription(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { let audio: Value = litellm_host_python::from_py_argument(&required_field(&call.bound, "audio")?)?; let options = value_route_options(&call.bound)?; @@ -59,9 +60,8 @@ pub(crate) fn transcription(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult< #[pyfunction] pub(crate) fn atranscription<'py>( py: Python<'py>, - call: Bound<'py, PyAny>, + call: NativeCall<'py>, ) -> PyResult> { - let call = super::NativeCall::extract(&call)?; let audio: Value = litellm_host_python::from_py_argument(&required_field(&call.bound, "audio")?)?; let options = value_route_options(&call.bound)?; diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs deleted file mode 100644 index 27f076b7cce..00000000000 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions.rs +++ /dev/null @@ -1,150 +0,0 @@ -mod host; - -use pyo3::types::{PyDict, PyTuple}; - -use crate::execution::{run_async, run_sync}; -use litellm_inference_chat::{ChatCompletionsRoute, Error, types::ChatCompletionsRequest}; -use litellm_llms_types::formats::chat_completions::ChatCompletionsResponse; -use pyo3::prelude::*; -use serde_json::{Map, Value}; - -use crate::{ - errors::route_error_to_pyerr, - marshal::{ - RouteOptions, messages_argument, optional_object_field, required_field, value_route_options, - }, -}; - -async fn execute( - http: Result, - secrets: std::sync::Arc, - messages: Vec, - optional_params: Map, - options: RouteOptions, -) -> Result { - let RouteOptions { - model, - api_key, - api_base, - custom_llm_provider, - extra_headers, - timeout, - } = options; - ChatCompletionsRoute::new(http?, crate::http::resources().auth.clone(), secrets) - .execute( - ChatCompletionsRequest { - model: &model, - messages: Value::Array(messages), - optional_params, - api_key: api_key.as_deref(), - api_base: api_base.as_deref(), - custom_llm_provider: custom_llm_provider.as_deref(), - extra_headers, - timeout, - }, - &(), - None, - ) - .await -} - -#[pyfunction] -pub(crate) fn chat_completions(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - let messages: Vec = messages_argument(&required_field(&call.bound, "messages")?)?; - let optional_params = - optional_object_field(&call.bound, "optional_params")?.unwrap_or_default(); - let options = value_route_options(&call.bound)?; - let http = crate::http::provider_client(py, &call.kwargs, false)?; - let secrets = crate::secrets::source(py)?; - run_sync( - py, - execute(http, secrets, messages, optional_params, options), - route_error_to_pyerr, - ) -} - -#[pyfunction] -pub(crate) fn achat_completions<'py>( - py: Python<'py>, - call: Bound<'py, PyAny>, -) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - let messages: Vec = messages_argument(&required_field(&call.bound, "messages")?)?; - let optional_params = - optional_object_field(&call.bound, "optional_params")?.unwrap_or_default(); - let options = value_route_options(&call.bound)?; - let http = crate::http::provider_client(py, &call.kwargs, true)?; - let secrets = crate::secrets::source(py)?; - run_async( - py, - execute(http, secrets, messages, optional_params, options), - route_error_to_pyerr, - ) -} - -fn run_public( - py: Python<'_>, - request: Bound<'_, PyAny>, - args: Bound<'_, PyTuple>, - kwargs: Bound<'_, PyDict>, - asynchronous: bool, -) -> PyResult> { - use super::inference::InferenceHost; - use litellm_callbacks_legacy_python::LoggingOperation; - let host = InferenceHost::new( - request.clone().unbind(), - "litellm.rust_bridge.chat_completions.route_host", - ); - let cache_call_type = if asynchronous { - "acompletion" - } else { - "completion" - }; - crate::cache::admit_native(py, &kwargs, cache_call_type)?; - let (arguments, hooks) = crate::routes::call_hooks( - py, - LoggingOperation::Completion, - &request, - &args, - &kwargs, - asynchronous, - )?; - crate::routes::run_public_call( - py, - arguments, - move |py, arguments, request| { - let route = ChatCompletionsRoute::new( - crate::http::provider_client(py, arguments, asynchronous)? - .map_err(crate::http::client_error)?, - crate::http::resources().auth.clone(), - crate::secrets::source(py)?, - ); - let (cache, cache_options) = - crate::cache::configured_native(py, arguments, cache_call_type)?; - let route = match cache { - Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new( - cache, - litellm_cache_response::CacheScope::Shared, - )), - None => route, - }; - Ok(route.machine(request, cache_options.policy)) - }, - host::ChatCompletionsPythonHost(host), - hooks, - asynchronous, - ) -} - -#[pyfunction] -pub(crate) fn completion(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - run_public(py, call.bound.into_any(), call.args, call.kwargs, false) -} - -#[pyfunction] -pub(crate) fn acompletion(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - run_public(py, call.bound.into_any(), call.args, call.kwargs, true) -} diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs index e7525cd3a91..1547a4daa93 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs @@ -1,6 +1,6 @@ use std::convert::Infallible; -use super::super::inference::InferenceHost; +use crate::routes::inference::InferenceHost; use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned}; use litellm_inference_chat::{Error, route::ChatCompletions, types::ChatCompletionsCall}; use pyo3::{ diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs new file mode 100644 index 00000000000..1146436bfe6 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs @@ -0,0 +1,71 @@ +mod host; + +use std::sync::Arc; + +use host::ChatCompletionsPythonHost; +use litellm_auth::AuthServices; +use litellm_cache_response::{CachePolicy, ScopedCache}; +use litellm_callbacks_legacy_python::LoggingOperation; +use litellm_host::{call::HostedMachine, protocol::Protocol}; +use litellm_inference_chat::{ChatCompletionsRoute, route::ChatCompletions}; +use litellm_secrets::source::SecretSource; +use pyo3::prelude::*; + +use super::{ + NativeCall, + inference::{InferenceHost, InferenceRoute, run_inference}, +}; + +fn run_chat_completions( + py: Python<'_>, + call: NativeCall<'_>, + asynchronous: bool, +) -> PyResult> { + let host = InferenceHost::new( + call.bound.clone().unbind(), + "litellm.rust_bridge.chat_completions.route_host", + ); + run_inference::( + py, + call, + asynchronous, + ChatCompletionsPythonHost(host), + ) +} + +#[pyfunction] +pub(crate) fn completion(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + run_chat_completions(py, call, false) +} + +#[pyfunction] +pub(crate) fn acompletion(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + run_chat_completions(py, call, true) +} + +impl InferenceRoute for ChatCompletionsRoute { + type Protocol = ChatCompletions; + const OPERATION: LoggingOperation = LoggingOperation::Completion; + const SYNC_CALL_TYPE: &'static str = "completion"; + const ASYNC_CALL_TYPE: &'static str = "acompletion"; + + fn new( + http: litellm_http::Client, + auth: Arc, + secrets: Arc, + ) -> Self { + Self::new(http, auth, secrets) + } + + fn with_cache(self, cache: ScopedCache) -> Self { + self.with_cache(cache) + } + + fn machine( + self, + call: ::Request, + policy: CachePolicy, + ) -> HostedMachine { + self.machine(call, policy) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/embeddings.rs b/litellm-rust/crates/python-bridge/src/routes/embeddings.rs index 1d34e3c21ff..2b184274041 100644 --- a/litellm-rust/crates/python-bridge/src/routes/embeddings.rs +++ b/litellm-rust/crates/python-bridge/src/routes/embeddings.rs @@ -1,17 +1,18 @@ use pyo3::prelude::*; +use super::NativeCall; use crate::errors::RustBridgeDeclined; #[pyfunction] -pub(crate) fn embedding(call: Bound<'_, PyAny>) -> PyResult> { - drop(super::NativeCall::extract(&call)?); +pub(crate) fn embedding(call: NativeCall<'_>) -> PyResult> { + drop(call); Err(RustBridgeDeclined::new_err( "native embeddings route is not implemented", )) } #[pyfunction] -pub(crate) fn aembedding(call: Bound<'_, PyAny>) -> PyResult> { +pub(crate) fn aembedding(call: NativeCall<'_>) -> PyResult> { embedding(call) } @@ -36,7 +37,8 @@ call = SimpleNamespace(args=(), kwargs={}, bound={'model':'test-model','input':' Some(&locals), ) .unwrap(); - let call = locals.get_item("call").unwrap().unwrap(); + let call: super::NativeCall<'_> = + locals.get_item("call").unwrap().unwrap().extract().unwrap(); let error = if asynchronous { super::aembedding(call) } else { diff --git a/litellm-rust/crates/python-bridge/src/routes/inference.rs b/litellm-rust/crates/python-bridge/src/routes/inference.rs index 28025b80b0f..0333aaeb04a 100644 --- a/litellm-rust/crates/python-bridge/src/routes/inference.rs +++ b/litellm-rust/crates/python-bridge/src/routes/inference.rs @@ -1,11 +1,19 @@ +use std::sync::Arc; + +use litellm_auth::AuthServices; +use litellm_cache_response::{CachePolicy, CacheScope, ScopedCache}; +use litellm_callbacks_legacy_python::LoggingOperation; use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; -use litellm_host_python::{from_py, lookup}; +use litellm_host::{call::HostedMachine, protocol::Protocol}; +use litellm_host_python::{PythonBinding, PythonHostCalls, from_py, present}; use litellm_http::transport::Error as TransportError; use litellm_inference::RouteError; +use litellm_secrets::source::SecretSource; use pyo3::{exceptions::PyValueError, prelude::*, types::PyDict}; use serde::Serialize; use serde_json::{Map, Value}; +use super::NativeCall; use crate::{ errors::{RustUpstreamError, route_error_to_pyerr}, marshal::{ @@ -15,7 +23,7 @@ use crate::{ }; pub(super) struct InferenceHost { - pub request: Py, + pub request: Py, module: &'static str, } @@ -26,7 +34,7 @@ pub(super) struct ProjectedCall { } impl InferenceHost { - pub fn new(request: Py, module: &'static str) -> Self { + pub fn new(request: Py, module: &'static str) -> Self { Self { request, module } } @@ -85,21 +93,7 @@ impl InferenceHost { arguments: &Bound<'py, PyDict>, name: &str, ) -> PyResult>> { - let request = self.request.bind(py); - if let Some(value) = lookup(arguments, request, name)? { - return Ok((!value.is_none()).then_some(value)); - } - if request.is_instance_of::() { - return Ok(None); - } - let parameter = request - .getattr("parameters")? - .call_method1("get", (name,))?; - if !parameter.is_none() { - return Ok(Some(parameter)); - } - let extra = request.getattr("kwargs")?.call_method1("get", (name,))?; - Ok((!extra.is_none()).then_some(extra)) + present(arguments, self.request.bind(py), name) } pub fn parameters( @@ -140,3 +134,62 @@ impl InferenceHost { Ok(PyErr::from_value(mapped)) } } + +pub(super) trait InferenceRoute: Sized + 'static { + type Protocol: Protocol; + const OPERATION: LoggingOperation; + const SYNC_CALL_TYPE: &'static str; + const ASYNC_CALL_TYPE: &'static str; + + fn new( + http: litellm_http::Client, + auth: Arc, + secrets: Arc, + ) -> Self; + fn with_cache(self, cache: ScopedCache) -> Self; + fn machine( + self, + call: ::Request, + policy: CachePolicy, + ) -> HostedMachine; +} + +pub(super) fn run_inference( + py: Python<'_>, + call: NativeCall<'_>, + asynchronous: bool, + host: H, +) -> PyResult> +where + R: InferenceRoute, + H: PythonBinding + PythonHostCalls + 'static, +{ + let call_type = if asynchronous { + R::ASYNC_CALL_TYPE + } else { + R::SYNC_CALL_TYPE + }; + crate::cache::admit_native(py, &call.kwargs, call_type)?; + let (arguments, hooks) = super::call_hooks(py, R::OPERATION, &call, asynchronous)?; + super::run_public_call( + py, + arguments, + move |py, arguments, request| { + let route = R::new( + crate::http::provider_client(py, arguments, asynchronous)? + .map_err(crate::http::client_error)?, + crate::http::resources().auth.clone(), + crate::secrets::source(py)?, + ); + let (cache, cache_options) = crate::cache::configured_native(py, arguments, call_type)?; + let route = match cache { + Some(cache) => route.with_cache(ScopedCache::new(cache, CacheScope::Shared)), + None => route, + }; + Ok(route.machine(request, cache_options.policy)) + }, + host, + hooks, + asynchronous, + ) +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index bde1cc7269c..841ef3081e3 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -2,7 +2,7 @@ use crate::cache::{CacheCall, Cached, PythonCache, Selection}; use litellm_host_python::{PythonHostCalls, PythonOwned}; use bytes::Bytes; -use litellm_host_python::{InvokeError, PythonBinding, from_py, lookup, to_py}; +use litellm_host_python::{InvokeError, PythonBinding, from_py, present, to_py}; use litellm_http::transport::Error as TransportError; use litellm_inference_messages::{ Error, MessagesCall, MessagesSettings, MessagesShaping, messages_body, @@ -89,12 +89,12 @@ fn native_error(py: Python<'_>, error: Error) -> PyResult { /// The Python side of the Messages route: projects the prepared arguments and builds the /// public response, chunks and exceptions. pub(super) struct MessagesPythonHost { - request: Py, + request: Py, cache: PythonCache, } impl MessagesPythonHost { - pub(super) fn new(request: Py, asynchronous: bool) -> Self { + pub(super) fn new(request: Py, asynchronous: bool) -> Self { Self { request, cache: PythonCache::new(asynchronous), @@ -107,9 +107,7 @@ impl MessagesPythonHost { arguments: &Bound<'_, PyDict>, ) -> PyResult> { let request = self.request.bind(py); - let argument = |name: &str| -> PyResult>> { - Ok(lookup(arguments, request, name)?.filter(|value| !value.is_none())) - }; + let argument = |name: &str| present(arguments, request, name); let string = |name: &str| -> PyResult> { argument(name)?.map(|value| value.extract()).transpose() }; @@ -153,8 +151,7 @@ impl MessagesPythonHost { ) -> PyResult>> { let request = self.request.bind(py); let mapping = |name: &str| -> PyResult>> { - lookup(arguments, request, name)? - .filter(|value| !value.is_none()) + present(arguments, request, name)? .map(|value| from_py(&value)) .transpose() }; @@ -169,8 +166,7 @@ impl MessagesPythonHost { py: Python<'_>, arguments: &Bound<'_, PyDict>, ) -> PyResult> { - lookup(arguments, self.request.bind(py), "provider_specific_header")? - .filter(|value| !value.is_none()) + present(arguments, self.request.bind(py), "provider_specific_header")? .map(|value| from_py(&value)) .transpose() } @@ -200,9 +196,9 @@ impl MessagesPythonHost { self.request .bind(py) .get_item("custom_llm_provider") - .and_then(|value| value.extract::>()) .ok() .flatten() + .and_then(|value| value.extract::>().ok().flatten()) .unwrap_or_else(|| "anthropic".into()) } diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index 318aae27121..6ed6656cd21 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -2,26 +2,13 @@ mod host; use host::MessagesPythonHost; use litellm_callbacks_legacy_python::LoggingOperation; -use pyo3::{ - prelude::*, - types::{PyDict, PyTuple}, -}; +use pyo3::prelude::*; -fn run_messages( - py: Python<'_>, - request: Bound<'_, PyAny>, - args: Bound<'_, PyTuple>, - kwargs: Bound<'_, PyDict>, - asynchronous: bool, -) -> PyResult> { - let (arguments, hooks) = crate::routes::call_hooks( - py, - LoggingOperation::Messages, - &request, - &args, - &kwargs, - asynchronous, - )?; +use super::NativeCall; + +fn run_messages(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult> { + let (arguments, hooks) = + crate::routes::call_hooks(py, LoggingOperation::Messages, &call, asynchronous)?; crate::routes::run_public_call( py, arguments, @@ -57,20 +44,18 @@ fn run_messages( }, )) }, - MessagesPythonHost::new(request.unbind(), asynchronous), + MessagesPythonHost::new(call.bound.unbind(), asynchronous), hooks, asynchronous, ) } #[pyfunction] -pub(crate) fn messages(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - run_messages(py, call.bound.into_any(), call.args, call.kwargs, false) +pub(crate) fn messages(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + run_messages(py, call, false) } #[pyfunction] -pub(crate) fn amessages(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - run_messages(py, call.bound.into_any(), call.args, call.kwargs, true) +pub(crate) fn amessages(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + run_messages(py, call, true) } diff --git a/litellm-rust/crates/python-bridge/src/routes/mod.rs b/litellm-rust/crates/python-bridge/src/routes/mod.rs index bf52fe3bfe8..646b7799730 100644 --- a/litellm-rust/crates/python-bridge/src/routes/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/mod.rs @@ -8,8 +8,7 @@ pub(crate) mod responses; pub(crate) mod token_counter; pub(crate) mod traces; -use litellm_callbacks_legacy_python::LoggingOperation; -use litellm_callbacks_legacy_python::{LegacyLogging, PublicCall}; +use litellm_callbacks_legacy_python::{LegacyLogging, LoggingOperation, PublicCall}; use litellm_host::{call::HostedCompletion, machine::Machine, protocol::Protocol}; use litellm_host_python::{HookChain, PythonBinding, PythonCallHooks, PythonHostCalls}; use pyo3::{ @@ -17,14 +16,16 @@ use pyo3::{ types::{PyDict, PyMapping, PyTuple}, }; -struct NativeCall<'py> { +pub(crate) struct NativeCall<'py> { args: Bound<'py, PyTuple>, kwargs: Bound<'py, PyDict>, bound: Bound<'py, PyDict>, } -impl<'py> NativeCall<'py> { - fn extract(call: &Bound<'py, PyAny>) -> PyResult { +impl<'py> FromPyObject<'_, 'py> for NativeCall<'py> { + type Error = PyErr; + + fn extract(call: Borrowed<'_, 'py, PyAny>) -> PyResult { Ok(Self { args: call.getattr("args")?.cast_into()?, kwargs: mapping_dict(&call.getattr("kwargs")?)?, @@ -46,12 +47,10 @@ fn mapping_dict<'py>(value: &Bound<'py, PyAny>) -> PyResult> fn call_hooks( py: Python<'_>, operation: LoggingOperation, - request: &Bound<'_, PyAny>, - args: &Bound<'_, PyTuple>, - kwargs: &Bound<'_, PyDict>, + call: &NativeCall<'_>, asynchronous: bool, ) -> PyResult<(Py, impl PythonCallHooks + use<>)> { - let call = PublicCall::capture(request, args, kwargs)?; + let call = PublicCall::capture(&call.bound, &call.args, &call.kwargs)?; let arguments = call.arguments(py); Ok(( arguments, @@ -99,6 +98,7 @@ mod tests { prelude::*, types::{PyDict, PyList}, }; + use rstest::rstest; fn value_call<'py>( py: Python<'py>, @@ -126,8 +126,10 @@ mod tests { .unwrap() } - #[test] - fn route_arguments_that_fail_to_convert_raise_value_error() { + #[rstest] + #[case::sync("transcription")] + #[case::asynchronous("atranscription")] + fn route_arguments_that_fail_to_convert_raise_value_error(#[case] name: &str) { Python::initialize(); Python::attach(|py| { let module = crate::native_module(py); @@ -151,48 +153,24 @@ value = Broken() .expect("locals should be readable") .expect("helper value should exist"); - for name in ["chat_completions", "achat_completions"] { - let error = module - .getattr(name) - .and_then(|function| { - function.call1((value_call(py, "messages", &broken, None),)) - }) - .expect_err("route should reject a value it cannot convert"); + let error = module + .getattr(name) + .and_then(|function| function.call1((value_call(py, "audio", &broken, None),))) + .expect_err("route should reject a value it cannot convert"); - assert!( - error.is_instance_of::(py), - "{name} surfaced {error} instead of ValueError" - ); - } + assert!( + error.is_instance_of::(py), + "{name} surfaced {error} instead of ValueError" + ); }); } - #[test] + #[rstest] fn sync_and_async_routes_apply_the_same_input_validation() { Python::initialize(); Python::attach(|py| { let module = crate::native_module(py); - let invalid_messages = PyDict::new(py); - let sync_chat_error = module - .getattr("chat_completions") - .and_then(|function| { - function.call1((value_call(py, "messages", &invalid_messages, None),)) - }) - .expect_err("sync chat should reject a non-list messages value"); - let async_chat_error = module - .getattr("achat_completions") - .and_then(|function| { - function.call1((value_call(py, "messages", &invalid_messages, None),)) - }) - .expect_err("async chat should reject a non-list messages value"); - - assert_eq!( - sync_chat_error.to_string(), - "ValueError: messages must be a list" - ); - assert_eq!(async_chat_error.to_string(), sync_chat_error.to_string()); - let invalid_headers = PyList::empty(py); let kwargs = PyDict::new(py); kwargs @@ -221,51 +199,43 @@ value = Broken() }); } - #[test] + #[rstest] + fn missing_and_explicit_none_optional_params_share_the_next_error() { + Python::initialize(); + Python::attach(|py| { + let module = crate::native_module(py); + let audio = PyDict::new(py); + let transcribe = |optional_params: Option>| { + let kwargs = PyDict::new(py); + if let Some(optional_params) = optional_params { + kwargs.set_item("optional_params", optional_params).unwrap(); + } + module + .getattr("transcription") + .and_then(|function| { + function.call1((value_call(py, "audio", &audio, Some(&kwargs)),)) + }) + .expect_err("model 'model' has no provider") + .to_string() + }; + + let omitted = transcribe(None); + assert_eq!(transcribe(Some(py.None().into_bound(py))), omitted); + assert_ne!(omitted, "ValueError: optional_params must be a dict"); + assert_eq!( + transcribe(Some(PyList::empty(py).into_any())), + "ValueError: optional_params must be a dict" + ); + }); + } + + #[rstest] fn route_input_validation_preserves_left_to_right_order() { Python::initialize(); Python::attach(|py| { let module = crate::native_module(py); let invalid = PyList::empty(py); - let chat_kwargs = PyDict::new(py); - chat_kwargs - .set_item("optional_params", &invalid) - .expect("kwargs should accept optional_params"); - chat_kwargs - .set_item("extra_headers", &invalid) - .expect("kwargs should accept extra_headers"); - let invalid_messages = PyDict::new(py); - let error = module - .getattr("chat_completions") - .and_then(|function| { - function.call1((value_call( - py, - "messages", - &invalid_messages, - Some(&chat_kwargs), - ),)) - }) - .expect_err("messages should be validated first"); - assert_eq!(error.to_string(), "ValueError: messages must be a list"); - - let valid_messages = PyList::empty(py); - let error = module - .getattr("chat_completions") - .and_then(|function| { - function.call1((value_call( - py, - "messages", - &valid_messages, - Some(&chat_kwargs), - ),)) - }) - .expect_err("optional_params should be validated before headers"); - assert_eq!( - error.to_string(), - "ValueError: optional_params must be a dict" - ); - let headers_kwargs = PyDict::new(py); headers_kwargs .set_item("extra_headers", &invalid) @@ -286,43 +256,4 @@ value = Broken() assert!(!error.to_string().contains("extra_headers")); }); } - - #[test] - fn missing_and_explicit_none_optional_params_share_the_next_error() { - Python::initialize(); - Python::attach(|py| { - let module = crate::native_module(py); - let messages = PyList::empty(py); - let headers = PyList::empty(py); - let omitted = PyDict::new(py); - omitted - .set_item("extra_headers", &headers) - .expect("kwargs should accept extra_headers"); - let explicit = PyDict::new(py); - explicit - .set_item("optional_params", py.None()) - .expect("kwargs should accept optional_params"); - explicit - .set_item("extra_headers", &headers) - .expect("kwargs should accept extra_headers"); - - let omitted_error = module - .getattr("chat_completions") - .and_then(|function| { - function.call1((value_call(py, "messages", &messages, Some(&omitted)),)) - }) - .expect_err("omitted optional_params should reach header validation"); - let explicit_error = module - .getattr("chat_completions") - .and_then(|function| { - function.call1((value_call(py, "messages", &messages, Some(&explicit)),)) - }) - .expect_err("None optional_params should reach header validation"); - assert_eq!( - omitted_error.to_string(), - "ValueError: extra_headers must be a dict" - ); - assert_eq!(explicit_error.to_string(), omitted_error.to_string()); - }); - } } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs index 7ddaba70937..ca683600d71 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/host.rs @@ -27,12 +27,12 @@ enum OcrHostData { /// document as it goes), acquires Azure AD tokens, and builds the public response and /// exception. pub(super) struct OcrPythonHost { - request: Py, + request: Py, data: OcrHostData, } impl OcrPythonHost { - pub(super) fn new(request: Py) -> Self { + pub(super) fn new(request: Py) -> Self { Self { request, data: OcrHostData::Unprojected, @@ -209,7 +209,7 @@ del provider .unwrap() .cast_into::() .unwrap(); - let mut host = OcrPythonHost::new(py.None()); + let mut host = OcrPythonHost::new(PyDict::new(py).unbind()); assert!(host.decode_request(py, &kwargs).unwrap().caller_token); locals.del_item("kwargs").unwrap(); drop(kwargs); diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 71b1054fc3c..4dcb2f4c53e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -9,10 +9,9 @@ use litellm_core_utils::settings::ProcessEnvironment; use litellm_host_python::to_py; use litellm_inference_ocr::provider_config; use litellm_llms::base_llm::ocr::settings::OcrSettings; -use pyo3::{ - prelude::*, - types::{PyDict, PyTuple}, -}; +use pyo3::prelude::*; + +use super::NativeCall; use crate::{ coercion::FieldSpec, @@ -29,21 +28,9 @@ const ENABLE_AZURE_AD_TOKEN_REFRESH: FieldSpec = Ok(field.exact_true()) }); -fn run_ocr( - py: Python<'_>, - request: Bound<'_, PyAny>, - args: Bound<'_, PyTuple>, - kwargs: Bound<'_, PyDict>, - asynchronous: bool, -) -> PyResult> { - let (arguments, hooks) = crate::routes::call_hooks( - py, - LoggingOperation::Ocr, - &request, - &args, - &kwargs, - asynchronous, - )?; +fn run_ocr(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult> { + let (arguments, hooks) = + crate::routes::call_hooks(py, LoggingOperation::Ocr, &call, asynchronous)?; crate::routes::run_public_call( py, arguments, @@ -61,7 +48,7 @@ fn run_ocr( let route = litellm_inference_ocr::OcrRoute::new(client); Ok(route.machine(request, None)) }, - OcrPythonHost::new(request.unbind()), + OcrPythonHost::new(call.bound.unbind()), hooks, asynchronous, ) @@ -81,15 +68,13 @@ fn project_provider_defaults(snapshot: &Snapshot<'_>) -> PyResult { } #[pyfunction] -pub(crate) fn ocr(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - run_ocr(py, call.bound.into_any(), call.args, call.kwargs, false) +pub(crate) fn ocr(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + run_ocr(py, call, false) } #[pyfunction] -pub(crate) fn aocr(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - run_ocr(py, call.bound.into_any(), call.args, call.kwargs, true) +pub(crate) fn aocr(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + run_ocr(py, call, true) } #[pyfunction] diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index ee7f6238510..7ae5a1f3ce2 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -1,5 +1,5 @@ use litellm_auth::SecretValue; -use litellm_host_python::from_py; +use litellm_host_python::{from_py, present}; use litellm_inference_ocr::{ types::{LiteLLMOcrRequest, OcrDocumentInput}, wire::{OcrWireRequest, consumed_optional_params, decode_document, decode_request_input}, @@ -22,13 +22,13 @@ pub(super) struct OcrHostHandles { } struct OcrArguments<'a, 'py> { - request: &'a Bound<'py, PyAny>, + bound: &'a Bound<'py, PyDict>, kwargs: &'a Bound<'py, PyDict>, } impl<'py> OcrArguments<'_, 'py> { fn lookup(&self, name: &str) -> PyResult> { - litellm_host_python::lookup(self.kwargs, self.request, name)? + litellm_host_python::lookup(self.kwargs, self.bound, name)? .ok_or_else(|| PyValueError::new_err(format!("missing argument: {name}"))) } @@ -58,7 +58,7 @@ impl<'py> OcrArguments<'_, 'py> { fn extra_headers(&self) -> PyResult>> { self.lookup("extra_headers")? .extract::>>()? - .map(|value| from_py(value.bind(self.request.py()))) + .map(|value| from_py(value.bind(self.bound.py()))) .transpose() } @@ -66,7 +66,7 @@ impl<'py> OcrArguments<'_, 'py> { Ok(self .lookup("timeout")? .extract::>>()? - .map(|value| python_timeout_seconds(self.request.py(), value)) + .map(|value| python_timeout_seconds(self.bound.py(), value)) .transpose()? .flatten()) } @@ -110,10 +110,10 @@ impl ProjectedDocument { } pub(super) fn project_request( - request: &Bound<'_, PyAny>, + bound: &Bound<'_, PyDict>, kwargs: &Bound<'_, PyDict>, ) -> PyResult<(LiteLLMOcrRequest, OcrHostHandles)> { - let arguments = OcrArguments { request, kwargs }; + let arguments = OcrArguments { bound, kwargs }; let model = arguments.model()?; let custom_llm_provider = arguments.custom_llm_provider()?; let document = ProjectedDocument::project(&arguments.document()?)?; @@ -122,7 +122,7 @@ pub(super) fn project_request( .map_err(ocr_error_to_pyerr)?; let names = specs.iter().map(|spec| spec.name).collect::>(); let optional_params = - project_optional_fields(names.iter().copied(), |name| kwargs.get_item(name))?; + project_optional_fields(names.iter().copied(), |name| present(kwargs, bound, name))?; let input_sources = request_input_sources( kwargs, names @@ -136,7 +136,7 @@ pub(super) fn project_request( let timeout_seconds = arguments.timeout_seconds()?; let wire = OcrWireRequest { model, - document: document.resolve(request.py())?, + document: document.resolve(bound.py())?, api_key, api_base, custom_llm_provider, @@ -169,11 +169,20 @@ mod tests { locals } + fn dict<'py>(locals: &Bound<'py, PyDict>, name: &str) -> Bound<'py, PyDict> { + locals + .get_item(name) + .unwrap() + .unwrap() + .cast_into::() + .unwrap() + } + fn arguments<'a, 'py>( - request: &'a Bound<'py, PyAny>, + bound: &'a Bound<'py, PyDict>, kwargs: &'a Bound<'py, PyDict>, ) -> OcrArguments<'a, 'py> { - OcrArguments { request, kwargs } + OcrArguments { bound, kwargs } } fn project_document(document: &Bound<'_, PyAny>) -> PyResult { @@ -204,141 +213,26 @@ sys.modules['litellm.rust_bridge.timeouts'] = timeouts } #[test] - fn kwargs_override_request_attributes_including_explicit_none() { + fn kwargs_override_bound_values_including_explicit_none() { Python::initialize(); Python::attach(|py| { let locals = eval( py, c" -class Request: - def __init__(self): - self.accesses = [] - def __getattribute__(self, name): - if name != 'accesses': - object.__getattribute__(self, 'accesses').append(name) - return object.__getattribute__(self, name) -request = Request() -request.model = 'from-request' -request.custom_llm_provider = 'mistral' +bound = {'model': 'from-bound', 'custom_llm_provider': 'mistral'} kwargs = {'model': 'from-kwargs', 'custom_llm_provider': None} ", ); - let request = locals.get_item("request").unwrap().unwrap(); - let kwargs = locals - .get_item("kwargs") - .unwrap() - .unwrap() - .cast_into::() - .unwrap(); - let arguments = arguments(&request, &kwargs); + let (bound, kwargs) = (dict(&locals, "bound"), dict(&locals, "kwargs")); + let arguments = arguments(&bound, &kwargs); assert_eq!(arguments.model().unwrap(), "from-kwargs"); assert_eq!(arguments.custom_llm_provider().unwrap(), None); - let accesses: Vec = request.getattr("accesses").unwrap().extract().unwrap(); - assert_eq!(accesses, Vec::::new()); }); } - #[test] - fn missing_kwargs_read_the_request_property_once() { - Python::initialize(); - Python::attach(|py| { - let locals = eval( - py, - c" -class Request: - def __init__(self): - self.reads = 0 - @property - def model(self): - self.reads += 1 - return 'mistral-ocr-latest' -request = Request() -kwargs = {} -", - ); - let request = locals.get_item("request").unwrap().unwrap(); - let kwargs = locals - .get_item("kwargs") - .unwrap() - .unwrap() - .cast_into::() - .unwrap(); - assert_eq!( - arguments(&request, &kwargs).model().unwrap(), - "mistral-ocr-latest" - ); - assert_eq!( - request.getattr("reads").unwrap().extract::().unwrap(), - 1 - ); - }); - } - - #[test] - fn request_property_exceptions_keep_their_identity() { - Python::initialize(); - Python::attach(|py| { - let locals = eval( - py, - c" -failure = LookupError('model failed') -class Request: - @property - def model(self): - raise failure -request = Request() -kwargs = {} -", - ); - let request = locals.get_item("request").unwrap().unwrap(); - let kwargs = locals - .get_item("kwargs") - .unwrap() - .unwrap() - .cast_into::() - .unwrap(); - let error = arguments(&request, &kwargs).model().unwrap_err(); - assert!( - error - .value(py) - .is(locals.get_item("failure").unwrap().unwrap()) - ); - }); - } - - #[test] - fn unused_raising_property_is_never_inspected() { - Python::initialize(); - Python::attach(|py| { - let locals = eval( - py, - c" -class Request: - @property - def unused(self): - raise RuntimeError('unused') - model = 'mistral-ocr-latest' - custom_llm_provider = None -request = Request() -kwargs = {} -", - ); - let request = locals.get_item("request").unwrap().unwrap(); - let kwargs = locals - .get_item("kwargs") - .unwrap() - .unwrap() - .cast_into::() - .unwrap(); - let arguments = arguments(&request, &kwargs); - assert_eq!(arguments.model().unwrap(), "mistral-ocr-latest"); - assert_eq!(arguments.custom_llm_provider().unwrap(), None); - }); - } - - /// A reader that rewrites the request while it runs shows which arguments projection - /// read before it and which after: every other argument is read first, and the read - /// happens exactly once. + /// A reader that rewrites the bound arguments while it runs shows which arguments + /// projection read before it and which after: every other argument is read first, and + /// the read happens exactly once. #[test] fn document_readers_are_read_once_after_every_other_argument() { Python::initialize(); @@ -347,37 +241,28 @@ kwargs = {} let locals = eval( py, c" -class Request: - model = 'mistral/mistral-ocr-latest' - custom_llm_provider = None - api_key = None - api_base = 'https://original.example.com' - extra_headers = {'x-source': 'original'} - timeout = 1 - @property - def document(self): - return document class Reader: reads = 0 def read(self): Reader.reads += 1 - Request.api_base = 'https://mutated.example.com' - Request.extra_headers = {'x-source': 'mutated'} - Request.timeout = 9 + bound['api_base'] = 'https://mutated.example.com' + bound['extra_headers'] = {'x-source': 'mutated'} + bound['timeout'] = 9 return b'abc' -document = {'type': 'file', 'file': Reader(), 'mime_type': 'application/pdf'} -request = Request() +bound = { + 'model': 'mistral/mistral-ocr-latest', + 'custom_llm_provider': None, + 'api_key': None, + 'api_base': 'https://original.example.com', + 'extra_headers': {'x-source': 'original'}, + 'timeout': 1, + 'document': {'type': 'file', 'file': Reader(), 'mime_type': 'application/pdf'}, +} kwargs = {} ", ); - let request = locals.get_item("request").unwrap().unwrap(); - let kwargs = locals - .get_item("kwargs") - .unwrap() - .unwrap() - .cast_into::() - .unwrap(); - let (projected, _) = project_request(&request, &kwargs).unwrap(); + let (projected, _) = + project_request(&dict(&locals, "bound"), &dict(&locals, "kwargs")).unwrap(); assert_eq!( py.eval(c"Reader.reads", Some(&locals), Some(&locals)) .unwrap() @@ -524,34 +409,26 @@ document = Document() }); } - fn request_and_kwargs<'py>( + fn bound_and_kwargs<'py>( py: Python<'py>, kwargs: &std::ffi::CStr, - ) -> (Bound<'py, PyAny>, Bound<'py, PyDict>) { + ) -> (Bound<'py, PyDict>, Bound<'py, PyDict>) { let locals = eval( py, c" -class Request: - model = 'mistral/mistral-ocr-latest' - custom_llm_provider = 'mistral' - document = {'type': 'document_url', 'document_url': 'https://example.com/request.pdf'} - api_key = None - api_base = 'https://request.example.com' - extra_headers = {'x-source': 'request'} - timeout = 1 -request = Request() +bound = { + 'model': 'mistral/mistral-ocr-latest', + 'custom_llm_provider': 'mistral', + 'document': {'type': 'document_url', 'document_url': 'https://example.com/bound.pdf'}, + 'api_key': None, + 'api_base': 'https://bound.example.com', + 'extra_headers': {'x-source': 'bound'}, + 'timeout': 1, +} ", ); py.run(kwargs, Some(&locals), Some(&locals)).unwrap(); - ( - locals.get_item("request").unwrap().unwrap(), - locals - .get_item("kwargs") - .unwrap() - .unwrap() - .cast_into::() - .unwrap(), - ) + (dict(&locals, "bound"), dict(&locals, "kwargs")) } #[test] @@ -559,7 +436,7 @@ request = Request() Python::initialize(); Python::attach(|py| { stub_timeout_conversion(py); - let (request, kwargs) = request_and_kwargs( + let (bound, kwargs) = bound_and_kwargs( py, c" kwargs = { @@ -575,7 +452,7 @@ kwargs = { } ", ); - let (projected, _) = project_request(&request, &kwargs).unwrap(); + let (projected, _) = project_request(&bound, &kwargs).unwrap(); assert_eq!( projected.optional_params.keys().collect::>(), ["pages"] @@ -584,12 +461,29 @@ kwargs = { }); } + #[rstest::rstest] + #[case::explicit_none_is_unset(c"bound['pages'] = [1]\nkwargs = {'pages': None}", None)] + #[case::bound_fallback(c"bound['pages'] = [1]\nkwargs = {}", Some(serde_json::json!([1])))] + #[case::keyword_wins(c"bound['pages'] = [1]\nkwargs = {'pages': [0]}", Some(serde_json::json!([0])))] + fn optional_params_read_through_bound_and_drop_none( + #[case] script: &std::ffi::CStr, + #[case] expected: Option, + ) { + Python::initialize(); + Python::attach(|py| { + stub_timeout_conversion(py); + let (bound, kwargs) = bound_and_kwargs(py, script); + let (projected, _) = project_request(&bound, &kwargs).unwrap(); + assert_eq!(projected.optional_params.get("pages").cloned(), expected); + }); + } + #[test] fn replacement_kwargs_project_provider_connection_and_timeout() { Python::initialize(); Python::attach(|py| { stub_timeout_conversion(py); - let (request, kwargs) = request_and_kwargs( + let (bound, kwargs) = bound_and_kwargs( py, c" kwargs = { @@ -602,7 +496,7 @@ kwargs = { } ", ); - let (projected, handles) = project_request(&request, &kwargs).unwrap(); + let (projected, handles) = project_request(&bound, &kwargs).unwrap(); assert_eq!(handles.provider, "azure_ai"); assert_eq!(projected.model, "mistral-ocr-latest"); assert_eq!( diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index 25d3fc07332..b22bbc6192b 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -1,6 +1,6 @@ use std::convert::Infallible; -use super::super::inference::InferenceHost; +use crate::routes::inference::InferenceHost; use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned}; use litellm_inference_responses::{Error, route::Responses, types::ResponsesCall}; use pyo3::{ diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs new file mode 100644 index 00000000000..a5b81b91752 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs @@ -0,0 +1,102 @@ +mod host; +mod websocket; + +use std::sync::Arc; + +use host::ResponsesPythonHost; +use litellm_auth::AuthServices; +use litellm_cache_response::{CachePolicy, ScopedCache}; +use litellm_callbacks_legacy_python::LoggingOperation; +use litellm_host::{call::HostedMachine, protocol::Protocol}; +use litellm_host_python::present; +use litellm_inference_responses::{ResponsesRoute, route::Responses}; +use litellm_secrets::source::SecretSource; +use pyo3::prelude::*; +use serde_json::Value; +pub(crate) use websocket::ResponsesWebSocketConnection; + +use super::{ + NativeCall, + inference::{InferenceHost, InferenceRoute, run_inference}, +}; +use crate::errors::RustBridgeDeclined; + +const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.responses.route_host"; + +fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> PyResult> { + if let Some(reason) = py + .import(ROUTE_HOST_MODULE)? + .getattr("decline_reason")? + .call1((&call.bound,))? + .extract::>()? + { + return Err(RustBridgeDeclined::new_err(reason)); + } + let argument = |name: &str| present(&call.kwargs, &call.bound, name); + let model = argument("model")? + .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("model is required"))? + .extract::()?; + let provider = argument("custom_llm_provider")? + .map(|value| value.extract::()) + .transpose()?; + if provider + .as_deref() + .is_some_and(|provider| provider != "openai") + || model + .strip_prefix("openai/") + .unwrap_or(&model) + .contains('/') + { + return Err(RustBridgeDeclined::new_err( + "native HTTP responses provider", + )); + } + if argument("stream")? + .map(|value| litellm_host_python::from_py::(&value)) + .transpose()? + .is_some_and(|value| value == Value::Bool(true)) + { + return Err(RustBridgeDeclined::new_err( + "native Python responses streaming", + )); + } + let host = InferenceHost::new(call.bound.clone().unbind(), ROUTE_HOST_MODULE); + run_inference::(py, call, asynchronous, ResponsesPythonHost(host)) +} + +#[pyfunction] +pub(crate) fn responses(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + run_responses(py, call, false) +} + +#[pyfunction] +pub(crate) fn aresponses(py: Python<'_>, call: NativeCall<'_>) -> PyResult> { + run_responses(py, call, true) +} + +impl InferenceRoute for ResponsesRoute { + type Protocol = Responses; + const OPERATION: LoggingOperation = LoggingOperation::Responses; + const SYNC_CALL_TYPE: &'static str = "responses"; + const ASYNC_CALL_TYPE: &'static str = "aresponses"; + + fn new( + http: litellm_http::Client, + auth: Arc, + secrets: Arc, + ) -> Self { + Self::new(http, auth, secrets) + } + + fn with_cache(self, cache: ScopedCache) -> Self { + self.with_cache(cache) + } + + fn machine( + self, + call: ::Request, + policy: CachePolicy, + ) -> HostedMachine { + self.machine(call, policy) + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/responses.rs b/litellm-rust/crates/python-bridge/src/routes/responses/websocket.rs similarity index 57% rename from litellm-rust/crates/python-bridge/src/routes/responses.rs rename to litellm-rust/crates/python-bridge/src/routes/responses/websocket.rs index 780f2e6929b..615a838a597 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/websocket.rs @@ -1,121 +1,12 @@ -mod host; - use litellm_inference_responses::websocket::ResponsesWebSocketConnection as RustResponsesWebSocketConnection; -use pyo3::{ - prelude::*, - types::{PyDict, PyTuple}, -}; +use pyo3::prelude::*; use serde_json::Value; use crate::{ - errors::{RustBridgeDeclined, route_error_to_pyerr}, + errors::route_error_to_pyerr, marshal::{marshal_headers, optional_timeout}, }; -fn run_public( - py: Python<'_>, - request: Bound<'_, PyAny>, - args: Bound<'_, PyTuple>, - kwargs: Bound<'_, PyDict>, - asynchronous: bool, -) -> PyResult> { - use super::inference::InferenceHost; - use litellm_callbacks_legacy_python::LoggingOperation; - let host = InferenceHost::new( - request.clone().unbind(), - "litellm.rust_bridge.responses.route_host", - ); - if let Some(reason) = py - .import("litellm.rust_bridge.responses.route_host")? - .getattr("decline_reason")? - .call1((&request,))? - .extract::>()? - { - return Err(RustBridgeDeclined::new_err(reason)); - } - let model = host - .argument(py, &kwargs, "model")? - .ok_or_else(|| pyo3::exceptions::PyValueError::new_err("model is required"))? - .extract::()?; - let provider = host - .argument(py, &kwargs, "custom_llm_provider")? - .map(|value| value.extract::()) - .transpose()?; - if provider - .as_deref() - .is_some_and(|provider| provider != "openai") - || model - .strip_prefix("openai/") - .unwrap_or(&model) - .contains('/') - { - return Err(RustBridgeDeclined::new_err( - "native HTTP responses provider", - )); - } - if host - .argument(py, &kwargs, "stream")? - .map(|value| litellm_host_python::from_py::(&value)) - .transpose()? - .is_some_and(|value| value == Value::Bool(true)) - { - return Err(RustBridgeDeclined::new_err( - "native Python responses streaming", - )); - } - let cache_call_type = if asynchronous { - "aresponses" - } else { - "responses" - }; - crate::cache::admit_native(py, &kwargs, cache_call_type)?; - let (arguments, hooks) = crate::routes::call_hooks( - py, - LoggingOperation::Responses, - &request, - &args, - &kwargs, - asynchronous, - )?; - crate::routes::run_public_call( - py, - arguments, - move |py, arguments, request| { - let route = litellm_inference_responses::ResponsesRoute::new( - crate::http::provider_client(py, arguments, asynchronous)? - .map_err(crate::http::client_error)?, - crate::http::resources().auth.clone(), - crate::secrets::source(py)?, - ); - let (cache, cache_options) = - crate::cache::configured_native(py, arguments, cache_call_type)?; - let route = match cache { - Some(cache) => route.with_cache(litellm_cache_response::ScopedCache::new( - cache, - litellm_cache_response::CacheScope::Shared, - )), - None => route, - }; - Ok(route.machine(request, cache_options.policy)) - }, - host::ResponsesPythonHost(host), - hooks, - asynchronous, - ) -} - -#[pyfunction] -pub(crate) fn responses(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - run_public(py, call.bound.into_any(), call.args, call.kwargs, false) -} - -#[pyfunction] -pub(crate) fn aresponses(py: Python<'_>, call: Bound<'_, PyAny>) -> PyResult> { - let call = super::NativeCall::extract(&call)?; - run_public(py, call.bound.into_any(), call.args, call.kwargs, true) -} - #[pyclass] pub(crate) struct ResponsesWebSocketConnection { inner: RustResponsesWebSocketConnection, diff --git a/litellm/rust_bridge/_native.pyi b/litellm/rust_bridge/_native.pyi index 5ca7b79d127..c83e4e5318c 100644 --- a/litellm/rust_bridge/_native.pyi +++ b/litellm/rust_bridge/_native.pyi @@ -110,12 +110,6 @@ def messages( def amessages( call: NativeCall, ) -> Coroutine[object, object, AnthropicMessagesResponse | AsyncIterator[bytes]]: ... -def chat_completions( - call: NativeCall, -) -> dict[str, object]: ... -def achat_completions( - call: NativeCall, -) -> Future[dict[str, object]]: ... @final class ResponsesWebSocketConnection: @@ -251,14 +245,12 @@ __all__ = [ "RustUpstreamError", "TokenCounter", "Tokenizer", - "achat_completions", "acompletion", "aembedding", "amessages", "aocr", "aresponses", "atranscription", - "chat_completions", "completion", "embedding", "gil_stats", diff --git a/litellm/rust_bridge/chat_completions/route_host.py b/litellm/rust_bridge/chat_completions/route_host.py index cb06ea8e213..d600495c919 100644 --- a/litellm/rust_bridge/chat_completions/route_host.py +++ b/litellm/rust_bridge/chat_completions/route_host.py @@ -37,16 +37,12 @@ def response(value: Mapping[str, object]) -> ModelResponse: return ModelResponse(**value) -def arguments(request: Mapping[str, object]) -> Mapping[str, object]: - return request - - def map_failure(error: Exception, request: Mapping[str, object]) -> Exception: provider: Final = optional_str(request.get("custom_llm_provider")) or str(request["model"]).partition("/")[0] return failures.map_native_failure( error, str(request["model"]), provider, - arguments(request), + request, optional_str(request.get("api_base")) or optional_str(request.get("base_url")), ) diff --git a/litellm/rust_bridge/messages/route_host.py b/litellm/rust_bridge/messages/route_host.py index 88a81fdae4a..69e27e78223 100644 --- a/litellm/rust_bridge/messages/route_host.py +++ b/litellm/rust_bridge/messages/route_host.py @@ -39,10 +39,6 @@ def stream_hidden_params(headers: Sequence[tuple[str, str]]) -> Mapping[str, obj return anthropic_messages_stream_hidden_params(httpx.Headers(list(headers))) -def arguments(request: Mapping[str, object]) -> Mapping[str, object]: - return request - - def map_failure(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: if getattr(error, "messages_request_error", False): return litellm.BadRequestError( @@ -51,7 +47,7 @@ def map_failure(error: Exception, request: Mapping[str, object], request_provide llm_provider=request_provider, ) return failures.map_native_failure( - error, str(request["model"]), request_provider, arguments(request), optional_str(request.get("api_base")) + error, str(request["model"]), request_provider, request, optional_str(request.get("api_base")) ) diff --git a/litellm/rust_bridge/ocr/route_host.py b/litellm/rust_bridge/ocr/route_host.py index a7eb0829f5c..8f563a381ef 100644 --- a/litellm/rust_bridge/ocr/route_host.py +++ b/litellm/rust_bridge/ocr/route_host.py @@ -12,7 +12,7 @@ from litellm.rust_bridge import failures from litellm.rust_bridge.failures import UpstreamFailure from litellm.rust_bridge.public_call import optional_str -__all__ = ("UpstreamFailure", "arguments", "map_failure", "response") +__all__ = ("UpstreamFailure", "map_failure", "response") _RESPONSE_ADAPTER: Final = TypeAdapter(dict[str, object]) @@ -27,10 +27,6 @@ def response(value: Mapping[str, object]) -> OCRResponse: return normalized -def arguments(request: Mapping[str, object]) -> Mapping[str, object]: - return request - - def map_failure(error: Exception, request: Mapping[str, object], request_provider: str) -> Exception: if getattr(error, "ocr_request_format_error", False): return litellm.UnsupportedParamsError( @@ -39,5 +35,5 @@ def map_failure(error: Exception, request: Mapping[str, object], request_provide llm_provider=request_provider, ) return failures.map_native_failure( - error, str(request["model"]), request_provider, arguments(request), optional_str(request.get("api_base")) + error, str(request["model"]), request_provider, request, optional_str(request.get("api_base")) ) diff --git a/litellm/rust_bridge/responses/route_host.py b/litellm/rust_bridge/responses/route_host.py index 00f24244f49..d8ad2a94b75 100644 --- a/litellm/rust_bridge/responses/route_host.py +++ b/litellm/rust_bridge/responses/route_host.py @@ -20,17 +20,13 @@ def response(value: Mapping[str, object]) -> ResponsesAPIResponse: return ResponsesAPIResponse.model_validate(value) -def arguments(request: Mapping[str, object]) -> Mapping[str, object]: - return request - - def map_failure(error: Exception, request: Mapping[str, object]) -> Exception: provider: Final = optional_str(request.get("custom_llm_provider")) or "openai" return failures.map_native_failure( error, str(request["model"]), provider, - arguments(request), + request, optional_str(request.get("api_base")) or optional_str(request.get("base_url")), ) diff --git a/tests/unit/rust_bridge/chat_completions/test_route_host.py b/tests/unit/rust_bridge/chat_completions/test_route_host.py index 92eed17de0f..402c032f340 100644 --- a/tests/unit/rust_bridge/chat_completions/test_route_host.py +++ b/tests/unit/rust_bridge/chat_completions/test_route_host.py @@ -4,7 +4,7 @@ from typing import Final import pytest import litellm -from litellm.rust_bridge.chat_completions.route_host import arguments, connection_defaults, response +from litellm.rust_bridge.chat_completions.route_host import connection_defaults, response from litellm.types.utils import ModelResponse @@ -35,12 +35,6 @@ def test_response_builds_the_public_model_response() -> None: assert built.usage.total_tokens == 5 -def test_arguments_are_the_public_kwargs_view() -> None: - kwargs: Final = MappingProxyType({"metadata": {"user_id": "u"}}) - - assert arguments(kwargs) is kwargs - - @pytest.mark.parametrize( ("provider", "global_key", "provider_key", "expected_key", "expected_base"), ( diff --git a/tests/unit/rust_bridge/messages/test_route_host.py b/tests/unit/rust_bridge/messages/test_route_host.py index dde76ee5e82..16b9634beb9 100644 --- a/tests/unit/rust_bridge/messages/test_route_host.py +++ b/tests/unit/rust_bridge/messages/test_route_host.py @@ -1,8 +1,7 @@ from types import MappingProxyType from typing import Final -from litellm.rust_bridge.messages.route_host import arguments, response -from litellm.rust_bridge.public_call import NativeCall +from litellm.rust_bridge.messages.route_host import response import pytest import litellm from litellm.rust_bridge.messages import route_host @@ -29,26 +28,6 @@ def test_response_is_a_detached_public_messages_dict() -> None: assert "_hidden_params" not in native -def test_arguments_preserve_the_bound_view() -> None: - kwargs: Final = MappingProxyType({"litellm_metadata": {"user_id": "u"}}) - request: Final = NativeCall( - args=(), - kwargs=kwargs, - bound={ - "model": "claude-sonnet-4-5", - "messages": [{"role": "user", "content": "hi"}], - "max_tokens": 16, - "stream": None, - "api_key": None, - "api_base": None, - "custom_llm_provider": "anthropic", - **kwargs, - }, - ) - - assert arguments(request.bound) is request.bound - - def test_settings_project_caller_configuration_without_resolving_a_model(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(litellm, "drop_params", False) monkeypatch.setattr(litellm, "reasoning_auto_summary", True) diff --git a/tests/unit/rust_bridge/native_route_wheel_test.py b/tests/unit/rust_bridge/native_route_wheel_test.py index 9e7523aa29e..268eca0f663 100644 --- a/tests/unit/rust_bridge/native_route_wheel_test.py +++ b/tests/unit/rust_bridge/native_route_wheel_test.py @@ -20,13 +20,6 @@ from typing import Final REQUEST_STARTED: Final = threading.Event() REQUEST_CANCELLED: Final = threading.Event() -ANTHROPIC_RESPONSE: Final = ( - b'{"id":"msg_native","type":"message","role":"assistant",' - b'"model":"claude-sonnet-4-5","content":[{"type":"text","text":"native-message"}],' - b'"stop_reason":"end_turn","stop_sequence":null,' - b'"usage":{"input_tokens":2,"output_tokens":3}}' -) - class NativeRouteServer(ThreadingHTTPServer): request_queue_size = 64 @@ -49,7 +42,7 @@ class NativeRouteHandler(BaseHTTPRequestHandler): return status: Final = 429 if outcome == "429" else 200 - response_body: Final = native_response(status, route) + response_body: Final = native_response(status) self.send_response(status) self.send_header("content-type", "application/json") @@ -78,32 +71,23 @@ def assert_native_request( headers: HTTPMessage, body: object, ) -> None: - if route not in {"transcription", "chat_completions"}: + if route != "transcription": raise AssertionError(f"unexpected route marker: {route!r}") if outcome not in {"success", "429", "hang"}: raise AssertionError(f"unexpected outcome marker: {outcome!r}") if not isinstance(body, dict): raise TypeError(f"{route} sent {type(body).__name__}, expected a JSON object") - if route == "transcription": - assert path == "/model/mistral.voxtral-mini-3b-2507/converse" - assert headers.get("authorization", "").startswith("AWS4-HMAC-SHA256 ") - assert headers.get("x-amz-date") - assert body["messages"][0]["content"][0]["audio"]["source"]["bytes"] == "AQI=" - assert "The audio language is en" in body["messages"][0]["content"][1]["text"] - return - assert path == "/v1/messages" - assert headers.get("x-api-key") == "sk-native" - assert body["model"] == "claude-sonnet-4-5" - assert body["max_tokens"] == 17 - assert body["messages"][0]["content"] == [{"type": "text", "text": "hello-from-chat"}] + assert path == "/model/mistral.voxtral-mini-3b-2507/converse" + assert headers.get("authorization", "").startswith("AWS4-HMAC-SHA256 ") + assert headers.get("x-amz-date") + assert body["messages"][0]["content"][0]["audio"]["source"]["bytes"] == "AQI=" + assert "The audio language is en" in body["messages"][0]["content"][1]["text"] -def native_response(status: int, route: str | None) -> bytes: +def native_response(status: int) -> bytes: if status == 429: return b'{"error":"native-rate-limit"}' - if route == "transcription": - return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}' - return ANTHROPIC_RESPONSE + return b'{"output":{"message":{"content":[{"text":"native-transcription"}]}}}' def load_native(native_path: Path) -> object: @@ -138,38 +122,23 @@ def route_kwargs(route: str, api_base: str, outcome: str) -> dict[str, object]: "language": "en", }, } - if route == "chat_completions": - return common | { - "model": "anthropic/claude-sonnet-4-5", - "messages": [{"role": "user", "content": "hello-from-chat"}], - "optional_params": {"max_tokens": 17}, - "api_key": "sk-native", - } raise AssertionError(f"unknown route: {route}") def assert_success(route: str, response: object) -> None: if not isinstance(response, dict): raise TypeError(f"{route} returned {type(response).__name__}, expected dict") - actual: Final = success_value(route, response) - expected: Final = "native-transcription" if route == "transcription" else "native-message" - if actual != expected: - raise AssertionError(f"{route} returned {actual!r}, expected {expected!r}") - - -def success_value(route: str, response: dict[object, object]) -> object: - if route == "transcription": - return response["text"] - return response["choices"][0]["message"]["content"] + if response["text"] != "native-transcription": + raise AssertionError(f"{route} returned {response['text']!r}, expected 'native-transcription'") def assert_rate_limit(route: str, error: BaseException) -> None: - if error.args != (429, native_response(429, route).decode()): + if error.args != (429, native_response(429).decode()): raise AssertionError(f"{route} returned the wrong 429 error: {error!r}") def exercise_sync(native: object, api_base: str) -> None: - for route in ("transcription", "chat_completions"): + for route in ("transcription",): function: Final = getattr(native, route) assert_success(route, function(route_call(route, api_base, "success"))) try: @@ -181,7 +150,7 @@ def exercise_sync(native: object, api_base: str) -> None: async def exercise_async(native: object, api_base: str) -> None: - for route in ("transcription", "chat_completions"): + for route in ("transcription",): function: Final = getattr(native, f"a{route}") assert_success(route, await function(route_call(route, api_base, "success"))) try: @@ -192,13 +161,11 @@ async def exercise_async(native: object, api_base: str) -> None: raise AssertionError(f"a{route} accepted a 429 response") responses: Final = await asyncio.wait_for( - asyncio.gather( - *(native.achat_completions(route_call("chat_completions", api_base, "success")) for _ in range(32)) - ), + asyncio.gather(*(native.atranscription(route_call("transcription", api_base, "success")) for _ in range(32))), timeout=15, ) for response in responses: - assert_success("chat_completions", response) + assert_success("transcription", response) def exercise_routes(native_path: Path, api_base: str) -> object: @@ -210,8 +177,8 @@ def exercise_routes(native_path: Path, api_base: str) -> object: def exercise_signal(native: object, api_base: str) -> int: try: - native.chat_completions( - route_call("chat_completions", api_base, "hang"), + native.transcription( + route_call("transcription", api_base, "hang"), ) except KeyboardInterrupt: sys.stdout.write("KeyboardInterrupt\n") diff --git a/tests/unit/rust_bridge/responses/test_route_host.py b/tests/unit/rust_bridge/responses/test_route_host.py index 1d67f2c368c..e110f16350b 100644 --- a/tests/unit/rust_bridge/responses/test_route_host.py +++ b/tests/unit/rust_bridge/responses/test_route_host.py @@ -5,8 +5,7 @@ import pytest from pydantic import ValidationError import litellm -from litellm.rust_bridge.responses.route_host import arguments, connection_defaults, map_failure, response -from litellm.rust_bridge.public_call import NativeCall +from litellm.rust_bridge.responses.route_host import connection_defaults, map_failure, response from litellm.types.llms.openai import ResponsesAPIResponse @@ -42,26 +41,6 @@ def test_response_rejects_a_payload_missing_required_fields() -> None: response(MappingProxyType({"object": "response"})) -def test_arguments_preserve_the_bound_view() -> None: - kwargs: Final = MappingProxyType({"litellm_metadata": {"user_id": "u"}}) - request: Final = NativeCall( - args=(), - kwargs=kwargs, - bound={ - "model": "gpt-4o", - "input": "hi", - "stream": None, - "api_key": None, - "api_base": None, - "custom_llm_provider": "openai", - "extra_headers": None, - **kwargs, - }, - ) - - assert arguments(request.bound) is request.bound - - @pytest.mark.parametrize( ("global_key", "provider_key", "expected"), (