mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-30 06:43:27 +00:00
fix(anthropic)!: send /v1/messages PDF documents to OpenAI-shaped providers as file parts (#47166)
* fix(anthropic): send /v1/messages PDF documents to OpenAI-shaped providers as file parts * refactor(anthropic): hold the chat completions document change for a later commit * fix(anthropic): send /v1/messages PDFs as file parts and keep file parts on azure_ai * fix(snowflake): turn file parts into document blocks for Cortex Claude * fix(prompt_security): sanitize chat file parts that carry file_data * fix: answer 400 or drop PDFs for models that cannot read them Models whose model_info or cost map entry sets supports_pdf_input to false, and Ollama, now get the PDF dropped under drop_params and a 400 from litellm without it. GigaChat uploads file parts, SAP keeps sending them as image_url data URLs, azure_ai passes dict content through, and annotations other than url_citation move to provider_specific_fields instead of failing the response. * fix(router): scope supports_pdf_input to its deployment and keep provider annotation keys Mark the azure_ai chat models in the cost map that cannot read PDFs, so drop_params drops the PDF and litellm answers its own 400 without it. Keep url_citation annotations whole, move only typed provider annotations to provider_specific_fields, and split them the same way when a stream is rebuilt. * fix(vertex_ai): send PDF file parts to Vertex partner models as image_url Vertex AI's OpenAI-compatible partner endpoint rejects the chat `file` part, so PDFs from /v1/messages reach those models as the image_url data URL they took before. Mark native DeepSeek chat models as unable to read PDFs, and treat a file part as a PDF by its format or filename when it has no data URL. * fix(cost-map): leave native DeepSeek PDF support unset /v1/messages reaches native DeepSeek through its own Anthropic endpoint, so the marks never applied there and would only change chat completions, which this PR does not otherwise touch. * fix(main): drop a user turn that held only a PDF under drop_params Dropping the PDF from a PDF-only user turn left an empty content list, which Foundry and OpenAI-shaped providers reject with a 400, so drop_params still failed for that request. The emptied user turn is now removed. * fix: read OCI PDFs, gate deployment base models, and stop empty drops OCI generic chat sends file parts as image data URLs again, so OCI Gemini keeps reading PDFs from /v1/messages and chat file parts. The PDF gate now reads model_info.base_model, drops PDFs under drop_params for azure_ai deployments that never declared PDF support (the old flattening behavior), and answers 400 when dropping the PDF leaves no user message. The cost map marks the Vertex partner text models that a live probe showed answering without reading the PDF. * test(integration): cover the PDF input gate, messages PDF file parts and provider annotations Wire cells for the supports_pdf_input check on chat, chat streaming, /v1/messages and the Responses bridge, its model_info, base_model, cost map and Ollama sources, drop_params at deployment and request level, azure_ai file parts, sibling deployment isolation, Anthropic SDK PDFs reaching azure as titled file parts, Gemini and Bedrock document controls, Prompt Security sanitizing a /v1/messages PDF, and OpenRouter file annotations live, cached and streamed * test(integration): cover Snowflake Claude and SAP in the messages PDF wire cells * fix(azure_ai): flatten non-PDF file parts to text as before and cover the GigaChat PDF upload * fix(main): PDF check reads the deployment's top-level base_model * fix(main): keep roles alternating after drop_params removes a PDF-only turn When ensure_alternating_roles is set, re-run role normalization on the messages left after the PDF check drops a turn, so two assistant turns never end up next to each other * test: type the PDF and annotation tests the PR added * test: type the GigaChat, SAP and Snowflake PDF tests
This commit is contained in:
Mateo Wang
GitHub
parent
c13ecb3085
commit
be6d082bdc
31 changed files with 2374 additions and 89 deletions
|
|
@ -19,6 +19,7 @@ from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
|||
from litellm.llms.ollama.common_utils import OllamaResponseIdNormalizer
|
||||
from litellm.types.llms.databricks import DatabricksTool
|
||||
from litellm.types.llms.openai import (
|
||||
ChatCompletionAnnotation,
|
||||
ChatCompletionReasoningItem,
|
||||
ChatCompletionThinkingBlock,
|
||||
ImageURLListItem,
|
||||
|
|
@ -59,6 +60,31 @@ _CHOICES_FIELDS: Final[frozenset] = frozenset(Choices.model_fields.keys())
|
|||
_MODEL_RESPONSE_FIELDS: Final[frozenset] = frozenset(ModelResponse.model_fields.keys()) | {"usage"}
|
||||
|
||||
|
||||
_ANNOTATION_ENTRIES: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def _is_provider_specific_annotation(annotation: object) -> bool:
|
||||
match annotation:
|
||||
case {"type": "url_citation"}:
|
||||
return False
|
||||
case {"type": _}:
|
||||
return True
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
def _annotation_entries(annotations: object) -> tuple[object, ...]:
|
||||
return _ANNOTATION_ENTRIES.validate_python(annotations) if isinstance(annotations, list) else ()
|
||||
|
||||
|
||||
def partition_annotations(annotations: object) -> tuple[object, tuple[object, ...]]:
|
||||
entries: Final = _annotation_entries(annotations)
|
||||
others: Final = tuple(annotation for annotation in entries if _is_provider_specific_annotation(annotation))
|
||||
if not others:
|
||||
return annotations, ()
|
||||
return [annotation for annotation in entries if not _is_provider_specific_annotation(annotation)] or None, others
|
||||
|
||||
|
||||
def _normalize_images_for_message(
|
||||
images: list[dict] | None,
|
||||
) -> list[ImageURLListItem] | None:
|
||||
|
|
@ -787,6 +813,9 @@ def convert_to_model_response_object(
|
|||
provider_specific_fields = dict(choice["message"].get("provider_specific_fields", None) or {})
|
||||
for f in choice["message"].keys() - _MESSAGE_FIELDS:
|
||||
provider_specific_fields[f] = choice["message"][f]
|
||||
annotations, other_annotations = partition_annotations(choice["message"].get("annotations", None))
|
||||
if other_annotations:
|
||||
provider_specific_fields["annotations"] = list(other_annotations)
|
||||
|
||||
# Handle reasoning models that display `reasoning_content` within `content`
|
||||
reasoning_content, content = extract_reasoning_content(choice["message"])
|
||||
|
|
@ -809,7 +838,9 @@ def convert_to_model_response_object(
|
|||
reasoning_content=reasoning_content,
|
||||
thinking_blocks=thinking_blocks,
|
||||
reasoning_items=choice["message"].get("reasoning_items", None),
|
||||
annotations=choice["message"].get("annotations", None),
|
||||
annotations=cast( # cast-ok: provider annotation dicts pass through whole, keeping keys such as url_citation.content
|
||||
list[ChatCompletionAnnotation] | None, annotations
|
||||
),
|
||||
images=_normalize_images_for_message(choice["message"].get("images", None)),
|
||||
)
|
||||
finish_reason = choice.get("finish_reason", None)
|
||||
|
|
|
|||
|
|
@ -424,6 +424,109 @@ def anthropic_image_source_to_openai_url(image_source: Mapping[str, object]) ->
|
|||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InlineDocument:
|
||||
filename: str
|
||||
file_data: str
|
||||
|
||||
|
||||
def _document_media_type(source: object) -> str:
|
||||
match source:
|
||||
case {"media_type": str() as media_type} if media_type:
|
||||
return media_type
|
||||
case _:
|
||||
return "application/pdf"
|
||||
|
||||
|
||||
def anthropic_inline_document(document: Mapping[str, object]) -> InlineDocument | None:
|
||||
source: Final = document.get("source")
|
||||
match source:
|
||||
case {"type": "base64", "data": str() as data} if data:
|
||||
title: Final = document.get("title")
|
||||
return InlineDocument(
|
||||
filename=title if isinstance(title, str) and title else "document.pdf",
|
||||
file_data=f"data:{_document_media_type(source)};base64,{data}",
|
||||
)
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def is_pdf_file_part(part: object) -> bool:
|
||||
match part:
|
||||
case {"type": "file", "file": {"file_data": str() as file_data}} if file_data.startswith("data:"):
|
||||
return file_data.startswith("data:application/pdf")
|
||||
case {"type": "file", "file": {"format": str() as file_format}}:
|
||||
return file_format == "application/pdf"
|
||||
case {"type": "file", "file": {"filename": str() as filename}}:
|
||||
return filename.lower().endswith(".pdf")
|
||||
case _:
|
||||
return False
|
||||
|
||||
|
||||
_CONTENT_PARTS: Final = TypeAdapter(tuple[object, ...])
|
||||
|
||||
|
||||
def content_parts(message: Mapping[str, object]) -> Sequence[object]:
|
||||
content: Final = message.get("content")
|
||||
return _CONTENT_PARTS.validate_python(content) if isinstance(content, list) else ()
|
||||
|
||||
|
||||
def _has_pdf_file_part(message: Mapping[str, object]) -> bool:
|
||||
return any(is_pdf_file_part(part) for part in content_parts(message))
|
||||
|
||||
|
||||
def messages_have_pdf_file_part(messages: Sequence[Mapping[str, object]]) -> bool:
|
||||
return any(_has_pdf_file_part(message) for message in messages)
|
||||
|
||||
|
||||
def _message_without_pdf_file_parts(message: AllMessageValues) -> AllMessageValues:
|
||||
if not _has_pdf_file_part(message):
|
||||
return message
|
||||
kept: Final = [part for part in content_parts(message) if not is_pdf_file_part(part)]
|
||||
return cast(AllMessageValues, {**message, "content": kept}) # cast-ok: same message with fewer content parts
|
||||
|
||||
|
||||
def _is_pdf_only_user_turn(message: AllMessageValues) -> bool:
|
||||
parts: Final = content_parts(message)
|
||||
return message["role"] == "user" and bool(parts) and all(is_pdf_file_part(part) for part in parts)
|
||||
|
||||
|
||||
def messages_without_pdf_file_parts(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]:
|
||||
return tuple(
|
||||
_message_without_pdf_file_parts(message) for message in messages if not _is_pdf_only_user_turn(message)
|
||||
)
|
||||
|
||||
|
||||
_CONTENT_PART: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _file_part_as_image_url(part: object) -> object:
|
||||
match part:
|
||||
case {"type": "file", "file": {"file_data": str() as file_data}} if file_data:
|
||||
fields: Final = _CONTENT_PART.validate_python(part)
|
||||
return {key: value for key, value in fields.items() if key != "file"} | {
|
||||
"type": "image_url",
|
||||
"image_url": {"url": file_data},
|
||||
}
|
||||
case _:
|
||||
return part
|
||||
|
||||
|
||||
def message_with_file_parts_as_image_urls(message: Mapping[str, object]) -> Mapping[str, object]:
|
||||
parts: Final = content_parts(message)
|
||||
if not parts:
|
||||
return message
|
||||
return {**message, "content": [_file_part_as_image_url(part) for part in parts]}
|
||||
|
||||
|
||||
def _typed_message_with_file_parts_as_image_urls(message: AllMessageValues) -> AllMessageValues:
|
||||
return cast(AllMessageValues, message_with_file_parts_as_image_urls(message)) # cast-ok: same message, image parts
|
||||
|
||||
|
||||
def messages_with_file_parts_as_image_urls(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]:
|
||||
return tuple(_typed_message_with_file_parts_as_image_urls(message) for message in messages)
|
||||
|
||||
|
||||
def _image_part_url(part: Mapping[str, object]) -> str | None:
|
||||
"""The image URL carried by one content part, whichever of the three dialects wrote it."""
|
||||
part_type: Final = part.get("type")
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ from litellm.litellm_core_utils.core_helpers import (
|
|||
)
|
||||
from litellm.litellm_core_utils.hidden_params import HIDDEN_PARAMS_ATTR, set_hidden_params
|
||||
from litellm.litellm_core_utils.llm_cost_calc.guardrail_cost import BEDROCK_TRACE_FIELDS, object_items, object_mapping
|
||||
from litellm.litellm_core_utils.llm_response_utils.convert_dict_to_response import partition_annotations
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_content_from_model_response, # pyright: ignore[reportUnknownVariableType] # response param accepts an untyped dict
|
||||
)
|
||||
|
|
@ -1700,10 +1701,17 @@ def stream_chunk_builder(
|
|||
# Merge annotations from ALL chunks — providers may spread
|
||||
# them across multiple streaming chunks or send them only in
|
||||
# the final chunk.
|
||||
all_annotations: Final[list] = []
|
||||
all_annotations: Final[list[object]] = []
|
||||
for ac in annotation_chunks:
|
||||
all_annotations.extend(ac["choices"][0]["delta"]["annotations"])
|
||||
response["choices"][0]["message"]["annotations"] = all_annotations
|
||||
message_annotations, provider_annotations = partition_annotations(all_annotations)
|
||||
if message_annotations is not None:
|
||||
response["choices"][0]["message"]["annotations"] = message_annotations
|
||||
if provider_annotations:
|
||||
response.choices[0].message.provider_specific_fields = {
|
||||
**(response.choices[0].message.provider_specific_fields or {}),
|
||||
"annotations": list(provider_annotations),
|
||||
}
|
||||
|
||||
audio_chunks: Final = [
|
||||
chunk
|
||||
|
|
@ -1766,7 +1774,10 @@ def stream_chunk_builder(
|
|||
|
||||
if combined_provider_fields:
|
||||
_choice = response.choices[0]
|
||||
_choice.message.provider_specific_fields = combined_provider_fields
|
||||
_choice.message.provider_specific_fields = {
|
||||
**(_choice.message.provider_specific_fields or {}),
|
||||
**combined_provider_fields,
|
||||
}
|
||||
|
||||
completion_output = get_content_from_model_response(response)
|
||||
|
||||
|
|
|
|||
|
|
@ -123,6 +123,7 @@ from openai.types.chat.chat_completion_chunk import Choice as OpenAIStreamingCho
|
|||
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
anthropic_image_source_to_openai_url,
|
||||
anthropic_inline_document,
|
||||
parse_tool_call_arguments,
|
||||
reasoning_content_from_thinking_blocks,
|
||||
with_prompt_cache_breakpoint,
|
||||
|
|
@ -185,6 +186,8 @@ from litellm.types.llms.openai import (
|
|||
AllMessageValues,
|
||||
ChatCompletionAssistantMessage,
|
||||
ChatCompletionAssistantToolCall,
|
||||
ChatCompletionFileObject,
|
||||
ChatCompletionFileObjectFile,
|
||||
ChatCompletionImageObject,
|
||||
ChatCompletionImageUrlObject,
|
||||
ChatCompletionRedactedThinkingBlock,
|
||||
|
|
@ -498,7 +501,9 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
for m in ordered_messages:
|
||||
user_message: ChatCompletionUserMessage | None = None
|
||||
tool_message_list: list[ChatCompletionToolMessage] = []
|
||||
new_user_content_list: list[ChatCompletionTextObject | ChatCompletionImageObject] = []
|
||||
new_user_content_list: list[
|
||||
ChatCompletionTextObject | ChatCompletionImageObject | ChatCompletionFileObject
|
||||
] = []
|
||||
if m["role"] == "system":
|
||||
system_message = self._translate_midturn_system_message_to_openai(m, model)
|
||||
if system_message is not None:
|
||||
|
|
@ -531,13 +536,8 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
self._add_prompt_cache_breakpoint_if_present(content, image_obj)
|
||||
)
|
||||
elif content.get("type") == "document":
|
||||
# Convert Anthropic document format (PDF, etc.) to OpenAI format
|
||||
source = content.get("source", {})
|
||||
openai_image_url = self._translate_anthropic_image_to_openai(cast(dict, source))
|
||||
|
||||
if openai_image_url:
|
||||
image_url_obj = ChatCompletionImageUrlObject(url=openai_image_url)
|
||||
doc_obj = ChatCompletionImageObject(type="image_url", image_url=image_url_obj)
|
||||
doc_obj = self._translate_anthropic_document_to_openai(content)
|
||||
if doc_obj is not None:
|
||||
self._add_cache_control_if_applicable(content, doc_obj, model)
|
||||
new_user_content_list.append(doc_obj)
|
||||
elif content.get("type") == "tool_result":
|
||||
|
|
@ -1357,7 +1357,7 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
text: Final = tool_result_text_part(block)
|
||||
return None if text is None else ChatCompletionTextObject(type="text", text=text)
|
||||
case "image" | "document":
|
||||
return self._tool_result_image_part(block.get("source"))
|
||||
return self._source_image_url_part(block.get("source"))
|
||||
case "tool_reference":
|
||||
return ChatCompletionToolReferenceObject(
|
||||
type="tool_reference", tool_name=str(block.get("tool_name") or "")
|
||||
|
|
@ -1365,7 +1365,18 @@ class LiteLLMAnthropicMessagesAdapter:
|
|||
case _:
|
||||
return None
|
||||
|
||||
def _tool_result_image_part(self, image_source: object) -> ChatCompletionImageObject | None:
|
||||
def _translate_anthropic_document_to_openai(
|
||||
self, document: Mapping[str, object]
|
||||
) -> ChatCompletionFileObject | ChatCompletionImageObject | None:
|
||||
inline_document: Final = anthropic_inline_document(document)
|
||||
if inline_document is None:
|
||||
return self._source_image_url_part(document.get("source"))
|
||||
return ChatCompletionFileObject(
|
||||
type="file",
|
||||
file=ChatCompletionFileObjectFile(filename=inline_document.filename, file_data=inline_document.file_data),
|
||||
)
|
||||
|
||||
def _source_image_url_part(self, image_source: object) -> ChatCompletionImageObject | None:
|
||||
if not isinstance(image_source, dict):
|
||||
return None
|
||||
openai_image_url = self._translate_anthropic_image_to_openai(image_source)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ from typing import Any, Final, cast
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
TOOL_RESULT_IMAGE_BOUNDARY,
|
||||
TOOL_RESULT_IMAGE_PLACEHOLDER,
|
||||
anthropic_inline_document,
|
||||
encrypted_reasoning_signature,
|
||||
responses_reasoning_items_from_thinking_blocks,
|
||||
with_prompt_cache_breakpoint,
|
||||
|
|
@ -106,27 +107,18 @@ class LiteLLMAnthropicToResponsesAPIAdapter:
|
|||
block: Mapping[str, object],
|
||||
) -> dict[str, str] | None: # mutable-ok: API message payload
|
||||
"""Convert an Anthropic document block to a Responses input_file part."""
|
||||
inline_document: Final = anthropic_inline_document(block)
|
||||
if inline_document is not None:
|
||||
return {
|
||||
"type": "input_file",
|
||||
"filename": inline_document.filename,
|
||||
"file_data": inline_document.file_data,
|
||||
}
|
||||
raw_source: Final = block.get("source")
|
||||
if not isinstance(raw_source, Mapping):
|
||||
return None
|
||||
source: Final = cast(Mapping[str, object], raw_source) # cast-ok: untrusted client payload
|
||||
source_type: Final = source.get("type")
|
||||
if source_type == "base64":
|
||||
data: Final = source.get("data")
|
||||
if not isinstance(data, str) or not data:
|
||||
return None
|
||||
raw_media_type: Final = source.get("media_type")
|
||||
media_type: Final = (
|
||||
raw_media_type if isinstance(raw_media_type, str) and raw_media_type else "application/pdf"
|
||||
)
|
||||
raw_title: Final = block.get("title")
|
||||
filename: Final = raw_title if isinstance(raw_title, str) and raw_title else "document.pdf"
|
||||
return {
|
||||
"type": "input_file",
|
||||
"filename": filename,
|
||||
"file_data": f"data:{media_type};base64,{data}",
|
||||
}
|
||||
if source_type == "url":
|
||||
if source.get("type") == "url":
|
||||
url: Final = source.get("url")
|
||||
if not isinstance(url, str) or not url:
|
||||
return None
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import copy
|
||||
import enum
|
||||
import re
|
||||
from collections.abc import AsyncIterator, Collection, Iterator
|
||||
from collections.abc import AsyncIterator, Collection, Iterator, Mapping
|
||||
from typing import TYPE_CHECKING, Final, cast
|
||||
|
||||
import httpx
|
||||
|
|
@ -10,10 +10,11 @@ from httpx import Response
|
|||
import litellm
|
||||
from litellm._logging import verbose_logger
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
audio_or_image_in_message_content,
|
||||
content_parts,
|
||||
convert_content_list_to_str,
|
||||
drop_tool_reference_parts_from_tool_messages,
|
||||
filter_value_from_dict,
|
||||
is_pdf_file_part,
|
||||
)
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.llms.azure_ai.common_utils import (
|
||||
|
|
@ -46,6 +47,20 @@ NON_OPENAI_SPEC_MESSAGE_FIELDS: Final = (
|
|||
"cache_control",
|
||||
)
|
||||
|
||||
LIST_ONLY_CONTENT_PART_TYPES: Final = frozenset({"image_url", "input_audio"})
|
||||
|
||||
|
||||
def _is_list_only_content_part(part: object) -> bool:
|
||||
match part:
|
||||
case {"type": str() as part_type} if part_type in LIST_ONLY_CONTENT_PART_TYPES:
|
||||
return True
|
||||
case _:
|
||||
return is_pdf_file_part(part)
|
||||
|
||||
|
||||
def _has_list_only_content_part(message: Mapping[str, object]) -> bool:
|
||||
return any(_is_list_only_content_part(part) for part in content_parts(message))
|
||||
|
||||
|
||||
class AzureAIChatCompletionResponseIterator(OpenAIChatCompletionResponseIterator):
|
||||
def chunk_parser(self, chunk: dict[str, object]) -> ModelResponseStream: # mutable-ok: mirrors override contract
|
||||
|
|
@ -246,7 +261,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
these with "Extra inputs are not permitted", which breaks multi-turn
|
||||
Anthropic-format clients that echo thinking blocks back as history.
|
||||
2. Transforms list content to a string.
|
||||
3. If message contains an image or audio, send as is (user-intended)
|
||||
3. If message contains an image, audio, or PDF file part, send as is (user-intended)
|
||||
|
||||
Operates on a deep copy so the caller's messages keep their thinking blocks
|
||||
and provider metadata, which a fallback to another provider still needs.
|
||||
|
|
@ -257,8 +272,7 @@ class AzureAIStudioConfig(OpenAIConfig):
|
|||
for field in NON_OPENAI_SPEC_MESSAGE_FIELDS:
|
||||
filter_value_from_dict(message_dict, field)
|
||||
|
||||
# Do nothing if the message contains an image or audio
|
||||
if audio_or_image_in_message_content(message):
|
||||
if _has_list_only_content_part(message):
|
||||
continue
|
||||
|
||||
texts = convert_content_list_to_str(message=message)
|
||||
|
|
|
|||
|
|
@ -38,6 +38,15 @@ else:
|
|||
|
||||
_EMPTY_FUNCTION: Final[Mapping[str, object]] = MappingProxyType({})
|
||||
_JSON_VALUE: Final = TypeAdapter(object)
|
||||
_CONTENT_PART: Final = TypeAdapter(Mapping[str, object])
|
||||
|
||||
|
||||
def _file_part_data(part: Mapping[str, object]) -> str | None:
|
||||
match part:
|
||||
case {"type": "file", "file": {"file_data": str() as file_data}} if file_data:
|
||||
return file_data
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def is_valid_json(value: str) -> bool:
|
||||
|
|
@ -295,7 +304,7 @@ class GigaChatConfig(BaseConfig):
|
|||
|
||||
def _transform_list_content(self, content: Sequence) -> tuple[str, Sequence[object]]:
|
||||
"""
|
||||
Extract text and image attachments from a multimodal message content list.
|
||||
Extract text, image and file attachments from a multimodal message content list.
|
||||
|
||||
Args:
|
||||
content: List of content parts (OpenAI multimodal format)
|
||||
|
|
@ -320,6 +329,10 @@ class GigaChatConfig(BaseConfig):
|
|||
file_id = self._upload_image(upload_url)
|
||||
if file_id:
|
||||
attachments.append(file_id)
|
||||
elif (file_data := _file_part_data(_CONTENT_PART.validate_python(part))) and (
|
||||
uploaded_file_id := self._upload_image(file_data)
|
||||
):
|
||||
attachments.append(uploaded_file_id)
|
||||
text: Final = "\n".join(texts) if texts else ""
|
||||
return text, attachments
|
||||
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import litellm
|
|||
from litellm._logging import verbose_logger
|
||||
from litellm.constants import DEFAULT_OCI_CHAT_MAX_TOKENS
|
||||
from litellm.litellm_core_utils.logging_utils import track_llm_api_timing
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
|
||||
from litellm.litellm_core_utils.streaming_handler import CustomStreamWrapper
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
|
||||
|
|
@ -681,7 +682,9 @@ class OCIChatConfig(BaseConfig):
|
|||
servingMode=serving_mode,
|
||||
chatRequest=OCIChatRequestPayload(
|
||||
apiFormat=vendor.value,
|
||||
messages=adapt_messages_to_generic_oci_standard(messages),
|
||||
messages=adapt_messages_to_generic_oci_standard(
|
||||
list(messages_with_file_parts_as_image_urls(messages))
|
||||
),
|
||||
**self._get_optional_params(vendor, optional_params, model),
|
||||
),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ import httpx
|
|||
from pydantic import BaseModel
|
||||
|
||||
import litellm
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import message_with_file_parts_as_image_urls
|
||||
from litellm.litellm_core_utils.url_utils import join_api_base
|
||||
from litellm.types.llms.openai import AllMessageValues
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
|
@ -104,7 +105,7 @@ def _message_role_mapping(role: str) -> type[ChatMessage]:
|
|||
def _messages_to_sap_template(messages: list[AllMessageValues]) -> list:
|
||||
template: Final = []
|
||||
for message in messages:
|
||||
folded_message = _fold_message_cache_control(message)
|
||||
folded_message = _fold_message_cache_control(message_with_file_parts_as_image_urls(message))
|
||||
message_role_class = _message_role_mapping(str(folded_message["role"]))
|
||||
validated_message = validate_dict(folded_message, message_role_class)
|
||||
template.append(validated_message)
|
||||
|
|
|
|||
|
|
@ -112,7 +112,7 @@ def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object:
|
|||
pdf/text data URIs become document blocks rather than malformed image blocks.
|
||||
"""
|
||||
image_url: Final = block.get("image_url")
|
||||
url: Final = image_url if isinstance(image_url, str) else _image_url_field(image_url, "url")
|
||||
url: Final = image_url if isinstance(image_url, str) else _str_field(image_url, "url")
|
||||
if not url:
|
||||
return block
|
||||
|
||||
|
|
@ -121,18 +121,31 @@ def _convert_image_url_to_anthropic(block: Mapping[str, object]) -> object:
|
|||
if select_anthropic_content_block_type_for_file(_data_uri_media_type(url)) == "document"
|
||||
else create_anthropic_image_param(
|
||||
image_url if isinstance(image_url, dict) else url,
|
||||
format=_image_url_field(image_url, "format"),
|
||||
format=_str_field(image_url, "format"),
|
||||
is_bedrock_invoke=True,
|
||||
)
|
||||
)
|
||||
return _with_cache_control(converted, block)
|
||||
|
||||
|
||||
def _convert_file_to_anthropic(block: Mapping[str, object]) -> object:
|
||||
file_data: Final = _str_field(block.get("file"), "file_data")
|
||||
if not file_data:
|
||||
return block
|
||||
return _with_cache_control(
|
||||
anthropic_process_openai_file_message({"type": "file", "file": {"file_data": file_data}}), block
|
||||
)
|
||||
|
||||
|
||||
def _with_cache_control(converted: Mapping[str, object], block: Mapping[str, object]) -> Mapping[str, object]:
|
||||
cache_control: Final = block.get("cache_control")
|
||||
if cache_control is None:
|
||||
return converted
|
||||
return {**converted, "cache_control": cache_control}
|
||||
|
||||
|
||||
def _image_url_field(image_url: object, key: str) -> str | None:
|
||||
value: Final = image_url.get(key) if isinstance(image_url, dict) else None
|
||||
def _str_field(container: object, key: str) -> str | None:
|
||||
value: Final = container.get(key) if isinstance(container, dict) else None
|
||||
return value if isinstance(value, str) else None
|
||||
|
||||
|
||||
|
|
@ -141,15 +154,23 @@ def _data_uri_media_type(url: str) -> str:
|
|||
return match.group(1) if match else ""
|
||||
|
||||
|
||||
def _convert_image_url_blocks_to_anthropic(content: object) -> object:
|
||||
def _convert_media_block_to_anthropic(block: object) -> object:
|
||||
mapping: Final = as_str_mapping(block)
|
||||
if mapping is None:
|
||||
return block
|
||||
match mapping.get("type"):
|
||||
case "image_url":
|
||||
return _convert_image_url_to_anthropic(mapping)
|
||||
case "file":
|
||||
return _convert_file_to_anthropic(mapping)
|
||||
case _:
|
||||
return block
|
||||
|
||||
|
||||
def _convert_media_blocks_to_anthropic(content: object) -> object:
|
||||
if not isinstance(content, list):
|
||||
return content
|
||||
return [
|
||||
_convert_image_url_to_anthropic(block)
|
||||
if isinstance(block, Mapping) and block.get("type") == "image_url"
|
||||
else block
|
||||
for block in content
|
||||
]
|
||||
return [_convert_media_block_to_anthropic(block) for block in content]
|
||||
|
||||
|
||||
def _convert_tool_result_to_anthropic(
|
||||
|
|
@ -391,7 +412,7 @@ class SnowflakeConfig(SnowflakeBaseConfig, OpenAIGPTConfig):
|
|||
conversation.append(
|
||||
{
|
||||
"role": role,
|
||||
"content": _convert_image_url_blocks_to_anthropic(content),
|
||||
"content": _convert_media_blocks_to_anthropic(content),
|
||||
}
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import httpx
|
|||
|
||||
import litellm
|
||||
from litellm import LlmProviders
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import messages_with_file_parts_as_image_urls
|
||||
from litellm.types.llms.vertex_ai import VertexPartnerProvider
|
||||
from litellm.utils import ModelResponse
|
||||
|
||||
|
|
@ -217,7 +218,7 @@ class VertexAIPartnerModels(VertexBase):
|
|||
return base_llm_http_handler.completion(
|
||||
model=model,
|
||||
stream=stream,
|
||||
messages=messages,
|
||||
messages=list(messages_with_file_parts_as_image_urls(messages)),
|
||||
acompletion=acompletion,
|
||||
api_base=api_base,
|
||||
model_response=model_response,
|
||||
|
|
|
|||
163
litellm/main.py
163
litellm/main.py
|
|
@ -108,6 +108,10 @@ from litellm.litellm_core_utils.mock_functions import (
|
|||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
get_content_from_model_response as get_content_from_model_response, # noqa: PLC0414 # public re-export
|
||||
)
|
||||
from litellm.litellm_core_utils.prompt_templates.common_utils import (
|
||||
messages_have_pdf_file_part,
|
||||
messages_without_pdf_file_parts,
|
||||
)
|
||||
from litellm.litellm_core_utils.provider_affinity import add_provider_affinity_header
|
||||
from litellm.litellm_core_utils.request_timeout_resolver import (
|
||||
get_configured_request_timeout,
|
||||
|
|
@ -191,11 +195,13 @@ from litellm.utils import (
|
|||
get_requester_metadata,
|
||||
get_secret,
|
||||
get_standard_openai_params,
|
||||
is_explicitly_disabled_factory,
|
||||
mock_completion_streaming_obj,
|
||||
pre_process_non_default_params,
|
||||
read_config_args,
|
||||
should_run_mock_completion,
|
||||
supports_httpx_timeout,
|
||||
supports_pdf_input,
|
||||
validate_and_fix_openai_messages,
|
||||
validate_and_fix_openai_tools,
|
||||
validate_and_fix_thinking_param,
|
||||
|
|
@ -393,6 +399,125 @@ def _resolve_control_options(kwargs: Mapping[str, object], model: str) -> Contro
|
|||
return assert_never(control)
|
||||
|
||||
|
||||
PROVIDERS_WITHOUT_DOCUMENT_INPUT: Final = frozenset({"ollama", "ollama_chat"})
|
||||
PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT: Final = frozenset({"azure_ai"})
|
||||
|
||||
|
||||
def _cost_map_pdf_input_support(model: str, custom_llm_provider: str) -> bool | None:
|
||||
if custom_llm_provider in PROVIDERS_WITHOUT_DOCUMENT_INPUT or is_explicitly_disabled_factory(
|
||||
model, custom_llm_provider, "supports_pdf_input"
|
||||
):
|
||||
return False
|
||||
if supports_pdf_input(model, custom_llm_provider):
|
||||
return True
|
||||
return None
|
||||
|
||||
|
||||
def _deployment_pdf_input_support(model: str, base_model: str, custom_llm_provider: str) -> bool | None:
|
||||
supports: Final = (
|
||||
_cost_map_pdf_input_support(model, custom_llm_provider),
|
||||
_cost_map_pdf_input_support(base_model, custom_llm_provider),
|
||||
)
|
||||
if False in supports:
|
||||
return False
|
||||
if True in supports:
|
||||
return True
|
||||
return None
|
||||
|
||||
|
||||
def _declared_pdf_input_support(
|
||||
model: str, custom_llm_provider: str, model_info: Mapping[str, object] | None, base_model: str | None
|
||||
) -> bool | None:
|
||||
match model_info:
|
||||
case {"supports_pdf_input": bool() as declared}:
|
||||
return declared
|
||||
case _ if base_model:
|
||||
return _deployment_pdf_input_support(model, base_model, custom_llm_provider)
|
||||
case _:
|
||||
return _cost_map_pdf_input_support(model, custom_llm_provider)
|
||||
|
||||
|
||||
def _pdf_input_unreadable(
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
model_info: Mapping[str, object] | None,
|
||||
base_model: str | None,
|
||||
drop_params: bool,
|
||||
) -> bool:
|
||||
match _declared_pdf_input_support(model, custom_llm_provider, model_info, base_model):
|
||||
case False:
|
||||
return True
|
||||
case None:
|
||||
return drop_params and custom_llm_provider in PROVIDERS_DROPPING_UNDECLARED_PDF_INPUT
|
||||
case True:
|
||||
return False
|
||||
|
||||
|
||||
def _messages_without_unreadable_pdfs(
|
||||
messages: Sequence[AllMessageValues],
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
model_info: Mapping[str, object] | None,
|
||||
base_model: str | None,
|
||||
drop_params: bool,
|
||||
) -> tuple[AllMessageValues, ...] | None:
|
||||
if not messages_have_pdf_file_part(messages) or not _pdf_input_unreadable(
|
||||
model, custom_llm_provider, model_info, base_model, drop_params
|
||||
):
|
||||
return None
|
||||
if not drop_params:
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
f"{custom_llm_provider}/{model} does not accept PDF input. "
|
||||
"Remove the PDF, or set drop_params=True to send the request without it"
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
readable_messages: Final = messages_without_pdf_file_parts(messages)
|
||||
if not any(message["role"] == "user" for message in readable_messages):
|
||||
raise litellm.BadRequestError(
|
||||
message=(
|
||||
f"{custom_llm_provider}/{model} does not accept PDF input, "
|
||||
"and no user message is left once the PDF is dropped"
|
||||
),
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
)
|
||||
return readable_messages
|
||||
|
||||
|
||||
def _messages_readable_by_model(
|
||||
messages: Sequence[AllMessageValues],
|
||||
model: str,
|
||||
custom_llm_provider: str,
|
||||
model_info: Mapping[str, object] | None,
|
||||
base_model: str | None,
|
||||
drop_params: bool,
|
||||
ensure_alternating_roles: bool | None,
|
||||
user_continue_message: ChatCompletionUserMessage | None,
|
||||
assistant_continue_message: ChatCompletionAssistantMessage | None,
|
||||
) -> tuple[AllMessageValues, ...] | None:
|
||||
pdf_free_messages: Final = _messages_without_unreadable_pdfs(
|
||||
messages=messages,
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_info=model_info,
|
||||
base_model=base_model,
|
||||
drop_params=drop_params,
|
||||
)
|
||||
if pdf_free_messages is None:
|
||||
return None
|
||||
return tuple(
|
||||
get_completion_messages(
|
||||
messages=list(pdf_free_messages),
|
||||
ensure_alternating_roles=ensure_alternating_roles or False,
|
||||
user_continue_message=user_continue_message,
|
||||
assistant_continue_message=assistant_continue_message,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
####### COMPLETION ENDPOINTS ################
|
||||
|
||||
|
||||
|
|
@ -4234,7 +4359,6 @@ def _complete_sagemaker(ctx: CompletionDispatchContext) -> _CompletionDispatchRe
|
|||
_ADDITIONAL_DROP_PARAMS_ADAPTER: Final = TypeAdapter(list[str])
|
||||
_OPTIONAL_PARAMS_ADAPTER: Final = TypeAdapter(dict[str, object])
|
||||
_OLLAMA_EMBEDDING_HEADERS_ADAPTER: Final = TypeAdapter(dict[str, str])
|
||||
_REQUEST_KWARGS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_MODEL_INFO_ADAPTER: Final = TypeAdapter(Mapping[str, object])
|
||||
_PROVIDER_ROUTING_HINTS: Final = TypeAdapter(ProviderRoutingHints)
|
||||
_BASE_URL_HINT: Final = TypeAdapter(BaseUrlRoutingHint)
|
||||
|
|
@ -5558,9 +5682,15 @@ def completion(
|
|||
)
|
||||
headers = kwargs.get("headers", None) or extra_headers
|
||||
|
||||
ensure_alternating_roles: Final[bool | None] = kwargs.get("ensure_alternating_roles", None)
|
||||
user_continue_message: Final[ChatCompletionUserMessage | None] = kwargs.get("user_continue_message", None)
|
||||
assistant_continue_message: ChatCompletionAssistantMessage | None = kwargs.get("assistant_continue_message", None)
|
||||
ensure_alternating_roles: Final = cast( # cast-ok: read from untyped kwargs
|
||||
bool | None, kwargs.get("ensure_alternating_roles", None)
|
||||
)
|
||||
user_continue_message: Final = cast( # cast-ok: read from untyped kwargs
|
||||
ChatCompletionUserMessage | None, kwargs.get("user_continue_message", None)
|
||||
)
|
||||
assistant_continue_message: Final = cast( # cast-ok: read from untyped kwargs
|
||||
ChatCompletionAssistantMessage | None, kwargs.get("assistant_continue_message", None)
|
||||
)
|
||||
headers = {} if headers is None else dict(headers)
|
||||
if extra_headers is not None:
|
||||
headers.update(extra_headers)
|
||||
|
|
@ -5928,15 +6058,17 @@ def completion(
|
|||
def optional_params_for_model_info(deployment_info: Mapping[str, object] | None) -> dict[str, object]:
|
||||
return get_optional_params(**mapping_args, model_info=deployment_info, **mapping_special_params)
|
||||
|
||||
optional_params = optional_params_for_model_info(
|
||||
validated_model_info: Final = (
|
||||
_MODEL_INFO_ADAPTER.validate_python(model_info) if isinstance(model_info, dict) else None
|
||||
)
|
||||
deployment_base_model: Final = base_model if isinstance(base_model, str) else None
|
||||
optional_params = optional_params_for_model_info(validated_model_info)
|
||||
fakes_stream: Final = partial(
|
||||
_completion_fakes_stream,
|
||||
model,
|
||||
custom_llm_provider,
|
||||
provider_config,
|
||||
base_model=base_model if isinstance(base_model, str) else None,
|
||||
base_model=deployment_base_model,
|
||||
bridges_to_responses_api=bridges_to_responses_api is True,
|
||||
)
|
||||
resolve_timeout: Final = partial(
|
||||
|
|
@ -5974,16 +6106,29 @@ def completion(
|
|||
functions_unsupported_model: Final[list[object]] = TypeAdapter(list[object]).validate_python(
|
||||
optional_params.pop("functions_unsupported_model")
|
||||
)
|
||||
request_kwargs: Final = _REQUEST_KWARGS_ADAPTER.validate_python(kwargs)
|
||||
messages = function_call_prompt(
|
||||
messages=messages,
|
||||
functions=functions_unsupported_model,
|
||||
model=model,
|
||||
llm_provider=custom_llm_provider,
|
||||
drop_params=litellm.drop_params is True
|
||||
or normalize_drop_params(request_kwargs.get("drop_params")) is True,
|
||||
drop_params=drop_unsupported_reasoning,
|
||||
)
|
||||
|
||||
readable_messages: Final = _messages_readable_by_model(
|
||||
messages=cast(list[AllMessageValues], messages), # cast-ok: completion types messages as a bare List
|
||||
model=model,
|
||||
custom_llm_provider=custom_llm_provider,
|
||||
model_info=validated_model_info,
|
||||
base_model=deployment_base_model,
|
||||
drop_params=drop_unsupported_reasoning,
|
||||
ensure_alternating_roles=ensure_alternating_roles,
|
||||
user_continue_message=user_continue_message,
|
||||
assistant_continue_message=assistant_continue_message,
|
||||
)
|
||||
messages = ( # rebind-ok: completion() rewrites messages step by step
|
||||
messages if readable_messages is None else list(readable_messages)
|
||||
)
|
||||
|
||||
# For logging - save the values of the litellm-specific params passed in
|
||||
requested_litellm_params: Final = get_litellm_params(
|
||||
acompletion=acompletion,
|
||||
|
|
|
|||
|
|
@ -4049,6 +4049,7 @@
|
|||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -12437,7 +12438,8 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_native_streaming": true
|
||||
"supports_native_streaming": true,
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/FLUX-1.1-pro": {
|
||||
"deprecation_date": "2027-01-05",
|
||||
|
|
@ -12502,6 +12504,7 @@
|
|||
"output_cost_per_token": 1.85e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12516,6 +12519,7 @@
|
|||
"output_cost_per_token": 3.828e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12531,6 +12535,7 @@
|
|||
"output_cost_per_token": 3.52e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12546,6 +12551,7 @@
|
|||
"output_cost_per_token": 4.84e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12560,6 +12566,7 @@
|
|||
"output_cost_per_token": 4.84e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12574,6 +12581,7 @@
|
|||
"output_cost_per_token": 7.26e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12594,6 +12602,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12616,6 +12625,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12638,6 +12648,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12660,6 +12671,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12687,6 +12699,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12703,6 +12716,7 @@
|
|||
"output_cost_per_token": 1.32e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12724,6 +12738,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12744,6 +12759,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12765,6 +12781,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12833,6 +12850,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12847,6 +12865,7 @@
|
|||
"output_cost_per_token": 7.1e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": {
|
||||
|
|
@ -12859,6 +12878,7 @@
|
|||
"output_cost_per_token": 1e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -12872,6 +12892,7 @@
|
|||
"output_cost_per_token": 7.8e-07,
|
||||
"source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -12884,6 +12905,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3.7e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/Meta-Llama-3.1-70B-Instruct": {
|
||||
|
|
@ -12895,6 +12917,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3.54e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/Phi-3-medium-128k-instruct": {
|
||||
|
|
@ -12906,6 +12929,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6.8e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12918,6 +12942,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6.8e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12930,6 +12955,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.2e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12942,6 +12968,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.2e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12954,6 +12981,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12966,6 +12994,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12978,6 +13007,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6.4e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12990,6 +13020,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.2e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -13002,6 +13033,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.2e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -13015,6 +13047,7 @@
|
|||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -13027,7 +13060,8 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/Phi-4-multimodal-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
|
|
@ -13041,6 +13075,7 @@
|
|||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_audio_input": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4-mini-reasoning": {
|
||||
|
|
@ -13052,7 +13087,8 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/Phi-4-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
|
|
@ -13064,6 +13100,7 @@
|
|||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
|
|
@ -13106,6 +13143,7 @@
|
|||
"output_cost_per_token": 1e-05,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/doc-intelligence/prebuilt-read": {
|
||||
|
|
@ -13144,6 +13182,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.4e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -13202,6 +13241,7 @@
|
|||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -13217,6 +13257,7 @@
|
|||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -13230,6 +13271,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 4.56e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/deepseek-v4-pro": {
|
||||
|
|
@ -13242,6 +13284,7 @@
|
|||
"output_cost_per_token": 3.48e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"cache_read_input_token_cost": 1.45e-07,
|
||||
|
|
@ -13257,6 +13300,7 @@
|
|||
"output_cost_per_token": 5.1e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"cache_read_input_token_cost": 2.8e-08,
|
||||
|
|
@ -13273,6 +13317,7 @@
|
|||
"output_cost_per_token": 1.32e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -13306,6 +13351,7 @@
|
|||
"output_cost_per_token": 1.5e-05,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
|
|
@ -13323,6 +13369,7 @@
|
|||
"output_cost_per_token_above_200k_tokens": 5e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -13343,6 +13390,7 @@
|
|||
"output_cost_per_token_above_200k_tokens": 1.2e-05,
|
||||
"source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -13361,6 +13409,7 @@
|
|||
"output_cost_per_token": 2.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -13379,6 +13428,7 @@
|
|||
"output_cost_per_token": 2.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -13395,6 +13445,7 @@
|
|||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
|
|
@ -13409,6 +13460,7 @@
|
|||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -13424,6 +13476,7 @@
|
|||
"output_cost_per_token": 1.5e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
|
|
@ -13436,7 +13489,8 @@
|
|||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.00971,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models"
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/jamba-instruct": {
|
||||
"deprecation_date": "2025-03-01",
|
||||
|
|
@ -13447,6 +13501,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 7e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/kimi-k2.5": {
|
||||
|
|
@ -13459,6 +13514,7 @@
|
|||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
|
|
@ -13482,6 +13538,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
|
|
@ -13498,6 +13555,7 @@
|
|||
"output_cost_per_token": 4e-08,
|
||||
"source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-large": {
|
||||
|
|
@ -13510,6 +13568,7 @@
|
|||
"output_cost_per_token": 1.2e-05,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-large-2407": {
|
||||
|
|
@ -13522,6 +13581,7 @@
|
|||
"output_cost_per_token": 6e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-large-latest": {
|
||||
|
|
@ -13533,6 +13593,7 @@
|
|||
"output_cost_per_token": 6e-06,
|
||||
"source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-large-3": {
|
||||
|
|
@ -13544,6 +13605,7 @@
|
|||
"output_cost_per_token": 1.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -13557,6 +13619,7 @@
|
|||
"output_cost_per_token": 2e-06,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-nemo": {
|
||||
|
|
@ -13568,7 +13631,8 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_function_calling": true
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/mistral-small": {
|
||||
"deprecation_date": "2025-07-31",
|
||||
|
|
@ -13580,6 +13644,7 @@
|
|||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-small-2503": {
|
||||
|
|
@ -13591,6 +13656,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-07,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -52006,6 +52072,7 @@
|
|||
],
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -52026,6 +52093,7 @@
|
|||
],
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -52777,6 +52845,7 @@
|
|||
"output_cost_per_token": 2.5e-06,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
|
|
@ -52794,6 +52863,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -52811,6 +52881,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -52989,6 +53060,7 @@
|
|||
"output_cost_per_token": 3.6e-07,
|
||||
"output_cost_per_token_batches": 1.8e-07,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"vertex_ai/openai/gpt-oss-20b-maas": {
|
||||
|
|
@ -53000,6 +53072,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-07,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"cache_read_input_token_cost": 7e-09,
|
||||
"input_cost_per_token_batches": 3.5e-08,
|
||||
|
|
@ -53131,6 +53204,7 @@
|
|||
"us-south1"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": {
|
||||
|
|
@ -53149,6 +53223,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
|
||||
|
|
@ -53164,6 +53239,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
|
||||
|
|
@ -53179,6 +53255,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/veo-3.1-generate-preview": {
|
||||
|
|
@ -67322,6 +67399,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -67336,6 +67414,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -67378,6 +67457,7 @@
|
|||
"output_cost_per_token": 1.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -67392,6 +67472,7 @@
|
|||
"output_cost_per_token": 3.1e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -67406,6 +67487,7 @@
|
|||
"output_cost_per_token": 5.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -67420,6 +67502,7 @@
|
|||
"output_cost_per_token": 6.25e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -67436,6 +67519,7 @@
|
|||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -67451,6 +67535,7 @@
|
|||
"output_cost_per_token": 3.2e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -67469,6 +67554,7 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -67478,7 +67564,8 @@
|
|||
"output_cost_per_token": 5.4e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/deepseek-v3-0324": {
|
||||
"deprecation_date": "2026-07-13",
|
||||
|
|
@ -67486,7 +67573,8 @@
|
|||
"output_cost_per_token": 4.56e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/deepseek-v3.1": {
|
||||
"deprecation_date": "2026-07-13",
|
||||
|
|
@ -67494,7 +67582,8 @@
|
|||
"output_cost_per_token": 4.94e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/grok-3": {
|
||||
"deprecation_date": "2026-05-01",
|
||||
|
|
@ -67502,7 +67591,8 @@
|
|||
"output_cost_per_token": 1.5e-05,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/grok-3-mini": {
|
||||
"deprecation_date": "2026-05-01",
|
||||
|
|
@ -67510,7 +67600,8 @@
|
|||
"output_cost_per_token": 1.27e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/grok-4-fast-non-reasoning": {
|
||||
"deprecation_date": "2026-05-01",
|
||||
|
|
@ -67518,7 +67609,8 @@
|
|||
"output_cost_per_token": 5e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/grok-4-fast-reasoning": {
|
||||
"deprecation_date": "2026-05-01",
|
||||
|
|
@ -67526,7 +67618,8 @@
|
|||
"output_cost_per_token": 5e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": {
|
||||
"input_cost_per_token": 7.2e-08,
|
||||
|
|
@ -73196,6 +73289,7 @@
|
|||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_regions": ["global"],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -73811,6 +73905,7 @@
|
|||
"output_cost_per_token": 3.5e-06,
|
||||
"source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -74865,6 +74960,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -75113,6 +75209,7 @@
|
|||
"output_cost_per_token_above_200k_tokens": 1.2e-05,
|
||||
"source": "https://docs.x.ai/developers/models/grok-4.7",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
|
|||
|
|
@ -647,9 +647,10 @@ class PromptSecurityGuardrail(CustomGuardrail):
|
|||
async def _process_document_item(self, item: dict, user_api_key_alias: str | None) -> dict:
|
||||
"""Process and sanitize document/file items."""
|
||||
doc_data: Final = item.get("document") or item.get("file") or item
|
||||
url_key: Final = "file_data" if isinstance(doc_data, dict) and not doc_data.get("url") else "url"
|
||||
|
||||
if isinstance(doc_data, dict):
|
||||
url = doc_data.get("url", "")
|
||||
url = doc_data.get(url_key) or ""
|
||||
doc_content = doc_data.get("data", "")
|
||||
else:
|
||||
url = doc_data if isinstance(doc_data, str) else ""
|
||||
|
|
@ -698,7 +699,7 @@ class PromptSecurityGuardrail(CustomGuardrail):
|
|||
if url.startswith("data:") and header:
|
||||
sanitized_url: Final = f"{header},{sanitized_encoded}"
|
||||
if isinstance(doc_data, dict):
|
||||
doc_data["url"] = sanitized_url
|
||||
doc_data[url_key] = sanitized_url
|
||||
elif isinstance(doc_data, dict):
|
||||
doc_data["data"] = sanitized_encoded
|
||||
|
||||
|
|
|
|||
|
|
@ -4147,7 +4147,12 @@ class CustomPricingLiteLLMParams(MirroredPricingParams):
|
|||
DEPLOYMENT_SCOPED_PRICING_FIELDS: Final[frozenset[str]] = frozenset({"off_peak_pricing"})
|
||||
|
||||
DEPLOYMENT_SCOPED_CAPABILITY_FIELDS: Final[frozenset[str]] = frozenset(
|
||||
{"supports_regex_lookaround", "supports_assistant_prefill", "supports_bedrock_native_structured_output"}
|
||||
{
|
||||
"supports_regex_lookaround",
|
||||
"supports_assistant_prefill",
|
||||
"supports_bedrock_native_structured_output",
|
||||
"supports_pdf_input",
|
||||
}
|
||||
)
|
||||
|
||||
DEPLOYMENT_SCOPED_TOKEN_COUNT_FIELDS: Final[frozenset[str]] = frozenset({"token_count_framing", "token_count_model"})
|
||||
|
|
|
|||
|
|
@ -4049,6 +4049,7 @@
|
|||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -12437,7 +12438,8 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_native_streaming": true
|
||||
"supports_native_streaming": true,
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/FLUX-1.1-pro": {
|
||||
"deprecation_date": "2027-01-05",
|
||||
|
|
@ -12502,6 +12504,7 @@
|
|||
"output_cost_per_token": 1.85e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12516,6 +12519,7 @@
|
|||
"output_cost_per_token": 3.828e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12531,6 +12535,7 @@
|
|||
"output_cost_per_token": 3.52e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12546,6 +12551,7 @@
|
|||
"output_cost_per_token": 4.84e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12560,6 +12566,7 @@
|
|||
"output_cost_per_token": 4.84e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12574,6 +12581,7 @@
|
|||
"output_cost_per_token": 7.26e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12594,6 +12602,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12616,6 +12625,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12638,6 +12648,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12660,6 +12671,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12687,6 +12699,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12703,6 +12716,7 @@
|
|||
"output_cost_per_token": 1.32e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12724,6 +12738,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12744,6 +12759,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -12765,6 +12781,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12833,6 +12850,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -12847,6 +12865,7 @@
|
|||
"output_cost_per_token": 7.1e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/Llama-4-Maverick-17B-128E-Instruct-FP8": {
|
||||
|
|
@ -12859,6 +12878,7 @@
|
|||
"output_cost_per_token": 1e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -12872,6 +12892,7 @@
|
|||
"output_cost_per_token": 7.8e-07,
|
||||
"source": "https://azure.microsoft.com/en-us/blog/introducing-the-llama-4-herd-in-azure-ai-foundry-and-azure-databricks/",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -12884,6 +12905,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3.7e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/Meta-Llama-3.1-70B-Instruct": {
|
||||
|
|
@ -12895,6 +12917,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3.54e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/Phi-3-medium-128k-instruct": {
|
||||
|
|
@ -12906,6 +12929,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6.8e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12918,6 +12942,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6.8e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12930,6 +12955,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.2e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12942,6 +12968,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.2e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12954,6 +12981,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12966,6 +12994,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12978,6 +13007,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 6.4e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -12990,6 +13020,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.2e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -13002,6 +13033,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.2e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -13015,6 +13047,7 @@
|
|||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": false
|
||||
},
|
||||
|
|
@ -13027,7 +13060,8 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/Phi-4-multimodal-instruct": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
|
|
@ -13041,6 +13075,7 @@
|
|||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_audio_input": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_vision": true
|
||||
},
|
||||
"azure_ai/Phi-4-mini-reasoning": {
|
||||
|
|
@ -13052,7 +13087,8 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/Phi-4-reasoning": {
|
||||
"deprecation_date": "2027-01-17",
|
||||
|
|
@ -13064,6 +13100,7 @@
|
|||
"output_cost_per_token": 5e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
|
|
@ -13106,6 +13143,7 @@
|
|||
"output_cost_per_token": 1e-05,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/doc-intelligence/prebuilt-read": {
|
||||
|
|
@ -13144,6 +13182,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 5.4e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -13202,6 +13241,7 @@
|
|||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -13217,6 +13257,7 @@
|
|||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -13230,6 +13271,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 4.56e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/deepseek-v4-pro": {
|
||||
|
|
@ -13242,6 +13284,7 @@
|
|||
"output_cost_per_token": 3.48e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"cache_read_input_token_cost": 1.45e-07,
|
||||
|
|
@ -13257,6 +13300,7 @@
|
|||
"output_cost_per_token": 5.1e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"cache_read_input_token_cost": 2.8e-08,
|
||||
|
|
@ -13273,6 +13317,7 @@
|
|||
"output_cost_per_token": 1.32e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -13306,6 +13351,7 @@
|
|||
"output_cost_per_token": 1.5e-05,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
|
|
@ -13323,6 +13369,7 @@
|
|||
"output_cost_per_token_above_200k_tokens": 5e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -13343,6 +13390,7 @@
|
|||
"output_cost_per_token_above_200k_tokens": 1.2e-05,
|
||||
"source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/grok/",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -13361,6 +13409,7 @@
|
|||
"output_cost_per_token": 2.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -13379,6 +13428,7 @@
|
|||
"output_cost_per_token": 2.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -13395,6 +13445,7 @@
|
|||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
|
|
@ -13409,6 +13460,7 @@
|
|||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -13424,6 +13476,7 @@
|
|||
"output_cost_per_token": 1.5e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/foundry-models/concepts/models-sold-directly-by-azure",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
|
|
@ -13436,7 +13489,8 @@
|
|||
"max_output_tokens": 8192,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 0.00971,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models"
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/jamba-instruct": {
|
||||
"deprecation_date": "2025-03-01",
|
||||
|
|
@ -13447,6 +13501,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 7e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/kimi-k2.5": {
|
||||
|
|
@ -13459,6 +13514,7 @@
|
|||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_video_input": true,
|
||||
"supports_vision": true,
|
||||
|
|
@ -13482,6 +13538,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
|
|
@ -13498,6 +13555,7 @@
|
|||
"output_cost_per_token": 4e-08,
|
||||
"source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.ministral-3b-2410-offer?tab=Overview",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-large": {
|
||||
|
|
@ -13510,6 +13568,7 @@
|
|||
"output_cost_per_token": 1.2e-05,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-large-2407": {
|
||||
|
|
@ -13522,6 +13581,7 @@
|
|||
"output_cost_per_token": 6e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-large-latest": {
|
||||
|
|
@ -13533,6 +13593,7 @@
|
|||
"output_cost_per_token": 6e-06,
|
||||
"source": "https://marketplace.microsoft.com/en/marketplace/apps/000-000.mistral-ai-large-2407-offer?tab=Overview",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-large-3": {
|
||||
|
|
@ -13544,6 +13605,7 @@
|
|||
"output_cost_per_token": 1.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -13557,6 +13619,7 @@
|
|||
"output_cost_per_token": 2e-06,
|
||||
"supports_assistant_prefill": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-nemo": {
|
||||
|
|
@ -13568,7 +13631,8 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 1.5e-07,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_function_calling": true
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/mistral-small": {
|
||||
"deprecation_date": "2025-07-31",
|
||||
|
|
@ -13580,6 +13644,7 @@
|
|||
"output_cost_per_token": 3e-06,
|
||||
"source": "https://learn.microsoft.com/en-us/azure/foundry/openai/concepts/retired-models",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"azure_ai/mistral-small-2503": {
|
||||
|
|
@ -13591,6 +13656,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 3e-07,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -52006,6 +52072,7 @@
|
|||
],
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -52026,6 +52093,7 @@
|
|||
],
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -52777,6 +52845,7 @@
|
|||
"output_cost_per_token": 2.5e-06,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_web_search": true
|
||||
},
|
||||
|
|
@ -52794,6 +52863,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -52811,6 +52881,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -52989,6 +53060,7 @@
|
|||
"output_cost_per_token": 3.6e-07,
|
||||
"output_cost_per_token_batches": 1.8e-07,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true
|
||||
},
|
||||
"vertex_ai/openai/gpt-oss-20b-maas": {
|
||||
|
|
@ -53000,6 +53072,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 2.5e-07,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"cache_read_input_token_cost": 7e-09,
|
||||
"input_cost_per_token_batches": 3.5e-08,
|
||||
|
|
@ -53131,6 +53204,7 @@
|
|||
"us-south1"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/qwen/qwen3-coder-480b-a35b-instruct-maas": {
|
||||
|
|
@ -53149,6 +53223,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/qwen/qwen3-next-80b-a3b-instruct-maas": {
|
||||
|
|
@ -53164,6 +53239,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/qwen/qwen3-next-80b-a3b-thinking-maas": {
|
||||
|
|
@ -53179,6 +53255,7 @@
|
|||
"global"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
"vertex_ai/veo-3.1-generate-preview": {
|
||||
|
|
@ -67322,6 +67399,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -67336,6 +67414,7 @@
|
|||
"mode": "chat",
|
||||
"output_cost_per_token": 1.2e-06,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -67378,6 +67457,7 @@
|
|||
"output_cost_per_token": 1.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -67392,6 +67472,7 @@
|
|||
"output_cost_per_token": 3.1e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -67406,6 +67487,7 @@
|
|||
"output_cost_per_token": 5.5e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -67420,6 +67502,7 @@
|
|||
"output_cost_per_token": 6.25e-07,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -67436,6 +67519,7 @@
|
|||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_parallel_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -67451,6 +67535,7 @@
|
|||
"output_cost_per_token": 3.2e-06,
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
},
|
||||
|
|
@ -67469,6 +67554,7 @@
|
|||
"supported_output_modalities": [
|
||||
"text"
|
||||
],
|
||||
"supports_pdf_input": false,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true
|
||||
},
|
||||
|
|
@ -67478,7 +67564,8 @@
|
|||
"output_cost_per_token": 5.4e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/deepseek-v3-0324": {
|
||||
"deprecation_date": "2026-07-13",
|
||||
|
|
@ -67486,7 +67573,8 @@
|
|||
"output_cost_per_token": 4.56e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/deepseek-v3.1": {
|
||||
"deprecation_date": "2026-07-13",
|
||||
|
|
@ -67494,7 +67582,8 @@
|
|||
"output_cost_per_token": 4.94e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/grok-3": {
|
||||
"deprecation_date": "2026-05-01",
|
||||
|
|
@ -67502,7 +67591,8 @@
|
|||
"output_cost_per_token": 1.5e-05,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/grok-3-mini": {
|
||||
"deprecation_date": "2026-05-01",
|
||||
|
|
@ -67510,7 +67600,8 @@
|
|||
"output_cost_per_token": 1.27e-06,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/grok-4-fast-non-reasoning": {
|
||||
"deprecation_date": "2026-05-01",
|
||||
|
|
@ -67518,7 +67609,8 @@
|
|||
"output_cost_per_token": 5e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"azure_ai/grok-4-fast-reasoning": {
|
||||
"deprecation_date": "2026-05-01",
|
||||
|
|
@ -67526,7 +67618,8 @@
|
|||
"output_cost_per_token": 5e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"mode": "chat",
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'"
|
||||
"source": "https://prices.azure.com/api/retail/prices?$filter=serviceName%20eq%20'Foundry%20Models'%20and%20armRegionName%20eq%20'eastus'%20and%20priceType%20eq%20'Consumption'",
|
||||
"supports_pdf_input": false
|
||||
},
|
||||
"bedrock/us-gov-west-1/nvidia.nemotron-nano-3-30b": {
|
||||
"input_cost_per_token": 7.2e-08,
|
||||
|
|
@ -73196,6 +73289,7 @@
|
|||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing",
|
||||
"supported_regions": ["global"],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
@ -73811,6 +73905,7 @@
|
|||
"output_cost_per_token": 3.5e-06,
|
||||
"source": "https://azure.microsoft.com/en-us/pricing/details/ai-foundry-models/microsoft/",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true
|
||||
|
|
@ -74865,6 +74960,7 @@
|
|||
"text"
|
||||
],
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": false,
|
||||
"supports_reasoning": true,
|
||||
"supports_tool_choice": true,
|
||||
|
|
@ -75113,6 +75209,7 @@
|
|||
"output_cost_per_token_above_200k_tokens": 1.2e-05,
|
||||
"source": "https://docs.x.ai/developers/models/grok-4.7",
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": false,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
|
|
|
|||
|
|
@ -0,0 +1,421 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import anthropic
|
||||
import pytest
|
||||
from integration._support.client import Gateway, Scenario, string_value
|
||||
from integration._support.pdf_document import LETTER, Page, document, encoded, pdf_bytes
|
||||
from integration._support.process import owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_API_KEY: Final = "synthetic-pdf-document-key"
|
||||
_ANSWER: Final = "The codeword is PELICAN-42."
|
||||
_TITLE: Final = "Quarterly report"
|
||||
_URL: Final = "https://example.com/brief.pdf"
|
||||
_SAP_TOKEN_TARGET: Final = "/oauth/token"
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_PARTS: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
|
||||
|
||||
def _pdf(text: str) -> str:
|
||||
return encoded(pdf_bytes((Page(text=text), LETTER)))
|
||||
|
||||
|
||||
def _chat_completion(identity: str, model: str) -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": model,
|
||||
"choices": [
|
||||
{"index": 0, "message": {"role": "assistant", "content": _ANSWER}, "finish_reason": "stop"}
|
||||
],
|
||||
"usage": {"prompt_tokens": 914, "completion_tokens": 9, "total_tokens": 923},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _only_body(wire: Wire) -> dict[str, JsonValue]:
|
||||
received: Final = tuple(
|
||||
request
|
||||
for request in wire.drain()
|
||||
if request.method == "POST" and not request.target.startswith(_SAP_TOKEN_TARGET)
|
||||
)
|
||||
assert len(received) == 1, received
|
||||
return _JSON_OBJECT.validate_json(received[0].body)
|
||||
|
||||
|
||||
def _user_parts(body: dict[str, JsonValue], key: str) -> list[dict[str, JsonValue]]:
|
||||
turns: Final = _PARTS.validate_python(body[key])
|
||||
assert len(turns) == 1, turns
|
||||
return _PARTS.validate_python(turns[0]["content" if key == "messages" else "parts"])
|
||||
|
||||
|
||||
def test_anthropic_sdk_pdf_documents_reach_azure_as_file_parts_named_by_title(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
backend: Final = f"gpt-pdf-{marker[:8]}"
|
||||
identity: Final = f"chatcmpl-{marker}"
|
||||
titled: Final = _pdf(f"PELICAN-42 {marker}")
|
||||
untitled: Final = _pdf(f"ORCHID-77 {marker}")
|
||||
question: Final = f"What codeword does each document carry? {marker}"
|
||||
with wire_server(lambda _: _chat_completion(identity, backend)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"azure/{backend}", api_base=wire.url, api_key=_API_KEY, api_version="2024-10-21"
|
||||
)
|
||||
client: Final = anthropic.Anthropic(base_url=str(gateway.client.base_url), api_key=gateway.key, max_retries=0)
|
||||
message: Final = client.messages.create(
|
||||
model=model,
|
||||
max_tokens=64,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "document",
|
||||
"title": _TITLE,
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": titled},
|
||||
},
|
||||
{
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": untitled},
|
||||
},
|
||||
{"type": "document", "source": {"type": "url", "url": _URL}},
|
||||
{"type": "text", "text": question},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert message.id == identity
|
||||
assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", _ANSWER)]
|
||||
assert _user_parts(_only_body(wire), "messages") == [
|
||||
{"type": "file", "file": {"filename": _TITLE, "file_data": f"data:application/pdf;base64,{titled}"}},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"filename": "document.pdf", "file_data": f"data:application/pdf;base64,{untitled}"},
|
||||
},
|
||||
{"type": "image_url", "image_url": {"url": _URL}},
|
||||
{"type": "text", "text": question},
|
||||
]
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Reader:
|
||||
deploy: Callable[[Scenario, Wire], str]
|
||||
reply: Callable[[Request], Reply]
|
||||
pdf_parts: Callable[[dict[str, JsonValue]], Sequence[JsonValue]]
|
||||
|
||||
|
||||
def _gemini_reply(_: Request) -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"candidates": [
|
||||
{"content": {"parts": [{"text": _ANSWER}], "role": "model"}, "finishReason": "STOP", "index": 0}
|
||||
],
|
||||
"usageMetadata": {"promptTokenCount": 300, "candidatesTokenCount": 9, "totalTokenCount": 309},
|
||||
"modelVersion": "gemini-2.5-flash",
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _bedrock_reply(_: Request) -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"output": {"message": {"role": "assistant", "content": [{"text": _ANSWER}]}},
|
||||
"stopReason": "end_turn",
|
||||
"usage": {"inputTokens": 300, "outputTokens": 9, "totalTokens": 309},
|
||||
"metrics": {"latencyMs": 1},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _anthropic_reply(_: Request) -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"id": "msg_pdf",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": "claude-sonnet-4-5",
|
||||
"content": [{"type": "text", "text": _ANSWER}],
|
||||
"stop_reason": "end_turn",
|
||||
"usage": {"input_tokens": 300, "output_tokens": 9},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _sap_reply(request: Request) -> Reply:
|
||||
if request.target.startswith(_SAP_TOKEN_TARGET):
|
||||
return Reply(body=json.dumps({"access_token": "scripted", "expires_in": 3600}).encode())
|
||||
match request.target:
|
||||
case "/v2/lm/deployments":
|
||||
deployment: Final = {
|
||||
"scenarioId": "orchestration",
|
||||
"configurationId": "orchestration-config",
|
||||
"deploymentUrl": f"http://{request.headers['host']}/sap-deployment",
|
||||
"createdAt": "2026-01-01T00:00:00Z",
|
||||
}
|
||||
return Reply(body=json.dumps({"resources": [deployment]}).encode())
|
||||
case "/v2/lm/configurations/orchestration-config":
|
||||
return Reply(body=json.dumps({"executableId": "orchestration"}).encode())
|
||||
case _:
|
||||
completion: Final = _JSON_OBJECT.validate_json(_chat_completion("chatcmpl-sap-pdf", "claude").body)
|
||||
return Reply(body=json.dumps({"final_result": completion}).encode())
|
||||
|
||||
|
||||
def _snowflake(scenario: Scenario, wire: Wire) -> str:
|
||||
return scenario.model(model="snowflake/claude-sonnet-4-5", api_base=wire.url, api_key=_API_KEY)
|
||||
|
||||
|
||||
def _sap(scenario: Scenario, wire: Wire) -> str:
|
||||
service_key: Final = {
|
||||
"clientid": "scripted",
|
||||
"clientsecret": _API_KEY,
|
||||
"url": wire.url,
|
||||
"serviceurls": {"AI_API_URL": wire.url},
|
||||
}
|
||||
return scenario.model(model="sap/anthropic--claude-4.5-sonnet", api_base=wire.url, api_key=json.dumps(service_key))
|
||||
|
||||
|
||||
def _snowflake_pdf_parts(body: dict[str, JsonValue]) -> Sequence[JsonValue]:
|
||||
return tuple(part.get("source") for part in _user_parts(body, "messages") if part.get("type") == "document")
|
||||
|
||||
|
||||
def _sap_pdf_parts(body: dict[str, JsonValue]) -> Sequence[JsonValue]:
|
||||
config: Final = _JSON_OBJECT.validate_python(body["config"])
|
||||
templating: Final = _JSON_OBJECT.validate_python(
|
||||
_JSON_OBJECT.validate_python(config["modules"])["prompt_templating"]
|
||||
)
|
||||
prompt: Final = _JSON_OBJECT.validate_python(templating["prompt"])
|
||||
turns: Final = _PARTS.validate_python(prompt["template"])
|
||||
assert len(turns) == 1, turns
|
||||
parts: Final = _PARTS.validate_python(turns[0]["content"])
|
||||
return tuple(
|
||||
_JSON_OBJECT.validate_python(part["image_url"]).get("url") for part in parts if part.get("type") == "image_url"
|
||||
)
|
||||
|
||||
|
||||
def _gemini(scenario: Scenario, wire: Wire) -> str:
|
||||
return scenario.model(model="gemini/gemini-2.5-flash", api_base=wire.url, api_key=_API_KEY)
|
||||
|
||||
|
||||
def _bedrock(scenario: Scenario, wire: Wire) -> str:
|
||||
return scenario.model(
|
||||
model="bedrock/converse/us.amazon.nova-2-lite-v1:0",
|
||||
api_key=None,
|
||||
aws_access_key_id="AKIDSYNTHETIC",
|
||||
aws_secret_access_key="synthetic-secret",
|
||||
aws_region_name="us-east-1",
|
||||
aws_bedrock_runtime_endpoint=wire.url,
|
||||
)
|
||||
|
||||
|
||||
def _gemini_pdf_parts(body: dict[str, JsonValue]) -> Sequence[JsonValue]:
|
||||
return tuple(part.get("inline_data") for part in _user_parts(body, "contents") if "inline_data" in part)
|
||||
|
||||
|
||||
def _bedrock_pdf_parts(body: dict[str, JsonValue]) -> Sequence[JsonValue]:
|
||||
documents: Final = tuple(
|
||||
_JSON_OBJECT.validate_python(part["document"]) for part in _user_parts(body, "messages") if "document" in part
|
||||
)
|
||||
return tuple({"format": item.get("format"), "source": item.get("source")} for item in documents)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("reader", "expected"),
|
||||
[
|
||||
pytest.param(
|
||||
_Reader(_gemini, _gemini_reply, _gemini_pdf_parts),
|
||||
lambda data: [{"mime_type": "application/pdf", "data": data}],
|
||||
id="gemini",
|
||||
),
|
||||
pytest.param(
|
||||
_Reader(_bedrock, _bedrock_reply, _bedrock_pdf_parts),
|
||||
lambda data: [{"format": "pdf", "source": {"bytes": data}}],
|
||||
id="bedrock-converse",
|
||||
),
|
||||
pytest.param(
|
||||
_Reader(_snowflake, _anthropic_reply, _snowflake_pdf_parts),
|
||||
lambda data: [{"type": "base64", "media_type": "application/pdf", "data": data}],
|
||||
id="snowflake-claude",
|
||||
),
|
||||
pytest.param(
|
||||
_Reader(_sap, _sap_reply, _sap_pdf_parts),
|
||||
lambda data: [f"data:application/pdf;base64,{data}"],
|
||||
id="sap-orchestration",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_messages_pdf_still_reaches_providers_that_read_pdfs_in_their_own_document_shape(
|
||||
gateway: Gateway, reader: _Reader, expected: Callable[[str], Sequence[JsonValue]]
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
data: Final = _pdf(f"PELICAN-42 {marker}")
|
||||
with wire_server(reader.reply) as wire, gateway.scenario() as scenario:
|
||||
model: Final = reader.deploy(scenario, wire)
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
document({"type": "base64", "media_type": "application/pdf", "data": data}, title=_TITLE),
|
||||
{"type": "text", "text": f"What is the codeword? {marker}"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
headers={"anthropic-version": "2023-06-01"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _JSON_OBJECT.validate_json(response.content)["content"] == [{"type": "text", "text": _ANSWER}]
|
||||
assert list(reader.pdf_parts(_only_body(wire))) == list(expected(data))
|
||||
|
||||
|
||||
def _prompt_security(job: str) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
match (request.method, request.target.partition("?")[0]):
|
||||
case ("POST", "/api/sanitizeFile"):
|
||||
return Reply(body=json.dumps({"jobId": job}).encode())
|
||||
case ("GET", "/api/sanitizeFile"):
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"status": "done",
|
||||
"content": "",
|
||||
"metadata": {"action": "block", "violations": ["Malicious File"]},
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
case _:
|
||||
return Reply(body=json.dumps({"result": {"prompt": {"action": "allow"}}}).encode())
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _delete_guardrail(gateway: Gateway, identity: str) -> None:
|
||||
response: Final = gateway.request("DELETE", f"/guardrails/{identity}")
|
||||
assert response.status_code == 200, response.text
|
||||
|
||||
|
||||
def test_prompt_security_still_sanitizes_a_messages_pdf_and_blocks_it_before_the_provider(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
pdf: Final = pdf_bytes((Page(text=f"PELICAN-42 {marker}"), LETTER))
|
||||
with (
|
||||
wire_server(lambda _: _chat_completion(f"chatcmpl-{marker}", "gpt-pdf")) as provider,
|
||||
wire_server(_prompt_security(f"job-{marker}")) as sink,
|
||||
gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(
|
||||
model=f"azure/gpt-pdf-{marker[:8]}", api_base=provider.url, api_key=_API_KEY, api_version="2024-10-21"
|
||||
)
|
||||
name: Final = f"pdf-prompt-security-{marker}"
|
||||
created: Final = gateway.post(
|
||||
"/guardrails",
|
||||
{
|
||||
"guardrail": {
|
||||
"guardrail_name": name,
|
||||
"litellm_params": {
|
||||
"guardrail": "prompt_security",
|
||||
"mode": "pre_call",
|
||||
"api_key": "synthetic-prompt-security-key",
|
||||
"api_base": sink.url,
|
||||
},
|
||||
}
|
||||
},
|
||||
)
|
||||
scenario.cleanups.callback(_delete_guardrail, gateway, string_value(created["guardrail_id"]))
|
||||
response: Final = gateway.request(
|
||||
"POST",
|
||||
"/v1/messages",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"guardrails": [name],
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
document({"type": "base64", "media_type": "application/pdf", "data": encoded(pdf)}),
|
||||
{"type": "text", "text": f"Summarize this PDF {marker}"},
|
||||
],
|
||||
}
|
||||
],
|
||||
},
|
||||
headers={"anthropic-version": "2023-06-01"},
|
||||
)
|
||||
assert response.status_code == 400, response.text
|
||||
assert "Malicious File" in response.text
|
||||
assert provider.drain() == ()
|
||||
uploads: Final = tuple(
|
||||
request for request in sink.drain() if (request.method, request.target) == ("POST", "/api/sanitizeFile")
|
||||
)
|
||||
assert len(uploads) == 1, uploads
|
||||
assert pdf in uploads[0].body
|
||||
|
||||
|
||||
def _gigachat_reply(file_id: str) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.target.endswith("/files"):
|
||||
return Reply(body=json.dumps({"id": file_id, "object": "file", "purpose": "general"}).encode())
|
||||
return _chat_completion(f"chatcmpl-{file_id}", "GigaChat-Pro")
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def test_anthropic_sdk_pdf_document_reaches_gigachat_as_an_uploaded_attachment(
|
||||
gateway: Gateway, tmp_path: Path
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
file_id: Final = f"file-{marker}"
|
||||
pdf: Final = pdf_bytes((Page(text=f"PELICAN-42 {marker}"), LETTER))
|
||||
with (
|
||||
wire_server(_gigachat_reply(file_id)) as wire,
|
||||
owned_proxy_process(gateway, tmp_path, {"GIGACHAT_ACCESS_TOKEN": _API_KEY}) as owned,
|
||||
owned.gateway.scenario() as scenario,
|
||||
):
|
||||
model: Final = scenario.model(model="gigachat/GigaChat-Pro", api_base=f"{wire.url}/v1", api_key=_API_KEY)
|
||||
client: Final = anthropic.Anthropic(
|
||||
base_url=str(owned.gateway.client.base_url), api_key=owned.gateway.key, max_retries=0
|
||||
)
|
||||
message: Final = client.messages.create(
|
||||
model=model,
|
||||
max_tokens=64,
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": encoded(pdf)},
|
||||
},
|
||||
{"type": "text", "text": f"What is the codeword? {marker}"},
|
||||
],
|
||||
}
|
||||
],
|
||||
)
|
||||
assert [(block.type, getattr(block, "text", None)) for block in message.content] == [("text", _ANSWER)]
|
||||
upload, chat = (request for request in wire.drain() if request.method == "POST")
|
||||
assert upload.target == "/v1/files", upload.target
|
||||
assert upload.headers["authorization"] == f"Bearer {_API_KEY}", sorted(upload.headers)
|
||||
assert pdf in upload.body
|
||||
assert b"application/pdf" in upload.body
|
||||
assert chat.target == "/v1/chat/completions", chat.target
|
||||
turns: Final = _PARTS.validate_python(_JSON_OBJECT.validate_json(chat.body)["messages"])
|
||||
assert turns[-1]["attachments"] == [file_id], chat.body
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable, Iterator
|
||||
from itertools import chain
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
from integration._support.client import Gateway
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from integration.caching.response_cache_case import await_entry, redis_store
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
from redis import Redis
|
||||
|
||||
_BACKEND: Final = "z-ai/glm-5.2"
|
||||
_API_KEY: Final = "synthetic-openrouter-key"
|
||||
_ANSWER: Final = "The report covers revenue."
|
||||
_URL_CITATION: Final[dict[str, JsonValue]] = {
|
||||
"type": "url_citation",
|
||||
"url_citation": {
|
||||
"url": "https://example.com/report",
|
||||
"title": "Report",
|
||||
"start_index": 0,
|
||||
"end_index": 10,
|
||||
"content": "cited passage",
|
||||
},
|
||||
}
|
||||
_FILE_ANNOTATION: Final[dict[str, JsonValue]] = {
|
||||
"type": "file",
|
||||
"file": {"hash": "sha-report", "name": "report.pdf", "content": [{"type": "text", "text": "parsed page one"}]},
|
||||
}
|
||||
_USAGE: Final[dict[str, JsonValue]] = {"prompt_tokens": 40, "completion_tokens": 6, "total_tokens": 46}
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_CHOICES: Final = TypeAdapter(list[dict[str, JsonValue]])
|
||||
|
||||
|
||||
def _completion(identity: str) -> bytes:
|
||||
return json.dumps(
|
||||
{
|
||||
"id": identity,
|
||||
"object": "chat.completion",
|
||||
"created": 1,
|
||||
"model": _BACKEND,
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": _ANSWER,
|
||||
"annotations": [_URL_CITATION, _FILE_ANNOTATION],
|
||||
},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": _USAGE,
|
||||
}
|
||||
).encode()
|
||||
|
||||
|
||||
def _frame(frame: dict[str, JsonValue]) -> bytes:
|
||||
return b"data: " + json.dumps(frame).encode() + b"\n\n"
|
||||
|
||||
|
||||
def _stream(identity: str) -> Reply:
|
||||
chunk: Final[dict[str, JsonValue]] = {
|
||||
"id": identity,
|
||||
"object": "chat.completion.chunk",
|
||||
"created": 1,
|
||||
"model": _BACKEND,
|
||||
}
|
||||
return Reply(
|
||||
content_type="text/event-stream",
|
||||
chunks=(
|
||||
_frame({**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": _ANSWER}}]}),
|
||||
_frame({**chunk, "choices": [{"index": 0, "delta": {"annotations": [_URL_CITATION, _FILE_ANNOTATION]}}]}),
|
||||
_frame({**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}], "usage": _USAGE}),
|
||||
b"data: [DONE]\n\n",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _responder(identity: str) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
if _JSON_OBJECT.validate_json(request.body).get("stream") is True:
|
||||
return _stream(identity)
|
||||
return Reply(body=_completion(identity))
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _chat_body(model: str, marker: str, stream: bool) -> dict[str, JsonValue]:
|
||||
return {
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": f"Summarize the report {marker}"}],
|
||||
"stream": stream,
|
||||
}
|
||||
|
||||
|
||||
def _posted(wire: Wire) -> int:
|
||||
return sum(1 for request in wire.drain() if request.method == "POST")
|
||||
|
||||
|
||||
def _message(response: httpx.Response) -> dict[str, JsonValue]:
|
||||
assert response.status_code == 200, response.text
|
||||
choices: Final = _CHOICES.validate_python(_JSON_OBJECT.validate_json(response.content)["choices"])
|
||||
return _JSON_OBJECT.validate_python(choices[0]["message"])
|
||||
|
||||
|
||||
def _assert_split(message: dict[str, JsonValue]) -> None:
|
||||
assert message["content"] == _ANSWER, message
|
||||
assert message["annotations"] == [_URL_CITATION], message
|
||||
assert _JSON_OBJECT.validate_python(message["provider_specific_fields"])["annotations"] == [_FILE_ANNOTATION]
|
||||
|
||||
|
||||
def test_file_annotation_reaches_the_caller_in_provider_specific_fields_and_survives_a_cache_hit(
|
||||
gateway: Gateway,
|
||||
) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
identity: Final = f"gen-{marker}"
|
||||
with wire_server(_responder(identity)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"openrouter/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
first: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, marker, stream=False))
|
||||
_assert_split(_message(first))
|
||||
assert _posted(wire) == 1
|
||||
await_entry(redis_store(), identity)
|
||||
cached: Final = gateway.request("POST", "/v1/chat/completions", _chat_body(model, marker, stream=False))
|
||||
assert cached.headers.get("x-litellm-cache-key"), dict(cached.headers)
|
||||
_assert_split(_message(cached))
|
||||
assert _posted(wire) == 0
|
||||
|
||||
|
||||
def _deltas(response: httpx.Response) -> Iterator[dict[str, JsonValue]]:
|
||||
for line in response.iter_lines():
|
||||
if not line.startswith("data: ") or line == "data: [DONE]":
|
||||
continue
|
||||
choices = _CHOICES.validate_python(_JSON_OBJECT.validate_json(line.removeprefix("data: "))["choices"])
|
||||
yield from (_JSON_OBJECT.validate_python(choice["delta"]) for choice in choices)
|
||||
|
||||
|
||||
def _streamed(gateway: Gateway, body: dict[str, JsonValue]) -> tuple[dict[str, JsonValue], ...]:
|
||||
with gateway.client.stream(
|
||||
"POST", "/v1/chat/completions", json=body, headers={"Authorization": f"Bearer {gateway.key}"}
|
||||
) as response:
|
||||
assert response.status_code == 200, response.read()
|
||||
return tuple(_deltas(response))
|
||||
|
||||
|
||||
def _listed(value: JsonValue) -> list[JsonValue]:
|
||||
return value if isinstance(value, list) else []
|
||||
|
||||
|
||||
def _provider_fields(delta: dict[str, JsonValue]) -> dict[str, JsonValue]:
|
||||
return _JSON_OBJECT.validate_python(delta.get("provider_specific_fields") or {})
|
||||
|
||||
|
||||
def _annotations(deltas: tuple[dict[str, JsonValue], ...]) -> list[JsonValue]:
|
||||
return list(chain.from_iterable(_listed(delta.get("annotations")) for delta in deltas))
|
||||
|
||||
|
||||
def _provider_annotations(deltas: tuple[dict[str, JsonValue], ...]) -> list[JsonValue]:
|
||||
return list(chain.from_iterable(_listed(_provider_fields(delta).get("annotations")) for delta in deltas))
|
||||
|
||||
|
||||
def _text(deltas: tuple[dict[str, JsonValue], ...]) -> str:
|
||||
return "".join(str(delta.get("content") or "") for delta in deltas)
|
||||
|
||||
|
||||
def _cached_message(store: Redis, key: str) -> dict[str, JsonValue]:
|
||||
raw: Final = store.get(key)
|
||||
assert isinstance(raw, bytes), key
|
||||
response: Final = _JSON_OBJECT.validate_json(raw)["response"]
|
||||
stored: Final = _JSON_OBJECT.validate_json(response) if isinstance(response, str) else response
|
||||
choices: Final = _CHOICES.validate_python(_JSON_OBJECT.validate_python(stored)["choices"])
|
||||
return _JSON_OBJECT.validate_python(choices[0]["message"])
|
||||
|
||||
|
||||
def test_streamed_annotations_reach_the_caller_as_sent_and_the_cached_stream_splits_them(gateway: Gateway) -> None:
|
||||
marker: Final = uuid.uuid4().hex
|
||||
identity: Final = f"gen-{marker}"
|
||||
with wire_server(_responder(identity)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(model=f"openrouter/{_BACKEND}", api_base=wire.url, api_key=_API_KEY)
|
||||
live: Final = _streamed(gateway, _chat_body(model, marker, stream=True))
|
||||
assert _text(live) == _ANSWER, live
|
||||
assert _annotations(live) == [_URL_CITATION, _FILE_ANNOTATION], live
|
||||
assert _posted(wire) == 1
|
||||
store: Final = redis_store()
|
||||
_assert_split(_cached_message(store, await_entry(store, identity)))
|
||||
replayed: Final = _streamed(gateway, _chat_body(model, marker, stream=True))
|
||||
assert _text(replayed) == _ANSWER, replayed
|
||||
assert _provider_annotations(replayed) == [_FILE_ANNOTATION], replayed
|
||||
assert _posted(wire) == 0
|
||||
390
tests/integration/providers/test_pdf_input_support_wire.py
Normal file
390
tests/integration/providers/test_pdf_input_support_wire.py
Normal file
|
|
@ -0,0 +1,390 @@
|
|||
import json
|
||||
import uuid
|
||||
from collections.abc import Callable, Mapping, Sequence
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from integration._support.client import Gateway, Scenario
|
||||
from integration._support.openai_wire import chat_reply
|
||||
from integration._support.pdf_document import (
|
||||
LETTER,
|
||||
chat_body,
|
||||
chat_file,
|
||||
messages_body,
|
||||
pdf_data_url,
|
||||
pdf_document,
|
||||
responses_body,
|
||||
responses_input_file,
|
||||
)
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue, TypeAdapter
|
||||
|
||||
_API_KEY: Final = "synthetic-pdf-gate-key"
|
||||
_ANSWER: Final = "The report covers revenue."
|
||||
_FILENAME: Final = "report.pdf"
|
||||
_PDF: Final = pdf_data_url((LETTER,))
|
||||
_JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue])
|
||||
_MESSAGES: Final = TypeAdapter(list[JsonValue])
|
||||
_COST_MAP_PATH: Final = Path(__file__).parents[3] / "litellm" / "model_prices_and_context_window_backup.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _Entrypoint:
|
||||
path: str
|
||||
body: Callable[[str, str], Mapping[str, JsonValue]]
|
||||
stream: bool = False
|
||||
provider: str = "hosted_vllm"
|
||||
|
||||
|
||||
_ENTRYPOINTS: Final = (
|
||||
pytest.param(
|
||||
_Entrypoint("/v1/chat/completions", lambda model, text: chat_body(model, [chat_file(_PDF, _FILENAME)], text)),
|
||||
id="chat-completions",
|
||||
),
|
||||
pytest.param(
|
||||
_Entrypoint(
|
||||
"/v1/chat/completions",
|
||||
lambda model, text: chat_body(model, [chat_file(_PDF, _FILENAME)], text, stream=True),
|
||||
stream=True,
|
||||
),
|
||||
id="chat-completions-stream",
|
||||
),
|
||||
pytest.param(
|
||||
_Entrypoint("/v1/messages", lambda model, text: messages_body(model, [pdf_document((LETTER,))], text)),
|
||||
id="messages",
|
||||
),
|
||||
pytest.param(
|
||||
_Entrypoint(
|
||||
"/v1/responses",
|
||||
lambda model, text: responses_body(model, [responses_input_file(_PDF, _FILENAME)], text),
|
||||
provider="lm_studio",
|
||||
),
|
||||
id="responses-bridge",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _azure_ai_chat_model_pdf_input(entry: JsonValue) -> bool | None:
|
||||
match entry:
|
||||
case {"litellm_provider": "azure_ai", "mode": "chat", "supports_pdf_input": bool() as supports}:
|
||||
return supports
|
||||
case _:
|
||||
return None
|
||||
|
||||
|
||||
def _azure_ai_chat_model(reads_pdfs: bool) -> str:
|
||||
entries: Final = _JSON_OBJECT.validate_json(_COST_MAP_PATH.read_bytes())
|
||||
matching: Final = sorted(
|
||||
name for name, entry in entries.items() if _azure_ai_chat_model_pdf_input(entry) is reads_pdfs
|
||||
)
|
||||
assert matching, f"the cost map marks no azure_ai chat model with supports_pdf_input {reads_pdfs}"
|
||||
return matching[0]
|
||||
|
||||
|
||||
def _ollama_reply(backend: str) -> Reply:
|
||||
return Reply(
|
||||
body=json.dumps(
|
||||
{
|
||||
"model": backend,
|
||||
"created_at": "2026-01-01T00:00:00Z",
|
||||
"message": {"role": "assistant", "content": _ANSWER},
|
||||
"done": True,
|
||||
"done_reason": "stop",
|
||||
"prompt_eval_count": 5,
|
||||
"eval_count": 3,
|
||||
}
|
||||
).encode()
|
||||
)
|
||||
|
||||
|
||||
def _responder(backend: str) -> Callable[[Request], Reply]:
|
||||
def respond(request: Request) -> Reply:
|
||||
if request.target.endswith("/api/chat"):
|
||||
return _ollama_reply(backend)
|
||||
body: Final = _JSON_OBJECT.validate_json(request.body)
|
||||
return chat_reply(f"chatcmpl-{uuid.uuid4().hex}", backend, _ANSWER, stream=body.get("stream") is True)
|
||||
|
||||
return respond
|
||||
|
||||
|
||||
def _post(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> httpx.Response:
|
||||
return gateway.request("POST", path, body, headers={"anthropic-version": "2023-06-01"})
|
||||
|
||||
|
||||
def _post_streaming(gateway: Gateway, path: str, body: Mapping[str, JsonValue]) -> httpx.Response:
|
||||
with gateway.client.stream(
|
||||
"POST", path, json=dict(body), headers={"Authorization": f"Bearer {gateway.key}"}
|
||||
) as response:
|
||||
response.read()
|
||||
return response
|
||||
|
||||
|
||||
def _send(gateway: Gateway, entrypoint: _Entrypoint, model: str, text: str) -> httpx.Response:
|
||||
body: Final = entrypoint.body(model, text)
|
||||
if entrypoint.stream:
|
||||
return _post_streaming(gateway, entrypoint.path, body)
|
||||
return _post(gateway, entrypoint.path, body)
|
||||
|
||||
|
||||
def _error_message(response: httpx.Response) -> str:
|
||||
match _JSON_OBJECT.validate_json(response.content):
|
||||
case {"error": {"message": str() as message}}:
|
||||
return message
|
||||
case _:
|
||||
raise AssertionError(response.text)
|
||||
|
||||
|
||||
def _assert_refused_before_the_provider(response: httpx.Response, wire: Wire, backend: str) -> None:
|
||||
assert response.status_code == 400, response.text
|
||||
message: Final = _error_message(response)
|
||||
assert "PDF" in message and backend in message, message
|
||||
assert wire.drain() == ()
|
||||
|
||||
|
||||
def _bodies(wire: Wire) -> tuple[dict[str, JsonValue], ...]:
|
||||
return tuple(_JSON_OBJECT.validate_json(request.body) for request in wire.drain() if request.method == "POST")
|
||||
|
||||
|
||||
def _only_body(wire: Wire) -> dict[str, JsonValue]:
|
||||
bodies: Final = _bodies(wire)
|
||||
assert len(bodies) == 1, bodies
|
||||
return bodies[0]
|
||||
|
||||
|
||||
def _assert_reaches_the_provider(gateway: Gateway, wire: Wire, model: str) -> None:
|
||||
response: Final = _post(gateway, "/v1/chat/completions", chat_body(model, [], f"text only {uuid.uuid4().hex}"))
|
||||
assert response.status_code == 200, response.text
|
||||
assert len(_bodies(wire)) == 1
|
||||
|
||||
|
||||
def _openai_compatible(
|
||||
scenario: Scenario,
|
||||
wire: Wire,
|
||||
backend: str,
|
||||
model_info: Mapping[str, JsonValue],
|
||||
provider: str = "hosted_vllm",
|
||||
**parameters: JsonValue,
|
||||
) -> str:
|
||||
return scenario.model(
|
||||
model=f"{provider}/{backend}",
|
||||
api_base=f"{wire.url}/v1",
|
||||
api_key=_API_KEY,
|
||||
model_info=model_info,
|
||||
**parameters,
|
||||
)
|
||||
|
||||
|
||||
def _backend() -> str:
|
||||
return f"pdf-gate-{uuid.uuid4().hex[:12]}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entrypoint", _ENTRYPOINTS)
|
||||
def test_pdf_to_a_deployment_marked_unable_to_read_it_is_refused_before_the_provider(
|
||||
gateway: Gateway, entrypoint: _Entrypoint
|
||||
) -> None:
|
||||
backend: Final = _backend()
|
||||
with wire_server(_responder(backend)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _openai_compatible(scenario, wire, backend, {"supports_pdf_input": False}, entrypoint.provider)
|
||||
response: Final = _send(gateway, entrypoint, model, f"Summarize the report {uuid.uuid4().hex}")
|
||||
_assert_refused_before_the_provider(response, wire, backend)
|
||||
_assert_reaches_the_provider(gateway, wire, model)
|
||||
|
||||
|
||||
def _base_model_deployment(scenario: Scenario, wire: Wire) -> tuple[str, str]:
|
||||
backend: Final = f"pdf-gate-{uuid.uuid4().hex[:12]}"
|
||||
model: Final = scenario.model(
|
||||
model=f"azure_ai/{backend}",
|
||||
api_base=wire.url,
|
||||
api_key=_API_KEY,
|
||||
model_info={"base_model": _azure_ai_chat_model(reads_pdfs=False)},
|
||||
)
|
||||
return model, backend
|
||||
|
||||
|
||||
def _litellm_params_base_model_deployment(scenario: Scenario, wire: Wire) -> tuple[str, str]:
|
||||
backend: Final = _backend()
|
||||
model: Final = scenario.model(
|
||||
model=f"azure_ai/{backend}",
|
||||
api_base=wire.url,
|
||||
api_key=_API_KEY,
|
||||
base_model=_azure_ai_chat_model(reads_pdfs=False),
|
||||
)
|
||||
return model, backend
|
||||
|
||||
|
||||
def _cost_map_deployment(scenario: Scenario, wire: Wire) -> tuple[str, str]:
|
||||
marked: Final = _azure_ai_chat_model(reads_pdfs=False)
|
||||
return scenario.model(model=marked, api_base=wire.url, api_key=_API_KEY), marked.removeprefix("azure_ai/")
|
||||
|
||||
|
||||
def _ollama_deployment(scenario: Scenario, wire: Wire) -> tuple[str, str]:
|
||||
backend: Final = _backend()
|
||||
return scenario.model(model=f"ollama_chat/{backend}", api_base=wire.url, api_key=None), backend
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"deploy",
|
||||
[
|
||||
pytest.param(_base_model_deployment, id="model-info-base-model"),
|
||||
pytest.param(_litellm_params_base_model_deployment, id="litellm-params-base-model"),
|
||||
pytest.param(_cost_map_deployment, id="cost-map"),
|
||||
pytest.param(_ollama_deployment, id="ollama"),
|
||||
],
|
||||
)
|
||||
def test_pdf_to_a_model_its_declaration_marks_unable_is_refused_before_the_provider(
|
||||
gateway: Gateway, deploy: Callable[[Scenario, Wire], tuple[str, str]]
|
||||
) -> None:
|
||||
with wire_server(_responder("scripted")) as wire, gateway.scenario() as scenario:
|
||||
model, backend = deploy(scenario, wire)
|
||||
response: Final = _post(
|
||||
gateway,
|
||||
"/v1/chat/completions",
|
||||
chat_body(model, [chat_file(_PDF, _FILENAME)], f"Summarize the report {uuid.uuid4().hex}"),
|
||||
)
|
||||
_assert_refused_before_the_provider(response, wire, backend)
|
||||
_assert_reaches_the_provider(gateway, wire, model)
|
||||
|
||||
|
||||
def _pdf_turns(question: str) -> list[JsonValue]:
|
||||
return [
|
||||
{"role": "user", "content": [chat_file(_PDF, _FILENAME), {"type": "text", "text": question}]},
|
||||
{"role": "assistant", "content": "It is a quarterly report."},
|
||||
{"role": "user", "content": [chat_file(_PDF, "appendix.pdf")]},
|
||||
{"role": "user", "content": "Who wrote it?"},
|
||||
]
|
||||
|
||||
|
||||
def _answer(response: httpx.Response) -> JsonValue:
|
||||
match _JSON_OBJECT.validate_json(response.content):
|
||||
case {"choices": [{"message": {"content": content}}]}:
|
||||
return content
|
||||
case _:
|
||||
raise AssertionError(response.text)
|
||||
|
||||
|
||||
def _turns_without_pdfs(question: str) -> list[JsonValue]:
|
||||
return [
|
||||
{"role": "user", "content": [{"type": "text", "text": question}]},
|
||||
{"role": "assistant", "content": "It is a quarterly report."},
|
||||
{"role": "user", "content": "Who wrote it?"},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("deployment_params", "request_params"),
|
||||
[
|
||||
pytest.param({"drop_params": True}, {}, id="deployment-drop-params"),
|
||||
pytest.param({}, {"drop_params": True}, id="request-drop-params"),
|
||||
],
|
||||
)
|
||||
def test_drop_params_sends_the_conversation_without_the_pdfs_a_model_cannot_read(
|
||||
gateway: Gateway, deployment_params: Mapping[str, JsonValue], request_params: Mapping[str, JsonValue]
|
||||
) -> None:
|
||||
backend: Final = _backend()
|
||||
question: Final = f"Summarize the report {uuid.uuid4().hex}"
|
||||
with wire_server(_responder(backend)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _openai_compatible(scenario, wire, backend, {"supports_pdf_input": False}, **deployment_params)
|
||||
response: Final = _post(
|
||||
gateway,
|
||||
"/v1/chat/completions",
|
||||
{"model": model, "max_tokens": 64, "messages": _pdf_turns(question), **request_params},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert _answer(response) == _ANSWER
|
||||
assert _only_body(wire)["messages"] == _turns_without_pdfs(question)
|
||||
|
||||
|
||||
def test_drop_params_refuses_a_request_left_with_no_user_turn_once_the_pdf_is_dropped(gateway: Gateway) -> None:
|
||||
backend: Final = _backend()
|
||||
with wire_server(_responder(backend)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = _openai_compatible(scenario, wire, backend, {"supports_pdf_input": False}, drop_params=True)
|
||||
response: Final = _post(
|
||||
gateway,
|
||||
"/v1/chat/completions",
|
||||
{
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"messages": [
|
||||
{"role": "system", "content": f"Answer from the attachment {uuid.uuid4().hex}"},
|
||||
{"role": "user", "content": [chat_file(_PDF, _FILENAME)]},
|
||||
],
|
||||
},
|
||||
)
|
||||
_assert_refused_before_the_provider(response, wire, backend)
|
||||
_assert_reaches_the_provider(gateway, wire, model)
|
||||
|
||||
|
||||
def _file_and_question(question: str) -> list[JsonValue]:
|
||||
return [{"role": "user", "content": [chat_file(_PDF, _FILENAME), {"type": "text", "text": question}]}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model_info", "deployment_params", "keeps_pdf"),
|
||||
[
|
||||
pytest.param({}, {}, True, id="undeclared"),
|
||||
pytest.param({}, {"drop_params": True}, False, id="undeclared-drop-params"),
|
||||
pytest.param({"supports_pdf_input": True}, {"drop_params": True}, True, id="declared-drop-params"),
|
||||
pytest.param(
|
||||
{},
|
||||
{"drop_params": True, "base_model": _azure_ai_chat_model(reads_pdfs=True)},
|
||||
True,
|
||||
id="base-model-reads-drop-params",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_azure_ai_model_outside_the_cost_map_gets_the_pdf_unless_drop_params_finds_no_declaration(
|
||||
gateway: Gateway, model_info: Mapping[str, JsonValue], deployment_params: Mapping[str, JsonValue], keeps_pdf: bool
|
||||
) -> None:
|
||||
backend: Final = _backend()
|
||||
question: Final = f"Summarize the report {uuid.uuid4().hex}"
|
||||
with wire_server(_responder(backend)) as wire, gateway.scenario() as scenario:
|
||||
model: Final = scenario.model(
|
||||
model=f"azure_ai/{backend}",
|
||||
api_base=wire.url,
|
||||
api_key=_API_KEY,
|
||||
model_info=model_info,
|
||||
**deployment_params,
|
||||
)
|
||||
response: Final = _post(
|
||||
gateway, "/v1/chat/completions", chat_body(model, [chat_file(_PDF, _FILENAME)], question)
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
expected: Final[list[JsonValue]] = (
|
||||
_file_and_question(question) if keeps_pdf else [{"role": "user", "content": question}]
|
||||
)
|
||||
assert _only_body(wire)["messages"] == expected
|
||||
|
||||
|
||||
def _sibling_pdf_request(gateway: Gateway, model: str) -> httpx.Response:
|
||||
return _post(
|
||||
gateway,
|
||||
"/v1/chat/completions",
|
||||
chat_body(model, [chat_file(_PDF, _FILENAME)], f"Summarize the report {uuid.uuid4().hex}"),
|
||||
)
|
||||
|
||||
|
||||
def _forwarded_file_parts(bodies: Sequence[dict[str, JsonValue]]) -> tuple[JsonValue, ...]:
|
||||
return tuple(_MESSAGES.validate_python(body["messages"])[0] for body in bodies)
|
||||
|
||||
|
||||
def test_pdf_opt_out_on_one_deployment_leaves_its_sibling_on_the_same_model_reading_pdfs(gateway: Gateway) -> None:
|
||||
backend: Final = _backend()
|
||||
burst: Final = 6
|
||||
with wire_server(_responder(backend)) as wire, gateway.scenario() as scenario:
|
||||
sibling: Final = _openai_compatible(scenario, wire, backend, {})
|
||||
marked: Final = _openai_compatible(scenario, wire, backend, {"supports_pdf_input": False})
|
||||
with ThreadPoolExecutor(max_workers=burst) as pool:
|
||||
responses: Final = tuple(pool.map(lambda _: _sibling_pdf_request(gateway, sibling), range(burst)))
|
||||
assert [response.status_code for response in responses] == [200] * burst, [r.text for r in responses]
|
||||
forwarded: Final = _forwarded_file_parts(_bodies(wire))
|
||||
assert len(forwarded) == burst, forwarded
|
||||
assert all(
|
||||
_MESSAGES.validate_python(_JSON_OBJECT.validate_python(message)["content"])[0] == chat_file(_PDF, _FILENAME)
|
||||
for message in forwarded
|
||||
), forwarded
|
||||
refused: Final = _sibling_pdf_request(gateway, marked)
|
||||
_assert_refused_before_the_provider(refused, wire, backend)
|
||||
|
|
@ -2968,3 +2968,54 @@ def test_convert_to_model_response_object_function_output():
|
|||
assert result.usage.completion_tokens_details == CompletionTokensDetailsWrapper(reasoning_tokens=0)
|
||||
|
||||
assert result._hidden_params is not None
|
||||
|
||||
|
||||
_URL_CITATION: Final = {
|
||||
"type": "url_citation",
|
||||
"url_citation": {
|
||||
"url": "https://example.com",
|
||||
"title": "Example",
|
||||
"content": "cited snippet",
|
||||
"start_index": 0,
|
||||
"end_index": 4,
|
||||
},
|
||||
}
|
||||
_FILE_ANNOTATION: Final = {
|
||||
"type": "file",
|
||||
"file": {"hash": "abc123", "name": "report.pdf", "content": [{"type": "text", "text": "parsed page"}]},
|
||||
}
|
||||
|
||||
|
||||
def _message_with_annotations(annotations: list[dict[str, object]]) -> Message:
|
||||
response: Final = {
|
||||
"id": "gen-1",
|
||||
"object": "chat.completion",
|
||||
"created": 1712697600,
|
||||
"model": "z-ai/glm-5.3",
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"finish_reason": "stop",
|
||||
"message": {"role": "assistant", "content": "done", "annotations": annotations},
|
||||
}
|
||||
],
|
||||
"usage": {"completion_tokens": 1, "prompt_tokens": 1, "total_tokens": 2},
|
||||
}
|
||||
return convert_to_model_response_object(
|
||||
response_object=response, model_response_object=ModelResponse(), response_type="completion"
|
||||
).choices[0].message
|
||||
|
||||
|
||||
def test_annotations_typed_other_than_url_citation_move_to_provider_specific_fields() -> None:
|
||||
untyped_annotation: Final = {"start_index": 0, "end_index": 4}
|
||||
message: Final = _message_with_annotations([_URL_CITATION, untyped_annotation, _FILE_ANNOTATION])
|
||||
|
||||
assert message.annotations == [_URL_CITATION, untyped_annotation]
|
||||
assert message.provider_specific_fields["annotations"] == [_FILE_ANNOTATION]
|
||||
|
||||
|
||||
def test_url_citation_annotations_keep_provider_keys() -> None:
|
||||
message: Final = _message_with_annotations([_URL_CITATION])
|
||||
|
||||
assert message.annotations == [_URL_CITATION]
|
||||
assert "annotations" not in (message.provider_specific_fields or {})
|
||||
|
|
|
|||
|
|
@ -5286,6 +5286,40 @@ def test_eager_input_streaming_tool_reaches_bedrock_converse_as_beta():
|
|||
assert data["toolConfig"]["tools"][0]["toolSpec"]["inputSchema"]["json"] == EAGER_INPUT_SCHEMA
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("title", "expected_filename"),
|
||||
[(None, "document.pdf"), ("Quarterly report", "Quarterly report")],
|
||||
)
|
||||
def test_user_pdf_document_reaches_openai_compatible_providers_as_file_part(
|
||||
title: str | None, expected_filename: str
|
||||
) -> None:
|
||||
document: Final = _base64_pdf_block() | ({"title": title} if title else {})
|
||||
request_messages: Final = _run_chat_completions_pipeline(
|
||||
[
|
||||
AnthropicMessagesUserMessageParam(
|
||||
role="user",
|
||||
content=[document, {"type": "text", "text": "What is the codeword?"}],
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
assert request_messages == [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "file",
|
||||
"file": {
|
||||
"filename": expected_filename,
|
||||
"file_data": f"data:application/pdf;base64,{TOOL_RESULT_PDF_B64}",
|
||||
},
|
||||
},
|
||||
{"type": "text", "text": "What is the codeword?"},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
_CHAT_BRIDGE_MODEL: Final = "azure/gpt-5.6"
|
||||
_CHAT_BRIDGE_API_BASE: Final = "https://tool-result-text.openai.azure.com"
|
||||
_CHAT_BRIDGE_URL: Final = r".*tool-result-text\.openai\.azure\.com/openai/deployments/gpt-5\.6/chat/completions.*"
|
||||
|
|
|
|||
|
|
@ -1101,6 +1101,58 @@ def test_map_azure_model_group(model_group_header, expected_model):
|
|||
assert config._map_azure_model_group(model_group_header) == expected_model
|
||||
|
||||
|
||||
def test_azure_ai_sends_file_parts_instead_of_flattening_them_to_text() -> None:
|
||||
file_part: Final = {
|
||||
"type": "file",
|
||||
"file": {"filename": "document.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"},
|
||||
}
|
||||
content: Final = [file_part, {"type": "text", "text": "What is the codeword?"}]
|
||||
|
||||
request: Final = AzureAIStudioConfig().transform_request(
|
||||
model="Kimi-K2.6",
|
||||
messages=[{"role": "user", "content": content}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["messages"] == [{"role": "user", "content": content}]
|
||||
|
||||
|
||||
def test_azure_ai_passes_dict_message_content_through_for_the_provider_to_judge() -> None:
|
||||
content: Final = {"type": "text", "text": "Say hi"}
|
||||
|
||||
request: Final = AzureAIStudioConfig().transform_request(
|
||||
model="gpt-5.6-sol",
|
||||
messages=[{"role": "user", "content": content}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["messages"] == [{"role": "user", "content": content}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"file",
|
||||
[
|
||||
{"filename": "notes.txt", "file_data": "data:text/plain;base64,aGVsbG8="},
|
||||
{"file_id": "file-abc"},
|
||||
],
|
||||
ids=["text_data_url", "file_id_only"],
|
||||
)
|
||||
def test_azure_ai_flattens_a_non_pdf_file_part_to_its_text_as_before(file: dict[str, str]) -> None:
|
||||
request: Final = AzureAIStudioConfig().transform_request(
|
||||
model="Llama-3.3-70B-Instruct",
|
||||
messages=[{"role": "user", "content": [{"type": "text", "text": "summarize"}, {"type": "file", "file": file}]}],
|
||||
optional_params={},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert request["messages"] == [{"role": "user", "content": "summarize"}]
|
||||
|
||||
|
||||
def test_translate_developer_role_hoists_a_later_developer_message_for_a_model_that_rejects_it(
|
||||
declare_mid_conversation_system: Callable[[str, bool], None],
|
||||
) -> None:
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ get_model_response_iterator, and get_error_class.
|
|||
"""
|
||||
|
||||
import json
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import httpx
|
||||
|
|
@ -512,6 +513,20 @@ class TestTransformMessages:
|
|||
assert result[0]["content"] == "look"
|
||||
assert "file-456" in result[0]["attachments"]
|
||||
|
||||
def test_list_content_with_pdf_file_part_uploads_it_as_an_attachment(self) -> None:
|
||||
pdf_data_url: Final = "data:application/pdf;base64,JVBERi0xLjQK"
|
||||
content: Final = [
|
||||
{"type": "text", "text": "summarize"},
|
||||
{"type": "file", "file": {"filename": "report.pdf", "file_data": pdf_data_url}},
|
||||
]
|
||||
with patch.object(self.config, "_upload_image", return_value="file-789") as upload:
|
||||
result: Final = self.config._transform_messages([
|
||||
{"role": "user", "content": content}
|
||||
])
|
||||
upload.assert_called_once_with(pdf_data_url)
|
||||
assert result[0]["content"] == "summarize"
|
||||
assert result[0]["attachments"] == ["file-789"]
|
||||
|
||||
|
||||
class TestTransformRequest:
|
||||
def setup_method(self):
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import datetime
|
||||
from typing import Final
|
||||
import httpx
|
||||
import pytest
|
||||
import json
|
||||
|
|
@ -2164,3 +2165,32 @@ class TestOpenAIPrefixMaxTokensKey:
|
|||
chat_request = _oci_chat_request("openai.gpt-5", {"max_tokens": 64})
|
||||
assert chat_request["maxCompletionTokens"] == 64
|
||||
assert "maxTokens" not in chat_request
|
||||
|
||||
|
||||
def test_transform_request_sends_a_pdf_file_part_as_an_image_data_url() -> None:
|
||||
pdf_data_url: Final = "data:application/pdf;base64,JVBERi0xLjQK"
|
||||
transformed_request: Final = OCIChatConfig().transform_request(
|
||||
model="google.gemini-2.5-flash",
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize this."},
|
||||
{"type": "file", "file": {"filename": "report.pdf", "file_data": pdf_data_url}},
|
||||
],
|
||||
}
|
||||
],
|
||||
optional_params={"oci_compartment_id": TEST_COMPARTMENT_ID},
|
||||
litellm_params={},
|
||||
headers={},
|
||||
)
|
||||
|
||||
assert transformed_request["chatRequest"]["messages"] == [
|
||||
{
|
||||
"role": "USER",
|
||||
"content": [
|
||||
{"type": "TEXT", "text": "Summarize this."},
|
||||
{"type": "IMAGE", "imageUrl": {"url": pdf_data_url}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
import warnings
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
|
|
@ -908,6 +910,22 @@ class TestCacheControl:
|
|||
assert "cache_control" not in content[0]
|
||||
assert content[1].get("cache_control") == {"type": "ephemeral"}
|
||||
|
||||
def test_pdf_file_part_reaches_sap_as_an_image_url_data_url(self) -> None:
|
||||
pdf_data_url: Final = "data:application/pdf;base64,JVBERi0xLjQK"
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize this."},
|
||||
{"type": "file", "file": {"filename": "report.pdf", "file_data": pdf_data_url}},
|
||||
],
|
||||
}
|
||||
]
|
||||
body: Final = self._transform("anthropic--claude-3-5-sonnet", messages)
|
||||
content: Final = self._template(body)[0]["content"]
|
||||
assert content[1]["type"] == "image_url"
|
||||
assert content[1]["image_url"]["url"] == pdf_data_url
|
||||
|
||||
def test_message_level_cache_control_folded_onto_trailing_image_block(self):
|
||||
messages = [
|
||||
{
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ import pytest
|
|||
import litellm
|
||||
from litellm import completion, acompletion
|
||||
from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler, HTTPHandler
|
||||
from litellm.llms.anthropic.messages.adapters.transformation import LiteLLMAnthropicMessagesAdapter
|
||||
from litellm.llms.snowflake.chat.transformation import SnowflakeConfig, SnowflakeStreamingHandler
|
||||
from litellm.types.utils import ModelResponse
|
||||
|
||||
|
|
@ -479,6 +480,35 @@ class TestSnowflakeCortexClaudeFixes:
|
|||
}
|
||||
]
|
||||
|
||||
def test_messages_api_pdf_reaches_cortex_claude_as_a_document_block(self) -> None:
|
||||
translated: Final = LiteLLMAnthropicMessagesAdapter().translate_anthropic_messages_to_openai(
|
||||
messages=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": "ZmFrZQ=="},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{"type": "text", "text": "What is the codeword?"},
|
||||
],
|
||||
}
|
||||
],
|
||||
model="snowflake/claude-sonnet-4-6",
|
||||
)
|
||||
|
||||
body: Final = self._transform(translated)
|
||||
|
||||
assert body["messages"][0]["content"] == [
|
||||
{
|
||||
"type": "document",
|
||||
"source": {"type": "base64", "media_type": "application/pdf", "data": "ZmFrZQ=="},
|
||||
"cache_control": {"type": "ephemeral"},
|
||||
},
|
||||
{"type": "text", "text": "What is the codeword?"},
|
||||
]
|
||||
|
||||
def test_multipart_tool_result_preserves_text_and_converts_image(self):
|
||||
body = self._transform(
|
||||
[
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ These tests verify that:
|
|||
|
||||
import json
|
||||
import os
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -422,3 +423,52 @@ async def test_vertex_ai_gemma_vision_passthrough():
|
|||
assert isinstance(content, list), f"Expected list content, got: {content}"
|
||||
image_parts = [p for p in content if p.get("type") == "image_url"]
|
||||
assert image_parts, f"No image_url part in forwarded message content: {content}"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_vertex_ai_gemma_pdf_file_part_is_sent_as_image_url() -> None:
|
||||
pdf_data_url: Final = "data:application/pdf;base64,JVBERi0xLjQKJcfsj6IKMSAwIG9iago8PD4+CmVuZG9iagp0cmFpbGVyCjw8Pj4KJSVFT0YK"
|
||||
messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "What does the PDF say?"},
|
||||
{"type": "file", "file": {"file_data": pdf_data_url, "filename": "report.pdf"}},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
mock_response: Final = MagicMock()
|
||||
mock_response.status_code = 200
|
||||
mock_response.headers = {}
|
||||
mock_response.json.return_value = _MOCK_RESPONSE_JSON
|
||||
|
||||
mock_vertexai: Final = MagicMock()
|
||||
mock_vertexai.preview = MagicMock()
|
||||
|
||||
with (
|
||||
patch("litellm.llms.custom_httpx.http_handler.AsyncHTTPHandler") as mock_http_handler,
|
||||
patch(
|
||||
"litellm.llms.vertex_ai.vertex_ai_partner_models.main.VertexAIPartnerModels._ensure_access_token",
|
||||
return_value=("fake-token", "test-project"),
|
||||
),
|
||||
patch.dict(
|
||||
"sys.modules",
|
||||
{"vertexai": mock_vertexai, "vertexai.preview": mock_vertexai.preview},
|
||||
),
|
||||
patch.dict(litellm.model_cost, _GEMMA_MODEL_COST_ENTRY, clear=False),
|
||||
):
|
||||
mock_http_handler.return_value.post = AsyncMock(return_value=mock_response)
|
||||
|
||||
await litellm.acompletion(
|
||||
model="vertex_ai/google/gemma-4-26b-a4b-it-maas",
|
||||
messages=messages,
|
||||
vertex_ai_project="test-project",
|
||||
)
|
||||
|
||||
body: Final = json.loads(mock_http_handler.return_value.post.call_args.kwargs["data"])
|
||||
user_msg: Final = next(m for m in body["messages"] if m["role"] == "user")
|
||||
assert user_msg["content"] == [
|
||||
{"type": "text", "text": "What does the PDF say?"},
|
||||
{"type": "image_url", "image_url": {"url": pdf_data_url}},
|
||||
]
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import asyncio
|
||||
import base64
|
||||
from collections.abc import Mapping, Sequence
|
||||
from typing import Final
|
||||
from unittest.mock import AsyncMock, patch
|
||||
|
||||
import pytest
|
||||
|
|
@ -8,6 +9,7 @@ from fastapi.exceptions import HTTPException
|
|||
from httpx import ReadTimeout, Request, Response
|
||||
|
||||
import litellm
|
||||
from litellm.llms.anthropic.chat.guardrail_translation.handler import AnthropicMessagesHandler
|
||||
from litellm.proxy._types import UserAPIKeyAuth
|
||||
from litellm.proxy.guardrails.guardrail_hooks.prompt_security.prompt_security import (
|
||||
PromptSecurityGuardrail,
|
||||
|
|
@ -1247,3 +1249,54 @@ async def test_check_tool_results_enabled(monkeypatch: pytest.MonkeyPatch):
|
|||
|
||||
assert "indirect_prompt_injection" in str(excinfo.value.detail)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_messages_api_pdf_reaches_file_sanitization_and_is_blocked() -> None:
|
||||
sanitize_url: Final = "https://test.prompt.security/api/sanitizeFile"
|
||||
upload: Final = AsyncMock(
|
||||
return_value=Response(200, json={"jobId": "pdf-job"}, request=Request("POST", sanitize_url))
|
||||
)
|
||||
poll: Final = AsyncMock(
|
||||
return_value=Response(
|
||||
200,
|
||||
json={"status": "done", "content": "", "metadata": {"action": "block", "violations": ["Malicious File"]}},
|
||||
request=Request("GET", sanitize_url),
|
||||
)
|
||||
)
|
||||
guardrail: Final = PromptSecurityGuardrail(
|
||||
api_key="test-key",
|
||||
api_base="https://test.prompt.security",
|
||||
guardrail_name="test-guard",
|
||||
event_hook="pre_call",
|
||||
default_on=True,
|
||||
)
|
||||
guardrail.poll_interval = 0
|
||||
pdf: Final = b"%PDF-1.4 pelican"
|
||||
request: Final = {
|
||||
"model": "gpt-5.6",
|
||||
"max_tokens": 64,
|
||||
"messages": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "document",
|
||||
"source": {
|
||||
"type": "base64",
|
||||
"media_type": "application/pdf",
|
||||
"data": base64.b64encode(pdf).decode(),
|
||||
},
|
||||
},
|
||||
{"type": "text", "text": "Summarize this PDF"},
|
||||
],
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with patch.object(guardrail.async_handler, "post", upload), patch.object(guardrail.async_handler, "get", poll):
|
||||
with pytest.raises(HTTPException) as blocked:
|
||||
await AnthropicMessagesHandler().process_input_messages(request, guardrail)
|
||||
|
||||
assert blocked.value.status_code == 400
|
||||
assert "Malicious File" in str(blocked.value.detail)
|
||||
assert upload.call_args.kwargs["files"]["file"][1] == pdf
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ import respx
|
|||
from openai import APITimeoutError, AsyncOpenAI, OpenAI
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai.types.chat.chat_completion import ChatCompletion
|
||||
from pydantic import TypeAdapter
|
||||
from pydantic import BaseModel, ConfigDict, TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import (
|
||||
|
|
@ -10200,6 +10200,309 @@ async def test_atranscription_ignores_a_non_string_base_url_when_api_base_is_set
|
|||
assert response.text == "four score"
|
||||
|
||||
|
||||
_PDF_FILE_PART: Final = {
|
||||
"type": "file",
|
||||
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"},
|
||||
}
|
||||
_SUMMARIZE_TEXT_PART: Final = {"type": "text", "text": "Summarize this."}
|
||||
|
||||
|
||||
def _pdf_messages() -> list[dict[str, object]]:
|
||||
return [{"role": "user", "content": [_SUMMARIZE_TEXT_PART, _PDF_FILE_PART]}]
|
||||
|
||||
|
||||
class _SentChatRequest(BaseModel):
|
||||
model_config = ConfigDict(frozen=True)
|
||||
|
||||
messages: list[dict[str, object]]
|
||||
|
||||
|
||||
def _sent_messages(respx_mock: respx.MockRouter) -> list[dict[str, object]]:
|
||||
request: Final[httpx.Request] = respx_mock.calls[0].request
|
||||
return _SentChatRequest.model_validate_json(request.content).messages
|
||||
|
||||
|
||||
def _mock_pdf_chat_route(respx_mock: respx.MockRouter, api_base: str) -> respx.Route:
|
||||
return respx_mock.post(url__regex=rf"{api_base}/chat/completions.*").mock(
|
||||
return_value=httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-pdf",
|
||||
"object": "chat.completion",
|
||||
"created": 1712697600,
|
||||
"model": "qa-pdf-model",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def test_drop_params_drops_a_pdf_the_model_cannot_read(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12347/v1"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
litellm.completion(
|
||||
model="openai/qa-text-only-model",
|
||||
messages=_pdf_messages(),
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
model_info={"supports_pdf_input": False},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert _sent_messages(respx_mock) == [{"role": "user", "content": [_SUMMARIZE_TEXT_PART]}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"pdf_file",
|
||||
[
|
||||
{"filename": "report.pdf", "file_data": "JVBERi0xLjQK"},
|
||||
{"file_id": "file-report", "format": "application/pdf"},
|
||||
{"file_id": "file-report", "filename": "Report.PDF"},
|
||||
],
|
||||
ids=["bare_base64_named_pdf", "file_id_with_pdf_format", "file_id_named_pdf"],
|
||||
)
|
||||
def test_drop_params_drops_a_pdf_file_part_without_a_data_url(
|
||||
respx_mock: respx.MockRouter, pdf_file: dict[str, str]
|
||||
) -> None:
|
||||
api_base: Final = "http://localhost:12350/v1"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
litellm.completion(
|
||||
model="openai/qa-text-only-model",
|
||||
messages=[{"role": "user", "content": [_SUMMARIZE_TEXT_PART, {"type": "file", "file": pdf_file}]}],
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
model_info={"supports_pdf_input": False},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert _sent_messages(respx_mock) == [{"role": "user", "content": [_SUMMARIZE_TEXT_PART]}]
|
||||
|
||||
|
||||
def test_drop_params_drops_a_user_turn_that_held_only_a_pdf(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12352/v1"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
litellm.completion(
|
||||
model="openai/qa-text-only-model",
|
||||
messages=[
|
||||
{"role": "user", "content": [_PDF_FILE_PART]},
|
||||
{"role": "user", "content": [_SUMMARIZE_TEXT_PART]},
|
||||
],
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
model_info={"supports_pdf_input": False},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert _sent_messages(respx_mock) == [{"role": "user", "content": [_SUMMARIZE_TEXT_PART]}]
|
||||
|
||||
|
||||
def test_drop_params_keeps_roles_alternating_after_dropping_a_pdf_only_turn(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12353/v1"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
litellm.completion(
|
||||
model="openai/qa-text-only-model",
|
||||
messages=[
|
||||
{"role": "user", "content": [_SUMMARIZE_TEXT_PART]},
|
||||
{"role": "assistant", "content": "Send the file."},
|
||||
{"role": "user", "content": [_PDF_FILE_PART]},
|
||||
{"role": "assistant", "content": "Got it."},
|
||||
{"role": "user", "content": [_SUMMARIZE_TEXT_PART]},
|
||||
],
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
model_info={"supports_pdf_input": False},
|
||||
drop_params=True,
|
||||
ensure_alternating_roles=True,
|
||||
)
|
||||
|
||||
sent_roles: Final = [message["role"] for message in _sent_messages(respx_mock)]
|
||||
assert sent_roles == ["user", "assistant", "user", "assistant", "user"]
|
||||
|
||||
|
||||
def test_drop_params_keeps_an_image_data_url_whose_filename_says_pdf(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12351/v1"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
image_part: Final = {
|
||||
"type": "file",
|
||||
"file": {"filename": "scan.pdf", "file_data": "data:image/png;base64,iVBORw0KGgo="},
|
||||
}
|
||||
|
||||
litellm.completion(
|
||||
model="openai/qa-text-only-model",
|
||||
messages=[{"role": "user", "content": [_SUMMARIZE_TEXT_PART, image_part]}],
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
model_info={"supports_pdf_input": False},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert _sent_messages(respx_mock) == [{"role": "user", "content": [_SUMMARIZE_TEXT_PART, image_part]}]
|
||||
|
||||
|
||||
def test_pdf_to_a_model_that_cannot_read_it_is_rejected_without_drop_params(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12348/v1"
|
||||
route: Final = _mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="does not accept PDF input"):
|
||||
litellm.completion(
|
||||
model="openai/qa-text-only-model",
|
||||
messages=_pdf_messages(),
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
model_info={"supports_pdf_input": False},
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
def test_pdf_to_a_model_with_unknown_pdf_support_is_sent_even_with_drop_params(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12349/v1"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
litellm.completion(
|
||||
model="openai/qa-unlisted-model",
|
||||
messages=_pdf_messages(),
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert _sent_messages(respx_mock) == _pdf_messages()
|
||||
|
||||
|
||||
def test_pdf_to_ollama_is_rejected_without_drop_params() -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match="does not accept PDF input"):
|
||||
litellm.completion(model="ollama_chat/llama3.2", messages=_pdf_messages(), api_base="http://127.0.0.1:9")
|
||||
|
||||
|
||||
def _azure_ai_model_without_pdf_input() -> str:
|
||||
return next(
|
||||
name
|
||||
for name, entry in litellm.model_cost.items()
|
||||
if entry.get("litellm_provider") == "azure_ai" and entry.get("supports_pdf_input") is False
|
||||
)
|
||||
|
||||
|
||||
def test_pdf_to_a_deployment_whose_base_model_cannot_read_it_is_rejected() -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match="does not accept PDF input"):
|
||||
litellm.completion(
|
||||
model="azure_ai/qa-custom-deployment",
|
||||
messages=_pdf_messages(),
|
||||
api_base="http://127.0.0.1:9",
|
||||
api_key="fake_azure_ai_api_key",
|
||||
model_info={"base_model": _azure_ai_model_without_pdf_input()},
|
||||
)
|
||||
|
||||
|
||||
def test_drop_params_drops_a_pdf_a_deployments_base_model_cannot_read(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12353"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
litellm.completion(
|
||||
model="azure_ai/qa-custom-deployment",
|
||||
messages=_pdf_messages(),
|
||||
api_base=api_base,
|
||||
api_key="fake_azure_ai_api_key",
|
||||
model_info={"base_model": _azure_ai_model_without_pdf_input()},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert _sent_messages(respx_mock) == [{"role": "user", "content": _SUMMARIZE_TEXT_PART["text"]}]
|
||||
|
||||
|
||||
def _azure_ai_model_with_pdf_input() -> str:
|
||||
return next(
|
||||
name
|
||||
for name, entry in litellm.model_cost.items()
|
||||
if entry.get("litellm_provider") == "azure_ai" and entry.get("supports_pdf_input") is True
|
||||
)
|
||||
|
||||
|
||||
def test_pdf_to_a_deployment_whose_base_model_param_cannot_read_it_is_rejected() -> None:
|
||||
with pytest.raises(litellm.BadRequestError, match="does not accept PDF input"):
|
||||
litellm.completion(
|
||||
model="azure_ai/qa-custom-deployment",
|
||||
messages=_pdf_messages(),
|
||||
api_base="http://127.0.0.1:9",
|
||||
api_key="fake_azure_ai_api_key",
|
||||
base_model=_azure_ai_model_without_pdf_input(),
|
||||
)
|
||||
|
||||
|
||||
def test_drop_params_keeps_a_pdf_a_deployments_base_model_param_reads(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12357"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
litellm.completion(
|
||||
model="azure_ai/qa-custom-deployment",
|
||||
messages=_pdf_messages(),
|
||||
api_base=api_base,
|
||||
api_key="fake_azure_ai_api_key",
|
||||
base_model=_azure_ai_model_with_pdf_input(),
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert _sent_messages(respx_mock) == _pdf_messages()
|
||||
|
||||
|
||||
def test_drop_params_rejects_a_request_whose_only_user_turn_was_a_pdf(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12354/v1"
|
||||
route: Final = _mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="no user message is left"):
|
||||
litellm.completion(
|
||||
model="openai/qa-text-only-model",
|
||||
messages=[
|
||||
{"role": "system", "content": "You read documents."},
|
||||
{"role": "user", "content": [_PDF_FILE_PART]},
|
||||
],
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
model_info={"supports_pdf_input": False},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert not route.called
|
||||
|
||||
|
||||
def test_drop_params_drops_a_pdf_an_unlisted_azure_ai_model_never_declared_it_reads(
|
||||
respx_mock: respx.MockRouter,
|
||||
) -> None:
|
||||
api_base: Final = "http://localhost:12355"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
litellm.completion(
|
||||
model="azure_ai/qa-unlisted-deployment",
|
||||
messages=_pdf_messages(),
|
||||
api_base=api_base,
|
||||
api_key="fake_azure_ai_api_key",
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert _sent_messages(respx_mock) == [{"role": "user", "content": _SUMMARIZE_TEXT_PART["text"]}]
|
||||
|
||||
|
||||
def test_drop_params_keeps_a_pdf_an_azure_ai_deployment_declares_it_reads(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12356"
|
||||
_mock_pdf_chat_route(respx_mock, api_base)
|
||||
|
||||
litellm.completion(
|
||||
model="azure_ai/qa-unlisted-deployment",
|
||||
messages=_pdf_messages(),
|
||||
api_base=api_base,
|
||||
api_key="fake_azure_ai_api_key",
|
||||
model_info={"supports_pdf_input": True},
|
||||
drop_params=True,
|
||||
)
|
||||
|
||||
assert _sent_messages(respx_mock) == _pdf_messages()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("stream", (False, True))
|
||||
@pytest.mark.parametrize("asynchronous", (False, True))
|
||||
@pytest.mark.parametrize("profile,custom_price", ((None, False), ("us.openai.gpt-6-sol", False), (None, True)))
|
||||
|
|
|
|||
|
|
@ -614,6 +614,63 @@ def test_capability_flag_stays_on_the_deployment_that_set_it(flag: str) -> None:
|
|||
_restore_model_cost_entries(model_keys)
|
||||
|
||||
|
||||
def test_pdf_opt_out_on_one_deployment_does_not_block_its_sibling(respx_mock: respx.MockRouter) -> None:
|
||||
api_base: Final = "http://localhost:12361/v1"
|
||||
backend_model: Final = "openai/qa-shared-pdf-backend"
|
||||
pdf_messages: Final = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": "Summarize this."},
|
||||
{
|
||||
"type": "file",
|
||||
"file": {"filename": "report.pdf", "file_data": "data:application/pdf;base64,JVBERi0xLjQK"},
|
||||
},
|
||||
],
|
||||
}
|
||||
]
|
||||
route: Final = respx_mock.post(url__regex=rf"{api_base}/chat/completions.*").mock(
|
||||
return_value=httpx.Response(
|
||||
status_code=200,
|
||||
json={
|
||||
"id": "chatcmpl-pdf",
|
||||
"object": "chat.completion",
|
||||
"created": 1712697600,
|
||||
"model": "qa-shared-pdf-backend",
|
||||
"choices": [{"index": 0, "message": {"role": "assistant", "content": "ok"}, "finish_reason": "stop"}],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 1, "total_tokens": 2},
|
||||
},
|
||||
)
|
||||
)
|
||||
model_keys: Final = {
|
||||
key: litellm.model_cost.get(key) for key in ("qa-pdf-marked", "qa-pdf-unmarked", backend_model)
|
||||
}
|
||||
try:
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": "pdf-blocked",
|
||||
"litellm_params": {"model": backend_model, "api_base": api_base, "api_key": "fake-key"},
|
||||
"model_info": {"id": "qa-pdf-marked", "supports_pdf_input": False},
|
||||
},
|
||||
{
|
||||
"model_name": "pdf-allowed",
|
||||
"litellm_params": {"model": backend_model, "api_base": api_base, "api_key": "fake-key"},
|
||||
"model_info": {"id": "qa-pdf-unmarked"},
|
||||
},
|
||||
],
|
||||
)
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match="does not accept PDF input"):
|
||||
router.completion(model="pdf-blocked", messages=pdf_messages)
|
||||
router.completion(model="pdf-allowed", messages=pdf_messages)
|
||||
|
||||
assert route.call_count == 1
|
||||
assert json.loads(route.calls[0].request.content)["messages"] == pdf_messages
|
||||
finally:
|
||||
_restore_model_cost_entries(model_keys)
|
||||
|
||||
|
||||
def test_token_count_settings_stay_on_the_deployment_that_set_them() -> None:
|
||||
"""A deployment's ``token_count_model`` and ``token_count_framing`` must not land on
|
||||
the shared ``{provider}/{model}`` key, or a sibling deployment of that model would count with them."""
|
||||
|
|
|
|||
|
|
@ -6,6 +6,8 @@ annotation chunk, losing any annotations that arrived in later chunks.
|
|||
This fix merges annotations from ALL chunks.
|
||||
"""
|
||||
|
||||
from typing import Final
|
||||
|
||||
from litellm import stream_chunk_builder
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
|
|
@ -189,3 +191,46 @@ def test_stream_chunk_builder_no_annotations():
|
|||
|
||||
message = response["choices"][0]["message"]
|
||||
assert not hasattr(message, "annotations") or message.annotations is None
|
||||
|
||||
|
||||
def test_stream_chunk_builder_moves_typed_provider_annotations_like_non_streaming() -> None:
|
||||
url_citation: Final = {
|
||||
"type": "url_citation",
|
||||
"url_citation": {"url": "https://example.com", "title": "Example", "start_index": 0, "end_index": 5},
|
||||
}
|
||||
file_annotation: Final = {"type": "file", "file": {"hash": "abc123", "name": "report.pdf"}}
|
||||
chunks: Final = [
|
||||
ModelResponseStream(
|
||||
id="chatcmpl-test",
|
||||
created=1700000000,
|
||||
model="test-model",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
finish_reason=None,
|
||||
index=0,
|
||||
delta=Delta(content="Hello", role="assistant", provider_specific_fields={"region": "eu"}),
|
||||
)
|
||||
],
|
||||
),
|
||||
ModelResponseStream(
|
||||
id="chatcmpl-test",
|
||||
created=1700000000,
|
||||
model="test-model",
|
||||
object="chat.completion.chunk",
|
||||
choices=[
|
||||
StreamingChoices(
|
||||
finish_reason="stop",
|
||||
index=0,
|
||||
delta=Delta(content=None, annotations=[url_citation, file_annotation]),
|
||||
)
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
response: Final = stream_chunk_builder(chunks=chunks)
|
||||
assert response is not None
|
||||
|
||||
message: Final = response["choices"][0]["message"]
|
||||
assert message.annotations == [url_citation]
|
||||
assert message.provider_specific_fields == {"region": "eu", "annotations": [file_annotation]}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue