mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-28 06:23:07 +00:00
fix(completion): send prompt_cache_retention to OpenAI instead of silently dropping it (#47480)
* fix(completion): send prompt_cache_retention to OpenAI instead of silently dropping it litellm.completion(), Router.completion() and the proxy's /v1/chat/completions accepted prompt_cache_retention and listed it as a supported OpenAI param, but completion() never handed it to get_optional_params, and the leftover-kwargs paths skip known OpenAI params, so it never reached the request body. Make it a named argument of completion() and acompletion() like prompt_cache_key and pass it on. A unit guard sends every OpenAI body param through completion() and fails when a param added to OPENAI_CHAT_COMPLETION_PARAMS has no case proving it reaches the wire, the same hole store and prompt_cache_key fell into in #33184. Live e2e tests capture the body OpenAI receives for the proxy, the SDK and the Router on both openai/gpt-5.5 and openai/responses/gpt-5.5. * chore(completion): type the param-forwarding unit test and keep the basedpyright count flat Every new completion() kwarg adds one unknown-argument error at each call site that splats an untyped dict into it; typing print_verbose's argument and Chat's params removes more than prompt_cache_retention added. * test: freeze the outbound-body model and read captured requests from the mock Addresses review: the e2e body model is frozen, the backend params are a tuple, and the unit test reads the request from the respx route instead of appending to a list.
This commit is contained in:
ryan-crabbe-berri
GitHub
parent
8d2bf426f5
commit
9bbf70edb6
6 changed files with 275 additions and 5 deletions
|
|
@ -400,7 +400,11 @@ class LiteLLM:
|
|||
|
||||
|
||||
class Chat:
|
||||
def __init__(self, params, router_obj: "Router | None"):
|
||||
def __init__(
|
||||
self,
|
||||
params: dict[str, object], # mutable-ok: pops the acompletion flag in place
|
||||
router_obj: "Router | None",
|
||||
):
|
||||
self.params = params
|
||||
if self.params.get("acompletion", False) is True:
|
||||
self.params.pop("acompletion")
|
||||
|
|
@ -480,6 +484,7 @@ async def acompletion(
|
|||
service_tier: str | None = None,
|
||||
store: bool | None = None,
|
||||
prompt_cache_key: str | None = None,
|
||||
prompt_cache_retention: str | None = None,
|
||||
# set api_base, api_version, api_key
|
||||
base_url: str | None = None,
|
||||
api_version: str | None = None,
|
||||
|
|
@ -657,6 +662,7 @@ async def acompletion(
|
|||
"service_tier": service_tier,
|
||||
"store": store,
|
||||
"prompt_cache_key": prompt_cache_key,
|
||||
"prompt_cache_retention": prompt_cache_retention,
|
||||
"extra_headers": extra_headers,
|
||||
"acompletion": True, # assuming this is a required parameter
|
||||
"thinking": thinking,
|
||||
|
|
@ -5356,6 +5362,7 @@ def completion(
|
|||
service_tier: str | None = None,
|
||||
store: bool | None = None,
|
||||
prompt_cache_key: str | None = None,
|
||||
prompt_cache_retention: str | None = None,
|
||||
# soon to be deprecated params by OpenAI
|
||||
functions: list | None = None,
|
||||
function_call: str | None = None,
|
||||
|
|
@ -5478,6 +5485,7 @@ def completion(
|
|||
service_tier=service_tier,
|
||||
store=store,
|
||||
prompt_cache_key=prompt_cache_key,
|
||||
prompt_cache_retention=prompt_cache_retention,
|
||||
base_url=base_url,
|
||||
api_version=api_version,
|
||||
api_key=api_key,
|
||||
|
|
@ -5843,6 +5851,7 @@ def completion(
|
|||
"service_tier": service_tier,
|
||||
"store": store,
|
||||
"prompt_cache_key": prompt_cache_key,
|
||||
"prompt_cache_retention": prompt_cache_retention,
|
||||
"allowed_openai_params": allowed_openai_params,
|
||||
"base_model": base_model,
|
||||
}
|
||||
|
|
@ -9143,7 +9152,7 @@ async def ahealth_check(
|
|||
|
||||
####### HELPER FUNCTIONS ################
|
||||
## Set verbose to true -> ```litellm.set_verbose = True```
|
||||
def print_verbose(print_statement):
|
||||
def print_verbose(print_statement: object):
|
||||
try:
|
||||
verbose_logger.debug(print_statement)
|
||||
if litellm.set_verbose:
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@
|
|||
- {id: llm.chat_completions.openai.vision.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: vision, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "gpt-4o vision; high usage"}
|
||||
- {id: llm.chat_completions.openai.prompt_cache_5m.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_5m, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Prompt caching cost optimization"}
|
||||
- {id: llm.chat_completions.openai.service_tier.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: nonstream, assertions: [works], source: "OpenAI service_tier param", rationale: "OpenAI scale-tier request option is forwarded and echoed"}
|
||||
- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention sent through the proxy, litellm.completion or Router.completion reaches the body sent to OpenAI /v1/chat/completions, so a caller asking for 24h prompt caching gets it instead of a silent drop"}
|
||||
- {id: llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: prompt_cache_24h, streaming: nonstream, assertions: [forwards_retention], source: "main.py completion()", fail_before_fix: proven, rationale: "prompt_cache_retention on an openai/responses/ model (chat completions in, Responses out) reaches the body sent to OpenAI /v1/responses"}
|
||||
- {id: llm.chat_completions.openai.service_tier.stream.echoes_served_tier, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: service_tier, streaming: stream, assertions: [works], source: "litellm_core_utils/streaming_handler.py", fail_before_fix: proven, rationale: "Every relayed stream chunk carries the service_tier OpenAI stamped on it, so a streaming caller can see which tier served the request"}
|
||||
- {id: llm.chat_completions.openai.thinking.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: thinking, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "o-series reasoning; emerging"}
|
||||
- {id: llm.chat_completions.openai.structured_output.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: structured_output, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "response_schema extraction"}
|
||||
|
|
|
|||
|
|
@ -81,6 +81,7 @@ LlmCapability = Literal[
|
|||
"pdf_input",
|
||||
"logprobs",
|
||||
"prompt_cache_1h",
|
||||
"prompt_cache_24h",
|
||||
"prompt_cache_5m",
|
||||
"response_headers",
|
||||
"service_tier",
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
"""Live e2e for model-specific request features: service_tier.
|
||||
"""Live e2e for model-specific request features: service_tier and prompt_cache_retention.
|
||||
|
||||
Each case asserts the feature took effect, not just a 200.
|
||||
|
||||
|
|
@ -11,26 +11,38 @@ avoided here because it is capacity-constrained and returns a transient 429 when
|
|||
flex resources are unavailable. Bedrock and Vertex do not accept service_tier, so
|
||||
that cell is OpenAI-only by design.
|
||||
|
||||
Prompt caching lives in test_cache_control.py.
|
||||
prompt_cache_retention is judged on the body OpenAI receives, captured by a live
|
||||
provider edge, not on the reply: OpenAI echoes "24h" on gpt-5.5 even when the
|
||||
request never carried the param, so an echo would pass with the param dropped.
|
||||
|
||||
Prompt caching hits and misses live in test_cache_control.py.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import queue
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from pydantic import BaseModel
|
||||
|
||||
from e2e_config import unique_marker
|
||||
from e2e_config import PROVIDER_EDGE_ADVERTISE_HOST, PROVIDER_EDGE_BIND_HOST, unique_marker
|
||||
from e2e_http import unwrap
|
||||
from e2e_metadata import Domain, Mode, Provider, Route, Subject, meta
|
||||
from lifecycle import ResourceManager
|
||||
from models import ChatBody, ChatMessage, LiteLLMParamsBody
|
||||
from passthrough_client import PassthroughClient
|
||||
from provider_edge import LiveEdge, start_provider_edge
|
||||
from proxy_client import ProxyClient
|
||||
|
||||
pytestmark = pytest.mark.e2e
|
||||
|
||||
SERVICE_TIER = "priority"
|
||||
OPENAI_BACKEND: Final = "openai/gpt-5.5"
|
||||
OPENAI_RESPONSES_BRIDGE_BACKEND: Final = "openai/responses/gpt-5.5"
|
||||
PROMPT_CACHE_RETENTION: Final = "24h"
|
||||
|
||||
|
||||
class TestServiceTier:
|
||||
|
|
@ -74,3 +86,153 @@ class TestServiceTier:
|
|||
f"service_tier not honored: sent {SERVICE_TIER!r}, response reported "
|
||||
f"{response.service_tier!r} ({response})"
|
||||
)
|
||||
|
||||
|
||||
class _OutboundRetention(BaseModel, frozen=True):
|
||||
prompt_cache_retention: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class _OpenAIBodyCapture:
|
||||
"""The raw bodies sent to one OpenAI path, seen by a live edge in front of
|
||||
api.openai.com. Pass ``observe`` as the edge's ``observe_request``."""
|
||||
|
||||
path: str
|
||||
_bodies: queue.SimpleQueue[bytes] = field(default_factory=queue.SimpleQueue)
|
||||
|
||||
def observe(self, url: str, headers: Mapping[str, str], body: bytes | None) -> None:
|
||||
if body is not None and url.split("?")[0].endswith(self.path):
|
||||
self._bodies.put(body)
|
||||
|
||||
def retentions(self) -> tuple[str | None, ...]:
|
||||
bodies: Final = tuple(self._bodies.get_nowait() for _ in range(self._bodies.qsize()))
|
||||
return tuple(_OutboundRetention.model_validate_json(body).prompt_cache_retention for body in bodies)
|
||||
|
||||
|
||||
def _openai_edge_api_base(resources: ResourceManager, capture: _OpenAIBodyCapture, *, reachable_from_proxy: bool) -> str:
|
||||
"""The proxy may run in a container, so its edge binds where the proxy can
|
||||
reach it; an in-process SDK call reaches the edge on loopback."""
|
||||
edge: Final = (
|
||||
start_provider_edge(
|
||||
LiveEdge(observe_request=capture.observe),
|
||||
bind_host=PROVIDER_EDGE_BIND_HOST,
|
||||
advertise_host=PROVIDER_EDGE_ADVERTISE_HOST,
|
||||
)
|
||||
if reachable_from_proxy
|
||||
else start_provider_edge(LiveEdge(observe_request=capture.observe))
|
||||
)
|
||||
resources.defer(edge.shutdown)
|
||||
return f"{edge.edge.api_base('openai')}/v1"
|
||||
|
||||
|
||||
def _retention_backends() -> tuple[object, ...]:
|
||||
"""gpt-5.5 straight to /v1/chat/completions, and the openai/responses/ bridge
|
||||
(chat completions in, Responses out) to /v1/responses."""
|
||||
return (
|
||||
pytest.param(
|
||||
OPENAI_BACKEND,
|
||||
"/chat/completions",
|
||||
marks=(
|
||||
pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.forwards_retention"),
|
||||
meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(OPENAI_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
),
|
||||
),
|
||||
id="chat_completions",
|
||||
),
|
||||
pytest.param(
|
||||
OPENAI_RESPONSES_BRIDGE_BACKEND,
|
||||
"/responses",
|
||||
marks=(
|
||||
pytest.mark.covers("llm.chat_completions.openai.prompt_cache_24h.nonstream.bridge_forwards_retention"),
|
||||
meta(
|
||||
Subject(
|
||||
domain=Domain.LLM_TRANSLATION,
|
||||
route=Route.CHAT_COMPLETIONS,
|
||||
providers=(Provider.OPENAI,),
|
||||
models=(OPENAI_RESPONSES_BRIDGE_BACKEND,),
|
||||
mode=Mode.NONSTREAM,
|
||||
)
|
||||
),
|
||||
),
|
||||
id="responses_bridge",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
class TestPromptCacheRetention:
|
||||
@pytest.mark.provider_edge_host
|
||||
@pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
|
||||
def test_proxy_chat_completions_forwards_prompt_cache_retention_to_openai(
|
||||
self, proxy: ProxyClient, resources: ResourceManager, backend: str, openai_path: str
|
||||
) -> None:
|
||||
capture: Final = _OpenAIBodyCapture(openai_path)
|
||||
api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=True)
|
||||
model: Final = f"e2e-cache-retention-{unique_marker()}"
|
||||
model_id: Final = proxy.create_model(
|
||||
model, LiteLLMParamsBody(model=backend, api_key="os.environ/OPENAI_API_KEY", api_base=api_base)
|
||||
)
|
||||
resources.defer(lambda: proxy.delete_model(model_id))
|
||||
|
||||
unwrap(
|
||||
proxy.chat(
|
||||
resources.key(),
|
||||
ChatBody(
|
||||
model=model,
|
||||
messages=[ChatMessage(role="user", content="reply with one word")],
|
||||
prompt_cache_retention=PROMPT_CACHE_RETENTION,
|
||||
),
|
||||
)
|
||||
)
|
||||
|
||||
assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
|
||||
f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
|
||||
def test_sdk_completion_forwards_prompt_cache_retention_to_openai(
|
||||
self, resources: ResourceManager, backend: str, openai_path: str
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
capture: Final = _OpenAIBodyCapture(openai_path)
|
||||
api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False)
|
||||
|
||||
litellm.completion(
|
||||
model=backend,
|
||||
messages=[{"role": "user", "content": "reply with one word"}],
|
||||
api_base=api_base,
|
||||
prompt_cache_retention=PROMPT_CACHE_RETENTION,
|
||||
)
|
||||
|
||||
assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
|
||||
f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(("backend", "openai_path"), _retention_backends())
|
||||
def test_router_completion_forwards_prompt_cache_retention_to_openai(
|
||||
self, resources: ResourceManager, backend: str, openai_path: str
|
||||
) -> None:
|
||||
import litellm
|
||||
|
||||
capture: Final = _OpenAIBodyCapture(openai_path)
|
||||
api_base: Final = _openai_edge_api_base(resources, capture, reachable_from_proxy=False)
|
||||
router: Final = litellm.Router(
|
||||
model_list=[{"model_name": "cache-retention", "litellm_params": {"model": backend, "api_base": api_base}}]
|
||||
)
|
||||
|
||||
router.completion( # pyright: ignore[reportUnknownMemberType] # Router.completion forwards an untyped **kwargs to litellm.completion
|
||||
model="cache-retention",
|
||||
messages=[{"role": "user", "content": "reply with one word"}],
|
||||
prompt_cache_retention=PROMPT_CACHE_RETENTION,
|
||||
)
|
||||
|
||||
assert capture.retentions() == (PROMPT_CACHE_RETENTION,), (
|
||||
f"one {openai_path} call to OpenAI must carry prompt_cache_retention={PROMPT_CACHE_RETENTION!r}"
|
||||
)
|
||||
|
|
|
|||
|
|
@ -364,6 +364,7 @@ class ChatBody(BaseModel):
|
|||
thinking: ThinkingParam | None = None
|
||||
service_tier: str | None = None
|
||||
prompt_cache_key: str | None = None
|
||||
prompt_cache_retention: str | None = None
|
||||
tools: Sequence[ChatTool | McpChatTool | GeminiUrlContextTool] | None = None
|
||||
tool_choice: str | None = None
|
||||
guardrails: list[str] | None = None
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ import respx
|
|||
from openai import APITimeoutError, AsyncOpenAI
|
||||
from openai._legacy_response import LegacyAPIResponse
|
||||
from openai.types.chat.chat_completion import ChatCompletion
|
||||
from pydantic import TypeAdapter
|
||||
|
||||
import litellm
|
||||
from litellm import (
|
||||
|
|
@ -3937,6 +3938,100 @@ async def test_acompletion_forwards_store_and_prompt_cache_key_to_openai():
|
|||
assert request_body["prompt_cache_key"] == "test-cache-key"
|
||||
|
||||
|
||||
_TOOL: Final[Mapping[str, object]] = {"type": "function", "function": {"name": "f", "parameters": {"type": "object", "properties": {}}}}
|
||||
|
||||
# Client settings ride in OPENAI_CHAT_COMPLETION_PARAMS but configure the HTTP client, never the body.
|
||||
_OPENAI_CLIENT_SETTINGS: Final = frozenset(
|
||||
{
|
||||
"api_base",
|
||||
"api_key",
|
||||
"api_version",
|
||||
"base_url",
|
||||
"default_headers",
|
||||
"deployment_id",
|
||||
"extra_headers",
|
||||
"max_retries",
|
||||
"organization",
|
||||
"request_timeout",
|
||||
"timeout",
|
||||
}
|
||||
)
|
||||
|
||||
# Accepted by completion() for other providers, never sent to OpenAI's /chat/completions body.
|
||||
_NOT_OPENAI_BODY_PARAMS: Final = frozenset({"include_server_side_tool_invocations", "thinking"})
|
||||
|
||||
# param -> (model that accepts it, the call kwargs that carry it)
|
||||
_OPENAI_BODY_PARAM_CASES: Final[Mapping[str, tuple[str, Mapping[str, object]]]] = {
|
||||
"audio": ("openai/gpt-4o", {"audio": {"voice": "alloy", "format": "wav"}}),
|
||||
"frequency_penalty": ("openai/gpt-4o", {"frequency_penalty": 0.5}),
|
||||
"function_call": ("openai/gpt-4o", {"function_call": "auto"}),
|
||||
"functions": ("openai/gpt-4o", {"functions": [_TOOL["function"]]}),
|
||||
"logit_bias": ("openai/gpt-4o", {"logit_bias": {"1": 1}}),
|
||||
"logprobs": ("openai/gpt-4o", {"logprobs": True}),
|
||||
"max_completion_tokens": ("openai/gpt-4o", {"max_completion_tokens": 7}),
|
||||
"max_tokens": ("openai/gpt-4o", {"max_tokens": 7}),
|
||||
"modalities": ("openai/gpt-4o", {"modalities": ["text"]}),
|
||||
"n": ("openai/gpt-4o", {"n": 2}),
|
||||
"parallel_tool_calls": ("openai/gpt-4o", {"tools": [_TOOL], "parallel_tool_calls": False}),
|
||||
"prediction": ("openai/gpt-4o", {"prediction": {"type": "content", "content": "x"}}),
|
||||
"presence_penalty": ("openai/gpt-4o", {"presence_penalty": 0.5}),
|
||||
"prompt_cache_key": ("openai/gpt-4o", {"prompt_cache_key": "k"}),
|
||||
"prompt_cache_retention": ("openai/gpt-5.5", {"prompt_cache_retention": "24h"}),
|
||||
"reasoning_effort": ("openai/gpt-5.5", {"reasoning_effort": "low"}),
|
||||
"response_format": ("openai/gpt-4o", {"response_format": {"type": "json_object"}}),
|
||||
"safety_identifier": ("openai/gpt-4o", {"safety_identifier": "s"}),
|
||||
"seed": ("openai/gpt-4o", {"seed": 3}),
|
||||
"service_tier": ("openai/gpt-4o", {"service_tier": "priority"}),
|
||||
"stop": ("openai/gpt-4o", {"stop": ["x"]}),
|
||||
"store": ("openai/gpt-4o", {"store": False}),
|
||||
"stream": ("openai/gpt-4o", {"stream": True}),
|
||||
"stream_options": ("openai/gpt-4o", {"stream": True, "stream_options": {"include_usage": True}}),
|
||||
"temperature": ("openai/gpt-4o", {"temperature": 0.5}),
|
||||
"tool_choice": ("openai/gpt-4o", {"tools": [_TOOL], "tool_choice": "auto"}),
|
||||
"tools": ("openai/gpt-4o", {"tools": [_TOOL]}),
|
||||
"top_logprobs": ("openai/gpt-4o", {"logprobs": True, "top_logprobs": 2}),
|
||||
"top_p": ("openai/gpt-4o", {"top_p": 0.5}),
|
||||
"user": ("openai/gpt-4o", {"user": "u1"}),
|
||||
"verbosity": ("openai/gpt-5.5", {"verbosity": "low"}),
|
||||
"web_search_options": ("openai/gpt-4o", {"web_search_options": {"search_context_size": "low"}}),
|
||||
}
|
||||
|
||||
|
||||
def test_every_openai_body_param_has_a_forwarding_case():
|
||||
body_params: Final = frozenset(litellm.OPENAI_CHAT_COMPLETION_PARAMS) - _OPENAI_CLIENT_SETTINGS - _NOT_OPENAI_BODY_PARAMS
|
||||
assert frozenset(_OPENAI_BODY_PARAM_CASES) == body_params, (
|
||||
"a param added to OPENAI_CHAT_COMPLETION_PARAMS needs a case in _OPENAI_BODY_PARAM_CASES proving "
|
||||
"completion() sends it to OpenAI; prompt_cache_retention (LIT-9758) and store/prompt_cache_key "
|
||||
"(#33184) were listed as supported but silently dropped"
|
||||
)
|
||||
|
||||
|
||||
_REQUEST_BODY: Final = TypeAdapter(dict[str, object])
|
||||
|
||||
|
||||
@pytest.mark.parametrize("param", sorted(_OPENAI_BODY_PARAM_CASES))
|
||||
def test_completion_sends_openai_body_param_to_openai(
|
||||
param: str, respx_mock: respx.MockRouter, openai_api_response: Mapping[str, object]
|
||||
):
|
||||
api_base: Final = "http://localhost:12346/v1"
|
||||
route: Final = respx_mock.post(f"{api_base}/chat/completions").mock(
|
||||
return_value=httpx.Response(status_code=200, json=openai_api_response)
|
||||
)
|
||||
model, call_kwargs = _OPENAI_BODY_PARAM_CASES[param]
|
||||
|
||||
litellm.completion(
|
||||
model=model,
|
||||
messages=[{"role": "user", "content": "hi"}],
|
||||
api_base=api_base,
|
||||
api_key="fake_openai_api_key",
|
||||
**call_kwargs,
|
||||
)
|
||||
|
||||
assert route.call_count == 1
|
||||
sent: Final = _REQUEST_BODY.validate_json(route.calls.last.request.content)
|
||||
assert sent.get(param) == call_kwargs[param], f"{param} did not reach the OpenAI request body: {sent}"
|
||||
|
||||
|
||||
def test_completion_omits_store_and_prompt_cache_key_when_not_passed():
|
||||
"""
|
||||
When store and prompt_cache_key are not passed, they must not appear in the
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue