mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(vertex_ai,azure_ai): forward compact-2026-09-04 to Azure AI Foundry, merge Vertex body betas, and redact compaction summaries
Azure AI Foundry needs the compact-2026-09-04 header for compaction, a replayed compaction block, and count tokens, so the beta map, the native compaction providers, and the Rust beta table now allow it there. Vertex keeps a client's body betas next to the ones litellm adds, as the Anthropic route does, and turn_off_message_logging now redacts compaction summaries too
This commit is contained in:
parent
c8097f9c37
commit
90b5ccb89c
17 changed files with 251 additions and 151 deletions
|
|
@ -208,7 +208,6 @@ impl AnthropicBeta {
|
|||
BetaProvider::Databricks,
|
||||
],
|
||||
Self::Compact20260904 => &[
|
||||
BetaProvider::AzureAi,
|
||||
BetaProvider::BedrockConverse,
|
||||
BetaProvider::Bedrock,
|
||||
BetaProvider::BedrockMantle,
|
||||
|
|
|
|||
|
|
@ -545,6 +545,11 @@ mod tests {
|
|||
&[("anthropic-beta", "web-search-2025-03-05")],
|
||||
&[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")]
|
||||
)]
|
||||
#[case::compaction_needs_the_compact_2026_09_04_beta(
|
||||
json!({"compaction": {"type": "summarize"}}),
|
||||
&[],
|
||||
&[("x-api-key", "k"), ("anthropic-beta", "compact-2026-09-04")]
|
||||
)]
|
||||
#[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])]
|
||||
#[case::safeguards_need_the_dangerous_tool_use_beta(
|
||||
json!({"safeguards": [{"type": "dangerous_tool_use"}]}),
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@
|
|||
"bash_20250124": null,
|
||||
"code-execution-2025-08-25": "code-execution-2025-08-25",
|
||||
"compact-2026-01-12": null,
|
||||
"compact-2026-09-04": "compact-2026-09-04",
|
||||
"computer-use-2025-01-24": "computer-use-2025-01-24",
|
||||
"computer-use-2025-11-24": "computer-use-2025-11-24",
|
||||
"context-1m-2025-08-07": "context-1m-2025-08-07",
|
||||
|
|
|
|||
|
|
@ -147,7 +147,11 @@ def _redacted_provider_specific_fields(fields: Mapping[str, object]) -> Mapping[
|
|||
return {
|
||||
**fields,
|
||||
**({"reasoning_content": REDACTED_BY_LITELLM} if fields.get("reasoning_content") is not None else {}),
|
||||
**{key: None for key in ("thinking_blocks", "reasoningContentBlocks", "citations") if key in fields},
|
||||
**{
|
||||
key: None
|
||||
for key in ("thinking_blocks", "reasoningContentBlocks", "citations", "compaction_blocks")
|
||||
if key in fields
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -119,7 +119,7 @@ _DECODED_JSON: Final = TypeAdapter(object)
|
|||
|
||||
_CLAUDE_CODE_USER_AGENT_PREFIXES: Final = ("claude-cli/", "claude-code/")
|
||||
|
||||
NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "vertex_ai"})
|
||||
NATIVE_COMPACTION_PROVIDERS: Final = frozenset({"anthropic", "azure_ai", "vertex_ai"})
|
||||
|
||||
|
||||
def is_anthropic_messages_url(url: str) -> bool:
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ from litellm.types.llms.anthropic_tool_search import get_tool_search_beta_header
|
|||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
from ...common_utils import (
|
||||
NATIVE_COMPACTION_PROVIDERS,
|
||||
AnthropicError,
|
||||
AnthropicModelInfo,
|
||||
merge_anthropic_beta_headers,
|
||||
|
|
@ -99,7 +100,7 @@ class AnthropicMessagesConfig(BaseAnthropicMessagesConfig):
|
|||
"tool_choice",
|
||||
"thinking",
|
||||
"context_management",
|
||||
*(("compaction",) if self._resolved_provider == "anthropic" else ()),
|
||||
*(("compaction",) if self._resolved_provider in NATIVE_COMPACTION_PROVIDERS else ()),
|
||||
"output_format",
|
||||
"inference_geo",
|
||||
"speed",
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ class AzureAIAnthropicCountTokensHandler(AzureAIAnthropicCountTokensConfig):
|
|||
headers: Final = self.get_required_headers(
|
||||
api_key=api_key,
|
||||
litellm_params=litellm_params,
|
||||
messages=messages,
|
||||
)
|
||||
if not {"x-api-key", "Authorization"} & headers.keys():
|
||||
raise AnthropicError(
|
||||
|
|
|
|||
|
|
@ -4,13 +4,16 @@ Azure AI Anthropic CountTokens API transformation logic.
|
|||
Extends the base Anthropic CountTokens transformation with Azure authentication.
|
||||
"""
|
||||
|
||||
from collections.abc import Sequence
|
||||
from typing import Final
|
||||
|
||||
from litellm.constants import ANTHROPIC_TOKEN_COUNTING_BETA_VERSION
|
||||
from litellm.llms.anthropic.common_utils import merge_anthropic_beta_headers, requires_native_compaction_beta
|
||||
from litellm.llms.anthropic.count_tokens.transformation import (
|
||||
AnthropicCountTokensConfig,
|
||||
)
|
||||
from litellm.llms.azure.common_utils import BaseAzureLLM
|
||||
from litellm.types.llms.anthropic import ANTHROPIC_BETA_HEADER_VALUES
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
||||
|
|
@ -26,6 +29,7 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
self,
|
||||
api_key: str | None,
|
||||
litellm_params: dict[str, object] | None = None,
|
||||
messages: Sequence[object] = (),
|
||||
) -> dict[str, str]:
|
||||
"""
|
||||
Get the required headers for the Azure AI Anthropic CountTokens API.
|
||||
|
|
@ -44,7 +48,12 @@ class AzureAIAnthropicCountTokensConfig(AnthropicCountTokensConfig):
|
|||
headers: Final = {
|
||||
"Content-Type": "application/json",
|
||||
"anthropic-version": "2023-06-01",
|
||||
"anthropic-beta": ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
|
||||
"anthropic-beta": merge_anthropic_beta_headers(
|
||||
ANTHROPIC_TOKEN_COUNTING_BETA_VERSION,
|
||||
ANTHROPIC_BETA_HEADER_VALUES.COMPACT_2026_09_04.value
|
||||
if requires_native_compaction_beta("azure_ai", {}, messages)
|
||||
else None,
|
||||
),
|
||||
**({"x-api-key": api_key} if api_key else {}),
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -183,7 +183,7 @@ class VertexAIAnthropicConfig(AnthropicConfig):
|
|||
_NON_DEFAULT_PARAMS.validate_python(data).get("anthropic_beta"), "anthropic_beta", drop_params
|
||||
)
|
||||
|
||||
body_betas: Final = (injected_betas or client_body_betas) - _VERTEX_HEADER_ONLY_BETAS
|
||||
body_betas: Final = (injected_betas | client_body_betas) - _VERTEX_HEADER_ONLY_BETAS
|
||||
if body_betas:
|
||||
data["anthropic_beta"] = sorted(body_betas)
|
||||
elif "anthropic_beta" in data:
|
||||
|
|
|
|||
|
|
@ -75,7 +75,6 @@ import inspect
|
|||
from typing import List, Optional
|
||||
from litellm.types.utils import LiteLLMCommonStrings
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def logging_obj():
|
||||
return LitellmLogging(
|
||||
|
|
@ -346,10 +345,8 @@ async def test_mcp_native_structured_replacement_must_match_returned_content(
|
|||
)
|
||||
logging_obj.dynamic_success_callbacks = [NativeReplacement()]
|
||||
returned = await logging_obj.async_post_mcp_tool_call_hook(
|
||||
kwargs={"original_response": result},
|
||||
response_obj=result,
|
||||
start_time=datetime.datetime.now(),
|
||||
end_time=datetime.datetime.now(),
|
||||
kwargs={"original_response": result}, response_obj=result,
|
||||
start_time=datetime.datetime.now(), end_time=datetime.datetime.now(),
|
||||
)
|
||||
assert returned is result
|
||||
assert result.content == [TextContent(type="text", text="native-safe" if same_content else "final-safe")]
|
||||
|
|
@ -358,6 +355,7 @@ async def test_mcp_native_structured_replacement_must_match_returned_content(
|
|||
assert "SECRET-1234" not in result.model_dump_json()
|
||||
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("mode", ["none", "wrapper", "exception"])
|
||||
@pytest.mark.parametrize("structured", [False, True])
|
||||
|
|
@ -375,10 +373,8 @@ async def test_mcp_direct_content_edit_invalidates_stale_structured_data(logging
|
|||
)
|
||||
logging_obj.dynamic_success_callbacks = [DirectRedactor()]
|
||||
returned = await logging_obj.async_post_mcp_tool_call_hook(
|
||||
kwargs={"original_response": result},
|
||||
response_obj=result,
|
||||
start_time=datetime.datetime.now(),
|
||||
end_time=datetime.datetime.now(),
|
||||
kwargs={"original_response": result}, response_obj=result,
|
||||
start_time=datetime.datetime.now(), end_time=datetime.datetime.now(),
|
||||
)
|
||||
assert returned is result
|
||||
assert result.content == [TextContent(type="text", text="[REDACTED]")]
|
||||
|
|
@ -2543,6 +2539,7 @@ async def test_completed_success_selects_one_implemented_hook(
|
|||
assert (thread_id == loop_thread) is (kind != "sync")
|
||||
|
||||
|
||||
|
||||
@pytest.mark.parametrize("for_async_request", [False, True])
|
||||
@pytest.mark.parametrize("private_first", [False, True])
|
||||
@pytest.mark.parametrize("response_kind", ["responses", "completion"])
|
||||
|
|
@ -3230,13 +3227,8 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g
|
|||
|
||||
class RecordingShadowLogger(ShadowEvalLogger):
|
||||
async def async_log_success_event(
|
||||
self,
|
||||
kwargs: Mapping[str, object],
|
||||
response_obj: object,
|
||||
start_time: object,
|
||||
end_time: object,
|
||||
*,
|
||||
guardrail_snapshot: GuardrailRequestSnapshot | None = None,
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object,
|
||||
end_time: object, *, guardrail_snapshot: GuardrailRequestSnapshot | None = None,
|
||||
) -> None:
|
||||
shadow_snapshots.append(guardrail_snapshot)
|
||||
await super().async_log_success_event(
|
||||
|
|
@ -3245,20 +3237,13 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g
|
|||
|
||||
class RecordingLogger(CustomLogger):
|
||||
async def async_log_success_event(
|
||||
self,
|
||||
kwargs: Mapping[str, object],
|
||||
response_obj: object,
|
||||
start_time: object,
|
||||
end_time: object,
|
||||
self, kwargs: Mapping[str, object], response_obj: object, start_time: object, end_time: object,
|
||||
) -> None:
|
||||
other_payloads.append(kwargs)
|
||||
|
||||
class LoggingGuardrail(CustomGuardrail):
|
||||
async def async_logging_hook(
|
||||
self,
|
||||
kwargs: dict[str, object],
|
||||
result: object,
|
||||
call_type: str,
|
||||
self, kwargs: dict[str, object], result: object, call_type: str,
|
||||
) -> tuple[dict[str, object], object]:
|
||||
hook_snapshots.append(logging_obj.shadow_eval_request_snapshot)
|
||||
if hook_mode == "raises":
|
||||
|
|
@ -3270,36 +3255,26 @@ async def test_shadow_snapshot_stays_private_and_is_invalidated_before_logging_g
|
|||
"user_api_key_hash": "test-key",
|
||||
}
|
||||
snapshot: Final = GuardrailRequestSnapshot.capture(
|
||||
{"messages": [{"role": "user", "content": "snapshot-only"}]},
|
||||
metadata,
|
||||
{"messages": [{"role": "user", "content": "snapshot-only"}]}, metadata,
|
||||
)
|
||||
assert snapshot is not None
|
||||
shadow: Final = RecordingShadowLogger(prisma_provider=no_prisma, jobs_cache=InMemoryCache())
|
||||
guardrail: Final = LoggingGuardrail(
|
||||
guardrail_name="late-mask",
|
||||
default_on=True,
|
||||
guardrail_name="late-mask", default_on=True,
|
||||
event_hook=GuardrailEventHooks.pre_call if hook_mode == "disabled" else GuardrailEventHooks.logging_only,
|
||||
)
|
||||
monkeypatch.setattr(litellm, "_async_success_callback", [])
|
||||
logging_obj: Final = LitellmLogging(
|
||||
model="test-model",
|
||||
messages=[],
|
||||
stream=stream,
|
||||
call_type="anthropic_messages",
|
||||
start_time=datetime.datetime.now(),
|
||||
litellm_call_id="private-snapshot",
|
||||
function_id="private-snapshot",
|
||||
model="test-model", messages=[], stream=stream, call_type="anthropic_messages",
|
||||
start_time=datetime.datetime.now(), litellm_call_id="private-snapshot", function_id="private-snapshot",
|
||||
dynamic_async_success_callbacks=[shadow, RecordingLogger(), guardrail],
|
||||
)
|
||||
logging_obj.update_messages([{"role": "user", "content": "logged input"}])
|
||||
logging_obj.update_environment_variables(litellm_params={"metadata": metadata}, optional_params={})
|
||||
logging_obj.shadow_eval_request_snapshot = snapshot
|
||||
payload: Final = {
|
||||
"id": "private-snapshot",
|
||||
"call_type": "anthropic_messages",
|
||||
"metadata": metadata,
|
||||
"model_group": "test-model",
|
||||
"model_parameters": {},
|
||||
"id": "private-snapshot", "call_type": "anthropic_messages", "metadata": metadata,
|
||||
"model_group": "test-model", "model_parameters": {},
|
||||
}
|
||||
|
||||
await logging_obj.async_success_handler(result=ModelResponse(), standard_logging_object=payload)
|
||||
|
|
@ -3756,9 +3731,7 @@ def test_sentry_event_scrubber_initialization(monkeypatch):
|
|||
call_args = mock_init.call_args[1]
|
||||
assert call_args["send_default_pii"] is False
|
||||
assert call_args["event_scrubber"].recursive is True
|
||||
assert {name.lower() for name in SENTRY_PII_DENYLIST} <= {
|
||||
name.lower() for name in call_args["event_scrubber"].denylist
|
||||
}
|
||||
assert {name.lower() for name in SENTRY_PII_DENYLIST} <= {name.lower() for name in call_args["event_scrubber"].denylist}
|
||||
assert call_args["before_send"] is call_args["before_send_transaction"]
|
||||
|
||||
|
||||
|
|
@ -3775,9 +3748,7 @@ def test_sentry_send_default_pii_opt_in(monkeypatch):
|
|||
|
||||
call_args = mock_init.call_args[1]
|
||||
assert call_args["send_default_pii"] is True
|
||||
assert not {name.lower() for name in SENTRY_PII_DENYLIST} & {
|
||||
name.lower() for name in call_args["event_scrubber"].denylist
|
||||
}
|
||||
assert not {name.lower() for name in SENTRY_PII_DENYLIST} & {name.lower() for name in call_args["event_scrubber"].denylist}
|
||||
|
||||
|
||||
def test_get_masked_values():
|
||||
|
|
@ -5047,10 +5018,12 @@ def test_reported_cost_respects_configured_pricing_and_additional_charges(
|
|||
else {}
|
||||
),
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
)
|
||||
logging_obj.model_call_details.update(
|
||||
{"additional_response_cost": additional_cost, "cache_hit": cache_hit}
|
||||
)
|
||||
logging_obj.model_call_details.update({"additional_response_cost": additional_cost, "cache_hit": cache_hit})
|
||||
logging_obj._provider_reported_cost = reported_cost
|
||||
response: Final = ModelResponse(
|
||||
model=pricing_model,
|
||||
|
|
@ -7002,11 +6975,8 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe
|
|||
|
||||
class SyncRecorder(CustomLogger):
|
||||
def log_success_event(
|
||||
self,
|
||||
kwargs: dict[str, object],
|
||||
response_obj: object,
|
||||
start_time: dt.datetime,
|
||||
end_time: dt.datetime,
|
||||
self, kwargs: dict[str, object], response_obj: object,
|
||||
start_time: dt.datetime, end_time: dt.datetime,
|
||||
) -> None:
|
||||
events.put((response_obj, kwargs.get("response_cost")))
|
||||
logged.set()
|
||||
|
|
@ -7039,11 +7009,11 @@ async def test_background_interaction_completion_rebills_after_in_progress_succe
|
|||
assert logging_obj.model_call_details["response_cost"] > 0
|
||||
assert logging_obj.model_call_details["standard_logging_object"]["total_tokens"] == 175
|
||||
|
||||
|
||||
assert events.qsize() == 2
|
||||
assert events.get_nowait() == (in_progress, None)
|
||||
assert events.get_nowait() == (completed, logging_obj.model_call_details["response_cost"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_background_interaction_completion_prices_the_settled_body_itself():
|
||||
"""
|
||||
|
|
@ -8077,9 +8047,7 @@ def test_debugging_log_with_json_logs_tolerates_missing_headers(logging_obj, mon
|
|||
logging_obj.litellm_request_debug = True
|
||||
|
||||
with patch("litellm.litellm_core_utils.litellm_logging.verbose_logger.warning") as warning:
|
||||
logging_obj._print_llm_call_debugging_log(
|
||||
api_base="https://api.openai.com/v1", headers=None, additional_args={}
|
||||
)
|
||||
logging_obj._print_llm_call_debugging_log(api_base="https://api.openai.com/v1", headers=None, additional_args={})
|
||||
|
||||
assert "https://api.openai.com/v1" in warning.call_args.kwargs["extra"]["api_base"]
|
||||
|
||||
|
|
@ -10216,6 +10184,8 @@ def test_get_assembled_streaming_response_bills_a_provider_reported_usage_cost()
|
|||
assert logging_obj.response_cost_calculator(result=assembled) == 0.0042
|
||||
|
||||
|
||||
|
||||
|
||||
def test_response_cost_calculator_prices_terminal_responses_event_from_its_response():
|
||||
logging_obj: Final = _responses_stream_logging_obj()
|
||||
inner_response: Final = ResponsesAPIResponse(
|
||||
|
|
@ -10445,7 +10415,9 @@ def test_standard_logging_retries():
|
|||
]
|
||||
)
|
||||
|
||||
with patch.object(customHandler, "log_failure_event", new=MagicMock()) as mock_client:
|
||||
with patch.object(
|
||||
customHandler, "log_failure_event", new=MagicMock()
|
||||
) as mock_client:
|
||||
try:
|
||||
router.completion(
|
||||
model="gpt-3.5-turbo",
|
||||
|
|
@ -10457,10 +10429,19 @@ def test_standard_logging_retries():
|
|||
pass
|
||||
|
||||
assert mock_client.call_count == 2
|
||||
assert mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"]["trace_id"] is not None
|
||||
assert (
|
||||
mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"]["trace_id"]
|
||||
== mock_client.call_args_list[1].kwargs["kwargs"]["standard_logging_object"]["trace_id"]
|
||||
mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"][
|
||||
"trace_id"
|
||||
]
|
||||
is not None
|
||||
)
|
||||
assert (
|
||||
mock_client.call_args_list[0].kwargs["kwargs"]["standard_logging_object"][
|
||||
"trace_id"
|
||||
]
|
||||
== mock_client.call_args_list[1].kwargs["kwargs"][
|
||||
"standard_logging_object"
|
||||
]["trace_id"]
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -10727,7 +10708,9 @@ def test_signoz_dispatch_requires_an_endpoint(monkeypatch):
|
|||
custom_logger_init_args={},
|
||||
)
|
||||
assert created is None
|
||||
assert not [cb for cb in logging_module._in_memory_loggers if getattr(cb, "callback_name", None) == "signoz"]
|
||||
assert not [
|
||||
cb for cb in logging_module._in_memory_loggers if getattr(cb, "callback_name", None) == "signoz"
|
||||
]
|
||||
finally:
|
||||
logging_module._in_memory_loggers.clear()
|
||||
monkeypatch.delenv("LITELLM_OTEL_V2", raising=False)
|
||||
|
|
@ -10869,11 +10852,7 @@ def test_standard_logging_payload_reports_the_deployment_entry_the_request_was_b
|
|||
}
|
||||
}
|
||||
logging_obj.update_environment_variables(
|
||||
model="gpt-5.4-nano",
|
||||
user="",
|
||||
optional_params={},
|
||||
litellm_params=litellm_params,
|
||||
custom_llm_provider="openai",
|
||||
model="gpt-5.4-nano", user="", optional_params={}, litellm_params=litellm_params, custom_llm_provider="openai"
|
||||
)
|
||||
response: Final = ModelResponse(
|
||||
model="gpt-5.4-nano",
|
||||
|
|
@ -11185,7 +11164,10 @@ def test_enterprise_alerting_loggers_falls_back_without_disabling_callback_contr
|
|||
)
|
||||
logging_module._enterprise_alerting_loggers.cache_clear()
|
||||
for module_name in tuple(sys.modules):
|
||||
if any(module_name == prefix or module_name.startswith(f"{prefix}.") for prefix in module_prefixes):
|
||||
if any(
|
||||
module_name == prefix or module_name.startswith(f"{prefix}.")
|
||||
for prefix in module_prefixes
|
||||
):
|
||||
monkeypatch.delitem(sys.modules, module_name, raising=False)
|
||||
monkeypatch.setattr(sys, "meta_path", [BlockPagerDutyFinder(), *sys.meta_path])
|
||||
|
||||
|
|
@ -11305,16 +11287,13 @@ def _vcr_outcome_gate(request, vcr):
|
|||
yield
|
||||
record_vcr_outcome(request, vcr)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture(loop_scope="function")
|
||||
async def drain_logging_worker(isolate_litellm_state: None) -> AsyncIterator[None]:
|
||||
yield
|
||||
await asyncio.wait_for(GLOBAL_LOGGING_WORKER.flush(), timeout=LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS)
|
||||
|
||||
|
||||
LOGGING_WORKER_DRAIN_TIMEOUT_SECONDS: Final = LOGGING_WORKER_MAX_TIME_PER_COROUTINE + 5.0
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def isolate_litellm_state():
|
||||
"""
|
||||
|
|
@ -11352,7 +11331,6 @@ def isolate_litellm_state():
|
|||
if attr in _DEFAULTS:
|
||||
setattr(litellm, attr, _DEFAULTS[attr])
|
||||
|
||||
|
||||
_LIST_ATTRS = (
|
||||
"callbacks",
|
||||
"success_callback",
|
||||
|
|
@ -11380,7 +11358,6 @@ _SCALAR_ATTRS = (
|
|||
|
||||
_DEFAULTS: dict = {}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def setup_and_teardown():
|
||||
"""
|
||||
|
|
@ -11403,7 +11380,6 @@ def setup_and_teardown():
|
|||
litellm.in_memory_llm_clients_cache.flush_cache()
|
||||
yield
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"response_obj,expected_values",
|
||||
|
|
@ -11451,7 +11427,6 @@ def test_get_usage(response_obj, expected_values):
|
|||
assert usage.completion_tokens == expected_values[1]
|
||||
assert usage.total_tokens == expected_values[2]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_usage_from_image_generation_response():
|
||||
"""
|
||||
|
|
@ -11499,7 +11474,6 @@ def test_get_usage_from_image_generation_response():
|
|||
assert usage.completion_tokens_details.image_tokens == 272
|
||||
assert usage.completion_tokens_details.text_tokens == 100
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_additional_headers():
|
||||
additional_headers = {
|
||||
|
|
@ -11540,12 +11514,10 @@ def test_get_additional_headers():
|
|||
assert additional_logging_headers.get("llm_provider-request-id") == "req_01F6CycZZPSHKRCCctcS1Vto"
|
||||
assert additional_logging_headers.get("llm_provider-anthropic-ratelimit-requests-reset") == "2024-10-29T23:57:40Z"
|
||||
|
||||
|
||||
def all_fields_present(standard_logging_metadata: StandardLoggingMetadata):
|
||||
for field in StandardLoggingMetadata.__annotations__.keys():
|
||||
assert field in standard_logging_metadata
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"metadata_key, metadata_value",
|
||||
|
|
@ -11577,7 +11549,6 @@ def test_get_standard_logging_metadata(metadata_key, metadata_value):
|
|||
# Assert that the specific metadata field is set correctly
|
||||
assert standard_logging_metadata[metadata_key] == metadata_value
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_standard_logging_metadata_user_api_key_hash():
|
||||
valid_hash = "a" * 64 # 64 character string
|
||||
|
|
@ -11585,7 +11556,6 @@ def test_get_standard_logging_metadata_user_api_key_hash():
|
|||
result = StandardLoggingPayloadSetup.get_standard_logging_metadata(metadata)
|
||||
assert result["user_api_key_hash"] == valid_hash
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_standard_logging_metadata_invalid_user_api_key():
|
||||
invalid_hash = "not_a_valid_hash"
|
||||
|
|
@ -11594,7 +11564,6 @@ def test_get_standard_logging_metadata_invalid_user_api_key():
|
|||
all_fields_present(result)
|
||||
assert result["user_api_key_hash"] is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_standard_logging_metadata_non_string_user_api_key():
|
||||
"""Non-string user_api_key should not be set as user_api_key_hash."""
|
||||
|
|
@ -11603,7 +11572,6 @@ def test_get_standard_logging_metadata_non_string_user_api_key():
|
|||
all_fields_present(result)
|
||||
assert result["user_api_key_hash"] is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_standard_logging_metadata_none_user_api_key():
|
||||
"""None user_api_key should not be set as user_api_key_hash."""
|
||||
|
|
@ -11612,7 +11580,6 @@ def test_get_standard_logging_metadata_none_user_api_key():
|
|||
all_fields_present(result)
|
||||
assert result["user_api_key_hash"] is None
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_standard_logging_metadata_invalid_keys():
|
||||
metadata = {
|
||||
|
|
@ -11626,7 +11593,6 @@ def test_get_standard_logging_metadata_invalid_keys():
|
|||
assert "invalid_key" not in result
|
||||
assert "another_invalid_key" not in result
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_cleanup_timestamps():
|
||||
"""Test cleanup_timestamps with different input types"""
|
||||
|
|
@ -11661,7 +11627,6 @@ def test_cleanup_timestamps():
|
|||
with pytest.raises(ValueError, match="start_time is required, got=invalid of type <class 'str'>"):
|
||||
StandardLoggingPayloadSetup.cleanup_timestamps("invalid", end_float, completion_float)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_model_cost_information():
|
||||
"""Test get_model_cost_information with different inputs"""
|
||||
|
|
@ -11694,7 +11659,6 @@ def test_get_model_cost_information():
|
|||
# assert all fields in StandardLoggingModelInformation are present
|
||||
assert all(field in result for field in StandardLoggingModelInformation.__annotations__)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_model_cost_information_custom_pricing_uses_base_model():
|
||||
result = StandardLoggingPayloadSetup.get_model_cost_information(
|
||||
|
|
@ -11706,7 +11670,6 @@ def test_get_model_cost_information_custom_pricing_uses_base_model():
|
|||
assert result["model_map_value"] is not None
|
||||
assert result["model_map_key"] != "invoke_test_claude"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_standard_logging_payload_uses_deployment_when_no_base_model():
|
||||
"""metadata["deployment"] is used for cost-map lookup when base_model is not set."""
|
||||
|
|
@ -11764,7 +11727,6 @@ def test_standard_logging_payload_uses_deployment_when_no_base_model():
|
|||
assert payload["model_map_information"]["model_map_value"] is not None
|
||||
assert payload["model_map_information"]["model_map_key"] != "invoke_test_claude"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_hidden_params():
|
||||
"""Test get_hidden_params with different inputs"""
|
||||
|
|
@ -11800,7 +11762,6 @@ def test_get_hidden_params():
|
|||
# assert all fields in StandardLoggingHiddenParams are present
|
||||
assert all(field in result for field in StandardLoggingHiddenParams.__annotations__)
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_final_response_obj():
|
||||
"""Test get_final_response_obj with different input types and redaction scenarios"""
|
||||
|
|
@ -11832,7 +11793,6 @@ def test_get_final_response_obj():
|
|||
# Reset litellm.turn_off_message_logging to its original value
|
||||
litellm.turn_off_message_logging = False
|
||||
|
||||
|
||||
def testget_standard_logging_payload_trace_id():
|
||||
"""Test get_standard_logging_payload_trace_id with different input scenarios"""
|
||||
# Test case 1: When litellm_trace_id is provided in litellm_params
|
||||
|
|
@ -11870,7 +11830,6 @@ def testget_standard_logging_payload_trace_id():
|
|||
assert result == "12345"
|
||||
assert isinstance(result, str)
|
||||
|
||||
|
||||
def testget_standard_logging_payload_trace_id_prioritizes_trace_id_when_flag_on(monkeypatch):
|
||||
"""With request_correlation_in_logs on, an explicit litellm_trace_id wins over litellm_session_id."""
|
||||
from unittest.mock import MagicMock
|
||||
|
|
@ -11885,7 +11844,6 @@ def testget_standard_logging_payload_trace_id_prioritizes_trace_id_when_flag_on(
|
|||
)
|
||||
assert result == "the-trace-id"
|
||||
|
||||
|
||||
def testget_standard_logging_payload_trace_id_prioritizes_session_id_when_flag_off(monkeypatch):
|
||||
"""With request_correlation_in_logs off (default), legacy behavior is preserved:
|
||||
litellm_session_id still wins over litellm_trace_id."""
|
||||
|
|
@ -11901,7 +11859,6 @@ def testget_standard_logging_payload_trace_id_prioritizes_session_id_when_flag_o
|
|||
)
|
||||
assert result == "the-session-id"
|
||||
|
||||
|
||||
def testget_standard_logging_payload_session_id_when_flag_on(monkeypatch):
|
||||
"""Test get_standard_logging_payload_session_id with different input scenarios, flag enabled"""
|
||||
from unittest.mock import MagicMock
|
||||
|
|
@ -11954,7 +11911,6 @@ def testget_standard_logging_payload_session_id_when_flag_on(monkeypatch):
|
|||
)
|
||||
assert result == ""
|
||||
|
||||
|
||||
def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch):
|
||||
"""When request_correlation_in_logs is off (default), session_id is always empty,
|
||||
even if litellm_session_id was explicitly supplied - preserves the pre-existing
|
||||
|
|
@ -11971,7 +11927,6 @@ def testget_standard_logging_payload_session_id_empty_when_flag_off(monkeypatch)
|
|||
)
|
||||
assert result == ""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def restore_verbose_logger_level() -> Iterator[None]:
|
||||
original_level: Final = verbose_logger.level
|
||||
|
|
@ -12014,7 +11969,6 @@ def test_truncate_standard_logging_payload():
|
|||
assert len(str(truncated["response"])) < 10_500
|
||||
assert len(str(truncated["error_str"])) < 10_500
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_truncate_standard_logging_payload_keeps_a_partial_payload_intact():
|
||||
"""A payload built with only some of its fields comes back with exactly those keys and values"""
|
||||
|
|
@ -12023,14 +11977,12 @@ def test_truncate_standard_logging_payload_keeps_a_partial_payload_intact():
|
|||
|
||||
assert _custom_logger.truncate_standard_logging_payload_content(partial_payload) == partial_payload
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_strip_trailing_slash():
|
||||
common_api_base = "https://api.test.com"
|
||||
assert StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base + "/") == common_api_base
|
||||
assert StandardLoggingPayloadSetup.strip_trailing_slash(common_api_base) == common_api_base
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_error_information():
|
||||
"""Test get_error_information with different types of exceptions"""
|
||||
|
|
@ -12067,7 +12019,6 @@ def test_get_error_information():
|
|||
assert result["llm_provider"] == "openai"
|
||||
assert result["error_message"] == "litellm.RateLimitError: Test error"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_get_response_time():
|
||||
"""Test get_response_time with different streaming scenarios"""
|
||||
|
|
@ -12103,7 +12054,6 @@ def test_get_response_time():
|
|||
# For streaming, should return completion_start_time - start_time
|
||||
assert response_time == 2.0
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"metadata, expected_requester_metadata",
|
||||
|
|
@ -12126,7 +12076,6 @@ def test_standard_logging_metadata_requester_metadata(metadata, expected_request
|
|||
result = StandardLoggingPayloadSetup.get_standard_logging_metadata(metadata)
|
||||
assert result["requester_metadata"] == expected_requester_metadata
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_cost_breakdown_in_standard_logging_payload():
|
||||
"""
|
||||
|
|
@ -12216,7 +12165,6 @@ def test_cost_breakdown_in_standard_logging_payload():
|
|||
|
||||
print("✅ Cost breakdown test passed!")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_cost_breakdown_missing_in_standard_logging_payload():
|
||||
"""
|
||||
|
|
@ -12277,7 +12225,6 @@ def test_cost_breakdown_missing_in_standard_logging_payload():
|
|||
|
||||
print("✅ Cost breakdown missing test passed!")
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
@pytest.mark.parametrize(
|
||||
"use_combined_usage_object",
|
||||
|
|
@ -12365,7 +12312,6 @@ def test_usage_dict_roundtrip_in_payload(use_combined_usage_object):
|
|||
assert usage_obj["completion_tokens"] == 58
|
||||
assert usage_obj["total_tokens"] == 100
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_standard_logging_payload_uses_actual_model_for_azure_router():
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -12414,7 +12360,6 @@ def test_standard_logging_payload_uses_actual_model_for_azure_router():
|
|||
assert payload is not None
|
||||
assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_standard_logging_payload_uses_actual_model_for_azure_router_with_underscore():
|
||||
from litellm.litellm_core_utils.litellm_logging import (
|
||||
|
|
@ -12463,7 +12408,6 @@ def test_standard_logging_payload_uses_actual_model_for_azure_router_with_unders
|
|||
assert payload is not None
|
||||
assert payload["model"] == "azure_ai/gpt-5-nano-2025-08-07"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_merge_litellm_metadata_basic():
|
||||
"""
|
||||
|
|
@ -12495,7 +12439,6 @@ def test_merge_litellm_metadata_basic():
|
|||
assert result["model_info"] == {"id": "model-123"}
|
||||
assert result["tags"] == ["tag1", "tag2"]
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_merge_litellm_metadata_precedence():
|
||||
"""
|
||||
|
|
@ -12522,7 +12465,6 @@ def test_merge_litellm_metadata_precedence():
|
|||
# litellm_metadata values should only be included if not in metadata
|
||||
assert result["model_group"] == "gpt-4-group"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_merge_litellm_metadata_skip_non_serializable():
|
||||
"""
|
||||
|
|
@ -12555,7 +12497,6 @@ def test_merge_litellm_metadata_skip_non_serializable():
|
|||
assert result["safe_field"] == "safe_value"
|
||||
assert result["model_group"] == "gpt-4-group"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_merge_litellm_metadata_empty_params():
|
||||
"""
|
||||
|
|
@ -12591,7 +12532,6 @@ def test_merge_litellm_metadata_empty_params():
|
|||
result = StandardLoggingPayloadSetup.merge_litellm_metadata(litellm_params)
|
||||
assert result == {}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_merge_litellm_metadata_bedrock_passthrough_scenario():
|
||||
"""
|
||||
|
|
@ -12651,10 +12591,8 @@ def test_merge_litellm_metadata_bedrock_passthrough_scenario():
|
|||
# Verify total number of fields (9 user fields + 4 model fields = 13)
|
||||
assert len(result) == 13
|
||||
|
||||
|
||||
service_logger = ServiceLogging()
|
||||
|
||||
|
||||
def setup_logging():
|
||||
return Logging(
|
||||
model="gpt-5.5",
|
||||
|
|
@ -12744,7 +12682,6 @@ def test_get_callback_name():
|
|||
# Test string callback
|
||||
assert logging._get_callback_name("callback_string") == "callback_string"
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_is_internal_litellm_proxy_callback():
|
||||
"""
|
||||
|
|
@ -12765,7 +12702,6 @@ def test_is_internal_litellm_proxy_callback():
|
|||
# Test string callback
|
||||
assert logging._is_internal_litellm_proxy_callback("callback_string") == False
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_should_run_sync_callbacks_for_async_calls():
|
||||
"""
|
||||
|
|
@ -12791,7 +12727,6 @@ def test_should_run_sync_callbacks_for_async_calls():
|
|||
litellm.success_callback = [PROXY_MaxIterationsHandler]
|
||||
assert logging._should_run_sync_callbacks_for_async_calls() == False
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("_vcr_outcome_gate", "drain_logging_worker", "isolate_litellm_state", "setup_and_teardown")
|
||||
def test_remove_internal_litellm_callbacks():
|
||||
logging = setup_logging()
|
||||
|
|
@ -12813,7 +12748,6 @@ def test_remove_internal_litellm_callbacks():
|
|||
assert PROXY_MaxIterationsHandler not in filtered
|
||||
assert PROXY_CacheControlCheck not in filtered
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_background_interaction_completion_logs_while_in_progress_handler_is_parked(monkeypatch):
|
||||
"""
|
||||
|
|
@ -12916,9 +12850,14 @@ class CompletionCustomHandler(
|
|||
assert metadata_value is None or isinstance(metadata_value, dict)
|
||||
if metadata_value is not None:
|
||||
if litellm.turn_off_message_logging is True:
|
||||
assert metadata_value["raw_request"] is LiteLLMCommonStrings.redacted_by_litellm.value
|
||||
assert (
|
||||
metadata_value["raw_request"]
|
||||
is LiteLLMCommonStrings.redacted_by_litellm.value
|
||||
)
|
||||
else:
|
||||
assert "raw_request" not in metadata_value or isinstance(metadata_value["raw_request"], str)
|
||||
assert "raw_request" not in metadata_value or isinstance(
|
||||
metadata_value["raw_request"], str
|
||||
)
|
||||
except Exception:
|
||||
print(f"Assertion Error: {traceback.format_exc()}")
|
||||
self.errors.append(traceback.format_exc())
|
||||
|
|
@ -12967,18 +12906,23 @@ class CompletionCustomHandler(
|
|||
assert isinstance(response_obj, litellm.ModelResponseStream)
|
||||
## KWARGS
|
||||
assert isinstance(kwargs["model"], str)
|
||||
assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict)
|
||||
assert isinstance(kwargs["messages"], list) and isinstance(
|
||||
kwargs["messages"][0], dict
|
||||
)
|
||||
assert isinstance(kwargs["optional_params"], dict)
|
||||
assert isinstance(kwargs["litellm_params"], dict)
|
||||
assert isinstance(kwargs["start_time"], (datetime, type(None)))
|
||||
assert isinstance(kwargs["stream"], bool)
|
||||
assert isinstance(kwargs["user"], (str, type(None)))
|
||||
assert (isinstance(kwargs["input"], list) and isinstance(kwargs["input"][0], dict)) or isinstance(
|
||||
kwargs["input"], (dict, str)
|
||||
)
|
||||
assert (
|
||||
isinstance(kwargs["input"], list)
|
||||
and isinstance(kwargs["input"][0], dict)
|
||||
) or isinstance(kwargs["input"], (dict, str))
|
||||
assert isinstance(kwargs["api_key"], (str, type(None)))
|
||||
assert (
|
||||
isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper))
|
||||
isinstance(
|
||||
kwargs["original_response"], (str, litellm.CustomStreamWrapper)
|
||||
)
|
||||
or inspect.isasyncgen(kwargs["original_response"])
|
||||
or inspect.iscoroutine(kwargs["original_response"])
|
||||
)
|
||||
|
|
@ -13011,7 +12955,9 @@ class CompletionCustomHandler(
|
|||
)
|
||||
## KWARGS
|
||||
assert isinstance(kwargs["model"], str)
|
||||
assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict)
|
||||
assert isinstance(kwargs["messages"], list) and isinstance(
|
||||
kwargs["messages"][0], dict
|
||||
)
|
||||
assert isinstance(kwargs["optional_params"], dict)
|
||||
assert isinstance(kwargs["litellm_params"], dict)
|
||||
assert isinstance(kwargs["litellm_params"]["api_base"], str)
|
||||
|
|
@ -13021,7 +12967,10 @@ class CompletionCustomHandler(
|
|||
assert isinstance(kwargs["user"], (str, type(None)))
|
||||
assert (
|
||||
isinstance(kwargs["input"], list)
|
||||
and (isinstance(kwargs["input"][0], dict) or isinstance(kwargs["input"][0], str))
|
||||
and (
|
||||
isinstance(kwargs["input"][0], dict)
|
||||
or isinstance(kwargs["input"][0], str)
|
||||
)
|
||||
) or isinstance(kwargs["input"], (dict, str))
|
||||
assert isinstance(kwargs["api_key"], (str, type(None)))
|
||||
assert isinstance(
|
||||
|
|
@ -13049,7 +12998,9 @@ class CompletionCustomHandler(
|
|||
assert response_obj == None
|
||||
## KWARGS
|
||||
assert isinstance(kwargs["model"], str)
|
||||
assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict)
|
||||
assert isinstance(kwargs["messages"], list) and isinstance(
|
||||
kwargs["messages"][0], dict
|
||||
)
|
||||
|
||||
assert isinstance(kwargs["optional_params"], dict)
|
||||
assert isinstance(kwargs["litellm_params"], dict)
|
||||
|
|
@ -13057,12 +13008,15 @@ class CompletionCustomHandler(
|
|||
assert isinstance(kwargs["start_time"], (datetime, type(None)))
|
||||
assert isinstance(kwargs["stream"], bool)
|
||||
assert isinstance(kwargs["user"], (str, type(None)))
|
||||
assert (isinstance(kwargs["input"], list) and isinstance(kwargs["input"][0], dict)) or isinstance(
|
||||
kwargs["input"], (dict, str)
|
||||
)
|
||||
assert (
|
||||
isinstance(kwargs["input"], list)
|
||||
and isinstance(kwargs["input"][0], dict)
|
||||
) or isinstance(kwargs["input"], (dict, str))
|
||||
assert isinstance(kwargs["api_key"], (str, type(None)))
|
||||
assert (
|
||||
isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper))
|
||||
isinstance(
|
||||
kwargs["original_response"], (str, litellm.CustomStreamWrapper)
|
||||
)
|
||||
or kwargs["original_response"] == None
|
||||
)
|
||||
assert isinstance(kwargs["additional_args"], (dict, type(None)))
|
||||
|
|
@ -13080,7 +13034,9 @@ class CompletionCustomHandler(
|
|||
assert isinstance(messages, list) and isinstance(messages[0], dict)
|
||||
## KWARGS
|
||||
assert isinstance(kwargs["model"], str)
|
||||
assert isinstance(kwargs["messages"], list) and isinstance(kwargs["messages"][0], dict)
|
||||
assert isinstance(kwargs["messages"], list) and isinstance(
|
||||
kwargs["messages"][0], dict
|
||||
)
|
||||
assert isinstance(kwargs["optional_params"], dict)
|
||||
assert isinstance(kwargs["litellm_params"], dict)
|
||||
assert isinstance(kwargs["start_time"], (datetime, type(None)))
|
||||
|
|
@ -13092,7 +13048,9 @@ class CompletionCustomHandler(
|
|||
|
||||
async def async_log_success_event(self, kwargs, response_obj, start_time, end_time):
|
||||
try:
|
||||
print("in async_log_success_event", kwargs, response_obj, start_time, end_time)
|
||||
print(
|
||||
"in async_log_success_event", kwargs, response_obj, start_time, end_time
|
||||
)
|
||||
self.states.append("async_success")
|
||||
## START TIME
|
||||
assert isinstance(start_time, datetime)
|
||||
|
|
@ -13121,7 +13079,9 @@ class CompletionCustomHandler(
|
|||
assert isinstance(kwargs["input"], (list, dict, str))
|
||||
assert isinstance(kwargs["api_key"], (str, type(None)))
|
||||
assert (
|
||||
isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper))
|
||||
isinstance(
|
||||
kwargs["original_response"], (str, litellm.CustomStreamWrapper)
|
||||
)
|
||||
or inspect.isasyncgen(kwargs["original_response"])
|
||||
or inspect.iscoroutine(kwargs["original_response"])
|
||||
)
|
||||
|
|
@ -13153,7 +13113,9 @@ class CompletionCustomHandler(
|
|||
assert isinstance(kwargs["input"], (list, str, dict))
|
||||
assert isinstance(kwargs["api_key"], (str, type(None)))
|
||||
assert (
|
||||
isinstance(kwargs["original_response"], (str, litellm.CustomStreamWrapper))
|
||||
isinstance(
|
||||
kwargs["original_response"], (str, litellm.CustomStreamWrapper)
|
||||
)
|
||||
or inspect.isasyncgen(kwargs["original_response"])
|
||||
or inspect.iscoroutine(kwargs["original_response"])
|
||||
or kwargs["original_response"] == None
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@ def _provider_fields() -> dict[str, object]:
|
|||
"thinking_blocks": [{"type": "thinking", "thinking": SECRET}],
|
||||
"reasoningContentBlocks": [{"reasoningText": {"text": SECRET}}],
|
||||
"citations": [{"cited_text": SECRET}],
|
||||
"compaction_blocks": [{"type": "compaction", "content": SECRET, "signature": "compaction-signature"}],
|
||||
"thought_signatures": [SIGNATURE],
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -4529,7 +4529,7 @@ _SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "summary so
|
|||
|
||||
@pytest.mark.parametrize(
|
||||
"provider,expected",
|
||||
[("anthropic", True), ("vertex_ai", True), ("azure_ai", False), ("bedrock", False), ("bedrock_converse", False)],
|
||||
[("anthropic", True), ("vertex_ai", True), ("azure_ai", True), ("bedrock", False), ("bedrock_converse", False)],
|
||||
)
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params,messages",
|
||||
|
|
@ -4547,7 +4547,7 @@ def test_requires_native_compaction_beta_per_provider(
|
|||
assert requires_native_compaction_beta(provider, optional_params, messages) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize("provider", ["anthropic", "vertex_ai"])
|
||||
@pytest.mark.parametrize("provider", ["anthropic", "azure_ai", "vertex_ai"])
|
||||
@pytest.mark.parametrize(
|
||||
"optional_params,messages",
|
||||
[
|
||||
|
|
|
|||
|
|
@ -215,3 +215,36 @@ async def test_count_uses_the_shared_api_base_chat_would_use(monkeypatch: pytest
|
|||
assert result is not None
|
||||
assert result.total_tokens == 6
|
||||
assert route.called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
("assistant_content", "expected_beta"),
|
||||
[
|
||||
(
|
||||
[{"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig-abc"}],
|
||||
"compact-2026-09-04,token-counting-2024-11-01",
|
||||
),
|
||||
("Use Recipe and Step tables.", "token-counting-2024-11-01"),
|
||||
],
|
||||
ids=("signed_compaction_block", "plain_turn"),
|
||||
)
|
||||
async def test_count_sends_the_compact_beta_only_when_the_history_replays_a_compaction_block(
|
||||
monkeypatch: pytest.MonkeyPatch, assistant_content: object, expected_beta: str
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "disable_aiohttp_transport", True)
|
||||
with respx.mock:
|
||||
route: Final = respx.post(COUNT_URL).mock(return_value=httpx.Response(200, json={"input_tokens": 376}))
|
||||
result: Final = await AzureAIAnthropicTokenCounter().count_tokens(
|
||||
model_to_use="claude-sonnet-5-5",
|
||||
messages=[
|
||||
{"role": "assistant", "content": assistant_content},
|
||||
{"role": "user", "content": "What tables did we pick?"},
|
||||
],
|
||||
contents=None,
|
||||
deployment={"litellm_params": {"api_key": "azure-key", "api_base": API_BASE}},
|
||||
request_model="azure_ai/claude-sonnet-5-5",
|
||||
)
|
||||
assert result is not None
|
||||
assert result.total_tokens == 376
|
||||
assert route.calls.last.request.headers["anthropic-beta"] == expected_beta
|
||||
|
|
|
|||
|
|
@ -596,3 +596,49 @@ async def test_outgoing_azure_request_keeps_an_object_output_format_under_drop_p
|
|||
body: Final = await _azure_request_body_for_output_format(output_format, drop_params=True)
|
||||
|
||||
assert body["output_format"] == output_format
|
||||
|
||||
|
||||
_SIGNED_COMPACTION_BLOCK: Final = {"type": "compaction", "content": "Summary of the chat so far.", "signature": "sig"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.usefixtures("local_beta_headers_config")
|
||||
@pytest.mark.parametrize(
|
||||
("messages", "params"),
|
||||
[
|
||||
([{"role": "user", "content": "Summarize the chat so far."}], {"compaction": {"type": "summarize"}}),
|
||||
(
|
||||
[{"role": "assistant", "content": [_SIGNED_COMPACTION_BLOCK]}, {"role": "user", "content": "Go on"}],
|
||||
{},
|
||||
),
|
||||
],
|
||||
ids=["compaction_param", "signed_compaction_block"],
|
||||
)
|
||||
async def test_outgoing_azure_compaction_request_carries_the_compact_2026_09_04_beta(
|
||||
messages: list[object], params: dict[str, object]
|
||||
) -> None:
|
||||
"""Foundry answers 400 "compaction: this parameter requires anthropic-beta: compact-2026-09-04" for the param
|
||||
and 400 "Input tag 'compaction' ... does not match" for a replayed block without the header, and 200 with it
|
||||
(live, claude-sonnet-5-5 on Azure AI Foundry, 2026-10-20)."""
|
||||
received: Final[list[httpx.Request]] = []
|
||||
|
||||
def foundry_answers(request: httpx.Request) -> httpx.Response:
|
||||
received.append(request)
|
||||
return _foundry_answer(request)
|
||||
|
||||
upstream: Final = AsyncHTTPHandler(transport=httpx.MockTransport(foundry_answers))
|
||||
try:
|
||||
await litellm.anthropic.messages.acreate(
|
||||
max_tokens=256,
|
||||
messages=messages,
|
||||
model="azure_ai/claude-sonnet-5-5",
|
||||
api_key="test-api-key",
|
||||
api_base="https://test-resource.services.ai.azure.com",
|
||||
client=upstream,
|
||||
**params,
|
||||
)
|
||||
finally:
|
||||
await upstream.close()
|
||||
|
||||
assert len(received) == 1, received
|
||||
assert received[0].headers["anthropic-beta"].split(",").count("compact-2026-09-04") == 1, dict(received[0].headers)
|
||||
|
|
|
|||
|
|
@ -5,10 +5,12 @@ sys.path.insert(
|
|||
0, os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../.."))
|
||||
)
|
||||
|
||||
from typing import Final
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
from litellm.anthropic_beta_headers_manager import update_headers_with_filtered_beta
|
||||
from litellm.llms.azure_ai.anthropic.transformation import AzureAnthropicConfig
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
|
|
@ -484,3 +486,31 @@ def test_chat_flagged_model_keeps_mid_conversation_system_role_in_place(local_mo
|
|||
"role": "system",
|
||||
"content": [{"type": "text", "text": "<system-reminder>Answer with exactly one word.</system-reminder>"}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("optional_params", "messages", "expected"),
|
||||
[
|
||||
({"compaction": {"type": "summarize"}}, [{"role": "user", "content": "Hello"}], True),
|
||||
(
|
||||
{},
|
||||
[
|
||||
{"role": "assistant", "content": [{"type": "compaction", "content": "Summary.", "signature": "sig"}]},
|
||||
{"role": "user", "content": "Go on"},
|
||||
],
|
||||
True,
|
||||
),
|
||||
({}, [{"role": "user", "content": "Hello"}], False),
|
||||
],
|
||||
ids=["compaction_param", "signed_compaction_block", "no_compaction"],
|
||||
)
|
||||
def test_azure_chat_compaction_sends_the_compact_2026_09_04_beta(
|
||||
local_beta_headers_config: None, optional_params: dict[str, object], messages: list[object], expected: bool
|
||||
) -> None:
|
||||
headers: Final = AzureAnthropicConfig().update_headers_with_optional_anthropic_beta(
|
||||
headers={}, optional_params=optional_params, messages=messages
|
||||
)
|
||||
|
||||
filtered: Final = update_headers_with_filtered_beta(headers=headers, provider="azure_ai")
|
||||
|
||||
assert ("compact-2026-09-04" in filtered.get("anthropic-beta", "").split(",")) is expected
|
||||
|
|
|
|||
|
|
@ -1307,21 +1307,29 @@ def test_vertex_ai_anthropic_client_sent_body_header_only_beta_moves_to_the_head
|
|||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"client_beta,header",
|
||||
"client_beta,body,header",
|
||||
[
|
||||
("context-1m-2025-08-07", "context-management-2025-06-27"),
|
||||
("inline-tools-2026-09-15", "context-management-2025-06-27,inline-tools-2026-09-15"),
|
||||
(
|
||||
"context-1m-2025-08-07",
|
||||
["context-1m-2025-08-07", "context-management-2025-06-27"],
|
||||
"context-management-2025-06-27",
|
||||
),
|
||||
(
|
||||
"inline-tools-2026-09-15",
|
||||
["context-management-2025-06-27"],
|
||||
"context-management-2025-06-27,inline-tools-2026-09-15",
|
||||
),
|
||||
],
|
||||
ids=["body_beta", "header_only_beta"],
|
||||
)
|
||||
def test_vertex_ai_anthropic_litellm_betas_replace_the_client_body_betas(
|
||||
local_beta_headers_config: None, client_beta: str, header: str
|
||||
def test_vertex_ai_anthropic_litellm_betas_merge_with_the_client_body_betas(
|
||||
local_beta_headers_config: None, client_beta: str, body: list[str], header: str
|
||||
) -> None:
|
||||
headers: Final[dict[str, str]] = {}
|
||||
|
||||
result: Final = _vertex_request({"context_management": _CLEAR_TOOL_USES, "anthropic_beta": [client_beta]}, headers)
|
||||
|
||||
assert result["anthropic_beta"] == ["context-management-2025-06-27"]
|
||||
assert result["anthropic_beta"] == body
|
||||
assert headers["anthropic-beta"] == header
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -83,7 +83,7 @@ class TestAnthropicBetaHeadersFiltering:
|
|||
filtered = filter_and_transform_beta_headers(
|
||||
beta_headers=all_headers, provider=provider
|
||||
)
|
||||
assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "vertex_ai"))
|
||||
assert ("compact-2026-09-04" in filtered) is (provider in ("anthropic", "azure_ai", "vertex_ai"))
|
||||
|
||||
for header in unsupported_headers:
|
||||
assert (
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue