mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-17 04:34:25 +00:00
d18fcb09d6
* fix(otel): detach post-response service spans by request phase, name redis spans by operation Service spans logged from the post-response phase (success callbacks, the response-cache write) now root their own trace linked to the request span even while the server span is still recording, instead of only when they happen to end after it. Redis service spans are named `redis <operation>`; the litellm call chain that issued them moves to the `litellm.service.caller` attribute via a typed `ServiceLoggerPayload.caller` field. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): keep the service caller on failure and legacy spans, test the production phase dispatch sites Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): mark anthropic messages stream cache write as post-response phase The /v1/messages streaming cache writer awaits async_add_cache inline instead of going through create_cache_write_task, so its redis span stayed parented under the request trace. Wrap the write in post_response_phase so it detaches like the chat completions write. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(anthropic): write the Messages stream cache in a background task after handoff Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: yassin <yassin@berri.ai> Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
54 lines
2 KiB
Python
54 lines
2 KiB
Python
"""
|
|
Internal request context for LiteLLM.
|
|
|
|
Provides a ContextVar-based mechanism for internal signals that must not
|
|
be settable from user input. Context variables are scoped to the current
|
|
asyncio task and cannot be injected via HTTP request bodies.
|
|
"""
|
|
|
|
from collections.abc import Generator
|
|
from contextlib import contextmanager
|
|
from contextvars import ContextVar
|
|
from datetime import datetime, timezone
|
|
from typing import Final
|
|
|
|
# When True, suppresses async logging and billing for internal sub-calls
|
|
# (e.g., emulated file-search steps that make nested LLM calls).
|
|
is_internal_call: Final[ContextVar[bool]] = ContextVar("is_internal_call", default=False)
|
|
|
|
# One request prices its totals, its per-token-type lines and the rates it reports on
|
|
# separate code paths. Each reads the clock for off-peak pricing, so without a pinned
|
|
# moment they can land on either side of a window boundary and disagree with each other.
|
|
_billing_time: Final[ContextVar[datetime | None]] = ContextVar("billing_time", default=None)
|
|
|
|
_post_response: Final[ContextVar[bool]] = ContextVar("post_response", default=False)
|
|
|
|
|
|
@contextmanager
|
|
def post_response_phase() -> Generator[None]:
|
|
"""Work the caller no longer waits for (success callbacks, response-cache writes), including tasks it spawns."""
|
|
token: Final = _post_response.set(True)
|
|
try:
|
|
yield
|
|
finally:
|
|
_post_response.reset(token)
|
|
|
|
|
|
def in_post_response_phase() -> bool:
|
|
return _post_response.get()
|
|
|
|
|
|
@contextmanager
|
|
def pinned_billing_time(moment: datetime) -> Generator[None]:
|
|
"""Price every rate lookup inside this block at ``moment`` rather than at each one's own clock read."""
|
|
token: Final = _billing_time.set(moment)
|
|
try:
|
|
yield
|
|
finally:
|
|
_billing_time.reset(token)
|
|
|
|
|
|
def current_billing_time() -> datetime:
|
|
"""The pinned billing moment, or now in UTC outside a pinned block."""
|
|
pinned: Final = _billing_time.get()
|
|
return pinned if pinned is not None else datetime.now(timezone.utc)
|