mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-22 10:40:46 +00:00
fix(otel): decide per-team Arize sampling from the trace id so every pod agrees
This commit is contained in:
parent
ca0ec627b6
commit
14bd4649f7
4 changed files with 357 additions and 61 deletions
|
|
@ -42,8 +42,8 @@ class OtelDestination(LiteLLMBaseModel):
|
|||
success_sampling_rate: float | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"Share of the request trees forwarded, 0.0..1.0, drawn once per request when its root span ends; "
|
||||
"``None`` forwards every one."
|
||||
"Share of the request trees forwarded, 0.0..1.0, decided from the trace id so every process "
|
||||
"serving the trace agrees; ``None`` forwards every one."
|
||||
),
|
||||
)
|
||||
error_sampling_rate: float | None = Field(
|
||||
|
|
|
|||
|
|
@ -1,11 +1,11 @@
|
|||
"""Provider / exporter factory + the Baggage span processor."""
|
||||
|
||||
import queue
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from hashlib import sha256
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, Literal
|
||||
|
||||
|
|
@ -512,9 +512,9 @@ def _is_sampled(destination: "OtelDestination") -> bool:
|
|||
return destination.success_sampling_rate is not None or destination.error_sampling_rate is not None
|
||||
|
||||
|
||||
#: A verdict is one request tree's draw for one destination at its rates: the exporter
|
||||
#: A verdict is one trace's keep or drop for one destination at its rates: the exporter
|
||||
#: ``cache_key`` leaves the rates out, so two destinations sharing an exporter with
|
||||
#: different rates would otherwise share one draw.
|
||||
#: different rates would otherwise share one verdict.
|
||||
_VerdictKey = tuple[int, object, float | None, float | None]
|
||||
|
||||
|
||||
|
|
@ -522,15 +522,19 @@ def _verdict_key(trace_id: int, destination: "OtelDestination") -> _VerdictKey:
|
|||
return (trace_id, destination.cache_key(), destination.success_sampling_rate, destination.error_sampling_rate)
|
||||
|
||||
|
||||
def _keeps_tree(destination: "OtelDestination", failed: bool, draw: Callable[[], float]) -> bool:
|
||||
"""Whether one draw against the destination's rate keeps a request tree, read the
|
||||
way the legacy Arize callback reads ``arize_success_sampling_rate`` /
|
||||
``arize_error_sampling_rate``: a tree with a failed span answers to the error
|
||||
rate, an unset rate keeps everything and ``0.0`` keeps nothing."""
|
||||
def _keeps_tree(destination: "OtelDestination", failed: bool, trace_id: int) -> bool:
|
||||
"""Whether the destination's rate keeps a request tree of ``trace_id``: a tree with
|
||||
a failed span answers to ``arize_error_sampling_rate``, the rest to
|
||||
``arize_success_sampling_rate``, an unset rate keeps everything and ``0.0`` keeps
|
||||
nothing.
|
||||
|
||||
The first eight bytes of a SHA-256 of the trace id are read against the rate, so
|
||||
every process serving a span of the trace reaches the same verdict."""
|
||||
rate: Final = destination.error_sampling_rate if failed else destination.success_sampling_rate
|
||||
if rate is None:
|
||||
return True
|
||||
return rate > 0.0 and draw() <= rate
|
||||
digest: Final = sha256(trace_id.to_bytes(16, "big")).digest()
|
||||
return int.from_bytes(digest[:8], "big") < round(rate * 2**64)
|
||||
|
||||
|
||||
def _in_scope(span: ReadableSpan, scope: "OtelSpanScope") -> bool:
|
||||
|
|
@ -667,10 +671,8 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
default_capture_by_backend: Mapping[str, str] = MappingProxyType({}),
|
||||
pending_drains: int = _MAX_PENDING_DRAINS,
|
||||
drain_pool: _DrainPool | None = None,
|
||||
sampling_draw: Callable[[], float] | None = None,
|
||||
) -> None:
|
||||
self._operator_sinks: Final = operator_sinks
|
||||
self._draw: Final = sampling_draw if sampling_draw is not None else random.random
|
||||
self._excluded_db_systems: Final = excluded_db_systems
|
||||
self._default_capture: Final = default_capture
|
||||
self._default_capture_by_backend: Final = default_capture_by_backend
|
||||
|
|
@ -750,7 +752,7 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
tree is held for the rest of its trace, or what the bounds made the fan-out decide
|
||||
early (a tree that outgrew its cap is decided on what it has, and so is the oldest
|
||||
tree once too many are waiting, so a span that never ends holds nothing back for
|
||||
ever). A failed span marks its tree before any such early draw. Once shutdown
|
||||
ever). A failed span marks its tree before any such early decision. Once shutdown
|
||||
has started flushing, nothing is held: a span ending then is decided at once,
|
||||
since no later flush would reach it.
|
||||
|
||||
|
|
@ -795,16 +797,16 @@ class TenantFanOutSpanProcessor(SpanProcessor):
|
|||
return self._decide_locked(trace_id)
|
||||
|
||||
def _decide_locked(self, trace_id: int) -> "tuple[tuple[ReadableSpan, OtelDestination], ...]":
|
||||
"""Draw once per destination for the held tree; the spans the draws keep, and a
|
||||
verdict remembered for the spans of this trace that start later."""
|
||||
"""Decide once per destination for the held tree; the spans the verdicts keep, and
|
||||
a verdict remembered for the spans of this trace that start later."""
|
||||
tree: Final = self._pending.pop(trace_id, None)
|
||||
if tree is None:
|
||||
return ()
|
||||
verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one draw per destination of this tree
|
||||
verdicts: dict[_VerdictKey, bool] = {} # mutable-ok: one verdict per destination of this tree
|
||||
for _, destination in tree.held:
|
||||
key = _verdict_key(trace_id, destination)
|
||||
if key not in verdicts:
|
||||
verdicts[key] = _keeps_tree(destination, tree.failed, self._draw)
|
||||
verdicts[key] = _keeps_tree(destination, tree.failed, trace_id)
|
||||
self._verdicts[key] = verdicts[key]
|
||||
while len(self._verdicts) > _MAX_REMEMBERED_VERDICTS:
|
||||
self._verdicts.popitem(last=False)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,248 @@
|
|||
"""A team's Arize sampling rate decides once per trace, so pods serving calls of one trace agree.
|
||||
|
||||
Two proxies share one database, as two pods behind a load balancer do. Each trace sends one call
|
||||
to each pod under the caller's ``traceparent``; the team's Arize space must receive both calls of a
|
||||
trace or neither.
|
||||
"""
|
||||
|
||||
import re
|
||||
import uuid
|
||||
from collections.abc import Iterator, Mapping
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, replace
|
||||
from pathlib import Path
|
||||
from typing import Final
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
from integration._support.client import (
|
||||
Gateway,
|
||||
Scenario,
|
||||
eventually,
|
||||
gateway_from_environment,
|
||||
object_value,
|
||||
until_asserted,
|
||||
)
|
||||
from integration._support.database import scratch_database
|
||||
from integration._support.otlp_sink import Span, SpanSinks, recorded_requests, recorded_spans
|
||||
from integration._support.process import owned_proxy_process
|
||||
from integration._support.wire import Reply, Request, Wire, wire_server
|
||||
from pydantic import JsonValue
|
||||
|
||||
MODEL: Final = "team-sampling"
|
||||
SERVER: Final = 2
|
||||
CALL_ID: Final = "litellm.call_id"
|
||||
OPERATION: Final = "gen_ai.operation.name"
|
||||
SPACE: Final = "space-sampled"
|
||||
CREDENTIALS: Final = {"arize_space_key": SPACE, "arize_api_key": "arize-sampled"}
|
||||
FAILING: Final = re.compile(rb"sampling-fail")
|
||||
TRACES_PER_CASE: Final = 16
|
||||
|
||||
|
||||
def _upstream(request: Request) -> Reply:
|
||||
if FAILING.search(request.body) is not None:
|
||||
return Reply(status=500, body=b'{"error":{"message":"scripted upstream failure","type":"server_error"}}')
|
||||
return Reply(
|
||||
body=(
|
||||
b'{"id":"chatcmpl-sampling","object":"chat.completion","created":1,"model":"gpt-4o-mini",'
|
||||
b'"choices":[{"index":0,"message":{"role":"assistant","content":"ok"},"finish_reason":"stop"}],'
|
||||
b'"usage":{"prompt_tokens":7,"completion_tokens":2,"total_tokens":9}}'
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Sent:
|
||||
trace_id: str
|
||||
parent_span_id: str
|
||||
call_id: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Pods:
|
||||
first: Gateway
|
||||
second: Gateway
|
||||
scenario: Scenario
|
||||
sinks: SpanSinks
|
||||
|
||||
def team_key(self, variables: Mapping[str, str]) -> str:
|
||||
team: Final = self.scenario.team()
|
||||
attached: Final = self.first.request(
|
||||
"POST", f"/team/{team}/callback", {"callback_name": "arize", "callback_vars": dict(variables)}
|
||||
)
|
||||
assert attached.status_code == 200, attached.text
|
||||
return self.scenario.key(team_id=team)
|
||||
|
||||
def send(self, pod: Gateway, key: str, trace_id: str, content: str = "hello") -> Sent:
|
||||
parent: Final = uuid.uuid4().hex[:16]
|
||||
call_id: Final = str(uuid.uuid4())
|
||||
body: Final[dict[str, JsonValue]] = {"model": MODEL, "messages": [{"role": "user", "content": content}]}
|
||||
response: Final = pod.client.post(
|
||||
"/v1/chat/completions",
|
||||
json=body,
|
||||
headers={
|
||||
"Authorization": f"Bearer {key}",
|
||||
"traceparent": f"00-{trace_id}-{parent}-01",
|
||||
"x-litellm-call-id": call_id,
|
||||
},
|
||||
)
|
||||
assert response.status_code == (500 if FAILING.search(content.encode()) else 200), response.text
|
||||
return Sent(trace_id, parent, call_id)
|
||||
|
||||
|
||||
def _space_span_ids(url: str) -> frozenset[str]:
|
||||
"""Span ids the Arize sink received in export requests sent with the team's space id."""
|
||||
return frozenset(
|
||||
span_id
|
||||
for request in recorded_requests(url)
|
||||
if {name.lower(): value for name, value in object_value(request.get("headers") or {}).items()}.get(
|
||||
"arize-space-id"
|
||||
)
|
||||
== SPACE
|
||||
for span_id in request.get("span_ids") or ()
|
||||
if isinstance(span_id, str)
|
||||
)
|
||||
|
||||
|
||||
def _roots(spans: tuple[Span, ...], sent: tuple[Sent, ...]) -> frozenset[str]:
|
||||
"""Call ids of ``sent`` whose server span is among ``spans``."""
|
||||
by_parent: Final = {(call.trace_id, call.parent_span_id): call.call_id for call in sent}
|
||||
return frozenset(
|
||||
by_parent[(span["trace_id"], span["parent_span_id"])]
|
||||
for span in spans
|
||||
if span["kind"] == SERVER and (span["trace_id"], span["parent_span_id"]) in by_parent
|
||||
)
|
||||
|
||||
|
||||
def _team_roots(pods: Pods, since: int, sent: tuple[Sent, ...]) -> frozenset[str]:
|
||||
owned: Final = _space_span_ids(pods.sinks.arize)
|
||||
spans: Final = recorded_spans(pods.sinks.arize, since)[1]
|
||||
return _roots(tuple(span for span in spans if span["span_id"] in owned), sent)
|
||||
|
||||
|
||||
def _logged(pods: Pods, since: int, sent: tuple[Sent, ...]) -> frozenset[str]:
|
||||
"""Call ids of ``sent`` whose model span reached the operator's unsampled copy."""
|
||||
spans: Final = recorded_spans(pods.sinks.operator, since)[1]
|
||||
return frozenset(
|
||||
str(span["attributes"][CALL_ID])
|
||||
for span in spans
|
||||
if OPERATION in span["attributes"] and CALL_ID in span["attributes"]
|
||||
) & {call.call_id for call in sent}
|
||||
|
||||
|
||||
def _trace_ids(case: str) -> tuple[str, ...]:
|
||||
"""Fixed RFC 4122 shaped trace ids, like the uuid4 hex a caller mints for ``traceparent``."""
|
||||
return tuple(
|
||||
uuid.uuid5(uuid.NAMESPACE_URL, f"team-sampling/{case}/{index}").hex for index in range(TRACES_PER_CASE)
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("variables", "content"),
|
||||
[({"arize_success_sampling_rate": "0.5"}, "hello"), ({"arize_error_sampling_rate": "0.5"}, "sampling-fail")],
|
||||
ids=["success-rate", "error-rate"],
|
||||
)
|
||||
def test_each_trace_reaches_the_team_space_whole_or_not_at_all(
|
||||
pods: Pods, variables: Mapping[str, str], content: str
|
||||
) -> None:
|
||||
sampled: Final = pods.team_key({**CREDENTIALS, **variables})
|
||||
barrier: Final = pods.team_key(CREDENTIALS)
|
||||
operator_since: Final = recorded_spans(pods.sinks.operator)[0]
|
||||
arize_since: Final = recorded_spans(pods.sinks.arize)[0]
|
||||
case: Final = "-".join(sorted(variables))
|
||||
calls: Final = tuple(
|
||||
pods.send(pod, sampled, trace_id, content) for trace_id in _trace_ids(case) for pod in (pods.first, pods.second)
|
||||
)
|
||||
eventually(
|
||||
lambda: _logged(pods, operator_since, calls),
|
||||
lambda logged: len(logged) == len(calls),
|
||||
seconds=60,
|
||||
)
|
||||
# The unsampled team shares the sampled team's exporter, which sends spans in the order they were handed to it,
|
||||
# so its calls arriving means every decided call of the sampled team has been sent too.
|
||||
fences: Final = tuple(pods.send(pod, barrier, uuid.uuid4().hex) for pod in (pods.first, pods.second))
|
||||
|
||||
def whole_or_nothing() -> dict[str, tuple[bool, bool]]:
|
||||
delivered: Final = _team_roots(pods, arize_since, calls + fences)
|
||||
assert {fence.call_id for fence in fences} <= delivered, "the unsampled team's calls have not arrived yet"
|
||||
by_trace: Final = {
|
||||
trace_id: (calls[2 * index].call_id in delivered, calls[2 * index + 1].call_id in delivered)
|
||||
for index, trace_id in enumerate(_trace_ids(case))
|
||||
}
|
||||
split: Final = {trace_id: kept for trace_id, kept in by_trace.items() if kept[0] != kept[1]}
|
||||
assert split == {}, f"pods disagreed on {len(split)} of {len(by_trace)} traces (first pod, second pod): {split}"
|
||||
return by_trace
|
||||
|
||||
verdicts: Final = until_asserted(whole_or_nothing, seconds=30)
|
||||
kept: Final = sum(1 for first, _ in verdicts.values() if first)
|
||||
assert 0 < kept < TRACES_PER_CASE, f"a rate of 0.5 kept {kept} of {TRACES_PER_CASE} traces"
|
||||
|
||||
|
||||
def _write_config(directory: Path, sinks: SpanSinks, provider: Wire) -> Path:
|
||||
config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text())
|
||||
config["model_list"] = [
|
||||
*config.get("model_list", []),
|
||||
{
|
||||
"model_name": MODEL,
|
||||
"litellm_params": {
|
||||
"model": "openai/gpt-4o-mini",
|
||||
"api_key": "integration-provider-key",
|
||||
"api_base": provider.url + "/v1",
|
||||
},
|
||||
},
|
||||
]
|
||||
config["litellm_settings"] = {
|
||||
**config.get("litellm_settings", {}),
|
||||
"callbacks": ["langfuse_otel"],
|
||||
"otel_tenant_destination_mode": "additive",
|
||||
"provider_url_destination_allowed_hosts": [urlparse(sinks.operator).netloc, urlparse(sinks.arize).netloc],
|
||||
}
|
||||
config["router_settings"] = {**config.get("router_settings", {}), "num_retries": 0}
|
||||
config["general_settings"] = {**config.get("general_settings", {}), "user_api_key_cache_ttl": 2}
|
||||
path: Final = directory / "team-sampling.yaml"
|
||||
path.write_text(yaml.safe_dump(config))
|
||||
return path
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _two_pods(provider: Wire, sinks: SpanSinks, directory: Path) -> Iterator[Pods]:
|
||||
environment: Final = {
|
||||
"LITELLM_OTEL_V2": "1",
|
||||
"OTEL_BSP_SCHEDULE_DELAY": "300",
|
||||
"MAPPER_NAMES": "genai,openinference",
|
||||
"LANGFUSE_HOST": sinks.operator,
|
||||
"LANGFUSE_PUBLIC_KEY": "pk-lf-operator",
|
||||
"LANGFUSE_SECRET_KEY": "sk-lf-operator",
|
||||
"ARIZE_SPACE_KEY": "space-operator",
|
||||
"ARIZE_API_KEY": "arize-operator",
|
||||
"ARIZE_HTTP_ENDPOINT": sinks.arize + "/v1/traces",
|
||||
}
|
||||
removed: Final = ("LITELLM_OTEL_TENANT_DESTINATION_MODE", "OTEL_TRACES_SAMPLER", "OTEL_TRACES_SAMPLER_ARG")
|
||||
config: Final = _write_config(directory, sinks, provider)
|
||||
for pod in ("first", "second"):
|
||||
(directory / pod).mkdir()
|
||||
with gateway_from_environment() as shared, scratch_database("team_sampling") as database_url:
|
||||
overrides: Final = {**environment, "DATABASE_URL": database_url}
|
||||
with (
|
||||
owned_proxy_process(
|
||||
shared, directory / "first", overrides, config=config, remove_environment=removed
|
||||
) as first,
|
||||
owned_proxy_process(
|
||||
shared, directory / "second", overrides, config=config, remove_environment=removed
|
||||
) as second,
|
||||
replace(first.gateway, database_url=database_url).scenario() as scenario,
|
||||
):
|
||||
yield Pods(first.gateway, second.gateway, scenario, sinks)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def provider() -> Iterator[Wire]:
|
||||
with wire_server(_upstream) as wire:
|
||||
yield wire
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def pods(provider: Wire, audit_sinks: SpanSinks, tmp_path_factory: pytest.TempPathFactory) -> Iterator[Pods]:
|
||||
with _two_pods(provider, audit_sinks, tmp_path_factory.mktemp("team-sampling")) as started:
|
||||
yield started
|
||||
|
|
@ -4,6 +4,7 @@ import asyncio
|
|||
import contextvars
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from base64 import b64encode
|
||||
from collections.abc import Mapping
|
||||
from dataclasses import replace
|
||||
|
|
@ -18,6 +19,7 @@ from opentelemetry.sdk.resources import Resource
|
|||
from opentelemetry.sdk.trace import ReadableSpan, TracerProvider
|
||||
from opentelemetry.sdk.trace.export import SimpleSpanProcessor
|
||||
from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter
|
||||
from opentelemetry.sdk.trace.id_generator import RandomIdGenerator
|
||||
from opentelemetry.trace import Status, StatusCode
|
||||
|
||||
import litellm
|
||||
|
|
@ -1165,6 +1167,22 @@ def arize_destination(**rates: str) -> OtelDestination:
|
|||
return destination
|
||||
|
||||
|
||||
#: Trace ids whose SHA-256 reads 0.40 and 0.80 of the way through its first eight bytes,
|
||||
#: so a rate of 0.5 keeps the first and drops the second.
|
||||
KEPT_AT_HALF: Final = 4
|
||||
DROPPED_AT_HALF: Final = 5
|
||||
|
||||
|
||||
class _FixedTraceId(RandomIdGenerator):
|
||||
"""Every trace the provider starts gets ``trace_id``."""
|
||||
|
||||
def __init__(self, trace_id: int) -> None:
|
||||
self._trace_id = trace_id
|
||||
|
||||
def generate_trace_id(self) -> int:
|
||||
return self._trace_id
|
||||
|
||||
|
||||
class TestTenantSampling:
|
||||
"""A team's ``arize_success_sampling_rate`` and ``arize_error_sampling_rate`` decide how
|
||||
much of its traffic the fan-out delivers to its space, the way they already decide what
|
||||
|
|
@ -1173,14 +1191,13 @@ class TestTenantSampling:
|
|||
NAMES = {"POST /v1/chat/completions", "auth /v1/chat/completions", "chat gpt-4"}
|
||||
|
||||
@staticmethod
|
||||
def _fan_out(dest_exporter, draw=None, global_exporter=None):
|
||||
provider = TracerProvider()
|
||||
def _fan_out(dest_exporter, trace_id=None, global_exporter=None):
|
||||
provider = TracerProvider() if trace_id is None else TracerProvider(id_generator=_FixedTraceId(trace_id))
|
||||
if global_exporter is not None:
|
||||
provider.add_span_processor(_OverriddenBackendFilter(SimpleSpanProcessor(global_exporter), "arize"))
|
||||
kwargs = {"processor_factory": lambda _d: SimpleSpanProcessor(dest_exporter)}
|
||||
if draw is not None:
|
||||
kwargs["sampling_draw"] = draw
|
||||
provider.add_span_processor(TenantFanOutSpanProcessor(**kwargs))
|
||||
provider.add_span_processor(
|
||||
TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter))
|
||||
)
|
||||
return provider
|
||||
|
||||
@staticmethod
|
||||
|
|
@ -1224,7 +1241,7 @@ class TestTenantSampling:
|
|||
|
||||
def test_an_unset_rate_exports_everything(self):
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
provider = self._fan_out(dest_exporter, draw=lambda: 0.99)
|
||||
provider = self._fan_out(dest_exporter, trace_id=DROPPED_AT_HALF)
|
||||
|
||||
self._run(provider, (arize_destination(),))
|
||||
|
||||
|
|
@ -1300,10 +1317,10 @@ class TestTenantSampling:
|
|||
|
||||
def test_a_span_that_ends_after_the_root_follows_the_requests_verdict(self):
|
||||
"""A post-call database write that starts after the whole request tree has ended
|
||||
goes where the rest of the tree went, without a second draw."""
|
||||
goes where the rest of the tree went, without a second verdict."""
|
||||
kept_exporter, dropped_exporter = InMemorySpanExporter(), InMemorySpanExporter()
|
||||
kept = self._fan_out(kept_exporter, draw=lambda: 0.1)
|
||||
dropped = self._fan_out(dropped_exporter, draw=lambda: 0.9)
|
||||
kept = self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF)
|
||||
dropped = self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
|
||||
|
||||
def late_tree(provider):
|
||||
|
|
@ -1330,11 +1347,11 @@ class TestTenantSampling:
|
|||
def test_a_late_span_whose_verdict_was_forgotten_is_decided_on_its_own_not_stranded(self):
|
||||
"""Once enough other requests have been decided to evict a request's verdict, a span
|
||||
of it that starts late (a post-call database write) opens and closes its own count,
|
||||
so it is decided on a draw of its own as soon as it ends instead of waiting in a tree
|
||||
nothing would ever close."""
|
||||
so it is decided on its own as soon as it ends instead of waiting in a tree nothing
|
||||
would ever close."""
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
provider = self._fan_out(dest_exporter, draw=lambda: 0.1)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
|
||||
provider = self._fan_out(dest_exporter)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
|
||||
|
||||
def run():
|
||||
set_request_destinations(destinations)
|
||||
|
|
@ -1353,18 +1370,17 @@ class TestTenantSampling:
|
|||
assert names[0] == "POST /v1/chat/completions"
|
||||
assert names[-1] == "postgres INSERT LiteLLM_SpendLogs", "exported as it ended, not held for a root"
|
||||
|
||||
def test_a_callers_traceparent_neither_steers_the_draw_nor_hides_the_root(self):
|
||||
"""The draw is random, not read from the trace id a caller may have chosen, and the
|
||||
server span under a remote parent still closes the tree."""
|
||||
def test_a_callers_traceparent_decides_by_its_trace_id_and_still_closes_the_tree(self):
|
||||
"""A request continuing a caller's trace is decided on that trace id, and the server
|
||||
span under a remote parent still closes the tree."""
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
draws = iter([0.9, 0.1])
|
||||
provider = self._fan_out(dest_exporter, draw=lambda: next(draws))
|
||||
provider = self._fan_out(dest_exporter)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
|
||||
|
||||
def run():
|
||||
set_request_destinations(destinations)
|
||||
tracer = get_tracer(provider, "litellm")
|
||||
for trace_id in (1, 2):
|
||||
for trace_id in (DROPPED_AT_HALF, KEPT_AT_HALF):
|
||||
remote = trace_api.SpanContext(
|
||||
trace_id=trace_id, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01)
|
||||
)
|
||||
|
|
@ -1377,12 +1393,12 @@ class TestTenantSampling:
|
|||
|
||||
kept = dest_exporter.get_finished_spans()
|
||||
assert [s.name for s in kept] == ["chat gpt-4", "POST /v1/chat/completions"]
|
||||
assert all(s.context.trace_id == 2 for s in kept), "the second request, whose draw of 0.1 passed"
|
||||
assert all(s.context.trace_id == KEPT_AT_HALF for s in kept)
|
||||
|
||||
def test_a_root_that_never_ends_holds_the_tree_only_up_to_the_bound(self):
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
provider = self._fan_out(dest_exporter, draw=lambda: 0.0)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
|
||||
provider = self._fan_out(dest_exporter)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
|
||||
|
||||
def run():
|
||||
set_request_destinations(destinations)
|
||||
|
|
@ -1397,8 +1413,8 @@ class TestTenantSampling:
|
|||
|
||||
def test_a_flood_of_waiting_trees_decides_the_oldest_on_what_it_has(self):
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
provider = self._fan_out(dest_exporter, draw=lambda: 0.0)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
|
||||
provider = self._fan_out(dest_exporter)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
|
||||
|
||||
def run():
|
||||
set_request_destinations(destinations)
|
||||
|
|
@ -1419,8 +1435,8 @@ class TestTenantSampling:
|
|||
only the sampled ones. A trace pushed out of that count is decided on what it has as
|
||||
its next span ends, root still open, rather than held until a bound gets to it."""
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
provider = self._fan_out(dest_exporter, draw=lambda: 0.0)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
|
||||
provider = self._fan_out(dest_exporter)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="1.0"),)
|
||||
|
||||
def run():
|
||||
set_request_destinations(destinations)
|
||||
|
|
@ -1443,14 +1459,12 @@ class TestTenantSampling:
|
|||
|
||||
def test_shutdown_decides_what_is_still_held(self):
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
fan_out = TenantFanOutSpanProcessor(
|
||||
processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter), sampling_draw=lambda: 0.0
|
||||
)
|
||||
fan_out = TenantFanOutSpanProcessor(processor_factory=lambda _d: SimpleSpanProcessor(dest_exporter))
|
||||
provider = TracerProvider()
|
||||
provider.add_span_processor(fan_out)
|
||||
|
||||
def run():
|
||||
set_request_destinations((arize_destination(arize_success_sampling_rate="0.5"),))
|
||||
set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),))
|
||||
tracer = get_tracer(provider, "litellm")
|
||||
with tracer.start_as_current_span("POST /v1/chat/completions"):
|
||||
with tracer.start_as_current_span("chat gpt-4"):
|
||||
|
|
@ -1476,14 +1490,12 @@ class TestTenantSampling:
|
|||
straggler.end()
|
||||
super().on_end(span)
|
||||
|
||||
fan_out = TenantFanOutSpanProcessor(
|
||||
processor_factory=lambda _d: _EndsAStragglerOnExport(dest_exporter), sampling_draw=lambda: 0.0
|
||||
)
|
||||
fan_out = TenantFanOutSpanProcessor(processor_factory=lambda _d: _EndsAStragglerOnExport(dest_exporter))
|
||||
provider = TracerProvider()
|
||||
provider.add_span_processor(fan_out)
|
||||
|
||||
def run():
|
||||
set_request_destinations((arize_destination(arize_success_sampling_rate="0.5"),))
|
||||
set_request_destinations((arize_destination(arize_success_sampling_rate="1.0"),))
|
||||
tracer = get_tracer(provider, "litellm")
|
||||
root = tracer.start_span("POST /v1/chat/completions", context=trace_api.Context())
|
||||
tracer.start_span("chat gpt-4", context=trace_api.set_span_in_context(root)).end()
|
||||
|
|
@ -1495,20 +1507,20 @@ class TestTenantSampling:
|
|||
|
||||
assert {s.name for s in dest_exporter.get_finished_spans()} == {"chat gpt-4", "embed ada"}
|
||||
|
||||
def test_a_zero_rate_drops_even_a_draw_of_zero(self):
|
||||
def test_a_zero_rate_drops_a_trace_any_other_rate_would_keep(self):
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
provider = self._fan_out(dest_exporter, draw=lambda: 0.0)
|
||||
provider = self._fan_out(dest_exporter, trace_id=KEPT_AT_HALF)
|
||||
|
||||
self._run(provider, (arize_destination(arize_success_sampling_rate="0.0"),))
|
||||
|
||||
assert dest_exporter.get_finished_spans() == ()
|
||||
|
||||
def test_the_draw_is_compared_to_the_rate_the_way_the_legacy_callback_compares_it(self):
|
||||
def test_the_trace_id_is_read_against_the_rate(self):
|
||||
dropped_exporter, kept_exporter = InMemorySpanExporter(), InMemorySpanExporter()
|
||||
destinations = (arize_destination(arize_success_sampling_rate="0.2"),)
|
||||
destinations = (arize_destination(arize_success_sampling_rate="0.5"),)
|
||||
|
||||
self._run(self._fan_out(dropped_exporter, draw=lambda: 0.3), destinations)
|
||||
self._run(self._fan_out(kept_exporter, draw=lambda: 0.2), destinations)
|
||||
self._run(self._fan_out(dropped_exporter, trace_id=DROPPED_AT_HALF), destinations)
|
||||
self._run(self._fan_out(kept_exporter, trace_id=KEPT_AT_HALF), destinations)
|
||||
|
||||
assert dropped_exporter.get_finished_spans() == ()
|
||||
assert {s.name for s in kept_exporter.get_finished_spans()} == self.NAMES
|
||||
|
|
@ -1527,6 +1539,40 @@ class TestTenantSampling:
|
|||
assert all(names == self.NAMES for names in by_trace.values())
|
||||
assert 0 < len(by_trace) < 64
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("rates", "failed"),
|
||||
[({"arize_success_sampling_rate": "0.5"}, False), ({"arize_error_sampling_rate": "0.5"}, True)],
|
||||
ids=["success-rate", "error-rate"],
|
||||
)
|
||||
def test_two_processes_serving_one_trace_reach_the_same_verdict(self, rates, failed):
|
||||
"""Two pods behind a load balancer each serve one call of a trace the caller
|
||||
continued with its ``traceparent``: the team's space gets both calls or neither,
|
||||
and a rate of 0.5 over UUID-shaped trace ids keeps some traces and drops others."""
|
||||
exporters = (InMemorySpanExporter(), InMemorySpanExporter())
|
||||
pods = tuple(self._fan_out(exporter) for exporter in exporters)
|
||||
destinations = (arize_destination(**rates),)
|
||||
trace_ids = tuple(uuid.uuid5(uuid.NAMESPACE_URL, f"pod/{index}").int for index in range(32))
|
||||
|
||||
def run(provider):
|
||||
set_request_destinations(destinations)
|
||||
tracer = get_tracer(provider, "litellm")
|
||||
for trace_id in trace_ids:
|
||||
remote = trace_api.SpanContext(
|
||||
trace_id=trace_id, span_id=1, is_remote=True, trace_flags=trace_api.TraceFlags(0x01)
|
||||
)
|
||||
parent = trace_api.set_span_in_context(trace_api.NonRecordingSpan(remote))
|
||||
with tracer.start_as_current_span("POST /v1/chat/completions", context=parent):
|
||||
with tracer.start_as_current_span("chat gpt-4") as span:
|
||||
if failed:
|
||||
span.set_status(Status(StatusCode.ERROR, "upstream 500"))
|
||||
|
||||
for provider in pods:
|
||||
in_fresh_context(run, provider)
|
||||
|
||||
first, second = ({s.context.trace_id for s in exporter.get_finished_spans()} for exporter in exporters)
|
||||
assert first == second
|
||||
assert 0 < len(first) < len(trace_ids)
|
||||
|
||||
def test_in_additive_mode_the_operators_own_copy_is_not_sampled(self, monkeypatch):
|
||||
"""The rates are the team's setting for the team's space; the operator's backbone keeps
|
||||
every span."""
|
||||
|
|
@ -1547,7 +1593,7 @@ class TestTenantSampling:
|
|||
|
||||
assert {s.name for s in dest_exporter.get_finished_spans()} == self.NAMES
|
||||
|
||||
def test_two_views_of_one_exporter_each_draw_at_their_own_rate(self):
|
||||
def test_two_views_of_one_exporter_each_answer_to_their_own_rate(self):
|
||||
"""Two destinations to the same space with different rates share an exporter, not a
|
||||
verdict: the 1.0 view gets the tree once and the 0.0 view adds nothing."""
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
|
|
@ -1582,7 +1628,7 @@ class TestTenantSampling:
|
|||
@pytest.mark.parametrize("bad", ["abc", "-0.1", "1.5", "nan"])
|
||||
def test_an_unusable_rate_exports_rather_than_dropping(self, bad):
|
||||
dest_exporter = InMemorySpanExporter()
|
||||
provider = self._fan_out(dest_exporter, draw=lambda: 0.99)
|
||||
provider = self._fan_out(dest_exporter, trace_id=DROPPED_AT_HALF)
|
||||
|
||||
self._run(provider, (arize_destination(arize_success_sampling_rate=bad),))
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue