From dc95de52097e7824ed7d6c9fb1d290453d1e35e1 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 13:52:32 -0700 Subject: [PATCH 01/20] feat(otel)!: let a team capture_message_content override the global setting for its OTel v2 destination (#44264) * feat(otel): let a team strip captured message content from its own OTel v2 destination Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): split the shared callback option conflict check and sync dashboard API types Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): address review feedback on team capture_message_content Classify capture_message_content as NotSecret in the credential slot registry, inject test exporters through register_exporter_factory instead of patching _exporter_from_spec, type the test helpers, and stop reassigning processor in build_tracer_provider Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): add integration cells for team capture_message_content Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): cover weave_otel and newrelic team destinations in capture_message_content cells Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): stop rejecting client-sent capture_message_content at the request body boundary Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * feat(otel): let a team capture_message_content override the global setting in both directions Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): keep a team's capture_message_content on its own OTel v2 destination The setting no longer rides the flattened team callback vars, so a sibling destination and a routed provider follow the global mode. Request bodies get the base 400 back, a failure-only entry rejects the setting, and entries for one backend must agree Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): type the capture_message_content callback validation tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): type the remaining capture_message_content test signatures Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * test(otel): mark the new capture_message_content test locals Final Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): default tenant capture to callback_settings.otel and keep CaptureMessageContent formatting bare Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(otel): keep CaptureMessageContent a plain str like main The str-to-Enum switch changed the public type: the config field's type, repr and model_dump, a constructor that now raised on unknown values, and unknown env values coerced to no_content. CaptureMessageContent is the plain str subclass main ships again, the field is a str, and a captures_span_content helper replaces the enum property. A test pins that a published preset's capture mode stays the default for an unowned destination, which is what main forwards. * test(otel): drop the module-level CaptureMessageContent import the rewritten test no longer uses * fix(lint): keep the merged branch within main's type budgets Freeze the capture-mode settings model (LIT015) and read stored logging entries through one typed sequence helper, so the conflicting-capture checks add no unknown types. * fix(otel): reject a destination capture_message_content while OTel v2 is off The legacy OTel callbacks never read the per-destination setting, so a team could register no_content and still export prompts and replies under the global capture mode. Registration now refuses the var unless the proxy runs with LITELLM_OTEL_V2=true, the same rule the Arize transport and New Relic routing vars follow. * test(otel): type the OTel v2 fixtures and the v2-off regression test * fix(otel): accept only no_content and span_only on a team or key destination OTel v2 writes captured content only to span attributes, so event_only behaved like no_content and span_and_event like span_only on a destination. The proxy-wide setting keeps all four values * docs(otel): say an omitted capture setting follows the proxy's own callback for that backend * test(otel): describe the general capture fallback in the unowned destination test --------- Co-authored-by: mrinal Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: yucheng-berri Co-authored-by: yucheng-berri <293926326+yucheng-berri@users.noreply.github.com> --- litellm/integrations/otel/README.md | 4 +- litellm/integrations/otel/langfuse_logger.py | 6 + litellm/integrations/otel/logger.py | 17 +- litellm/integrations/otel/model/config.py | 59 +- .../integrations/otel/model/destination.py | 19 +- .../integrations/otel/plumbing/providers.py | 187 ++++- litellm/integrations/otel/plumbing/routing.py | 3 +- .../integrations/otel/presets/destinations.py | 2 + litellm/integrations/otel/presets/newrelic.py | 7 +- .../initialize_dynamic_callback_params.py | 8 + litellm/proxy/_types.py | 6 +- .../callback_config_validation.py | 81 +- litellm/proxy/litellm_pre_call_utils.py | 39 +- .../team_callback_endpoints.py | 13 + litellm/types/utils.py | 22 + tests/integration/_support/otlp_sink.py | 5 +- .../test_otel_team_capture_message_content.py | 720 ++++++++++++++++++ .../integrations/otel/test_langfuse_logger.py | 34 +- .../otel/test_otel_v2_destinations.py | 582 +++++++++++++- .../integrations/otel/test_otel_v2_logger.py | 59 +- .../otel/test_otel_v2_sources_of_truth.py | 13 + .../test_callback_config_validation.py | 115 +++ .../test_team_callback_endpoints.py | 60 ++ tests/unit/types/test_utils.py | 40 + ui/litellm-dashboard/src/lib/http/schema.d.ts | 3 + 25 files changed, 2019 insertions(+), 85 deletions(-) create mode 100644 tests/integration/observability/test_otel_team_capture_message_content.py create mode 100644 tests/unit/types/test_utils.py diff --git a/litellm/integrations/otel/README.md b/litellm/integrations/otel/README.md index 38f96f954f1..4b963ca3415 100644 --- a/litellm/integrations/otel/README.md +++ b/litellm/integrations/otel/README.md @@ -295,7 +295,9 @@ nothing here imports outside it: - [`config.py`](./model/config.py) — `OpenTelemetryV2Config`, a pydantic-settings model that reads `OTEL_*` / `LITELLM_OTEL_*` env vars, plus the feature gate. `capture_span_content` gates whether prompt/response bodies may be written as - span attributes; it defaults **off** (`no_content`). The Baggage allowlists are + span attributes; it defaults **off** (`no_content`). A team or key can override + the global mode per OTel v2 destination with `capture_message_content`; omitted + values follow the proxy's own callback for that backend, else the global mode. The Baggage allowlists are configurable, not hard-coded: set `LITELLM_OTEL_BAGGAGE_PROMOTED_KEYS` / `LITELLM_OTEL_BAGGAGE_METADATA_KEYS` / `LITELLM_OTEL_BAGGAGE_TEAM_METADATA_KEYS` (comma-separated) as env vars, or diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py index d029b153c52..8f6a7d5eb51 100644 --- a/litellm/integrations/otel/langfuse_logger.py +++ b/litellm/integrations/otel/langfuse_logger.py @@ -51,6 +51,10 @@ class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): response: "AsyncIterator[ModelResponseStream]", request_data: Mapping[str, object], ) -> "AsyncGenerator[ModelResponseStream, None]": + if not self._capture_span_content(): + async for chunk in response: + yield chunk + return relayed: Final[list[ModelResponseStream]] = [] # mutable-ok: relayed as they arrive, assembled at end of stream async for chunk in response: relayed.append(chunk) @@ -58,6 +62,8 @@ class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: + if not self._capture_span_content(): + return root: Final = request_root_span() if root is None or not root.is_recording(): return diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 498ebe655da..d8dccc39c13 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -58,6 +58,7 @@ from litellm.integrations.otel.plumbing.context import ( is_recordable_span, mcp_message_transport_span, post_response_root, + request_destinations, request_root_http_route, request_root_span, resolve_internal_call_span_context, @@ -82,6 +83,7 @@ from litellm.integrations.otel.plumbing.providers import ( resolve_meter_provider, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache +from litellm.types.utils import captures_span_content if TYPE_CHECKING: from opentelemetry.metrics import MeterProvider @@ -254,6 +256,11 @@ class OpenTelemetryV2(CustomLogger): return None return GenAIEventRecorder(get_event_logger(provider, LITELLM_TRACER_NAME), provider.resource) + def _capture_span_content(self) -> bool: + return self.config.capture_span_content or any( + captures_span_content(destination.capture_message_content) for destination in request_destinations() + ) + # ====================================================================== # # Proxy global registration # ====================================================================== # @@ -433,7 +440,7 @@ class OpenTelemetryV2(CustomLogger): return False payload: Final = cast("StandardLoggingPayload", raw_payload) data: Final = MCPToolCallSpanData.from_standard_logging_payload( - payload, capture_content=self.config.capture_span_content + payload, capture_content=self._capture_span_content() ) # A stray LLM carrier from a ``pre_call`` that mis-fired for this id would # otherwise linger until evicted; drop it so it's neither leaked nor closed @@ -477,7 +484,7 @@ class OpenTelemetryV2(CustomLogger): return False payload: Final = cast("StandardLoggingPayload", raw_payload) data: Final = MCPListToolsSpanData.from_standard_logging_payload( - payload, capture_content=self.config.capture_span_content + payload, capture_content=self._capture_span_content() ) if data.identity.call_id: self._release_carrier(self._open_llm_calls.pop(data.identity.call_id, None)) @@ -569,7 +576,7 @@ class OpenTelemetryV2(CustomLogger): return None data: Final = LLMCallSpanData.from_standard_logging_payload( payload, - capture_content=self.config.capture_span_content, + capture_content=self._capture_span_content(), time_to_first_chunk_seconds=call.time_to_first_chunk_seconds, request_route=request_root_http_route(), request_purpose=call.purpose, @@ -1076,6 +1083,6 @@ def build_otel_v2_logger( def _logger_class(config: OpenTelemetryV2Config) -> type[OpenTelemetryV2]: if "langfuse" not in config.mapper_names: return OpenTelemetryV2 - from litellm.integrations.otel.langfuse_logger import LangfuseContentOpenTelemetryV2, LangfuseOpenTelemetryV2 + from litellm.integrations.otel.langfuse_logger import LangfuseContentOpenTelemetryV2 - return LangfuseContentOpenTelemetryV2 if config.capture_span_content else LangfuseOpenTelemetryV2 + return LangfuseContentOpenTelemetryV2 diff --git a/litellm/integrations/otel/model/config.py b/litellm/integrations/otel/model/config.py index 3cf49ea2f11..55ef28bf470 100644 --- a/litellm/integrations/otel/model/config.py +++ b/litellm/integrations/otel/model/config.py @@ -16,19 +16,17 @@ from litellm.integrations.otel.model.baggage import ( ) from litellm.integrations.otel.model.spans import POSTGRESQL, db_system from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import OtelSpanScope +from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VALUES, + CaptureMessageContent, + OtelSpanScope, + captures_span_content, +) #: Master feature-flag env var. The logger is inert until this is truthy. OTEL_V2_ENV: Final = "LITELLM_OTEL_V2" -class CaptureMessageContent(str): - NO_CONTENT = "no_content" - SPAN_ONLY = "span_only" - EVENT_ONLY = "event_only" - SPAN_AND_EVENT = "span_and_event" - - class ExporterOwner(str, Enum): """The preset that contributed an exporter. Values match the callback names in ``presets.PRESET_BY_CALLBACK`` so per-request dynamic-credential routing @@ -297,9 +295,7 @@ class OpenTelemetryV2Config(BaseSettings): here are lower_snake_case; normalizing at the boundary keeps both spellings working and lets every downstream comparison stay exact. """ - if isinstance(value, str): - return value.lower() - return value + return _normalized_capture_message_content(value) @field_validator("langfuse_span_scope", mode="before") @classmethod @@ -371,22 +367,39 @@ class OpenTelemetryV2Config(BaseSettings): @property def capture_span_content(self) -> bool: - """Whether prompt/response content may be stamped as span attributes. - - Defaults off (``no_content``): an operator must opt in before message - bodies leave the process, so a user request can never force its prompt - or completion into the configured backend while capture is disabled. - """ - return self.capture_message_content in ( - CaptureMessageContent.SPAN_ONLY, - CaptureMessageContent.SPAN_AND_EVENT, - ) + """Whether the global setting permits prompt/response span attributes.""" + return captures_span_content(self.capture_message_content) @classmethod def from_env(cls) -> "OpenTelemetryV2Config": return cls() +def _normalized_capture_message_content(value: object) -> object: + if isinstance(value, str): + return value.lower() + return value + + +class _CaptureMessageContentSettings(BaseSettings): + model_config = SettingsConfigDict(extra="ignore", frozen=True) + + capture_message_content: str = Field( + default=CaptureMessageContent.NO_CONTENT, + validation_alias=AliasChoices("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT"), + ) + + @field_validator("capture_message_content", mode="before") + @classmethod + def _normalize_capture_message_content(cls, value: object) -> object: + return _normalized_capture_message_content(value) + + +def capture_message_content_from_env() -> str: + """The global capture mode, read without building a settings model that rereads every other env var""" + return _CaptureMessageContentSettings().capture_message_content + + _EXCLUDED_SERVICES_INPUT: Final[TypeAdapter[str | tuple[object, ...]]] = TypeAdapter(str | tuple[object, ...]) @@ -395,6 +408,10 @@ def excluded_db_systems_from(value: object) -> frozenset[str]: return _normalize_excluded_services(excluded_service_names(value)) +def parse_capture_message_content(value: object) -> str | None: + return value if isinstance(value, str) and value in CAPTURE_MESSAGE_CONTENT_VALUES else None + + def excluded_service_names(value: object) -> frozenset[str]: """Read a YAML list or comma-separated string of service names, logging and dropping unusable input so a malformed value cannot stop the OTel logger from being built""" diff --git a/litellm/integrations/otel/model/destination.py b/litellm/integrations/otel/model/destination.py index 56b3123660c..b0c22dea292 100644 --- a/litellm/integrations/otel/model/destination.py +++ b/litellm/integrations/otel/model/destination.py @@ -11,7 +11,7 @@ from urllib.parse import quote from pydantic import ConfigDict, Field from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.utils import OtelSpanScope +from litellm.types.utils import OtelSpanScope, captures_span_content class OtelDestination(LiteLLMBaseModel): @@ -50,6 +50,17 @@ class OtelDestination(LiteLLMBaseModel): default=None, description="Same, for the requests with a failed span in their tree; ``None`` forwards every one.", ) + capture_message_content: str | None = Field( + default=None, + description=( + "An explicit mode overrides the global capture policy for this destination. " + "Omitted follows the global setting." + ), + ) + + def captures_content(self, default: str) -> bool: + setting: Final = self.capture_message_content + return captures_span_content(default if setting is None else setting) def header_string(self) -> str: """Render headers as the ``k=v,k2=v2`` form an ``ExporterSpec`` expects. @@ -64,9 +75,9 @@ class OtelDestination(LiteLLMBaseModel): def cache_key(self) -> tuple[str, tuple[tuple[str, str], ...], tuple[tuple[str, str], ...], str | None]: """Identity for processor reuse, so one destination means one exporter. - ``span_scope`` and the sampling rates are left out on purpose: they decide which - spans reach the processor, not how the processor exports them, so a full and an - ``llm_only`` view of the same account share one exporter. + ``span_scope``, the sampling rates and ``capture_message_content`` are left out on + purpose: they decide which spans reach the processor and what they carry, not how + the processor exports them, so two views of the same account share one exporter. """ return ( self.endpoint, diff --git a/litellm/integrations/otel/plumbing/providers.py b/litellm/integrations/otel/plumbing/providers.py index 37a8f1951d9..4e7d4f231ed 100644 --- a/litellm/integrations/otel/plumbing/providers.py +++ b/litellm/integrations/otel/plumbing/providers.py @@ -5,7 +5,7 @@ import random import threading import time from collections import OrderedDict -from collections.abc import Callable, Iterable, Mapping, Sequence +from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from types import MappingProxyType from typing import TYPE_CHECKING, Final, Literal @@ -40,8 +40,19 @@ from opentelemetry.util.types import Attributes, AttributeValue from litellm._logging import verbose_logger from litellm._version import version as litellm_version -from litellm.integrations.otel.mappers.langfuse import LANGFUSE_TRACE_NAME -from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec, OpenTelemetryV2Config +from litellm.integrations.otel.mappers.langfuse import ( + LANGFUSE_OBSERVATION_INPUT, + LANGFUSE_OBSERVATION_OUTPUT, + LANGFUSE_TRACE_NAME, +) +from litellm.integrations.otel.model.config import ( + CaptureMessageContent, + ExporterOwner, + ExporterSpec, + OpenTelemetryV2Config, + capture_message_content_from_env, + captures_span_content, +) from litellm.integrations.otel.model.semconv import ( DB, MCP, @@ -385,6 +396,26 @@ _CAPTURED_HEADER_PREFIXES: Final = ("http.request.header.", "http.response.heade # ``?key=`` query parameter. _URL_KEYS: Final = frozenset({"http.url", "http.target", "url.full"}) _URL_QUERY_KEY: Final = "url.query" +# Prompt and response content, in every vocabulary the mappers write it in once the +# global capture policy collects it: GenAI semconv (MCP tool-call arguments and results +# included), Langfuse, OpenInference's blobs and per-index messages, Langtrace, Weave. +_MESSAGE_CONTENT_KEYS: Final = frozenset( + { + GenAI.INPUT_MESSAGES, + GenAI.OUTPUT_MESSAGES, + GenAI.SYSTEM_INSTRUCTIONS, + GenAI.TOOL_CALL_ARGUMENTS, + GenAI.TOOL_CALL_RESULT, + LANGFUSE_OBSERVATION_INPUT, + LANGFUSE_OBSERVATION_OUTPUT, + "input.value", + "output.value", + "llm.prompts", + "llm.completions", + "weave.output", + } +) +_INDEXED_MESSAGE_PREFIXES: Final = ("llm.input_messages.", "llm.output_messages.") class _SpanView(ReadableSpan): @@ -415,6 +446,19 @@ class _SpanView(ReadableSpan): ) +def _is_message_content(key: str) -> bool: + return key in _MESSAGE_CONTENT_KEYS or key.startswith(_INDEXED_MESSAGE_PREFIXES) + + +def without_message_content(span: ReadableSpan) -> ReadableSpan: + """``span`` without its prompt and response content, everything else left as it was.""" + attributes: Final = span.attributes or _NO_ATTRIBUTES + if not any(_is_message_content(key) for key in attributes): + return span + kept: Final = MappingProxyType({key: value for key, value in attributes.items() if not _is_message_content(key)}) + return _SpanView(span, span.resource, kept, span.events, span.status, parent=span.parent) + + def _is_database_span(attributes: Mapping[str, AttributeValue]) -> bool: return any(key in attributes for key in _DB_SYSTEM_KEYS) @@ -553,7 +597,11 @@ def _tenant_resource(resource: Resource, defaults: Mapping[str, str], extra: Map return filled.merge(Resource(extra)) if extra else filled -def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> ReadableSpan: +def _for_destination( + span: ReadableSpan, + destination: "OtelDestination", + default_capture: str, +) -> ReadableSpan: """The view of ``span`` a tenant destination receives. A span the tenant's own call produced keeps its error text. Every other span is @@ -564,8 +612,10 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read exception it raised and names the operator's guardrail endpoint. Stack traces walk the operator's install and come off every span, as do the headers the operator captures on the server span, whose request side holds the caller's bearer token, - and the query string of the request URL, which can hold the same key. The span - itself stays, so the tenant still gets the whole trace tree. + and the query string of the request URL, which can hold the same key. A + destination whose effective setting disables span content loses the prompt and + response content too. The span itself stays, so the tenant still gets the whole + trace tree. """ extra: Final = destination.resource_attributes defaults: Final = destination.resource_defaults @@ -573,11 +623,12 @@ def _for_destination(span: ReadableSpan, destination: "OtelDestination") -> Read database: Final = _is_database_span(attributes) owned: Final = _is_tenant_owned_span(attributes) unreachable: Final = _guardrail_unreachable(attributes) + redacted: Final = not destination.captures_content(default_capture) kept: Final = MappingProxyType( { key: _without_query(key, value) for key, value in attributes.items() - if _tenant_visible(key, database, owned, unreachable) + if _tenant_visible(key, database, owned, unreachable) and not (redacted and _is_message_content(key)) } ) recorded: Final = span.events @@ -612,6 +663,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): shutdown_drain_seconds: float = _SHUTDOWN_DRAIN_SECONDS, operator_sinks: 'Mapping[_SinkKey, "OtelSpanScope"]' = MappingProxyType({}), excluded_db_systems: frozenset[str] = frozenset(), + default_capture: str = CaptureMessageContent.NO_CONTENT, + default_capture_by_backend: Mapping[str, str] = MappingProxyType({}), pending_drains: int = _MAX_PENDING_DRAINS, drain_pool: _DrainPool | None = None, sampling_draw: Callable[[], float] | None = None, @@ -619,6 +672,8 @@ class TenantFanOutSpanProcessor(SpanProcessor): self._operator_sinks: Final = operator_sinks self._draw: Final = sampling_draw if sampling_draw is not None else random.random self._excluded_db_systems: Final = excluded_db_systems + self._default_capture: Final = default_capture + self._default_capture_by_backend: Final = default_capture_by_backend self._drain_seconds: Final = shutdown_drain_seconds self._lock: Final = threading.Condition() self._closed = False # guarded by ``_lock``: an unlocked read races the teardown it gates @@ -661,12 +716,22 @@ class TenantFanOutSpanProcessor(SpanProcessor): for held_span, held_destination in self._settle(trace_id, failed): self._forward(held_span, held_destination) + def _default_capture_for(self, destination: "OtelDestination") -> str: + backend: Final = destination.callback_name + return ( + self._default_capture_by_backend.get(backend, self._default_capture) if backend else self._default_capture + ) + def _forward(self, span: ReadableSpan, destination: "OtelDestination") -> None: processor: Final = self._acquire(destination) if processor is None: return try: - processor.on_end(_scoped(_for_destination(span, destination), destination.span_scope)) + processor.on_end( + _scoped( + _for_destination(span, destination, self._default_capture_for(destination)), destination.span_scope + ) + ) except Exception as exc: # noqa: BLE001 # one destination's failure must not cost the others their span verbose_logger.debug("OTel V2 fan-out: forwarding to %s failed: %s", destination.endpoint, exc) finally: @@ -963,14 +1028,15 @@ class _OverriddenBackendFilter(SpanProcessor): Wrapping is the only place this works: ``SynchronousMultiSpanProcessor.on_end`` ignores return values, so a sibling processor can never veto the export. - Under ``additive`` mode nothing is suppressed, so the wrapper passes every span - straight through and the operator keeps its copy. + Under ``additive`` mode spans are not suppressed, though global content policy + still strips message content when disabled. ``scope`` narrows what the exporter receives independently of that: under ``llm_only`` the model-call spans go through as trace roots and the rest of the tree is held back, unless a destination of the request names ``sink``, the account this exporter writes to, with a wider scope: the fan-out then delivers the rest of - the tree there and the model call keeps its place in it. + the tree there and the model call keeps its place in it. A destination naming + ``sink`` sets this exporter's content policy; otherwise it follows global capture. """ def __init__( @@ -979,11 +1045,15 @@ class _OverriddenBackendFilter(SpanProcessor): owner: str | None, scope: "OtelSpanScope" = "full", sink: _SinkKey | None = None, + global_captures: bool = True, + filter_content: bool = True, ) -> None: self._inner: Final = inner self._owner: Final = owner self._scope: Final = scope self._sink: Final = sink + self._global_captures: Final = global_captures + self._filter_content: Final = filter_content def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: self._inner.on_start(span, parent_context) @@ -991,7 +1061,28 @@ class _OverriddenBackendFilter(SpanProcessor): def on_end(self, span: ReadableSpan) -> None: if self._owner in suppressed_backends() or not _in_scope(span, self._scope): return - self._inner.on_end(_scoped(span, self._account_scope())) + scoped: Final = _scoped(span, self._account_scope()) + self._inner.on_end( + without_message_content(scoped) if self._filter_content and not self._account_captures_content() else scoped + ) + + def _account_captures_content(self) -> bool: + """Whether a destination of this request that names ``sink`` captures content. + + The fan-out skips a destination this exporter already writes to, so this copy is + the one that account gets and has to honor the destination's content mode. + """ + explicit: Final = next( + ( + destination.capture_message_content + for destination in request_destinations() + if self._sink is not None + and destination.capture_message_content is not None + and _sink_key(destination.endpoint, destination.headers) == self._sink + ), + None, + ) + return captures_span_content(explicit) if explicit is not None else self._global_captures def _account_scope(self) -> "OtelSpanScope": if self._scope == "full" or self._sink is None: @@ -1010,6 +1101,25 @@ class _OverriddenBackendFilter(SpanProcessor): return self._inner.force_flush(timeout_millis) +class _MessageContentFilter(SpanProcessor): + """Strip prompt and response content from every span before ``inner`` exports it.""" + + def __init__(self, inner: SpanProcessor) -> None: + self._inner: Final = inner + + def on_start(self, span: SDKSpan, parent_context: Context | None = None) -> None: + self._inner.on_start(span, parent_context) + + def on_end(self, span: ReadableSpan) -> None: + self._inner.on_end(without_message_content(span)) + + def shutdown(self) -> None: + self._inner.shutdown() + + def force_flush(self, timeout_millis: int = 30000) -> bool: + return self._inner.force_flush(timeout_millis) + + def build_span_exporter(config: OpenTelemetryV2Config) -> SpanExporter: """Build a single exporter from the top-level config fields. @@ -1261,12 +1371,23 @@ def build_resource(config: OpenTelemetryV2Config) -> Resource: return Resource.create(attributes) +def _spec_processor( + spec: ExporterSpec, use_simple_processor: bool | None, strip_message_content: bool +) -> SpanProcessor: + exporting: Final = _processor_for( + _exporter_from_spec(spec), + (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), + ) + return _MessageContentFilter(exporting) if strip_message_content else exporting + + def build_tracer_provider( config: OpenTelemetryV2Config, exporter: SpanExporter | None = None, baggage_processor: SpanProcessor | None = None, use_simple_processor: bool | None = None, tenant_overrides: bool = False, + strip_message_content: bool = False, ) -> TracerProvider: """Build the shared :class:`TracerProvider`. @@ -1285,6 +1406,9 @@ def build_tracer_provider( ``config.langfuse_span_scope`` narrows the exporter owned by ``langfuse_otel`` alone; a collector or any other backend in the same config keeps the full tree. + ``strip_message_content`` removes prompt and response content before every + exporter, for a routed provider whose spans can carry content another + destination asked for. """ provider: Final = TracerProvider(resource=build_resource(config)) if baggage_processor is None: @@ -1300,17 +1424,23 @@ def build_tracer_provider( for spec in config.exporters: if spec.requires_headers and not spec.headers: continue - exp = _exporter_from_spec(spec) - processor = _processor_for( - exp, - (spec.use_simple_processor if spec.use_simple_processor is not None else use_simple_processor), - ) + processor = _spec_processor(spec, use_simple_processor, strip_message_content) owner = spec.owner.value if tenant_overrides and spec.owner is not None else None scope = _operator_scope(config, spec) sink = _sink_key(spec.endpoint, parse_headers(spec.headers)) if _exports_to_the_wire(spec) else None provider.add_span_processor( - _OverriddenBackendFilter(processor, owner, scope, sink) - if owner is not None or scope != "full" + _OverriddenBackendFilter( + processor, + owner, + scope, + sink, + config.capture_span_content, + tenant_overrides, + ) + if owner is not None + or scope != "full" + or sink is not None + or (tenant_overrides and not config.capture_span_content) else processor ) return provider @@ -1340,10 +1470,29 @@ def attach_tenant_fan_out( TenantFanOutSpanProcessor( operator_sinks=operator_sink_scopes(*configs), excluded_db_systems=excluded_db_systems, + default_capture=_operator_capture_mode(configs), + default_capture_by_backend=MappingProxyType(dict(_owned_capture_modes(configs))), ) ) +def _operator_capture_mode(configs: Sequence[OpenTelemetryV2Config]) -> str: + """The mode of the operator's generic ``otel`` callback, so ``callback_settings.otel`` counts as the global default""" + return next( + (config.capture_message_content for config in configs if any(spec.owner is None for spec in config.exporters)), + capture_message_content_from_env(), + ) + + +def _owned_capture_modes( + configs: Sequence[OpenTelemetryV2Config], +) -> Iterator[tuple[str, str]]: + for config in configs: + for spec in config.exporters: + if spec.owner is not None: + yield spec.owner.value, config.capture_message_content + + def deliverable_destinations( destinations: Iterable["OtelDestination"], provider: trace.TracerProvider | None = None, diff --git a/litellm/integrations/otel/plumbing/routing.py b/litellm/integrations/otel/plumbing/routing.py index d7b1cadfc92..eb90b9035c5 100644 --- a/litellm/integrations/otel/plumbing/routing.py +++ b/litellm/integrations/otel/plumbing/routing.py @@ -282,7 +282,8 @@ class TenantTracerCache: self._providers.move_to_end(cache_key) return cached built: Final = build_tracer_provider( - self._routed_config(credential_headers, project_headers, endpoint, service_name) + self._routed_config(credential_headers, project_headers, endpoint, service_name), + strip_message_content=not self._config.capture_span_content, ) self._providers[cache_key] = built return built diff --git a/litellm/integrations/otel/presets/destinations.py b/litellm/integrations/otel/presets/destinations.py index 91341a57634..c4273fbb337 100644 --- a/litellm/integrations/otel/presets/destinations.py +++ b/litellm/integrations/otel/presets/destinations.py @@ -187,6 +187,7 @@ def destination_for( callback_name: str, params: StandardCallbackDynamicParams, service_name: str | None = None, + capture_message_content: str | None = None, ) -> OtelDestination | None: """The destination ``params`` names for ``callback_name``, or ``None``. @@ -217,6 +218,7 @@ def destination_for( callback_name=callback_name, protocol=protocol, span_scope=_span_scope(callback_name, params), + capture_message_content=capture_message_content, success_sampling_rate=success_sampling_rate, error_sampling_rate=error_sampling_rate, ) diff --git a/litellm/integrations/otel/presets/newrelic.py b/litellm/integrations/otel/presets/newrelic.py index 771b3f643c8..9d8d04d8f0d 100644 --- a/litellm/integrations/otel/presets/newrelic.py +++ b/litellm/integrations/otel/presets/newrelic.py @@ -9,6 +9,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict from litellm._logging import verbose_logger from litellm.integrations.otel.model.config import ( + CaptureMessageContent, ExporterOwner, ExporterSpec, OpenTelemetryV2Config, @@ -66,7 +67,11 @@ def newrelic_preset( # New Relic ingests the OTLP GenAI semantic conventions natively. "mapper_names": ensure_mappers(base.mapper_names, "genai"), **( - {"capture_message_content": ("span_only" if settings.record_content else "no_content")} + { + "capture_message_content": ( + CaptureMessageContent.SPAN_ONLY if settings.record_content else CaptureMessageContent.NO_CONTENT + ) + } if settings.record_content is not None else {} ), diff --git a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py index d2ba17e169a..2efeead6bb2 100644 --- a/litellm/litellm_core_utils/initialize_dynamic_callback_params.py +++ b/litellm/litellm_core_utils/initialize_dynamic_callback_params.py @@ -6,6 +6,7 @@ from typing import Any, Final from litellm.types.utils import ( ARIZE_OTLP_PROTOCOLS, + CAPTURE_MESSAGE_CONTENT_VALUES, OTEL_SPAN_SCOPES, TRUSTED_CALLBACK_VARS_FIELD, StandardCallbackDynamicParams, @@ -91,6 +92,13 @@ def validate_arize_otlp_protocol_value(value: str) -> None: raise ValueError(f"Invalid arize_otlp_protocol {value!r}: must be one of {sorted(ARIZE_OTLP_PROTOCOLS)}") +def validate_capture_message_content_value(value: str) -> None: + if value not in CAPTURE_MESSAGE_CONTENT_VALUES: + raise ValueError( + f"Invalid capture_message_content {value!r}: must be one of {sorted(CAPTURE_MESSAGE_CONTENT_VALUES)}" + ) + + # Hardcoded list of supported callback params to avoid runtime inspection issues with TypedDict _supported_callback_params: Final[tuple[str, ...]] = ( "langfuse_public_key", diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 0715a4cae97..39892bcc29c 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -33,6 +33,7 @@ from litellm._uuid import uuid from litellm.constants import DEFAULT_STAGGER_WINDOW_SECONDS, MCP_STDIO_ALLOWED_COMMANDS from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_arize_otlp_protocol_value, + validate_capture_message_content_value, validate_langfuse_environment_value, validate_langfuse_span_scope_value, validate_no_callback_env_reference, @@ -79,6 +80,7 @@ from litellm.types.router import AllowedModelRegion, RouterErrors, UpdateRouterC from litellm.types.router_weights import validate_router_settings_dict from litellm.types.secret_managers.main import KeyManagementSystem from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VAR, AzureSpillover, CallTypes, CostBreakdown, @@ -2202,7 +2204,7 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): @classmethod def validate_callback_vars(cls, values): callback_vars: Final = values.get("callback_vars", {}) - valid_keys: Final = set(StandardCallbackDynamicParams.__annotations__.keys()) + valid_keys: Final = {*StandardCallbackDynamicParams.__annotations__, CAPTURE_MESSAGE_CONTENT_VAR} for key, value in callback_vars.items(): if key not in valid_keys: raise ValueError(f"Invalid callback variable: {key}. Must be one of {valid_keys}") @@ -2214,6 +2216,8 @@ class AddTeamCallback(LiteLLMPydanticObjectBase): validate_langfuse_span_scope_value(callback_vars[key]) if key == "arize_otlp_protocol": validate_arize_otlp_protocol_value(callback_vars[key]) + if key == CAPTURE_MESSAGE_CONTENT_VAR: + validate_capture_message_content_value(callback_vars[key]) return values diff --git a/litellm/proxy/common_utils/callback_config_validation.py b/litellm/proxy/common_utils/callback_config_validation.py index 5b9fb2e3977..4956da853c6 100644 --- a/litellm/proxy/common_utils/callback_config_validation.py +++ b/litellm/proxy/common_utils/callback_config_validation.py @@ -10,6 +10,8 @@ from collections.abc import Mapping, Sequence from types import MappingProxyType from typing import Final +from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VAR + _NEWRELIC_CALLBACK: Final = "newrelic" _NEWRELIC_VAR_PREFIX: Final = "newrelic_" _LANGFUSE_OTEL_CALLBACK: Final = "langfuse_otel" @@ -36,6 +38,9 @@ def callback_config_error( ) if langfuse_error is not None: return langfuse_error + capture_error: Final = _capture_message_content_error(callback_name, callback_vars, callback_type) + if capture_error is not None: + return capture_error if callback_name != _NEWRELIC_CALLBACK: return None return _newrelic_config_error(callback_vars) @@ -80,6 +85,32 @@ def _langfuse_span_scope_error(callback_name: str | None, callback_vars: Mapping return None +def _capture_message_content_error( + callback_name: str | None, callback_vars: Mapping[str, str], callback_type: str | None +) -> str | None: + value: Final = callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR) + if value is None: + return None + from litellm.integrations.otel.model.config import is_otel_v2_enabled + from litellm.integrations.otel.presets.destinations import destination_capable_backends + from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( + validate_capture_message_content_value, + ) + + supported: Final = sorted(destination_capable_backends()) + if callback_name not in supported: + return f"{CAPTURE_MESSAGE_CONTENT_VAR} applies to the OTel v2 callbacks {supported} only, not {callback_name!r}" + if not is_otel_v2_enabled(): + return f"Per-destination {CAPTURE_MESSAGE_CONTENT_VAR} requires the proxy to run with LITELLM_OTEL_V2=true." + try: + validate_capture_message_content_value(value) + except ValueError as e: + return str(e) + if callback_type == "failure": + return f"{CAPTURE_MESSAGE_CONTENT_VAR} needs callback_type 'success' or 'success_and_failure'" + return None + + # Which credential family a dynamic variable belongs to. The families are the # integrations that share one account: every langfuse_* variable configures the # same Langfuse project whether it rides the classic callback or the OTel one, @@ -190,20 +221,41 @@ def conflicting_span_scope_error( ) +def conflicting_capture_error( + callback_name: str, + callback_vars: Mapping[str, str] | None, + stored_entries: Sequence[tuple[str | None, Mapping[str, str]]], +) -> str | None: + incoming: Final = None if callback_vars is None else callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR) + if incoming is None: + return None + return next( + ( + f"{CAPTURE_MESSAGE_CONTENT_VAR} is already set to {stored!r} by another {callback_name} entry. " + f"Every {callback_name} entry shares one value: remove that entry or send the same value." + for name, entry_vars in stored_entries + if name == callback_name and (stored := entry_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)) not in (None, incoming) + ), + None, + ) + + def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str | None: """Validate every ``logging`` entry of a team/key metadata payload.""" if not metadata: return None - entries: Final = metadata.get("logging") - if not isinstance(entries, Sequence) or isinstance(entries, (str, bytes)): + entries: Final = _object_sequence(metadata.get("logging")) + if entries is None: return None entry_vars: Final = tuple(_entry_callback_vars(entry) for entry in entries) + named_vars: Final = tuple(zip((_entry_callback_name(entry) for entry in entries), entry_vars)) return next( ( error for error in ( *(_logging_entry_error(entry) for entry in entries), *(conflicting_span_scope_error(entry_vars[i], entry_vars[:i]) for i in range(len(entry_vars))), + *(_conflicting_entry_capture_error(named_vars[i], named_vars[:i]) for i in range(len(named_vars))), ) if error is not None ), @@ -211,6 +263,31 @@ def logging_metadata_config_error(metadata: Mapping[str, object] | None) -> str ) +def stored_capture_entries(logging_entries: object) -> tuple[tuple[str | None, Mapping[str, str]], ...]: + """The callback name and vars of every stored ``logging`` entry, for ``conflicting_capture_error``.""" + entries: Final = _object_sequence(logging_entries) or () + return tuple((_entry_callback_name(entry), _entry_callback_vars(entry)) for entry in entries) + + +def _object_sequence(value: object) -> Sequence[object] | None: + return value if isinstance(value, Sequence) and not isinstance(value, (str, bytes)) else None + + +def _entry_callback_name(entry: object) -> str | None: + match entry: + case {"callback_name": str() as callback_name}: + return callback_name + case _: + return None + + +def _conflicting_entry_capture_error( + entry: tuple[str | None, Mapping[str, str]], earlier: Sequence[tuple[str | None, Mapping[str, str]]] +) -> str | None: + callback_name, callback_vars = entry + return None if callback_name is None else conflicting_capture_error(callback_name, callback_vars, earlier) + + def _entry_callback_vars(entry: object) -> Mapping[str, str]: callback_vars: Final = entry.get("callback_vars") if isinstance(entry, Mapping) else None if not isinstance(callback_vars, Mapping): diff --git a/litellm/proxy/litellm_pre_call_utils.py b/litellm/proxy/litellm_pre_call_utils.py index 0d693b85ce8..fdc2e6d94b8 100644 --- a/litellm/proxy/litellm_pre_call_utils.py +++ b/litellm/proxy/litellm_pre_call_utils.py @@ -227,6 +227,7 @@ from litellm.types.litellm_params import AGENTIC_LOOP_KWARG_NAMES from litellm.types.llms.anthropic import ANTHROPIC_API_HEADERS from litellm.types.services import ServiceTypes from litellm.types.utils import ( + CAPTURE_MESSAGE_CONTENT_VAR, CustomPricingLiteLLMParams, LlmProviders, ProviderSpecificHeader, @@ -975,6 +976,8 @@ def convert_key_logging_metadata_to_callback( team_callback_settings_obj.callbacks.append(data.callback_name) for var, value in data.callback_vars.items(): + if var == CAPTURE_MESSAGE_CONTENT_VAR: + continue # New Relic routing reads these from the trusted-vars overlay with no # callback-name check, so scope them to the newrelic entry: a team that # put newrelic_* under a different callback never asked for New Relic and @@ -1162,7 +1165,6 @@ def resolve_tenant_otel_destinations( span still routes to the tenant's credentials the way it did then. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled - from litellm.integrations.otel.presets.destinations import destination_for if not is_otel_v2_enabled(): return () @@ -1186,23 +1188,28 @@ def resolve_tenant_otel_destinations( for name in dict.fromkeys( callback.callback_name for callback in callbacks if callback.callback_type != "failure" ) - if ( - destination := destination_for( - name, - _tenant_otel_params( - MappingProxyType( - { - var: value - for callback in callbacks - if callback.callback_name == name - for var, value in callback.callback_vars.items() - } - ) - ), - _tenant_service_name(user_api_key_dict), + if (destination := _tenant_destination(name, callbacks, _tenant_service_name(user_api_key_dict))) is not None + ) + + +def _tenant_destination( + name: str, callbacks: Sequence[AddTeamCallback], service_name: str | None +) -> "OtelDestination | None": + from litellm.integrations.otel.model.config import parse_capture_message_content + from litellm.integrations.otel.presets.destinations import destination_for + + callback_vars: Final = MappingProxyType( + dict( + itertools.chain.from_iterable( + callback.callback_vars.items() for callback in callbacks if callback.callback_name == name ) ) - is not None + ) + return destination_for( + name, + _tenant_otel_params(callback_vars), + service_name, + parse_capture_message_content(callback_vars.get(CAPTURE_MESSAGE_CONTENT_VAR)), ) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 91fd447fb8d..3908ab3c927 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -31,8 +31,10 @@ from litellm.proxy._types import ( from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, + conflicting_capture_error, conflicting_span_scope_error, cross_entry_family_error, + stored_capture_entries, ) from litellm.proxy.common_utils.callback_utils import ( # noqa: F401 # legacy module exports _CALLBACK_VAR_ENCRYPTED_PREFIX, # pyright: ignore[reportPrivateUsage,reportUnusedImport] # backwards-compatible package export @@ -288,6 +290,9 @@ async def add_team_callbacks( - langfuse_host: The host for the Langfuse callback - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + - capture_message_content: For OTel v2 callbacks, "no_content" or "span_only". + An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. + "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out - gcs_bucket_name: The name of the GCS bucket - gcs_path_service_account: The path to the GCS service account - langsmith_api_key: The API key for the Langsmith callback @@ -351,11 +356,19 @@ async def add_team_callbacks( # Decrypted, because the checks compare the incoming values against # the stored ones and the credentials are encrypted at rest. decrypted_logging: Final = decrypt_callback_vars(team_metadata).get("logging") + stored_capture: Final = stored_capture_entries(decrypted_logging) stored_entries: Final = decrypted_logging if isinstance(decrypted_logging, list) else () stored_entry_vars: Final = [entry.get("callback_vars") or {} for entry in stored_entries] scope_error: Final = conflicting_span_scope_error(data.callback_vars, stored_entry_vars) if scope_error is not None: raise _callback_config_error(scope_error) + capture_error: Final = conflicting_capture_error( + data.callback_name, + data.callback_vars, + stored_capture, + ) + if capture_error is not None: + raise _callback_config_error(capture_error) # One entry has to own a credential family end to end. The entries are # flattened into one dict before a request reads them, so an entry # naming only a destination would pair with a key written on another diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 4a962e5b3bd..d6751610e7a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3726,6 +3726,28 @@ ArizeOtlpProtocol = Literal["grpc", "http/protobuf"] ARIZE_OTLP_PROTOCOLS: Final[frozenset[str]] = frozenset(get_args(ArizeOtlpProtocol)) +CAPTURE_MESSAGE_CONTENT_VAR: Final = "capture_message_content" + + +class CaptureMessageContent(str): + NO_CONTENT = "no_content" + SPAN_ONLY = "span_only" + EVENT_ONLY = "event_only" + SPAN_AND_EVENT = "span_and_event" + + +# The values a team or key destination may set. OTel v2 writes content only to span attributes, +# so the event modes stay global-only settings +CAPTURE_MESSAGE_CONTENT_VALUES: Final[frozenset[str]] = frozenset( + {CaptureMessageContent.NO_CONTENT, CaptureMessageContent.SPAN_ONLY} +) + + +def captures_span_content(mode: str | None) -> bool: + """Whether a capture mode puts prompt and response content on spans""" + return mode in (CaptureMessageContent.SPAN_ONLY, CaptureMessageContent.SPAN_AND_EVENT) + + class StandardCallbackDynamicParams(TypedDict, total=False): # Langfuse dynamic params langfuse_public_key: str | None diff --git a/tests/integration/_support/otlp_sink.py b/tests/integration/_support/otlp_sink.py index eeabe9d886f..a4d2ea30959 100644 --- a/tests/integration/_support/otlp_sink.py +++ b/tests/integration/_support/otlp_sink.py @@ -186,6 +186,7 @@ class _Handler(BaseHTTPRequestHandler): "count": len(recorded), "host": self.headers.get("host", ""), "headers": dict(self.headers), + "span_ids": [span["span_id"] for span in recorded], } ) self._send_json({"recorded": len(recorded)}, status=self.state.status) @@ -466,9 +467,7 @@ def _await_sink(url: str) -> None: def owned_grpc_sink(directory: Path) -> Iterator[GrpcSink]: http_port: Final = _free_port() grpc_port: Final = _free_port() - with _spawn_sink( - directory, "otlp-grpc-sink.log", ["--port", str(http_port), "--grpc-port", str(grpc_port)] - ): + with _spawn_sink(directory, "otlp-grpc-sink.log", ["--port", str(http_port), "--grpc-port", str(grpc_port)]): control_url: Final = f"http://127.0.0.1:{http_port}" _await_sink(control_url) yield GrpcSink(url=f"http://127.0.0.1:{grpc_port}", control_url=control_url) diff --git a/tests/integration/observability/test_otel_team_capture_message_content.py b/tests/integration/observability/test_otel_team_capture_message_content.py new file mode 100644 index 00000000000..84da9a40486 --- /dev/null +++ b/tests/integration/observability/test_otel_team_capture_message_content.py @@ -0,0 +1,720 @@ +import json +import re +import uuid +from collections.abc import Generator, Iterator, Mapping +from concurrent.futures import ThreadPoolExecutor +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, Literal +from urllib.parse import urlparse + +import httpx +import pytest +import yaml +from integration._support.client import Gateway, Scenario, eventually, gateway_from_environment, object_value +from integration._support.otlp_sink import ( + ConnectSink, + Span, + SpanSinks, + configure_sink, + owned_connect_sink, + recorded_requests, + recorded_spans, +) +from integration._support.process import OwnedProxy, owned_proxy_process +from integration._support.wire import Reply, Request, Wire, wire_server +from pydantic import JsonValue, TypeAdapter + +MARKER: Final = re.compile(rb"cmc-(?:tool-|fail-)?[0-9a-f]{32}") +JSON: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +SERVER: Final = 2 +CALL_ID: Final = "litellm.call_id" +OPERATION: Final = "gen_ai.operation.name" +INPUT_TOKENS: Final = "gen_ai.usage.input_tokens" +Endpoint = Literal["chat", "responses", "messages"] +Capture = Literal["no_content", "span_only"] + + +def _marker(kind: str = "") -> str: + return f"cmc-{kind}{uuid.uuid4().hex}" + + +def _usage() -> dict[str, JsonValue]: + return {"prompt_tokens": 7, "completion_tokens": 2, "total_tokens": 9} + + +def _chat_reply(marker: str, stream: bool) -> Reply: + identity: Final = f"chatcmpl-{uuid.uuid4().hex}" + if marker.startswith("cmc-tool-"): + call: Final[dict[str, JsonValue]] = { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": json.dumps({"query": marker})}, + } + message: Final[dict[str, JsonValue]] = {"role": "assistant", "content": None, "tool_calls": [call]} + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [{"index": 0, "message": message, "finish_reason": "tool_calls"}], + "usage": _usage(), + } + ).encode() + ) + if not stream: + return Reply( + body=json.dumps( + { + "id": identity, + "object": "chat.completion", + "created": 1, + "model": "gpt-4o-mini", + "choices": [ + {"index": 0, "message": {"role": "assistant", "content": marker}, "finish_reason": "stop"} + ], + "usage": _usage(), + } + ).encode() + ) + chunk: Final = {"id": identity, "object": "chat.completion.chunk", "created": 1, "model": "gpt-4o-mini"} + deltas: Final[tuple[dict[str, JsonValue], ...]] = ( + {**chunk, "choices": [{"index": 0, "delta": {"role": "assistant", "content": marker}}]}, + {**chunk, "choices": [{"index": 0, "delta": {}, "finish_reason": "stop"}]}, + {**chunk, "choices": [], "usage": _usage()}, + ) + return Reply( + content_type="text/event-stream", + chunks=(*(b"data: " + json.dumps(delta).encode() + b"\n\n" for delta in deltas), b"data: [DONE]\n\n"), + ) + + +def _responses_reply(marker: str, stream: bool) -> Reply: + identity: Final = uuid.uuid4().hex + response: Final[dict[str, JsonValue]] = { + "id": f"resp_{identity}", + "object": "response", + "created_at": 1, + "status": "completed", + "model": "gpt-4o-mini", + "output": [ + { + "id": f"msg_{identity}", + "type": "message", + "role": "assistant", + "status": "completed", + "content": [{"type": "output_text", "text": marker, "annotations": []}], + } + ], + "usage": {"input_tokens": 7, "output_tokens": 2, "total_tokens": 9}, + } + if not stream: + return Reply(body=json.dumps(response).encode()) + events: Final[tuple[dict[str, JsonValue], ...]] = ( + { + "type": "response.created", + "sequence_number": 0, + "response": {**response, "status": "in_progress", "output": []}, + }, + { + "type": "response.output_text.delta", + "sequence_number": 1, + "item_id": f"msg_{identity}", + "output_index": 0, + "content_index": 0, + "delta": marker, + }, + {"type": "response.completed", "sequence_number": 2, "response": response}, + ) + return Reply( + content_type="text/event-stream", + chunks=tuple(f"event: {event['type']}\ndata: {json.dumps(event)}\n\n".encode() for event in events), + ) + + +def _upstream(request: Request) -> Reply: + found: Final = MARKER.search(request.body) + if found is None: + return Reply(status=404, body=b'{"error":"no marker"}') + marker: Final = found.group(0).decode() + if marker.startswith("cmc-fail-"): + return Reply(status=500, body=b'{"error":{"message":"scripted upstream failure","type":"server_error"}}') + stream: Final = object_value(JSON.validate_json(request.body)).get("stream") is True + if request.target.endswith("/responses"): + return _responses_reply(marker, stream) + return _chat_reply(marker, stream) + + +def _body(model: str, endpoint: Endpoint, marker: str, stream: bool) -> tuple[str, dict[str, JsonValue]]: + if endpoint == "responses": + return "/v1/responses", {"model": model, "input": marker, "stream": stream} + messages: Final[list[JsonValue]] = [{"role": "user", "content": marker}] + if endpoint == "messages": + return "/v1/messages", {"model": model, "max_tokens": 16, "messages": messages, "stream": stream} + return "/v1/chat/completions", {"model": model, "messages": messages, "stream": stream} + + +@dataclass(frozen=True, slots=True) +class Sent: + marker: str + trace_id: str + call_id: str + status: int + text: str + + +@dataclass(frozen=True, slots=True) +class Cursors: + operator: int + tenant: int + arize: int + newrelic: int + + +@dataclass(frozen=True, slots=True) +class Rig: + proxy: Gateway + owned: OwnedProxy + scenario: Scenario + model: str + upstream: Wire + sinks: SpanSinks + newrelic: ConnectSink + + def cursors(self) -> Cursors: + self.upstream.drain() + return Cursors( + recorded_spans(self.sinks.operator)[0], + recorded_spans(self.sinks.tenant)[0], + recorded_spans(self.sinks.arize)[0], + recorded_spans(self.newrelic.control_url)[0], + ) + + def upstream_hits(self, marker: str) -> int: + return sum(1 for request in self.upstream.drain() if marker.encode() in request.body) + + def langfuse(self, name: str, host: str | None = None) -> dict[str, str]: + return { + "langfuse_public_key": f"pk-lf-{name}", + "langfuse_secret_key": f"sk-lf-{name}", + "langfuse_host": host or self.sinks.tenant, + } + + def attach(self, team: str, callback: str, variables: Mapping[str, str], kind: str | None = None) -> httpx.Response: + body: Final[dict[str, JsonValue]] = {"callback_name": callback, "callback_vars": dict(variables)} + return self.proxy.request( + "POST", f"/team/{team}/callback", body if kind is None else {**body, "callback_type": kind} + ) + + def team_key( + self, + capture: Capture | None, + name: str, + *, + callback: str = "langfuse_otel", + variables: Mapping[str, str] | None = None, + kind: str | None = None, + ) -> tuple[str, str]: + team: Final = self.scenario.team() + base: Final = dict(variables) if variables is not None else self.langfuse(name) + attached: Final = self.attach( + team, callback, base if capture is None else {**base, "capture_message_content": capture}, kind + ) + assert attached.status_code == 200, attached.text + return team, self.scenario.key(team_id=team) + + def send( + self, + key: str, + endpoint: Endpoint = "chat", + stream: bool = False, + *, + marker: str | None = None, + extra: Mapping[str, JsonValue] | None = None, + ) -> Sent: + tagged: Final = marker or _marker() + path, body = _body(self.model, endpoint, tagged, stream) + trace_id: Final = uuid.uuid4().hex + call_id: Final = str(uuid.uuid4()) + headers: Final = { + "Authorization": f"Bearer {key}", + "traceparent": f"00-{trace_id}-{uuid.uuid4().hex[:16]}-01", + "x-litellm-call-id": call_id, + } + with self.proxy.client.stream("POST", path, json={**body, **(extra or {})}, headers=headers) as response: + text: Final = response.read().decode() + return Sent(tagged, trace_id, call_id, response.status_code, text) + + +def _served(sent: Sent) -> Sent: + assert sent.status == 200, sent.text + assert sent.marker in sent.text, sent.text + return sent + + +Credential = tuple[str, str] + + +def _sent_with(url: str, credential: Credential) -> frozenset[str]: + """Span ids ``url`` received in export requests whose ``credential`` header matched.""" + header, value = credential + return frozenset( + span_id + for request in recorded_requests(url) + if {name.lower(): seen for name, seen in object_value(request.get("headers") or {}).items()}.get(header) + == value + for span_id in request.get("span_ids") or () + if isinstance(span_id, str) + ) + + +def _of_request(url: str, since: int, sent: Sent, credential: Credential | None = None) -> tuple[Span, ...]: + """Spans of ``sent``'s trace at ``url``, narrowed to one account's export requests when ``credential`` is set.""" + spans: Final = recorded_spans(url, since)[1] + traces: Final = {sent.trace_id} | { + span["trace_id"] for span in spans if span["attributes"].get(CALL_ID) == sent.call_id + } + owned: Final = None if credential is None else _sent_with(url, credential) + return tuple(span for span in spans if span["trace_id"] in traces and (owned is None or span["span_id"] in owned)) + + +def _carries(span: Span, marker: str) -> bool: + return marker in json.dumps(span["attributes"]) + + +def _model_spans(spans: tuple[Span, ...]) -> tuple[Span, ...]: + return tuple(span for span in spans if OPERATION in span["attributes"]) + + +def _names(spans: tuple[Span, ...]) -> list[str]: + return sorted(span["name"] for span in spans) + + +def _with_content(url: str, since: int, sent: Sent) -> tuple[Span, ...]: + """The copy of ``sent`` at ``url`` once its model span arrived and some span carries the marker.""" + return eventually( + lambda: _of_request(url, since, sent), + lambda spans: bool(_model_spans(spans)) and any(_carries(span, sent.marker) for span in spans), + seconds=40, + ) + + +def _arrived( + url: str, since: int, sent: Sent, expected: frozenset[str], credential: Credential | None = None +) -> tuple[Span, ...]: + """The copy of ``sent`` at ``url`` once every span name in ``expected`` and a usage-bearing model span landed.""" + spans: Final = eventually( + lambda: _of_request(url, since, sent, credential), + lambda found: ( + expected <= {span["name"] for span in found} + and any(INPUT_TOKENS in span["attributes"] for span in _model_spans(found)) + ), + seconds=40, + ) + assert all( + span["attributes"].get(INPUT_TOKENS) == 7 for span in _model_spans(spans) if INPUT_TOKENS in span["attributes"] + ) + return spans + + +def _assert_redacted_twin( + rig: Rig, sent: Sent, cursors: Cursors, sink: str, since: int, credential: Credential | None = None +) -> tuple[Span, ...]: + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + carried: Final = frozenset(span["name"] for span in operator if _carries(span, sent.marker)) + tenant: Final = _arrived(sink, since, sent, carried, credential) + leaked: Final = [span["name"] for span in tenant if _carries(span, sent.marker)] + assert leaked == [], f"{leaked} carried content; tenant saw {_names(tenant)}" + model: Final = _model_spans(tenant)[0]["attributes"] + assert "gen_ai.input.messages" not in model and "gen_ai.output.messages" not in model, sorted(model) + assert model.get("gen_ai.request.model"), sorted(model) + return tenant + + +def _assert_content_twin(rig: Rig, sent: Sent, cursors: Cursors) -> None: + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + carried: Final = frozenset(span["name"] for span in operator if _carries(span, sent.marker)) + tenant: Final = _arrived(rig.sinks.tenant, cursors.tenant, sent, carried) + kept: Final = frozenset(span["name"] for span in tenant if _carries(span, sent.marker)) + assert carried <= kept, f"content lost on {sorted(carried - kept)}" + + +@pytest.mark.parametrize( + ("endpoint", "stream"), + [("chat", False), ("chat", True), ("responses", False), ("messages", True)], + ids=["chat-unary", "chat-stream", "responses-unary", "messages-stream"], +) +def test_no_content_team_keeps_its_trace_and_metadata_without_content( + rig: Rig, endpoint: Endpoint, stream: bool +) -> None: + _, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key, endpoint, stream)) + assert rig.upstream_hits(sent.marker) == 1 + _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + + +@pytest.mark.parametrize("capture", ["span_only", None], ids=["span_only", "omitted"]) +def test_span_only_and_omitted_teams_keep_the_globally_captured_content(rig: Rig, capture: Capture | None) -> None: + _, key = rig.team_key(capture, "b") + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + _assert_content_twin(rig, sent, cursors) + + +def test_tool_call_arguments_never_reach_a_no_content_team(rig: Rig) -> None: + _, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + tool: Final[dict[str, JsonValue]] = { + "type": "function", + "function": {"name": "lookup", "parameters": {"type": "object", "properties": {"query": {"type": "string"}}}}, + } + sent: Final = rig.send(key, marker=_marker("tool-"), extra={"tools": [tool]}) + assert sent.status == 200 and sent.marker in sent.text, sent.text + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + assert any(sent.marker in str(span["attributes"].get("gen_ai.output.messages", "")) for span in operator), ( + "operator copy lacks the tool-call arguments" + ) + tenant: Final = _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + assert "lookup" in json.dumps([span["attributes"] for span in tenant]), "tool definitions are metadata and stay" + + +def test_cache_hit_twin_of_a_no_content_team_is_redacted(rig: Rig) -> None: + _, key = rig.team_key("no_content", "a") + marker: Final = _marker() + _served(rig.send(key, marker=marker)) + assert rig.upstream_hits(marker) == 1 + cursors: Final = rig.cursors() + hit: Final = eventually( + lambda: _served(rig.send(key, marker=marker)), lambda _: rig.upstream_hits(marker) == 0, seconds=20 + ) + operator: Final = eventually( + lambda: _of_request(rig.sinks.operator, cursors.operator, hit), + lambda spans: any(span["kind"] == SERVER for span in spans), + seconds=40, + ) + expected: Final = frozenset(span["name"] for span in operator if span["kind"] == SERVER) + tenant: Final = eventually( + lambda: _of_request(rig.sinks.tenant, cursors.tenant, hit), + lambda spans: expected <= {span["name"] for span in spans}, + seconds=40, + ) + assert [span["name"] for span in tenant if _carries(span, marker)] == [], _names(tenant) + + +def test_a_failure_only_entry_rejects_the_setting(rig: Rig) -> None: + team: Final = rig.scenario.team() + response: Final = rig.attach( + team, "langfuse_otel", {**rig.langfuse("fail"), "capture_message_content": "no_content"}, "failure" + ) + assert response.status_code == 400, response.text + assert "success_and_failure" in response.text, response.text + + +def test_key_level_logging_with_no_content_is_redacted(rig: Rig) -> None: + logging: Final[list[JsonValue]] = [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {**rig.langfuse("key"), "capture_message_content": "no_content"}, + } + ] + key: Final = rig.scenario.key(metadata={"logging": logging}) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + + +def test_same_account_destination_redacts_the_shared_copy_only_for_that_team(rig: Rig) -> None: + _, restricted = rig.team_key("no_content", "operator", variables=rig.langfuse("operator", host=rig.sinks.operator)) + _, other = rig.team_key("span_only", "b") + cursors: Final = rig.cursors() + shared: Final = _served(rig.send(restricted)) + assert rig.upstream_hits(shared.marker) == 1 + copy: Final = eventually( + lambda: _of_request(rig.sinks.operator, cursors.operator, shared), + lambda spans: any(INPUT_TOKENS in span["attributes"] for span in _model_spans(spans)), + seconds=40, + ) + assert [span["name"] for span in copy if _carries(span, shared.marker)] == [], _names(copy) + assert len([span for span in copy if span["kind"] == SERVER and span["name"].startswith("POST ")]) == 1, _names( + copy + ) + _assert_content_twin(rig, _served(rig.send(other)), cursors) + + +def test_arize_destination_drops_openinference_content(rig: Rig) -> None: + _, key = rig.team_key( + "no_content", "arize", callback="arize", variables={"arize_space_key": "space-a", "arize_api_key": "arize-a"} + ) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = _with_content(rig.sinks.operator, cursors.operator, sent) + assert any(sent.marker in str(span["attributes"].get("input.value", "")) for span in operator), ( + "operator copy has no OpenInference content to strip" + ) + tenant: Final = _assert_redacted_twin( + rig, sent, cursors, rig.sinks.arize, cursors.arize, ("arize-space-id", "space-a") + ) + model: Final = _model_spans(tenant)[0]["attributes"] + assert not any(name.startswith(("llm.input_messages.", "llm.output_messages.")) for name in model), sorted(model) + assert "input.value" not in model and "output.value" not in model, sorted(model) + assert model.get("llm.model_name"), sorted(model) + + +def test_weave_destination_drops_its_content(rig: Rig) -> None: + _, key = rig.team_key( + "no_content", + "weave", + callback="weave_otel", + variables={"wandb_api_key": "wandb-a", "weave_project_id": "team/a"}, + ) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = eventually( + lambda: _of_request(rig.sinks.arize, cursors.arize, sent, ("project_id", "operator/weave")), + lambda spans: any(sent.marker in str(span["attributes"].get("weave.output", "")) for span in spans), + seconds=40, + ) + assert operator, "operator weave copy has no weave.output to strip" + tenant: Final = _assert_redacted_twin(rig, sent, cursors, rig.sinks.arize, cursors.arize, ("project_id", "team/a")) + assert not any("weave.output" in span["attributes"] for span in tenant), _names(tenant) + + +def test_newrelic_destination_drops_genai_content(rig: Rig) -> None: + _, key = rig.team_key("no_content", "newrelic", callback="newrelic", variables={"newrelic_api_key": "nr-team-a"}) + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + tenant: Final = _assert_redacted_twin( + rig, sent, cursors, rig.newrelic.control_url, cursors.newrelic, ("api-key", "nr-team-a") + ) + hosts: Final = { + object_value(request.get("headers") or {}).get("Host") + for request in recorded_requests(rig.newrelic.control_url) + if object_value(request.get("headers") or {}).get("api-key") == "nr-team-a" + } + assert hosts == {"otlp.nr-data.net"}, hosts + assert tenant + + +def test_span_only_team_lifts_global_no_content_only_for_its_destination(dark_rig: Rig) -> None: + """The sibling Arize entry gets a fan-out destination and the SigNoz entry rides the routed tracer.""" + team, key = dark_rig.team_key("span_only", "b") + siblings: Final = ( + dark_rig.attach(team, "arize", {"arize_space_key": "space-sibling", "arize_api_key": "arize-sibling"}), + dark_rig.attach( + team, + "signoz", + {"signoz_ingestion_endpoint": dark_rig.sinks.arize + "/v1/traces", "signoz_ingestion_key": "signoz-team"}, + ), + ) + assert [response.status_code for response in siblings] == [200, 200], [response.text for response in siblings] + cursors: Final = dark_rig.cursors() + sent: Final = _served(dark_rig.send(key)) + assert dark_rig.upstream_hits(sent.marker) == 1 + operator: Final = _arrived(dark_rig.sinks.operator, cursors.operator, sent, frozenset()) + tenant: Final = _with_content(dark_rig.sinks.tenant, cursors.tenant, sent) + sibling: Final = _arrived( + dark_rig.sinks.arize, + cursors.arize, + sent, + frozenset(), + ("arize-space-id", "space-sibling"), + ) + signoz: Final = _arrived( + dark_rig.sinks.arize, + cursors.arize, + sent, + frozenset(), + ("signoz-ingestion-key", "signoz-team"), + ) + assert any(_carries(span, sent.marker) for span in tenant), _names(tenant) + assert not any(_carries(span, sent.marker) for span in operator), _names(operator) + assert not any(_carries(span, sent.marker) for span in sibling), _names(sibling) + assert not any(_carries(span, sent.marker) for span in signoz), _names(signoz) + + +def test_an_unsupported_value_fails_registration(rig: Rig) -> None: + team: Final = rig.scenario.team() + response: Final = rig.attach(team, "langfuse_otel", {**rig.langfuse("a"), "capture_message_content": "bogus"}) + assert response.status_code == 422, response.text + assert "Invalid capture_message_content 'bogus'" in response.text, response.text + + +def test_classic_langfuse_rejects_the_setting(rig: Rig) -> None: + team: Final = rig.scenario.team() + response: Final = rig.attach(team, "langfuse", {**rig.langfuse("a"), "capture_message_content": "no_content"}) + assert response.status_code == 400, response.text + assert "capture_message_content" in response.text, response.text + + +def test_a_second_entry_for_the_same_backend_cannot_disagree(rig: Rig) -> None: + team, _ = rig.team_key("no_content", "a") + response: Final = rig.attach(team, "langfuse_otel", {"capture_message_content": "span_only"}, "success_and_failure") + assert response.status_code == 400, response.text + assert "already set to 'no_content' by another langfuse_otel entry" in response.text, response.text + + +def test_replacing_the_registration_switches_the_team_to_content(rig: Rig) -> None: + team, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + _assert_redacted_twin(rig, _served(rig.send(key)), cursors, rig.sinks.tenant, cursors.tenant) + removed: Final = rig.proxy.request("DELETE", f"/team/{team}/callback/langfuse_otel") + assert removed.status_code == 200, removed.text + replaced: Final = rig.attach(team, "langfuse_otel", {**rig.langfuse("a"), "capture_message_content": "span_only"}) + assert replaced.status_code == 200, replaced.text + + def attempt() -> bool: + since: Final = rig.cursors() + sent: Final = _served(rig.send(key)) + operator: Final = _with_content(rig.sinks.operator, since.operator, sent) + carried: Final = frozenset(span["name"] for span in operator if _carries(span, sent.marker)) + tenant: Final = _arrived(rig.sinks.tenant, since.tenant, sent, carried) + return any(_carries(span, sent.marker) for span in tenant) + + assert eventually(attempt, bool, seconds=60) + + +def test_request_metadata_cannot_lift_the_team_restriction(rig: Rig) -> None: + _, key = rig.team_key("no_content", "a") + cursors: Final = rig.cursors() + sent: Final = _served(rig.send(key, extra={"metadata": {"capture_message_content": "span_only"}})) + assert rig.upstream_hits(sent.marker) == 1 + _assert_redacted_twin(rig, sent, cursors, rig.sinks.tenant, cursors.tenant) + + +def test_stalled_tenant_during_a_mixed_burst_keeps_serving_and_each_teams_policy(rig: Rig) -> None: + _, restricted = rig.team_key("no_content", "a") + _, open_key = rig.team_key("span_only", "b") + shapes: Final[tuple[tuple[Endpoint, bool], ...]] = ( + ("chat", False), + ("chat", True), + ("responses", False), + ("messages", True), + ) + plan: Final = [(restricted if index % 2 else open_key, *shapes[index % len(shapes)]) for index in range(30)] + cursors: Final = rig.cursors() + configure_sink(rig.sinks.tenant, paused=True) + try: + with ThreadPoolExecutor(max_workers=10) as pool: + sent: Final = list(pool.map(lambda job: rig.send(job[0], job[1], job[2]), plan)) + finally: + configure_sink(rig.sinks.tenant, paused=False) + assert [result.status for result in sent] == [200] * 30, [result.text for result in sent if result.status != 200] + for (key, _, _), result in zip(plan, sent, strict=True): + roots = [ + span + for span in _with_content(rig.sinks.operator, cursors.operator, result) + if span["kind"] == SERVER and span["name"].startswith("POST ") + ] + assert len(roots) == 1, roots + if key == restricted: + _assert_redacted_twin(rig, result, cursors, rig.sinks.tenant, cursors.tenant) + else: + _assert_content_twin(rig, result, cursors) + + +def _write_config(directory: Path, sinks: SpanSinks, name: str, callbacks: tuple[str, ...]) -> Path: + config: Final = yaml.safe_load(Path("tests/integration/proxy_config.yaml").read_text()) + config["litellm_settings"] = { + **config.get("litellm_settings", {}), + "callbacks": list(callbacks), + "otel_tenant_destination_mode": "additive", + "provider_url_destination_allowed_hosts": [ + urlparse(sinks.tenant).netloc, + urlparse(sinks.operator).netloc, + urlparse(sinks.arize).netloc, + ], + } + config["general_settings"] = {**config.get("general_settings", {}), "user_api_key_cache_ttl": 2} + path: Final = directory / f"{name}.yaml" + path.write_text(yaml.safe_dump(config)) + return path + + +@contextmanager +def _started( + provider: Wire, + sinks: SpanSinks, + newrelic: ConnectSink, + directory: Path, + capture: str | None, + callbacks: tuple[str, ...] = ("langfuse_otel",), +) -> Generator[Rig]: + environment: Final = { + "LITELLM_OTEL_V2": "1", + "OTEL_BSP_SCHEDULE_DELAY": "300", + "MAPPER_NAMES": "genai,openinference", + "LANGFUSE_HOST": sinks.operator, + "LANGFUSE_PUBLIC_KEY": "pk-lf-operator", + "LANGFUSE_SECRET_KEY": "sk-lf-operator", + "ARIZE_SPACE_KEY": "space-operator", + "ARIZE_API_KEY": "arize-operator", + "ARIZE_HTTP_ENDPOINT": sinks.arize + "/v1/traces", + "WANDB_API_KEY": "wandb-operator", + "WANDB_PROJECT_ID": "operator/weave", + "WANDB_HOST": sinks.arize, + "SIGNOZ_INGESTION_ENDPOINT": sinks.arize + "/v1/traces", + "SIGNOZ_INGESTION_KEY": "signoz-operator", + "HTTPS_PROXY": newrelic.proxy_url, + "NO_PROXY": "127.0.0.1,localhost", + "REQUESTS_CA_BUNDLE": newrelic.ca_pem, + **({} if capture is None else {"OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT": capture}), + } + with ( + gateway_from_environment() as gateway, + owned_proxy_process( + gateway, + directory, + environment, + config=_write_config(directory, sinks, "capture", callbacks), + remove_environment=( + "OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", + "LITELLM_OTEL_TENANT_DESTINATION_MODE", + ), + workers=2, + ) as owned, + owned.gateway.scenario() as scenario, + ): + yield Rig( + owned.gateway, owned, scenario, scenario.model(api_base=provider.url + "/v1"), provider, sinks, newrelic + ) + + +@pytest.fixture(scope="module") +def provider() -> Iterator[Wire]: + with wire_server(_upstream) as wire: + yield wire + + +@pytest.fixture(scope="module") +def newrelic_sink(tmp_path_factory: pytest.TempPathFactory) -> Iterator[ConnectSink]: + with owned_connect_sink(tmp_path_factory.mktemp("newrelic-sink")) as sink: + yield sink + + +@pytest.fixture(scope="module") +def rig( + provider: Wire, audit_sinks: SpanSinks, newrelic_sink: ConnectSink, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Rig]: + directory: Final = tmp_path_factory.mktemp("capture-span-only") + with _started(provider, audit_sinks, newrelic_sink, directory, "span_only") as started: + yield started + + +@pytest.fixture(scope="module") +def dark_rig( + provider: Wire, audit_sinks: SpanSinks, newrelic_sink: ConnectSink, tmp_path_factory: pytest.TempPathFactory +) -> Iterator[Rig]: + with _started( + provider, + audit_sinks, + newrelic_sink, + tmp_path_factory.mktemp("capture-no-content"), + None, + ("langfuse_otel", "signoz"), + ) as started: + yield started diff --git a/tests/unit/integrations/otel/test_langfuse_logger.py b/tests/unit/integrations/otel/test_langfuse_logger.py index aca9dcc8a5e..ff2060d91c2 100644 --- a/tests/unit/integrations/otel/test_langfuse_logger.py +++ b/tests/unit/integrations/otel/test_langfuse_logger.py @@ -16,6 +16,7 @@ import litellm # noqa: E402 from litellm.caching.dual_cache import DualCache # noqa: E402 from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger # noqa: E402 from litellm.integrations.otel.model.config import OpenTelemetryV2Config, is_otel_v2_enabled # noqa: E402 +from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 from litellm.integrations.otel.model.spans import LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole # noqa: E402 from litellm.integrations.otel.plumbing import context as otel_context # noqa: E402 from litellm.integrations.otel.plumbing import providers # noqa: E402 @@ -155,6 +156,25 @@ def test_chat_request_stamps_root_observation_input_and_output(): assert [(turn["role"], turn["content"]) for turn in output] == [("assistant", "pong")] +def test_content_on_destination_stamps_langfuse_root_io_when_global_capture_is_off() -> None: + logger, exporter = _logger(capture="no_content") + destination: Final = OtelDestination( + endpoint="https://team.example.com/api/public/otel", + callback_name="langfuse_otel", + capture_message_content="span_only", + ) + token: Final = otel_context.set_request_destinations((destination,)) + try: + response: Final = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + _run_request(logger, CHAT_DATA, "acompletion", response) + finally: + otel_context.reset_request_destinations(token) + + attrs: Final = _root_attrs(exporter) + assert json.loads(attrs[INPUT_ATTR]) == [{"role": "user", "content": "ping"}] + assert json.loads(attrs[OUTPUT_ATTR]) == [{"role": "assistant", "content": "pong"}] + + def test_responses_request_folds_instructions_into_input_and_stamps_output_items(): logger, exporter = _logger() data = {"model": "gpt-5.4-mini", "instructions": "be terse", "input": "ping"} @@ -461,7 +481,9 @@ def test_a_request_without_trace_controls_stamps_none_of_them(): logger, exporter = _logger() root_attrs, generation_attrs = _run_named_request( - logger, exporter, {"metadata": {"user_api_key_team_id": "t1", "tags": []}, "proxy_server_request": {"headers": {}}} + logger, + exporter, + {"metadata": {"user_api_key_team_id": "t1", "tags": []}, "proxy_server_request": {"headers": {}}}, ) assert set(TRACE_CONTROL_ATTRS).isdisjoint(root_attrs) @@ -472,7 +494,7 @@ def test_a_request_without_trace_controls_stamps_none_of_them(): ("capture", "mappers"), [("no_content", ("genai", "langfuse")), ("span_only", ("genai",))], ) -def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(capture, mappers): +def test_langfuse_root_io_requires_global_or_destination_capture(capture: str, mappers: tuple[str, ...]) -> None: logger, exporter = _logger(capture=capture, mappers=mappers) _run_request(logger, CHAT_DATA, "acompletion", ModelResponse()) @@ -484,13 +506,13 @@ def test_factory_keeps_the_base_logger_unless_langfuse_content_capture_is_on(cap ("capture", "mappers", "relays_streams"), [ ("span_only", ("genai", "langfuse"), True), - ("no_content", ("genai", "langfuse"), False), + ("no_content", ("genai", "langfuse"), True), ("span_only", ("genai",), False), ], ) -def test_only_langfuse_content_capture_takes_proxy_streams_off_the_fast_path( - monkeypatch, capture, mappers, relays_streams -): +def test_langfuse_mapper_relays_streams_for_per_request_content_capture( + monkeypatch: pytest.MonkeyPatch, capture: str, mappers: tuple[str, ...], relays_streams: bool +) -> None: logger, _ = _logger(capture=capture, mappers=mappers) monkeypatch.setattr(litellm, "callbacks", [logger]) diff --git a/tests/unit/integrations/otel/test_otel_v2_destinations.py b/tests/unit/integrations/otel/test_otel_v2_destinations.py index 2d7e0745039..5c9cab88784 100644 --- a/tests/unit/integrations/otel/test_otel_v2_destinations.py +++ b/tests/unit/integrations/otel/test_otel_v2_destinations.py @@ -5,6 +5,7 @@ import contextvars import time from base64 import b64encode from collections.abc import Mapping +from dataclasses import replace from datetime import datetime, timezone from functools import reduce from types import MappingProxyType @@ -13,13 +14,14 @@ from typing import Final import pytest from opentelemetry import trace as trace_api from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace import ReadableSpan, TracerProvider from opentelemetry.sdk.trace.export import SimpleSpanProcessor from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import Status, StatusCode import litellm from litellm.integrations.custom_logger import CustomLogger +from litellm.integrations.otel import GenAIOperation from litellm.integrations.otel import logger as otel_logger from litellm.integrations.otel.logger import ( OpenTelemetryV2, @@ -27,13 +29,24 @@ from litellm.integrations.otel.logger import ( fan_out_provider, publish_global_otel_v2_provider, ) +from litellm.integrations.otel.mappers import resolve_mappers from litellm.integrations.otel.model.config import ( + CaptureMessageContent, ExporterOwner, ExporterSpec, OpenTelemetryV2Config, is_otel_v2_enabled, ) from litellm.integrations.otel.model.destination import OtelDestination +from litellm.integrations.otel.model.payloads import ( + LLMCallSpanData, + LLMRequestParams, + LLMUsage, + MCPToolCallSpanData, + RequestIdentity, + ServerInfo, + ToolDefinition, +) from litellm.integrations.otel.plumbing import providers as otel_providers from litellm.integrations.otel.plumbing.context import ( destination_backends, @@ -44,9 +57,11 @@ from litellm.integrations.otel.plumbing.providers import ( TenantFanOutSpanProcessor, _OverriddenBackendFilter, _sink_key, + attach_tenant_fan_out, build_tracer_provider, deliverable_destinations, operator_sink_scopes, + register_exporter_factory, ) from litellm.integrations.otel.plumbing.routing import TenantTracerCache, get_tracer from litellm.integrations.otel.presets.arize import arize_preset @@ -55,7 +70,7 @@ from litellm.integrations.otel.presets.destinations import ( destination_for, ) from litellm.integrations.otel.presets.langfuse import langfuse_preset -from litellm.proxy._types import AddTeamCallback, UserAPIKeyAuth +from litellm.proxy._types import AddTeamCallback, TeamCallbackMetadata, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( convert_key_logging_metadata_to_callback, resolve_tenant_otel_destinations, @@ -4228,6 +4243,569 @@ class TestTenantHostSsrfGuard: assert sum("provider_url_destination_allowed_hosts" in record.message for record in caplog.records) == 1 +_ALL_MAPPERS: Final = ("genai", "legacy", "openinference", "langfuse", "weave", "langtrace") +_SECRET: Final = "SECRET-MARKER" +_MODEL_CALL_WITH_CONTENT: Final = LLMCallSpanData( + operation=GenAIOperation.CHAT, + provider="openai", + request_model="gpt-4o", + response_model="gpt-4o-2024", + response_id="resp_1", + request_params=LLMRequestParams(temperature=0.5, max_tokens=128), + usage=LLMUsage(input_tokens=12, output_tokens=8, total_tokens=20), + finish_reasons=("tool_calls",), + error=None, + response_cost=0.001, + server=ServerInfo("api.openai.com", 443), + identity=RequestIdentity(call_id="c1", team_id="t1"), + tools=(ToolDefinition(name="lookup", description="Find a city", parameters_json='{"type":"object"}'),), + messages_in=( + {"role": "system", "content": f"system {_SECRET}"}, + {"role": "user", "content": f"prompt {_SECRET}"}, + ), + choices_out=( + { + "finish_reason": "tool_calls", + "message": { + "role": "assistant", + "content": f"answer {_SECRET}", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": {"name": "lookup", "arguments": f'{{"city": "{_SECRET}"}}'}, + } + ], + }, + }, + ), +) +_MODEL_CALL_WITHOUT_CONTENT: Final = replace(_MODEL_CALL_WITH_CONTENT, messages_in=(), choices_out=()) +_TOOL_CALL_WITH_CONTENT: Final = MCPToolCallSpanData( + operation=GenAIOperation.EXECUTE_TOOL, + method="tools/call", + tool_name="lookup", + server_name="maps", + server_address="maps.local", + server_port=443, + session_id="s1", + arguments_json=f'{{"city": "{_SECRET}"}}', + result_json=f'{{"weather": "{_SECRET}"}}', + error=None, + response_cost=None, + identity=RequestIdentity(call_id="c2", team_id="t1"), +) +_TOOL_CALL_WITHOUT_CONTENT: Final = replace(_TOOL_CALL_WITH_CONTENT, arguments_json=None, result_json=None) + +NO_CONTENT_DEST = OtelDestination( + endpoint="http://team-a.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic YQ=="}), + callback_name="langfuse_otel", + capture_message_content="no_content", +) +INHERITING_DEST = OtelDestination( + endpoint="http://team-b.local/v1", + headers=MappingProxyType({"api_key": "k", "arize-space-id": "s"}), + callback_name="arize", +) + + +def mapped(data: LLMCallSpanData | MCPToolCallSpanData) -> Mapping[str, object]: + """What the configured mappers write on the span, the way the logger stamps it.""" + return MappingProxyType(reduce(lambda acc, mapper: {**acc, **mapper.map(data)}, resolve_mappers(_ALL_MAPPERS), {})) + + +def recorded(attributes: Mapping[str, object]) -> dict[str, object]: + """``attributes`` the way the SDK records them, sequences frozen into tuples.""" + return {key: tuple(value) if isinstance(value, list) else value for key, value in attributes.items()} + + +def model_call_tree(provider: TracerProvider, attributes: Mapping[str, object]) -> None: + tracer: Final = get_tracer(provider, "litellm") + with tracer.start_as_current_span("POST /v1/chat/completions"): + with tracer.start_as_current_span("chat gpt-4o") as llm: + llm.set_attributes(attributes) + llm.add_event("gen_ai.content.first_chunk", {"gen_ai.response.model": "gpt-4o-2024"}) + + +def by_name(exporter: InMemorySpanExporter) -> dict[str, ReadableSpan]: + return {span.name: span for span in exporter.get_finished_spans()} + + +def carries_content(span: ReadableSpan) -> bool: + return any(_SECRET in str(value) for value in span.attributes.values()) + + +class TestCaptureMessageContent: + """A destination's explicit capture mode overrides the global default.""" + + @staticmethod + def _fan_out( + operator: InMemorySpanExporter, + exporters: Mapping[str, InMemorySpanExporter], + default_capture: CaptureMessageContent = CaptureMessageContent.SPAN_ONLY, + ) -> TracerProvider: + provider: Final = TracerProvider() + provider.add_span_processor(SimpleSpanProcessor(operator)) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda d: SimpleSpanProcessor(exporters[d.endpoint]), + default_capture=default_capture, + ) + ) + return provider + + @staticmethod + def _run( + provider: TracerProvider, destinations: tuple[OtelDestination, ...], attributes: Mapping[str, object] + ) -> None: + def run() -> None: + set_request_destinations(destinations) + model_call_tree(provider, attributes) + + in_fresh_context(run) + + @pytest.mark.parametrize( + ("setting", "content_exported"), + [ + (None, True), + ("no_content", False), + ("span_only", True), + ("event_only", False), + ("span_and_event", True), + ], + ) + def test_a_destination_setting_controls_its_copy_and_omission_uses_the_global_default( + self, setting: str | None, content_exported: bool + ) -> None: + attributes: Final = mapped(_MODEL_CALL_WITH_CONTENT) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + capture_message_content=setting, + ) + operator, tenant = InMemorySpanExporter(), InMemorySpanExporter() + + self._run(self._fan_out(operator, {destination.endpoint: tenant}), (destination,), attributes) + + exported: Final = by_name(tenant)["chat gpt-4o"] + assert carries_content(exported) is content_exported + assert carries_content(by_name(operator)["chat gpt-4o"]) + assert exported.attributes["gen_ai.request.model"] == "gpt-4o" + assert exported.attributes["gen_ai.usage.input_tokens"] == 12 + assert exported.attributes["gen_ai.usage.output_tokens"] == 8 + + def test_a_span_only_team_lifts_global_no_content_only_for_its_destination(self) -> None: + operator_exporter, team_exporter, sibling_exporter = ( + InMemorySpanExporter(), + InMemorySpanExporter(), + InMemorySpanExporter(), + ) + kind: Final = "lit8244_global_no_content_capture" + register_exporter_factory(kind, lambda _spec: operator_exporter) + config: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.NO_CONTENT, + exporters=[ExporterSpec(kind=kind)], + ) + provider: Final = build_tracer_provider(config, use_simple_processor=True, tenant_overrides=True) + team: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + capture_message_content="span_only", + ) + sibling: Final = INHERITING_DEST + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda destination: SimpleSpanProcessor( + team_exporter if destination.endpoint == team.endpoint else sibling_exporter + ), + default_capture=config.capture_message_content, + ) + ) + + self._run( + provider, + (team, sibling), + mapped(_MODEL_CALL_WITH_CONTENT), + ) + + assert carries_content(by_name(team_exporter)["chat gpt-4o"]) + assert not any(carries_content(span) for span in operator_exporter.get_finished_spans()) + assert not any(carries_content(span) for span in sibling_exporter.get_finished_spans()) + + @pytest.mark.parametrize( + ("with_content", "without_content"), + [ + (_MODEL_CALL_WITH_CONTENT, _MODEL_CALL_WITHOUT_CONTENT), + (_TOOL_CALL_WITH_CONTENT, _TOOL_CALL_WITHOUT_CONTENT), + ], + ids=["model_call", "mcp_tool_call"], + ) + def test_no_content_removes_exactly_what_global_capture_adds_in_every_mapper_vocabulary( + self, + with_content: LLMCallSpanData | MCPToolCallSpanData, + without_content: LLMCallSpanData | MCPToolCallSpanData, + ) -> None: + captured, uncaptured = mapped(with_content), mapped(without_content) + assert frozenset(captured) - frozenset(uncaptured), "the fixture must exercise captured content" + operator, tenant = InMemorySpanExporter(), InMemorySpanExporter() + + self._run(self._fan_out(operator, {NO_CONTENT_DEST.endpoint: tenant}), (NO_CONTENT_DEST,), captured) + + exported: Final = by_name(tenant)["chat gpt-4o"] + assert dict(exported.attributes) == recorded(uncaptured), "only content goes, every other attribute stays" + assert not carries_content(exported) + + def test_restricting_one_destination_leaves_the_original_span_and_the_other_destination_alone(self) -> None: + attributes: Final = mapped(_MODEL_CALL_WITH_CONTENT) + operator, restricted, inheriting = InMemorySpanExporter(), InMemorySpanExporter(), InMemorySpanExporter() + exporters: Final = {NO_CONTENT_DEST.endpoint: restricted, INHERITING_DEST.endpoint: inheriting} + + self._run(self._fan_out(operator, exporters), (NO_CONTENT_DEST, INHERITING_DEST), attributes) + + original: Final = by_name(operator) + kept: Final = by_name(inheriting) + stripped: Final = by_name(restricted) + assert dict(original["chat gpt-4o"].attributes) == recorded(attributes) + assert dict(kept["chat gpt-4o"].attributes) == recorded(attributes) + assert not carries_content(stripped["chat gpt-4o"]) + for copy in (kept, stripped): + for name, span in copy.items(): + assert span.context == original[name].context, "same trace id and span id" + assert span.parent == original[name].parent, "same place in the tree" + assert [e.name for e in copy["chat gpt-4o"].events] == [e.name for e in original["chat gpt-4o"].events] + + def test_an_omitted_setting_and_span_only_export_the_same_span(self) -> None: + attributes: Final = mapped(_MODEL_CALL_WITH_CONTENT) + span_only: Final = INHERITING_DEST.model_copy( + update={"capture_message_content": CaptureMessageContent.SPAN_ONLY} + ) + omitted_exporter, span_only_exporter = InMemorySpanExporter(), InMemorySpanExporter() + + for destination, exporter in ((INHERITING_DEST, omitted_exporter), (span_only, span_only_exporter)): + self._run( + self._fan_out(InMemorySpanExporter(), {destination.endpoint: exporter}), (destination,), attributes + ) + + assert {n: dict(s.attributes) for n, s in by_name(omitted_exporter).items()} == { + n: dict(s.attributes) for n, s in by_name(span_only_exporter).items() + } + + @pytest.mark.parametrize( + ("globally_captured", "setting", "content_exported"), + [ + (True, "no_content", False), + (True, None, True), + (False, "span_only", True), + (False, None, False), + ], + ) + def test_the_operators_exporter_on_the_same_account_honors_the_teams_setting_once( + self, + monkeypatch: pytest.MonkeyPatch, + globally_captured: bool, + setting: str | None, + content_exported: bool, + ) -> None: + """The fan-out skips a destination the operator already writes to, so the operator's + copy is the one that account receives and must not bypass the team's restriction.""" + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", "additive", raising=False) + shared: Final = InMemorySpanExporter() + provider: Final = TracerProvider() + provider.add_span_processor( + _OverriddenBackendFilter( + SimpleSpanProcessor(shared), + "langfuse_otel", + "full", + TestRoutingMode.OPERATOR_SINK, + global_captures=globally_captured, + ) + ) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=MappingProxyType({TestRoutingMode.OPERATOR_SINK: "full"}), + ) + ) + destination: Final = OtelDestination( + endpoint=TestRoutingMode.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + capture_message_content=setting, + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + + model_calls: Final = [s for s in shared.get_finished_spans() if s.name == "chat gpt-4o"] + assert len(model_calls) == 1, "the same account received the span twice" + assert carries_content(model_calls[0]) is content_exported + + @pytest.mark.parametrize("mode", ["override", "additive"]) + def test_an_operator_collector_on_the_teams_account_does_not_bypass_no_content( + self, monkeypatch: pytest.MonkeyPatch, mode: str + ) -> None: + monkeypatch.setattr(litellm, "otel_tenant_destination_mode", mode, raising=False) + shared: Final = InMemorySpanExporter() + kind: Final = f"lit8244_collector_{mode}" + register_exporter_factory(kind, lambda _spec: shared) + config: Final = OpenTelemetryV2Config( + exporters=[ + ExporterSpec( + kind=kind, + endpoint=TestRoutingMode.OPERATOR_SINK[0], + headers="authorization=Basic op", + ) + ] + ) + provider: Final = build_tracer_provider(config, use_simple_processor=True, tenant_overrides=True) + provider.add_span_processor( + TenantFanOutSpanProcessor( + processor_factory=lambda _d: SimpleSpanProcessor(shared), + operator_sinks=operator_sink_scopes(config), + ) + ) + destination: Final = OtelDestination( + endpoint=TestRoutingMode.SAME_ACCOUNT_ENDPOINT, + headers=MappingProxyType({"Authorization": "Basic op"}), + callback_name="langfuse_otel", + capture_message_content="no_content", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + + model_calls: Final = [s for s in shared.get_finished_spans() if s.name == "chat gpt-4o"] + assert model_calls, "the account must receive the model call before its content can be judged absent" + assert not any(carries_content(span) for span in model_calls) + + @pytest.mark.parametrize( + ("global_capture", "content_exported"), + [(CaptureMessageContent.NO_CONTENT, False), (CaptureMessageContent.SPAN_ONLY, True)], + ) + def test_a_routed_provider_exports_content_only_when_the_global_setting_captures( + self, global_capture: str, content_exported: bool + ) -> None: + """A sibling destination's span_only makes the logger collect content, and a request + routed to this callback's team credentials must not carry it under a global no_content.""" + exporters: dict[ExporterOwner | None, InMemorySpanExporter] = {} + kind: Final = f"lit8244_routed_{global_capture}" + register_exporter_factory(kind, lambda spec: exporters.setdefault(spec.owner, InMemorySpanExporter())) + config: Final = OpenTelemetryV2Config( + capture_message_content=global_capture, + exporters=[ + ExporterSpec( + kind=kind, + endpoint="http://op.local", + owner=ExporterOwner.LANGFUSE_OTEL, + use_simple_processor=True, + ), + ExporterSpec(kind=kind), + ], + ) + cache: Final = TenantTracerCache(config, "langfuse_otel", "litellm") + + route: Final = cache.route_for( + get_tracer(TracerProvider(), "litellm"), + {"langfuse_public_key": "pk-team", "langfuse_secret_key": "sk-team"}, + ) + with route.tracer.start_as_current_span("chat gpt-4o") as llm: + llm.set_attributes(mapped(_MODEL_CALL_WITH_CONTENT)) + cache.release(route.provider) + + assert route.detached is True + for exporter in (exporters[ExporterOwner.LANGFUSE_OTEL], exporters[None]): + copy = by_name(exporter)["chat gpt-4o"] + assert carries_content(copy) is content_exported + assert copy.attributes["gen_ai.usage.input_tokens"] == 12 + + @pytest.mark.parametrize("first", ["newrelic", "langfuse_otel"]) + def test_an_omitted_setting_follows_its_own_callbacks_mode_whatever_the_callback_order( + self, monkeypatch: pytest.MonkeyPatch, first: str + ) -> None: + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + tenant: Final = InMemorySpanExporter() + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, "otlp_http", lambda _spec: tenant) + langfuse: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.SPAN_ONLY, + exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)], + ) + newrelic: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.NO_CONTENT, + exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.NEWRELIC)], + ) + provider: Final = TracerProvider() + attach_tenant_fan_out(provider, *((newrelic, langfuse) if first == "newrelic" else (langfuse, newrelic))) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + provider.force_flush() + + assert carries_content(by_name(tenant)["chat gpt-4o"]) + + def test_an_omitted_setting_follows_the_capture_mode_set_in_callback_settings_otel( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + tenant: Final = InMemorySpanExporter() + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, "otlp_http", lambda _spec: tenant) + operator: Final = OpenTelemetryV2Config(**{"capture_message_content": "span_only", "exporter": "in_memory"}) + provider: Final = TracerProvider() + attach_tenant_fan_out(provider, operator) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + provider.force_flush() + + assert carries_content(by_name(tenant)["chat gpt-4o"]), "the YAML value is the global default, not only the env" + + def test_a_published_presets_capture_mode_is_the_default_for_an_unowned_destination( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A destination for a backend the proxy does not export to itself falls back to the published logger's mode, + here New Relic with record_content on, so it keeps the content.""" + monkeypatch.delenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", raising=False) + tenant: Final = InMemorySpanExporter() + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, "otlp_http", lambda _spec: tenant) + newrelic: Final = OpenTelemetryV2Config( + capture_message_content=CaptureMessageContent.SPAN_ONLY, + exporters=[ + ExporterSpec(kind="console"), + ExporterSpec(kind="in_memory", owner=ExporterOwner.NEWRELIC), + ], + ) + provider: Final = TracerProvider() + attach_tenant_fan_out(provider, newrelic) + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + ) + + self._run(provider, (destination,), mapped(_MODEL_CALL_WITH_CONTENT)) + provider.force_flush() + + assert carries_content(by_name(tenant)["chat gpt-4o"]) + + @pytest.mark.usefixtures("allow_test_hosts") + @pytest.mark.parametrize("setting", ["no_content", "span_only", None]) + def test_the_setting_rides_the_teams_destination_and_omission_stays_omitted( + self, monkeypatch: pytest.MonkeyPatch, setting: str | None + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + callback_vars: Final = { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + } + if setting is not None: + callback_vars["capture_message_content"] = setting + auth: Final = UserAPIKeyAuth( + team_metadata={ + "logging": [ + {"callback_name": "langfuse_otel", "callback_type": "success", "callback_vars": callback_vars}, + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_api_key": "k", "arize_space_id": "s"}, + }, + ] + } + ) + + destinations: Final = {d.callback_name: d for d in resolve_tenant_otel_destinations(auth)} + + assert destinations["langfuse_otel"].capture_message_content == setting + assert destinations["arize"].capture_message_content is None, "the setting stays on its own callback" + + @pytest.mark.usefixtures("allow_test_hosts") + @pytest.mark.parametrize("setting", ["event_only", "span_and_event"]) + def test_a_stored_event_mode_is_skipped_like_any_invalid_entry( + self, monkeypatch: pytest.MonkeyPatch, setting: str + ) -> None: + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + auth: Final = UserAPIKeyAuth( + team_metadata={ + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": { + "langfuse_public_key": "pk-team", + "langfuse_secret_key": "sk-team", + "langfuse_host": "http://team.local", + "capture_message_content": setting, + }, + }, + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"arize_api_key": "k", "arize_space_id": "s"}, + }, + ] + } + ) + + destinations: Final = [d.callback_name for d in resolve_tenant_otel_destinations(auth)] + + assert destinations == ["arize"] + + @pytest.mark.parametrize("value", ["full", "NO_CONTENT", "", "true", "event_only", "span_and_event"]) + def test_an_unsupported_value_fails_registration(self, value: str) -> None: + with pytest.raises( + ValueError, + match=r"Invalid capture_message_content .*\['no_content', 'span_only'\]", + ): + AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "capture_message_content": value, + }, + ) + + @pytest.mark.parametrize("value", ["no_content", "span_only"]) + def test_a_supported_value_is_stored_as_given(self, value: str) -> None: + saved: Final = AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={"langfuse_public_key": "pk", "langfuse_secret_key": "sk", "capture_message_content": value}, + ) + + assert saved.callback_vars["capture_message_content"] == value + + def test_a_team_wide_callback_vars_map_cannot_carry_the_setting(self) -> None: + """The flattened map is shared by every callback, so a value there would apply to all of them.""" + with pytest.raises(ValueError, match="Invalid callback variable: capture_message_content"): + TeamCallbackMetadata( + success_callback=["langfuse_otel", "signoz"], callback_vars={"capture_message_content": "span_only"} + ) + + def test_flattening_the_entries_leaves_the_setting_on_its_own_entry(self) -> None: + flattened: Final = convert_key_logging_metadata_to_callback( + AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={"langfuse_public_key": "pk", "capture_message_content": "span_only"}, + ), + None, + ) + + assert flattened.callback_vars == {"langfuse_public_key": "pk"} + + class TestArizeProjectRouting: """Arize rejects an export that names no project (400: set x-project-name, arize.project.name, openinference.project.name, or model_id), so an Arize diff --git a/tests/unit/integrations/otel/test_otel_v2_logger.py b/tests/unit/integrations/otel/test_otel_v2_logger.py index 3b1dd592d08..6a9a963948e 100644 --- a/tests/unit/integrations/otel/test_otel_v2_logger.py +++ b/tests/unit/integrations/otel/test_otel_v2_logger.py @@ -9,11 +9,13 @@ hooks, proxy SERVER span lifecycle (start + setters), parent-context resolution import ast import asyncio import contextlib +import contextvars import os import re from dataclasses import dataclass from datetime import datetime, timedelta, timezone from pathlib import Path +from types import MappingProxyType from typing import Final from unittest.mock import patch @@ -43,7 +45,8 @@ from litellm.integrations.otel import ( # noqa: E402 OpenTelemetryV2Config, ) from litellm.integrations.otel.logger import OpenTelemetryV2 # noqa: E402 -from litellm.integrations.otel.model.config import ExporterSpec # noqa: E402 +from litellm.integrations.otel.model.config import CaptureMessageContent, ExporterSpec # noqa: E402 +from litellm.integrations.otel.model.destination import OtelDestination # noqa: E402 from litellm.integrations.otel.model.spans import ( # noqa: E402 LITELLM_PROXY_REQUEST_SPAN_NAME, SpanRole, @@ -55,6 +58,7 @@ from litellm.integrations.otel.plumbing.context import ( # noqa: E402 reset_mcp_message_transport_span, set_mcp_message_trace_carrier, set_mcp_message_transport_span, + set_request_destinations, set_request_root_span, ) from litellm.types.utils import AUTOROUTER_CLASSIFIER_CALL_ORIGIN # noqa: E402 @@ -617,8 +621,6 @@ def _mcp_payload(**overrides): def _logger_capturing(): - from litellm.integrations.otel.model.config import CaptureMessageContent - cfg = OpenTelemetryV2Config( exporter="in_memory", legacy_compat=False, @@ -684,6 +686,57 @@ def test_mcp_tool_call_captures_io_when_enabled(): assert "21" in span.attributes["gen_ai.tool.call.result"] +@pytest.mark.parametrize( + ("capture_mode", "captures_content"), + [ + (None, False), + (CaptureMessageContent.NO_CONTENT, False), + (CaptureMessageContent.SPAN_ONLY, True), + (CaptureMessageContent.EVENT_ONLY, False), + (CaptureMessageContent.SPAN_AND_EVENT, True), + ], +) +def test_only_a_request_destination_can_turn_on_llm_payload_content( + capture_mode: CaptureMessageContent | None, captures_content: bool +) -> None: + config: Final = OpenTelemetryV2Config( + exporter="in_memory", + capture_message_content=CaptureMessageContent.NO_CONTENT, + ) + exporter: Final = InMemorySpanExporter() + logger: Final = OpenTelemetryV2( + config=config, tracer_provider=providers.build_tracer_provider(config, exporter=exporter) + ) + payload: Final = _payload( + messages=[{"role": "user", "content": "prompt marker"}], + response={ + "id": "resp_1", + "model": "gpt-4o-2024", + "choices": [{"finish_reason": "stop", "message": {"role": "assistant", "content": "answer marker"}}], + }, + ) + kwargs: Final = { + **_kwargs(payload), + "standard_callback_dynamic_params": {"capture_message_content": "span_only"}, + } + destination: Final = OtelDestination( + endpoint="http://team.local/api/public/otel", + headers=MappingProxyType({"Authorization": "Basic dA=="}), + callback_name="langfuse_otel", + capture_message_content=capture_mode, + ) + + def emit() -> None: + set_request_destinations((destination,)) + _emit_llm(logger, kwargs) + + contextvars.copy_context().run(emit) + + (span,) = exporter.get_finished_spans() + assert ("gen_ai.input.messages" in span.attributes) is captures_content + assert ("gen_ai.output.messages" in span.attributes) is captures_content + + def test_mcp_tool_call_failure_marks_error(): logger, exporter = _logger() payload = _mcp_payload( diff --git a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py index ea0416cecbd..318859cb3ed 100644 --- a/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py +++ b/tests/unit/integrations/otel/test_otel_v2_sources_of_truth.py @@ -1692,6 +1692,19 @@ def test_capture_message_content_normalizer_only_touches_strings(): OpenTelemetryV2Config(capture_message_content=123) +def test_unknown_capture_message_content_env_is_kept_as_a_plain_string_that_captures_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setenv("OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT", "Invalid-Mode") + + config: Final = OpenTelemetryV2Config() + + assert type(config.capture_message_content) is str + assert config.capture_message_content == "invalid-mode" + assert config.capture_span_content is False + assert config.model_dump()["capture_message_content"] == "invalid-mode" + + def test_v2_flag_is_off_by_default(monkeypatch): monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) is_otel_v2_enabled.cache_clear() diff --git a/tests/unit/proxy/common_utils/test_callback_config_validation.py b/tests/unit/proxy/common_utils/test_callback_config_validation.py index d10a96e2aef..a6544e56c57 100644 --- a/tests/unit/proxy/common_utils/test_callback_config_validation.py +++ b/tests/unit/proxy/common_utils/test_callback_config_validation.py @@ -1,7 +1,11 @@ +from collections.abc import Iterator +from typing import Final + import pytest from litellm.proxy.common_utils.callback_config_validation import ( callback_config_error, + conflicting_capture_error, conflicting_span_scope_error, cross_entry_family_error, logging_metadata_config_error, @@ -259,3 +263,114 @@ def test_arize_sampling_rates_are_not_family_credentials(): stored = [{"arize_api_key": "k1", "arize_success_sampling_rate": "0.5"}] assert cross_entry_family_error({"arize_success_sampling_rate": "0.1"}, stored) is None assert cross_entry_family_error({"arize_error_sampling_rate": "0.5"}, stored) is None + + +@pytest.fixture +def otel_v2_on(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.parametrize("callback_name", ["langfuse_otel", "arize", "weave_otel", "newrelic"]) +@pytest.mark.parametrize("value", ["no_content", "span_only"]) +def test_capture_message_content_is_accepted_on_every_otel_v2_destination(callback_name: str, value: str) -> None: + assert callback_config_error(callback_name, {"capture_message_content": value}) is None + + +def test_capture_message_content_is_rejected_while_otel_v2_is_off(monkeypatch: pytest.MonkeyPatch) -> None: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.delenv("LITELLM_OTEL_V2", raising=False) + is_otel_v2_enabled.cache_clear() + try: + error: Final = callback_config_error("langfuse_otel", {"capture_message_content": "no_content"}) + finally: + is_otel_v2_enabled.cache_clear() + assert error is not None and "capture_message_content" in error and "LITELLM_OTEL_V2" in error + + +@pytest.mark.parametrize("callback_name", ["langfuse", "datadog", "otel", "arize_phoenix", None]) +def test_capture_message_content_is_rejected_where_it_would_never_take_effect(callback_name: str | None) -> None: + error: Final = callback_config_error(callback_name, {"capture_message_content": "no_content"}) + assert error is not None and "capture_message_content" in error and "langfuse_otel" in error + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.parametrize("value", ["full", "event_only", "span_and_event"]) +def test_an_unsupported_capture_message_content_is_rejected_on_key_logging_metadata(value: str) -> None: + error: Final = logging_metadata_config_error( + { + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"capture_message_content": value}, + } + ] + } + ) + assert error is not None and "Invalid capture_message_content" in error + + +@pytest.mark.usefixtures("otel_v2_on") +def test_each_entry_keeps_its_own_capture_message_content() -> None: + error: Final = logging_metadata_config_error( + { + "logging": [ + { + "callback_name": "langfuse_otel", + "callback_type": "success", + "callback_vars": {"capture_message_content": "span_only"}, + }, + { + "callback_name": "arize", + "callback_type": "success", + "callback_vars": {"capture_message_content": "no_content"}, + }, + ] + } + ) + assert error is None + + +@pytest.mark.usefixtures("otel_v2_on") +def test_a_failure_only_entry_rejects_capture_message_content() -> None: + error: Final = callback_config_error("langfuse_otel", {"capture_message_content": "no_content"}, "failure") + assert error is not None and "capture_message_content" in error and "success_and_failure" in error + + +@pytest.mark.parametrize( + ("stored", "rejected"), + [ + ([("langfuse_otel", {"capture_message_content": "no_content"})], True), + ([("langfuse_otel", {"capture_message_content": "span_only"})], False), + ([("langfuse_otel", {"langfuse_public_key": "pk"})], False), + ([("newrelic", {"capture_message_content": "no_content"})], False), + ], +) +def test_entries_for_one_backend_share_one_capture_message_content( + stored: list[tuple[str, dict[str, str]]], rejected: bool +) -> None: + """The backend's entries merge into one destination, so a second value would silently win or lose.""" + error: Final = conflicting_capture_error("langfuse_otel", {"capture_message_content": "span_only"}, stored) + assert (error is not None) is rejected + + +@pytest.mark.usefixtures("otel_v2_on") +def test_key_logging_entries_for_one_backend_may_not_disagree_on_capture_message_content() -> None: + def entry(callback_type: str, capture: str) -> dict[str, str | dict[str, str]]: + return { + "callback_name": "langfuse_otel", + "callback_type": callback_type, + "callback_vars": {"capture_message_content": capture}, + } + + error: Final = logging_metadata_config_error( + {"logging": [entry("success", "no_content"), entry("success_and_failure", "span_only")]} + ) + assert error is not None and "already set to 'no_content' by another langfuse_otel entry" in error diff --git a/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py index f576bac4742..ad5b6903ece 100644 --- a/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_team_callback_endpoints.py @@ -7,6 +7,7 @@ redacted audit rows for callback mutations. """ import json +from collections.abc import Iterator from typing import Final from unittest.mock import AsyncMock, MagicMock, Mock, patch @@ -1712,3 +1713,62 @@ def test_add_team_callback_accepts_arize_sampling_rate_vars(): ) assert data.callback_vars["arize_success_sampling_rate"] == "0.5" assert data.callback_vars["arize_error_sampling_rate"] == "1.0" + + +@pytest.mark.asyncio +async def test_add_team_callbacks_rejects_capture_message_content_on_a_non_otel_v2_callback(patched_prisma): + data: Final = AddTeamCallback( + callback_name="langfuse", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "capture_message_content": "no_content", + }, + ) + with pytest.raises(HTTPException) as exc: + await add_team_callbacks( + data=data, + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + assert exc.value.status_code == 400 + assert "capture_message_content" in str(exc.value.detail) + patched_prisma.db.litellm_teamtable.update.assert_not_called() + + +@pytest.fixture +def otel_v2_on(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + from litellm.integrations.otel.model.config import is_otel_v2_enabled + + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + yield + is_otel_v2_enabled.cache_clear() + + +@pytest.mark.usefixtures("otel_v2_on") +@pytest.mark.asyncio +@pytest.mark.parametrize("capture", ["no_content", "span_only"]) +async def test_add_team_callbacks_stores_capture_message_content_in_the_existing_callback_metadata( + patched_prisma, capture: str +) -> None: + data: Final = AddTeamCallback( + callback_name="langfuse_otel", + callback_type="success", + callback_vars={ + "langfuse_public_key": "pk", + "langfuse_secret_key": "sk", + "capture_message_content": capture, + }, + ) + await add_team_callbacks( + data=data, + http_request=Mock(spec=Request), + team_id="team-victim", + user_api_key_dict=_admin_auth(), + ) + patched_prisma.db.litellm_teamtable.update.assert_awaited_once() + stored: Final = json.loads(patched_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + assert [entry["callback_vars"].get("capture_message_content") for entry in stored["logging"]] == [capture] diff --git a/tests/unit/types/test_utils.py b/tests/unit/types/test_utils.py new file mode 100644 index 00000000000..09063d894e7 --- /dev/null +++ b/tests/unit/types/test_utils.py @@ -0,0 +1,40 @@ +import pytest + +from litellm.types.utils import CAPTURE_MESSAGE_CONTENT_VALUES, CaptureMessageContent, captures_span_content + + +@pytest.mark.parametrize( + "mode", + [ + CaptureMessageContent.NO_CONTENT, + CaptureMessageContent.SPAN_ONLY, + CaptureMessageContent.EVENT_ONLY, + CaptureMessageContent.SPAN_AND_EVENT, + ], +) +def test_capture_message_content_members_are_plain_strings(mode: str) -> None: + assert type(mode) is str + assert (str(mode), f"{mode}", repr(mode)) == (mode, mode, repr(str(mode))) + + +def test_a_destination_may_only_set_the_span_capture_modes() -> None: + assert CAPTURE_MESSAGE_CONTENT_VALUES == {"no_content", "span_only"} + + +def test_capture_message_content_constructor_accepts_any_string() -> None: + assert CaptureMessageContent("custom") == "custom" + + +@pytest.mark.parametrize( + ("mode", "captures"), + [ + ("no_content", False), + ("span_only", True), + ("event_only", False), + ("span_and_event", True), + ("bogus", False), + (None, False), + ], +) +def test_captures_span_content(mode: str | None, captures: bool) -> None: + assert captures_span_content(mode) is captures diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 731a92d8b98..47df4593a65 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -17246,6 +17246,9 @@ export interface paths { * - langfuse_host: The host for the Langfuse callback * - langfuse_environment: The tracing environment for the Langfuse callback (lowercase; falls back to LANGFUSE_TRACING_ENVIRONMENT) * - langfuse_span_scope: For langfuse_otel, "full" (default) sends the whole request trace, "llm_only" sends only the model-call spans + * - capture_message_content: For OTel v2 callbacks, "no_content" or "span_only". + * An explicit value overrides the global setting for this destination; omitted follows the proxy's own callback for that backend, else the global setting. + * "span_only" puts prompt and response content on the destination's spans, "no_content" leaves it out * - gcs_bucket_name: The name of the GCS bucket * - gcs_path_service_account: The path to the GCS service account * - langsmith_api_key: The API key for the Langsmith callback From c9d706adb5d387807d9d03fd9d90c49ef8a66847 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:58:07 -0700 Subject: [PATCH 02/20] refactor(types): move key request models into litellm.types.proxy.management_endpoints.key_requests (#45810) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): annotate GenerateRequestBase containers and check_max_budget Part of LIT-9423 * refactor(types): move GenerateRequestBase and LiteLLM_ObjectPermissionBase into litellm.types.proxy.management_endpoints.request_base Part of LIT-9423 * refactor(types): annotate key request containers and drop_blank_team_id Part of LIT-9423 * refactor(types): move key request models into litellm.types.proxy.management_endpoints.key_requests Part of LIT-9423 * fix(types): keep MappingProxyType exported from litellm.proxy._types for star importers Part of LIT-9423 * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main --- litellm/proxy/_types.py | 144 +++--------------- .../key_management_endpoints.py | 8 +- .../management_endpoints/key_requests.py | 138 +++++++++++++++++ 3 files changed, 161 insertions(+), 129 deletions(-) create mode 100644 litellm/types/proxy/management_endpoints/key_requests.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 39892bcc29c..4cbf4f15d6d 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -3,7 +3,7 @@ import json import os from collections.abc import Callable, Mapping, Sequence from datetime import datetime -from types import MappingProxyType +from types import MappingProxyType as MappingProxyType # noqa: PLC0414 # public re-export from typing import TYPE_CHECKING, Annotated, Any, Final, Literal, NamedTuple, TypeAlias import httpx @@ -1165,6 +1165,18 @@ from litellm.models.team import BudgetLimitEntry as BudgetLimitEntry # noqa: E4 from litellm.types.object_permission import ( # noqa: E402 ObjectPermissionDict as ObjectPermissionDict, ) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + AllowedVectorStoreIndexItem as AllowedVectorStoreIndexItem, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + GenerateKeyRequest as GenerateKeyRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + KeyRequestBase as KeyRequestBase, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + LiteLLMKeyType as LiteLLMKeyType, # noqa: PLC0414 # public re-export +) from litellm.types.proxy.management_endpoints.request_base import ( # noqa: E402 # re-export after the definitions above GenerateRequestBase as GenerateRequestBase, # noqa: PLC0414 # public re-export ) @@ -1173,71 +1185,6 @@ from litellm.types.proxy.management_endpoints.request_base import ( # noqa: E40 ) -class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): - index_name: str - index_permissions: list[Literal["read", "write"]] - - -class KeyRequestBase(GenerateRequestBase): - key: str | None = None - tpd_limit: int | None = None - default_estimated_output_tokens: PositiveInt | None = None - default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None - budget_id: str | None = None - end_user_budget_id: str | None = None - tags: list[str] | None = None - disable_global_guardrails: bool | None = None - enable_prompt_caching: bool | None = None - throttle_on_budget_exceeded: bool | None = None - enforced_params: list[str] | None = None - allowed_routes: list[Any] | None = [] - allowed_passthrough_routes: list[object] | None = None - denied_passthrough_routes: list[str] | None = None - allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None - rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( - None # raise an error if 'guaranteed_throughput' is set and we're overallocating rpm - ) - tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( - None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm - ) - router_settings: UpdateRouterConfig | None = None - access_group_ids: list[str] | None = None - - -class LiteLLMKeyType(str, enum.Enum): - """ - Enum for key types that determine what routes a key can access - """ - - LLM_API = "llm_api" # Can call LLM API routes (chat/completions, embeddings, etc.) - MANAGEMENT = "management" # Can call management routes (user/team/key management) - READ_ONLY = "read_only" # Can only call info/read routes - DEFAULT = "default" # Uses default allowed routes - - -class GenerateKeyRequest(KeyRequestBase): - soft_budget: float | None = None - send_invite_email: bool | None = None - key_type: LiteLLMKeyType | None = Field( - default=LiteLLMKeyType.DEFAULT, - description="Type of key that determines default allowed routes.", - ) - auto_rotate: bool | None = Field(default=False, description="Whether this key should be automatically rotated") - rotation_interval: str | None = Field( - default=None, - description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True", - ) - organization_id: str | None = None - project_id: str | None = None - - @field_validator("team_id", "organization_id", "project_id", mode="before") - @classmethod - def treat_cleared_id_as_unset(cls, v: object) -> object: - if v == "": - return None - return v - - class GenerateKeyResponse(KeyRequestBase): key: str key_name: str | None = None @@ -1280,65 +1227,12 @@ class GenerateKeyResponse(KeyRequestBase): return values -def _is_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: raw before-validator payload - return isinstance(value, Mapping) - - -class UpdateKeyRequest(KeyRequestBase): - # Note: the defaults of all Params here MUST BE NONE - # else they will get overwritten - duration: str | None = None - spend: float | None = None - soft_budget: float | None = None - metadata: dict[Any, Any] | None = None - temp_budget_increase: float | None = None - temp_budget_expiry: datetime | None = None - auto_rotate: bool | None = None - rotation_interval: str | None = None - organization_id: str | None = None - - project_id: str | None = Field( - default=None, - description="Omit to retain the project, or send null to detach. Assigning a different project is not supported.", - ) - - @model_validator(mode="before") - @classmethod - def drop_blank_team_id(cls, values: object) -> object: - if _is_mapping(values) and values.get("team_id") == "": - return MappingProxyType({k: v for k, v in values.items() if k != "team_id"}) - return values - - @field_validator("organization_id", mode="before") - @classmethod - def treat_cleared_organization_id_as_unset(cls, v: object) -> object: - if v == "": - return None - return v - - @model_validator(mode="after") - def validate_temp_budget(self) -> "UpdateKeyRequest": - if self.temp_budget_increase is not None or self.temp_budget_expiry is not None: - if self.temp_budget_increase is None or self.temp_budget_expiry is None: - raise ValueError("temp_budget_increase and temp_budget_expiry must be set together") - return self - - @model_validator(mode="after") - def validate_key_identifier(self) -> "UpdateKeyRequest": - if self.key is None and self.key_alias is None: - raise ValueError("either key or key_alias must be provided") - return self - - -class RegenerateKeyRequest(GenerateKeyRequest): - # This needs to be different from UpdateKeyRequest, because "key" is optional for this - key: str | None = None - new_key: str | None = None - duration: str | None = None - spend: float | None = None - metadata: dict[Any, Any] | None = None - new_master_key: str | None = None - grace_period: str | None = None # Duration to keep old key valid (e.g. "24h", "2d"); None = immediate revoke +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + RegenerateKeyRequest as RegenerateKeyRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.key_requests import ( # noqa: E402 # re-export after the definitions above + UpdateKeyRequest as UpdateKeyRequest, # noqa: PLC0414 # public re-export +) class ResetSpendRequest(LiteLLMPydanticObjectBase): diff --git a/litellm/types/proxy/management_endpoints/key_management_endpoints.py b/litellm/types/proxy/management_endpoints/key_management_endpoints.py index 05b63100ea4..f4348391710 100644 --- a/litellm/types/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/types/proxy/management_endpoints/key_management_endpoints.py @@ -5,14 +5,14 @@ from pydantic import ConfigDict, model_validator from typing_extensions import ReadOnly, TypedDict from litellm.models.verification_token import LiteLLM_VerificationToken -from litellm.proxy._types import ( +from litellm.types.llms.base import LiteLLMBaseModel, LiteLLMPydanticObjectBase +from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains +from litellm.types.proxy.management_endpoints.key_requests import ( GenerateKeyRequest, - LiteLLM_ObjectPermissionBase, RegenerateKeyRequest, UpdateKeyRequest, ) -from litellm.types.llms.base import LiteLLMBaseModel, LiteLLMPydanticObjectBase -from litellm.types.proxy.management_endpoints.internal_user_endpoints import InsensitiveContains +from litellm.types.proxy.management_endpoints.request_base import LiteLLM_ObjectPermissionBase class KeyTokenWhere(TypedDict): diff --git a/litellm/types/proxy/management_endpoints/key_requests.py b/litellm/types/proxy/management_endpoints/key_requests.py new file mode 100644 index 00000000000..baa02b68d98 --- /dev/null +++ b/litellm/types/proxy/management_endpoints/key_requests.py @@ -0,0 +1,138 @@ +import enum +from collections.abc import Mapping +from datetime import datetime +from types import MappingProxyType +from typing import Any, Literal # noqa: TID251 # allowed_routes and metadata stay Any for existing callers + +from pydantic import Field, PositiveInt, field_validator, model_validator +from typing_extensions import TypeIs # noqa: TID251 # narrows the raw payload a before-validator receives + +from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase +from litellm.types.router import UpdateRouterConfig +from litellm.types.utils import LiteLLMPydanticObjectBase + + +class AllowedVectorStoreIndexItem(LiteLLMPydanticObjectBase): + index_name: str + index_permissions: list[Literal["read", "write"]] + + +class KeyRequestBase(GenerateRequestBase): + key: str | None = None + tpd_limit: int | None = None + default_estimated_output_tokens: PositiveInt | None = None + default_estimated_output_tokens_per_model: Mapping[str, PositiveInt] | None = None + budget_id: str | None = None + end_user_budget_id: str | None = None + tags: list[str] | None = None + disable_global_guardrails: bool | None = None + enable_prompt_caching: bool | None = None + throttle_on_budget_exceeded: bool | None = None + enforced_params: list[str] | None = None + allowed_routes: list[Any] | None = [] + allowed_passthrough_routes: list[object] | None = None + denied_passthrough_routes: list[str] | None = None + allowed_vector_store_indexes: list[AllowedVectorStoreIndexItem] | None = None + rpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( + None # raise an error if 'guaranteed_throughput' is set and we're overallocating rpm + ) + tpm_limit_type: Literal["guaranteed_throughput", "best_effort_throughput", "dynamic"] | None = ( + None # raise an error if 'guaranteed_throughput' is set and we're overallocating tpm + ) + router_settings: UpdateRouterConfig | None = None + access_group_ids: list[str] | None = None + + +class LiteLLMKeyType(str, enum.Enum): + """ + Enum for key types that determine what routes a key can access + """ + + LLM_API = "llm_api" # Can call LLM API routes (chat/completions, embeddings, etc.) + MANAGEMENT = "management" # Can call management routes (user/team/key management) + READ_ONLY = "read_only" # Can only call info/read routes + DEFAULT = "default" # Uses default allowed routes + + +class GenerateKeyRequest(KeyRequestBase): + soft_budget: float | None = None + send_invite_email: bool | None = None + key_type: LiteLLMKeyType | None = Field( + default=LiteLLMKeyType.DEFAULT, + description="Type of key that determines default allowed routes.", + ) + auto_rotate: bool | None = Field(default=False, description="Whether this key should be automatically rotated") + rotation_interval: str | None = Field( + default=None, + description="How often to rotate this key (e.g., '30d', '90d'). Required if auto_rotate=True", + ) + organization_id: str | None = None + project_id: str | None = None + + @field_validator("team_id", "organization_id", "project_id", mode="before") + @classmethod + def treat_cleared_id_as_unset(cls, v: object) -> object: + if v == "": + return None + return v + + +def _is_mapping(value: object) -> TypeIs[Mapping[object, object]]: # guard-ok: raw before-validator payload + return isinstance(value, Mapping) + + +class UpdateKeyRequest(KeyRequestBase): + # Note: the defaults of all Params here MUST BE NONE + # else they will get overwritten + duration: str | None = None + spend: float | None = None + soft_budget: float | None = None + metadata: dict[Any, Any] | None = None + temp_budget_increase: float | None = None + temp_budget_expiry: datetime | None = None + auto_rotate: bool | None = None + rotation_interval: str | None = None + organization_id: str | None = None + + project_id: str | None = Field( + default=None, + description="Omit to retain the project, or send null to detach. Assigning a different project is not supported.", + ) + + @model_validator(mode="before") + @classmethod + def drop_blank_team_id(cls, values: object) -> object: + if _is_mapping(values) and values.get("team_id") == "": + return MappingProxyType({k: v for k, v in values.items() if k != "team_id"}) + return values + + @field_validator("organization_id", mode="before") + @classmethod + def treat_cleared_organization_id_as_unset(cls, v: object) -> object: + if v == "": + return None + return v + + @model_validator(mode="after") + def validate_temp_budget(self) -> "UpdateKeyRequest": + if self.temp_budget_increase is not None or self.temp_budget_expiry is not None: + if self.temp_budget_increase is None or self.temp_budget_expiry is None: + raise ValueError("temp_budget_increase and temp_budget_expiry must be set together") + return self + + @model_validator(mode="after") + def validate_key_identifier(self) -> "UpdateKeyRequest": + if self.key is None and self.key_alias is None: + raise ValueError("either key or key_alias must be provided") + return self + + +class RegenerateKeyRequest(GenerateKeyRequest): + # This needs to be different from UpdateKeyRequest, because "key" is optional for this + key: str | None = None + new_key: str | None = None + duration: str | None = None + spend: float | None = None + metadata: dict[Any, Any] | None = None + new_master_key: str | None = None + grace_period: str | None = None # Duration to keep old key valid (e.g. "24h", "2d"); None = immediate revoke From 513e7a89ac17e5301ceb57fa30e078454cacee7f Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:58:11 -0700 Subject: [PATCH 03/20] refactor(types): move user request models into litellm.types.proxy.management_endpoints.user_requests (#45813) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): annotate GenerateRequestBase containers and check_max_budget Part of LIT-9423 * refactor(types): move GenerateRequestBase and LiteLLM_ObjectPermissionBase into litellm.types.proxy.management_endpoints.request_base Part of LIT-9423 * refactor(types): annotate user request metadata and check_user_info validators Part of LIT-9423 * refactor(types): move user request models into litellm.types.proxy.management_endpoints.user_requests Part of LIT-9423 * fix(types): restore main's _types.py content alongside the request_base move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * style(types): merge request_base re-exports into one import block Part of LIT-9423 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 84 ++++--------------- .../internal_user_endpoints.py | 6 +- .../management_endpoints/user_requests.py | 82 ++++++++++++++++++ 3 files changed, 100 insertions(+), 72 deletions(-) create mode 100644 litellm/types/proxy/management_endpoints/user_requests.py diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4cbf4f15d6d..f43ce4dc211 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -1684,41 +1684,12 @@ class ListSkillsRequest(LiteLLMPydanticObjectBase): offset: int | None = 0 -class NewUserRequestTeam(LiteLLMPydanticObjectBase): - team_id: str - max_budget_in_team: float | None = None - user_role: Literal["user", "admin"] = "user" - - -class NewUserRequest(GenerateRequestBase): - max_budget: float | None = None - user_email: str | None = None - user_alias: str | None = None - user_role: ( - Literal[ - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - LitellmUserRoles.INTERNAL_USER, - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, - ] - | None - ) = None - teams: list[str] | list[NewUserRequestTeam] | None = None - auto_create_key: bool = True # flag used for returning a key as part of the /user/new response - send_invite_email: bool | None = None - sso_user_id: str | None = None - organizations: list[str] | None = None - password: str | None = None - - @field_validator("password") - @classmethod - def password_not_supported(cls, value: str | None) -> str | None: - if value is not None: - raise ValueError( - "password cannot be set via /user/new. Users set their own password through an " - "invitation link (POST /invitation/new)." - ) - return value +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + NewUserRequest as NewUserRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + NewUserRequestTeam as NewUserRequestTeam, # noqa: PLC0414 # public re-export +) class NewUserResponse(GenerateKeyResponse): @@ -1740,36 +1711,12 @@ class NewUserResponse(GenerateKeyResponse): updated_at: datetime | None = None -class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest - # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model - password: str | None = Field(default=None, repr=False) - spend: float | None = None - metadata: dict[object, object] | None = None - user_alias: str | None = None - user_role: ( - Literal[ - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - LitellmUserRoles.INTERNAL_USER, - LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, - ] - | None - ) = None - max_budget: float | None = None - - -class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail): - # Note: the defaults of all Params here MUST BE NONE - # else they will get overwritten - user_id: str | None = None - user_email: str | None = None - - @model_validator(mode="before") - @classmethod - def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: - if values.get("user_id") is None and values.get("user_email") is None: - raise ValueError("Either user id or user email must be provided") - return values +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + UpdateUserRequest as UpdateUserRequest, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + UpdateUserRequestNoUserIDorEmail as UpdateUserRequestNoUserIDorEmail, # noqa: PLC0414 # public re-export +) class ChangePasswordRequest(LiteLLMPydanticObjectBase): @@ -3632,10 +3579,6 @@ class LiteLLM_UserTableFiltered(LiteLLMBaseModel): # done to avoid exposing sen user_email: str | None = None -class LiteLLM_UserTableWithKeyCount(LiteLLM_UserTable): - key_count: int = 0 - - from litellm.models.access_group import ( # noqa: E402 LiteLLM_AccessGroupTable as LiteLLM_AccessGroupTable, ) @@ -3649,6 +3592,9 @@ from litellm.models.spend_logs import ( # noqa: E402 LiteLLM_SpendLogs as LiteLLM_SpendLogs, ) from litellm.models.tag import LiteLLM_TagTable as LiteLLM_TagTable # noqa: E402 +from litellm.types.proxy.management_endpoints.user_requests import ( # noqa: E402 # re-export after the definitions above + LiteLLM_UserTableWithKeyCount as LiteLLM_UserTableWithKeyCount, # noqa: PLC0414 # public re-export +) AUDIT_ACTIONS = Literal["created", "updated", "deleted", "blocked", "unblocked", "rotated", "kill_switch_fired"] diff --git a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py index 1fd3e7dca53..febd4485d10 100644 --- a/litellm/types/proxy/management_endpoints/internal_user_endpoints.py +++ b/litellm/types/proxy/management_endpoints/internal_user_endpoints.py @@ -4,14 +4,14 @@ from typing import Any, Final, Literal from pydantic import ConfigDict, Field, field_validator from typing_extensions import ReadOnly, TypedDict -from litellm.proxy._types import ( +from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse +from litellm.types.proxy.management_endpoints.user_requests import ( LiteLLM_UserTableWithKeyCount, NewUserRequest, UpdateUserRequest, UpdateUserRequestNoUserIDorEmail, ) -from litellm.types.llms.base import LiteLLMBaseModel -from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse MAX_BULK_DELETE_USERS: Final = 500 diff --git a/litellm/types/proxy/management_endpoints/user_requests.py b/litellm/types/proxy/management_endpoints/user_requests.py new file mode 100644 index 00000000000..49dda5995bc --- /dev/null +++ b/litellm/types/proxy/management_endpoints/user_requests.py @@ -0,0 +1,82 @@ +from collections.abc import Mapping +from typing import Literal + +from pydantic import Field, field_validator, model_validator + +from litellm.models.user import LiteLLM_UserTable +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.management_endpoints.request_base import GenerateRequestBase +from litellm.types.utils import LiteLLMPydanticObjectBase + + +class NewUserRequestTeam(LiteLLMPydanticObjectBase): + team_id: str + max_budget_in_team: float | None = None + user_role: Literal["user", "admin"] = "user" + + +class NewUserRequest(GenerateRequestBase): + max_budget: float | None = None + user_email: str | None = None + user_alias: str | None = None + user_role: ( + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + | None + ) = None + teams: list[str] | list[NewUserRequestTeam] | None = None + auto_create_key: bool = True # flag used for returning a key as part of the /user/new response + send_invite_email: bool | None = None + sso_user_id: str | None = None + organizations: list[str] | None = None + password: str | None = None + + @field_validator("password") + @classmethod + def password_not_supported(cls, value: str | None) -> str | None: + if value is not None: + raise ValueError( + "password cannot be set via /user/new. Users set their own password through an " + "invitation link (POST /invitation/new)." + ) + return value + + +class UpdateUserRequestNoUserIDorEmail(GenerateRequestBase): # shared with BulkUpdateUserRequest + # repr=False keeps the plaintext out of management-endpoint alerts, which str() the request model + password: str | None = Field(default=None, repr=False) + spend: float | None = None + metadata: dict[object, object] | None = None + user_alias: str | None = None + user_role: ( + Literal[ + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + LitellmUserRoles.INTERNAL_USER, + LitellmUserRoles.INTERNAL_USER_VIEW_ONLY, + ] + | None + ) = None + max_budget: float | None = None + + +class UpdateUserRequest(UpdateUserRequestNoUserIDorEmail): + # Note: the defaults of all Params here MUST BE NONE + # else they will get overwritten + user_id: str | None = None + user_email: str | None = None + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + +class LiteLLM_UserTableWithKeyCount(LiteLLM_UserTable): + key_count: int = 0 From 2011004d0adf28c45d0bc2194b3cdba67f666327 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 13:58:46 -0700 Subject: [PATCH 04/20] refactor(types): move MemberDeleteRequest into litellm.types.proxy.management_endpoints.team_endpoints (#45814) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): annotate GenerateRequestBase containers and check_max_budget Part of LIT-9423 * refactor(types): move GenerateRequestBase and LiteLLM_ObjectPermissionBase into litellm.types.proxy.management_endpoints.request_base Part of LIT-9423 * refactor(types): annotate user request metadata and check_user_info validators Part of LIT-9423 * refactor(types): move user request models into litellm.types.proxy.management_endpoints.user_requests Part of LIT-9423 * refactor(types): move MemberDeleteRequest into litellm.types.proxy.management_endpoints.team_endpoints Part of LIT-9423 * fix(types): restore main's _types.py content alongside the request_base move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * style(types): merge request_base re-exports into one import block Part of LIT-9423 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 13 +++---------- .../management_endpoints/team_endpoints.py | 19 +++++++++++++++---- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index f43ce4dc211..4abb93454c7 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -4370,16 +4370,9 @@ class OrganizationAddMemberResponse(LiteLLMPydanticObjectBase): updated_organization_memberships: list[LiteLLM_OrganizationMembershipTable] -class MemberDeleteRequest(LiteLLMPydanticObjectBase): - user_id: str | None = None - user_email: str | None = None - - @model_validator(mode="before") - @classmethod - def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: - if values.get("user_id") is None and values.get("user_email") is None: - raise ValueError("Either user id or user email must be provided") - return values +from litellm.types.proxy.management_endpoints.team_endpoints import ( # noqa: E402 # re-export after the definitions above + MemberDeleteRequest as MemberDeleteRequest, # noqa: PLC0414 # public re-export +) class MemberUpdateResponse(LiteLLMPydanticObjectBase): diff --git a/litellm/types/proxy/management_endpoints/team_endpoints.py b/litellm/types/proxy/management_endpoints/team_endpoints.py index 796bcae37bc..065205dcad8 100644 --- a/litellm/types/proxy/management_endpoints/team_endpoints.py +++ b/litellm/types/proxy/management_endpoints/team_endpoints.py @@ -1,16 +1,15 @@ +from collections.abc import Mapping from typing import Any, Final, Literal from pydantic import ConfigDict, Field, field_validator, model_validator from litellm.models.team import LiteLLM_DeletedTeamTable, LiteLLM_TeamTable, Member from litellm.models.team_membership import LiteLLM_TeamMembership -from litellm.proxy._types import ( - KeyManagementRoutes, - MemberDeleteRequest, -) from litellm.proxy.common_utils.timezone_utils import budget_duration_error from litellm.types.llms.base import LiteLLMBaseModel +from litellm.types.proxy.auth.user_roles import KeyManagementRoutes from litellm.types.proxy.management_endpoints.management_v1 import ResourceResponse +from litellm.types.utils import LiteLLMPydanticObjectBase TeamIdSearchMatch = Literal["exact", "prefix"] @@ -19,6 +18,18 @@ MAX_BULK_TEAM_MEMBER_DELETES: Final = 500 MAX_BULK_TEAM_MEMBER_BUDGET_UPDATES: Final = 500 +class MemberDeleteRequest(LiteLLMPydanticObjectBase): + user_id: str | None = None + user_email: str | None = None + + @model_validator(mode="before") + @classmethod + def check_user_info(cls, values: Mapping[str, object]) -> Mapping[str, object]: + if values.get("user_id") is None and values.get("user_email") is None: + raise ValueError("Either user id or user email must be provided") + return values + + class GetTeamMemberPermissionsRequest(LiteLLMBaseModel): """Request to get the team member permissions for a team""" From 38bdda9ca7c2f7b31d8d133a3d09f4c8a8d694b3 Mon Sep 17 00:00:00 2001 From: Soham Chatterjee <96soham96@gmail.com> Date: Sat, 10 Oct 2026 23:01:17 +0200 Subject: [PATCH 05/20] feat(scaledown): add native text provider support (#44168) * Add ScaleDown model pricing Registers the five ScaleDown models in the pricing map. ScaleDown bills on input tokens only, so output_cost_per_token is zero for every entry. The classify and decisions models bill at $0.04 per million input tokens, and extract, summarize, and compress bill at $0.05 per million. Co-Authored-By: Claude Opus 5 * Price all ScaleDown models at $0.04 per million and drop unverified context limits Every ScaleDown model now bills $0.04 per million input tokens. The 32000-token max_input_tokens/max_output_tokens values were not confirmed against the service, so they are removed rather than registered. Co-Authored-By: Claude Sonnet 5.5 * Price all ScaleDown models at $0.05 per million input tokens This is the standard rate in ScaleDown's billing and matches the usage dashboard. The Decisions usage.cost field is not used for pricing. Co-Authored-By: Claude Sonnet 5.5 * Add regression tests for the ScaleDown pricing entries Co-Authored-By: Claude Sonnet 5.5 * Cite the source of the ScaleDown rate in the pricing tests and type them Co-Authored-By: Claude Sonnet 5.5 * Add ScaleDown as a chat provider ScaleDown serves five models over two wire protocols. extract, summarize, and compress speak OpenAI chat on /v1/chat/completions, while classify and decisions implement the Jev Decisions API on /v1/scaledown, taking a state object plus a questions map and returning one typed answer per question. The provider authenticates with x-api-key rather than a bearer token, and the decisions models need a non-chat request body, so it routes through the HTTP handler instead of the OpenAI passthrough. Malformed decisions requests are rejected locally with a message naming the problem, so callers do not have to decode an opaque 422 from upstream. Decisions bills one model call per question, so the cost upstream reports is passed through rather than re-estimated from token counts. Co-Authored-By: Claude Opus 5 * Route ScaleDown domain models to the native API and stop trusting usage.cost The /v1/chat/completions and /v1/models routes return 403 on api.scaledown.xyz, so extract, summarize and compress now call the native /extract, /summarization/abstractive and /compress/raw/ endpoints. The default api_base is the bare host; a trailing /v1 is stripped and re-added only for decisions. The Decisions usage.cost field was about 600x the dashboard amount, so it is no longer passed through; cost is computed from input tokens at the registered rate. Native responses are returned unchanged as JSON, including the _value/_span_anchor wrappers on nested extraction. They carry no output token count, so completion_tokens is reported as 0 and documented as unmeasured. Co-Authored-By: Claude Sonnet 5.5 * Move ScaleDown translation into llms/ and address review findings - Dispatch moves to llms/scaledown/chat/handler.py; main.py keeps a thin call and get_llm_provider_logic.py no longer has a ScaleDown branch, since the config resolves key and base URL itself. - questions passed via extra_body are merged and validated after the shared merge (sign_request), so the documented extra_body path works. - extra_body can no longer change the upstream model, and decisions state may carry a document but not text, so guardrails always see the text. - A custom api_base requires its own api_key; the environment key is only sent to the default host or SCALEDOWN_API_BASE. - Image content parts become the decisions document; max_completion_tokens maps to max_tokens; stream is served as one chunk for every model. Co-Authored-By: Claude Sonnet 5.5 * Refuse prompt-bearing extra_body keys and empty-key fallback to the env credential extra_body is merged after guardrails inspect the messages, so each model now accepts only option keys there (extract threshold/top_n, compress compression_rate, decisions questions/state); text, instructions, prompt, context and model are rejected. An empty api_key no longer lets the SCALEDOWN_API_KEY from the environment go to a custom api_base. Co-Authored-By: Claude Sonnet 5.5 * Read compress token usage from the real response shape /compress/raw/ nests the per-prompt counts under results and reports the total as a top-level input_tokens; the provider read a top-level original_prompt_tokens that is not there, so compress reported zero tokens and zero cost. Co-Authored-By: Claude Sonnet 5.5 * Accept the router's max_retries and word the extra_body error clearly The proxy adds max_retries to every call, which map_openai_params rejected, so every proxied ScaleDown request failed with a 400. It is now accepted and not forwarded. Co-Authored-By: Claude Sonnet 5.5 * Return extracted fields from extract, follow schema refs, and meet the type-discipline gate - extract content is now the extracted fields (span anchors dropped, wrappers unwrapped), so it validates against the caller's response_format; the raw payload stays on _hidden_params['scaledown_response']. - Local $ref/$defs in the response_format schema are resolved. - The adapter is rewritten with Final locals, read-only annotations, one raising helper and a typed handler; check_type_discipline reports no violations. Co-Authored-By: Claude Sonnet 5.5 * Fix stream_options, schema-guided extract cleaning, document input and multi-image handling - stream_options is accepted and not sent upstream, so include_usage streams work. - Extract output is cleaned against the requested entities, so a field named _value or *_span_anchor is no longer dropped. - Extract and summarize send an image or PDF as the native document fields. - More than one image is rejected instead of silently using the first. - The cost test derives its rate from the cost map. Co-Authored-By: Claude Sonnet 5.5 * Cut recursive schema refs at the cycle and cap the expanded entity count A small response_format schema with a self-referencing $def expanded to 4^16 leaves before any request was sent. Refs that point back into themselves are now cut off at that point, and the total expanded entities are capped at 1000 so repeated acyclic refs are rejected with a clear error. Co-Authored-By: Claude Sonnet 5.5 * Bound ref chains and schema nesting so long schemas fail cleanly Resolving a chain of thousands of distinct $refs recursed until RecursionError. Ref resolution is now an iterative walk capped at 32 hops, and nested properties are capped at 32 levels; both raise a clear 400 instead. Co-Authored-By: Claude Sonnet 5.5 * fix(scaledown): support native classify and preserve extraction results * test(scaledown): include provider regressions in CI shard * fix(scaledown): pass registry typing and bounded recursion checks * docs(scaledown): capture provider setup before and after * fix(scaledown): preserve nullable schemas and reject malformed requests * fix(scaledown): isolate provider dispatch from existing providers * fix(scaledown): preserve the shared dispatch return contract * fix(scaledown): preserve failure logging and reject lossy schemas * fix(scaledown): keep timeout typing annotation on its expression * fix(scaledown): resolve escaped extraction schema references * fix(scaledown): annotate the bounded reference walk * chore(scaledown): remove unused loop suppression --------- Co-authored-by: Claude Opus 5 Co-authored-by: moe-berri --- .github/assets/scaledown/provider-after.jpg | Bin 0 -> 110207 bytes .github/assets/scaledown/provider-before.jpg | Bin 0 -> 111487 bytes .github/workflows/test-unit.yml | 1 + README.md | 1 + litellm/__init__.py | 10 +- litellm/_lazy_imports_registry.py | 5 + litellm/constants.py | 1 + litellm/llms/scaledown/__init__.py | 3 + litellm/llms/scaledown/chat/__init__.py | 3 + litellm/llms/scaledown/chat/transformation.py | 648 +++++++++ litellm/main.py | 35 + ...odel_prices_and_context_window_backup.json | 46 + .../provider_endpoints_support_backup.json | 18 + .../provider_create_fields.json | 18 + litellm/types/utils.py | 1 + litellm/utils.py | 1 + model_prices_and_context_window.json | 46 + provider_endpoints_support.json | 18 + .../code_coverage_tests/recursive_detector.py | 1 + tests/unit/llms/scaledown/__init__.py | 0 tests/unit/llms/scaledown/chat/__init__.py | 0 .../test_scaledown_chat_transformation.py | 1213 +++++++++++++++++ tests/unit/test_scaledown_model_prices.py | 46 + 23 files changed, 2114 insertions(+), 1 deletion(-) create mode 100644 .github/assets/scaledown/provider-after.jpg create mode 100644 .github/assets/scaledown/provider-before.jpg create mode 100644 litellm/llms/scaledown/__init__.py create mode 100644 litellm/llms/scaledown/chat/__init__.py create mode 100644 litellm/llms/scaledown/chat/transformation.py create mode 100644 tests/unit/llms/scaledown/__init__.py create mode 100644 tests/unit/llms/scaledown/chat/__init__.py create mode 100644 tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py create mode 100644 tests/unit/test_scaledown_model_prices.py diff --git a/.github/assets/scaledown/provider-after.jpg b/.github/assets/scaledown/provider-after.jpg new file mode 100644 index 0000000000000000000000000000000000000000..bcf1c59c96fb5d5360d4667c35508e4a75d8f8d7 GIT binary patch literal 110207 zcmeFZ30PCvmOmP_91(F~P(V<@iLqo|yQo zTfem>pC#j)*~vgNlQ;Q9vm*|u}rwr{@KwtL5p?K}7E z-m^z~x3sj(-UIt&_U_*+Exk`+-~NMg^78U~WEBr7$Q?Q$ColI~BT`#|I^S&DwQbuj zIT>jgx&Px!@*cE*=a$}WpsiBJL0k4qZQU;=X$GkQa&D9Q;|2QDCA9^Rar=&)yLL+h z1qk~U`Voc^QaS3A`&SALMc(Z|s+ zxcl(&uH6cXhYl;9IC<*y8BIeYW0P~|O|N`sWo=_?_x;UVU}qOsH+Qdl-afv5F#o`y z;E>R;@Q5crMaRTGeHNFJ`g2-(#xI#!uk#BEi;7F$lvY*a2{pvpcXf?T%`L5MAKN?n z2L`D_!=FcJEcV2g$*JkDGqc>~mDRO%9)Dx=H@&0)J^y_Dn__>b*M2~+E#G{z^_v~P z=_R!#6u7tU|7P2#v%(9+k^Ow)?q^d}kS-Mu#$oIEx$V{MVPf)tv2tw%~guODML z_9d9*3u+v0(#VoHZkGh~LHCse^l@z-b9V}}Dsxi;0%dF*j`W9iB^91)Ifd4mj*?y% z@prLxdq|odH_s?b+N*7 z|C46ci9_r&9eW`I)|J2da;m5&HE{`|@>jCium|#k5c7}s_ELVq26qkXK7aW-<$01j z$o-6g{l%T(FCT!B6$^MiicWfpyE`-GaJ*FPQ&96Raak_w@B!Cf1Kjpax;k~P|Ck#W z6P{o5>`U6vxnA$gM5@zu*IDPRUo8-YgC@zos`n0P$LE#ybr!xS**&^cR$FgV%Zpx|>0gRM@zNiIDK)2}<&?W3GpvJ9(_Lkkdi!(%1Voeay z3M>JAF98)n`D3~m>jd8LZd z5O}G8xWvQGVbtM8g5AY!a!&utZ?IkFwCxBJ@OSw)aJ+(i{bYQvPu`xRz~s1#bdT=F;Hya!cIlfA`+*8^_0pci-exT<8BH z0qxD|X&gJlH4M4pw4&H_T|4Hu1mxZJOBaq9GC-UZD^%1vHlV^Kplw@4`Cs@T)Xqx2 zNt0Md=`DpJBJIdt$7szsw`~YZ-Ic{GH+w zMpgjTPu*s3 zd2+=seg04k1Nb6geT~-m@HZcY#-0R0i@@<$aYx>M%)Bc}0_u~2ZV}VeDoGhY+jw3E?{bYPajIIQ< zdc`HS`CYl1NiOv1E8!;z2<5;TEVMRKSMQL3#xp3({^@G#^7Rr>Z#ud5f^0w8Mgn^5 zDFL;Ye#U%uUXg$u>O^o>L27usx=vYSB2x#w5=C2~JYv=i$F45XFl@C1bI3%)UHW^|pA6_< zX*x_=IoPczIzRc2mEVMD6d@j7vK+%f`KQ{NDAK5l{14m@<_;`|db1WhO4+0QjrWn^ z)H?QD2=lB5lHndYzs70%u-PV*vtcjAB857ynPKS8YwscJp3bZ{fS)nLB~ZyH6* zgdtvW`6|yDaaJwY|9Pf^`;!MzQw+iPQs^0k!? zR>8_mU#x59Nom=kj5$x`9YVSI@$WW!2Qperjww0nAfZ=lJpz|V0S}gl@@xtR47ohf zeRefo)@e!FwFe&u${0@7D`@I$!T_fAd}_b|otJ^5Mr4Q1w&*l7GAc0xL|r6hA`(y8EeE0xGC-`IN`` zUUtJ6=O}J9s}LRB79}tE4j5EYd|*|-+xn+66wdc*e^n;@7r!i|T| z@1kGP_&qiaoZA-K_RZoCDewK^A#IZa%Zw?&j*-JFnnchRMF~g_ZAwGg3?5DcBvq^! zVI#Bs%U%#ONaD=46Lk`hz-XxkO1PEI;SK_WTMr zm>@Pf@^6}|%~!KFSj$G%;d)3anMd=kC8Ss(d7;{v^-tvZqwj4_;oNh)mu@}$3Db0l z*uBy_-F$1E`#gW$aX3woKVxr835q%c=X#CqKK=T~zrN374uy?%en|3?8ycoXGgcE+ z6e5BtIfmITfyPWL_o>y2M1_d4Jo$yBXmgi9_{oJ7tHMQR8XeQ%EChP_1e7%`YR0VQ zz@PISB%ocgpX+yg*ady*aHH1G!%GI?{&bfqK{t3oeEMsKdvnBnljY6_8o8INXx(4%*`bJ)nvF=n zCJ=5bwtiQe;SPSg!mfp=&e|8nUr(goBaD*rMU(aPZ1F~!noJcj4o{B{TYSwCz@=3} zy5CsS1IV4?>UM#}0%aZ0`5A5_YCtQ`mUeaE$~*GLu@_|hJ` zmmWO|g-D+|G$FETVqy1y{zigQQvk_jQOMLEI-`{L(0|Ob7q52sD_Ws}vJbvFZ*mY; zbJ^IZHjx1?LeN9XnP<@}77JTS4SO&6Zk#sBZ{oN zg;$*ER<%EP)V-Cwle3da0q^ObDhnTpl895z?=9!{6}}A^v!ke zm_uDD+OT)@-6jU5B3GB`dghU7GH0n(6fi5J6e+!M^HSeEZd;SqJG`qAoOrja%Ksp~ zQ{`K%jY`J;3P#2fsssKB4Z$+H#Rq{#t(!W)zW=HQIlUYC~86nLHz>H=%#)o{?< zv;3IW*#R*y9I={s_=&o*?2w8v0oMNK^GSW1S}V{djPe>&QK-Uy#kFZB=AN#s!Vj$K zZE4z@jFr*1HQMHi!TYS-b50zMdOh6{G9E-vUwS8YDAhlKGc7Aq{cz4nj zV52aPhboMPJ*Fv_)X(VZSbd?pduQIEYhY%1CW66-v1d0CR;YG?2ETu0g7OS~6m`cW zw3Fpl!4-ZbH_b)jm;27@xAB69w zAjMkB0nkj!SmE6kjsz4Rgptecna}y2d0n8XCDk2ynlK-7HtVZqoeNK{m-w z1*}^{XrcaJ9=ND;zaP@d+gn&Q0%6Xke8LvSs}xj15QwVqybvR3W@h#z{ar}SLfMat zT2J_IxEUYvzQISzxU_DrdL9`Fmi{t;TP%*VwyyQ|0dIh7gq4+}OubHsbG-aoO_H}l zvAK(B%4O^W^Rx4bcE-XktWYa!1K7w)kK3!oDakfk?)1OyI+uI?aifoPZEF>b*ZB@Z zgU~7ChP|o_+$$3{Pb_5IU@p1(!*}|02=>&0g*}z+>JDbxy{&FJBUy9(qz+*MKN=U7 zF_G8wGnpHVe|uY~5^Kk4VE2%CwACzXLqivx5-(C1vJ~zDHqSj1FS9Dzasrz%`w2^X zgts}ZH8tdKIFm({Cn%Zl$;Kg!Z>2Lmc>S4@RqDg`WeF&x;zgp@szI~+T2l1t5qr16 z@@+-gMYQ(t;PaQqZl5m9d3&C90Y$6Ou4cRmU&@^`Kra~~~DTRD~t&cAQc zy>do=V}v{05@K;|NCNVc;<~hDRXh=b^TPrL!)`XE*A1rsU^fjT6)YpDX-IQ=jNd(H z#o-=MG6Q=QqNoCEe)Yt)Brb&&ob(fZVb51Z_xV0r zIBktVSx**sXBH%Go?94kHbS1w5czhi`1(sgvaiMcJ#Xv<$AnH%?!eCWI$!H1yab^E zJ7$M&I=PmR3C;`+pmkyClds?V3D1e%SSX=x53~AEXV|;e94`AQv5uQUzU=O+;D!Uk z6mXbbaKlZQU1E)~PLA14JccIyA=fZV4Rl-CQTnSw&H84}^W+h%7CxELu5RdGh7%Ar z$jA&tAVzipWyk49jtoG73H+?yv3yQ^fn%Nj0P8lVL}-~e8Frf4&+u4?v-VC%cT5Xe z*uac?#P5r0VJE44^^~P>M7h-^I%%Iss5;MVW$P~A+Oze!t~AOSy0Th!(r?{F!6)_8ve;*X=wicOuxVC3%{`g_ zk`hM{nU1Zj{|GrUIS_mXH|=qP-}alB@aD^>Rh=oxz}sS0RtGgpYHeb+^9N+ z*z-P#)Kd+acdi&dS2)Kr;Tn7^1^1_%6&vt9KQ(=T$_r0m=iDc!e-g@Ym75msSYXQ` z2C(%|xXTy0c=@humF-`;r0)NFZwGF*f`+Y^N-({3?26Znj?MQ`%II zNVSy^(jjmzcSqJ zjo=H?*T}bOpPeEmLWuTN-lL_TlQGfnHlqCp)8(A3K;bfHL=+8g3J6D9t;y!{BwTMk02*27D!o zx{)#EU9p?fT1KSZXcJY#CeBb;m@xD#Mj7V+#;e@Uz?%Ll0qrUzfD_W(GDJlJYRFNM z>1&DsLGP(cyf2oI6CZ^?5cl<>E{Qt{obP9sHH|9h>GHqLp&I5txb2<=!s@tcm%bPa)3(N7_Z+VTc+!riiZ!`PpGFVWVm0A?(tL_U;& z_5;RYHd;P&OT(vRrLyHFtwvDh;9n1EG|-B+-Jvv z<3e@`b>39NGCVKi+u9R@U8jx68apw+eS+olR)Kl_o%gqI zHnNk5fM3)6&NwQkcvfNaCoK)#;z?(3#)|GFV+9=~vJqbx5R+^qpnQdT$Mujt8-ULkEKD39{WPHc(7-TZvh1NXE}Fq3BkQrG$qM+O<)EG@1kll1$M-bN>iA^% z(lhj_Q9o5<({03`eCxwo^KAm|4oYdm+rXKS8E74-#kl{V5 zMwSL?FiZYAx3M1qPP8*YRpu=Y7r6YKonyncOqzI9mg>+~uZ}fv(607jFHQh}9tnJ% zq8%9^&kNWfgT-qR1mpdhbWB_Kv?vHF+6D_aobK@lSvTW6!Q>kq%EZ))xK2;}7=Ig6 zUM&HY17Ly`!OpuBaDvYRD6;(LkuiVMdot@dpZ?B;{A->kk}VJ^to-LO zPZ-U+NgCp_*tt-AExp^=&e263^EGj_zfpS^T#K=&?Z^&wQ6uA}A=U9P7-U|b69X5#v=S2esD zq<|%oOXsUoRee_S+hn%X%G_kD7V45PMf|`f6V+o;=SmuCP3+)%9_nLvg0q*}E7K@- zq>QQZq2-19IG>#S6z@1=jP8li1!7f2pr;O}jXfM|7`0__0$f{wRzAF8md?GqUwuMELq9<%Oe?+ zv@WLRY6*E?*lPexY_TwSJ1{}29q=KGslX;PiK{|z1B0dL#4CRJ^8)9I--) zky(Gx6zrUQ@$)U^3<`tZy>9Av2?)dW7;lR&Gq-#*8=qZ^EbtrA@CK(M7Yp3>*I*Gy zc17J@(E7`JvhI_KQO^@~UAxa0yt{|X_slykuGo9+$`@ezi#xM1ffk z2t$id*4%~Ir}zfnM&2ek?QKK2oTt6&=P$_7a;ku7B7la}XS0^(Ap&QiW}gPI+;X>m z*NbnpW$}DaC+`# z}mxC@6Q1E!!l6YzkGnrGQqhh3Op-vu!_P9i-Lt?j!Qjjl~O5Fo!+oaVMJ3 zZ8#Tt-r&7yOSecTjyY0i3QzM?^Fh1aPgmraKK{EX)?SZR)g4vb)X$wp)U{~F^8^jW ztl+j(!^jM{?vl0pv|+`g^@x*s@@YwNLNEfZx{%!VhUF3%AYO>@mRo$^%3;Q8*PnUi z4l{jtsB9s%Gbu6viB(<58hIUe@jnAVLGsty1B)6dt1T`P&=G9fbz0Tgv)^AgF$-CA z2Q?0%WDEJx`3MiYx70nBv$W;0^8m0(dWUwu%RgufaSOKU3mT*(4h>vxEe4+ z%fnYDZ)1Ae#8>h=Y9*k&Bw;ML>*AcN`O})b241Q35#3J56t=y`IUU-ha3=Rq+SLlt zRNELbBID1+ZPFe{K$dD7dkd~315;=B#wa73V;n~#|QL^*OZtS zLBPTkOecS;aZigTzw$jTnrs-=(ng4viQ5a8%Ppq?82DJaiYJz@7T(is3%m>o$!$-Q@0%+|urd9;_YQkI89lg*R0 z>=c$72$a}h`@jE27v)l$Ht)JZT+4G{>sqh7A0XF*mnVG$U7ZOyQbVAsCf`dQunoaF z2OBer-z&O2XhVvyFWo!}IL$cHU9Hwmo?FJEkTjAZ11bLV_e;{85Th-ZkAMO((rA^h zlwB5Tex7?d^A6h+@;!8Lb)pEcPFy+D7x( zZTmn>{q(FT!J{)ZFB4NYn(Hr;lS&b*^cOx}+%a&Uru>dP@BoOqjd*Wh!a9>O$>0sj zh#Z|F?MF+*BCrscf{AJ2+m?pa^^$=m@m$&{CHlV8#Y0)e(T0eCMdfZA+qZ*yM-%d6 zBD4Lzq!=OVOS|f~#s%b$GiB%YjeNkooFVqQsnA*ieO2Oj2p(ljNg&vL1fY&n3`G)QVgbwoR6-s>#e)C=%ch^{dtWxDg`W*m-WZ0VdLa2qy;u zo6(h{E)T&yN>Zk1-hN+VyhqD%yr199p2z-teTPA@(413~rf%=e%2EAvMcLYOtntuD z8hKfW<+OOYwb>|E@zOu#9!&lEcmrJ(0d}s=Mk*LE^2RdQA?zT z1mqw2Gd|&7I2k!+fcUhONk*<`0W<4~npFLHj!UDvb3$=*R?g6`yxeOQk>6n8^asH@ zO*T`SBF2C|yTKw3)P~vpwIBWrXOVn&vtKX0F5bOuJsP_B%y^N`AE2;~^w$Fmx&8o# zMy6Bd=ko9f`ZR_@oMiB@?s1^@yrwMhV8l;1VIe6tC73o^dq5e{QkOY~iJaU(`otw3y@My5 zx-?)&PlptF*q>wFwHyN%&!QFkg0XX23pO@YgL?O~gK^#6We@EFm}3c<4~s=K+Up&pzfjlpH+*p1F*q5#jZZ`{-+M8ZN#W)w`LCsTcBQ?oj`i_L`F4*$8_O@Ww2?X9~x#K*`dU)I$vfyZZVydhLU2NljU|oSkdnMeZjG z%i^^a*o^NUm@b#)^Iao`8swy&kIDCCLm>BO6m)Q!8(vPvm7;tk57kYo8ID6aihicE zKUJ0$RdN-ZDV(S9B;{F8CD#SF!{9V>kIcZ8=_YAIc%2b)DT+Z|Js}5bv9j8o?;jT> zcU|i5e9&v`GdD2CC!9Azs)*d{+d2U7cE3=HQxO*o0ak$PiM(#=hw7c??yo>+J z>Kmj4G{LZ(!(%qj!t(x9u_T~4m;LQyD%nk*%;UMazfekh#D?ykA#27Anepuj4b3KJ zZEMGHD$kA!$Fh?jYT5k$rl>mT#yV0k`=b_2RkgYgxfV}~5{JAABDzRrTh_aJJ@*oC zRT^q=5(H6J=9vJS1YX;!K&=)JT>mDW$z1^`9uOd&ddWSAn~^e_G@c6B7P5yBRui(Y zyV@)uTsaV23OfBd4)l@Idw*@Gk-VSt86-~JeW4ko-K~1qraOr>`uTO!rRst4fwVhr z-jzgwq2j?G2eiY$md2Lx{!@gn|26>L^&ALuGj~|Oaxxza13doJF7<09Kl912eeJ|0 zJ@Ez6kbGa?@0=ca3|tQh(;D=G-ZPpVigd)Hh zz$oCHF3MZ}UV}nYpscLPiiGY2E6^=*FR+ctE5R(W`$VtcfaSanP-$AQLD#oI7o?#Z zo3UcCjvCGQFEzm8zQ5GS|4WTqztvFwQ;k#W(Z9dE@8aKxOZyvfFaHPPV(^GRDVX=W zxI`<^f9d7_=i>ek)c|iBjgf}kj+(T-b5S0N#N~FdoY^OR63EcA7!nXU^853*fM?bW zp8BQxU`N09i*{0|00)WwR`ai(`~BscQ%leMC%yca&xZNE@&9~W{QtAYufu1{8V?>1 zFC(R!3%fL1{$Dn*{${TI4-MXbz2m>R_Wi3r{%BeLZ(XV#|H>Bq*?Ixy;6Jfm|GWmU z8TmKcsrJuo=l{wz@{b%}`3|8KH;ym+fkzu25f7W2c@YT=Vo#R!{x($%&BuL8Tdc1$ z{ge4ITzR@P0_hxUo7-2kQ8wl4_hq9-NOw0DFV2Q+9pLzVC_lyCk+g_95%n5odM(8$ zC)6!mfkv|vK21M29F$^ra)^d~hl%B5Ph#RtOhcP;T$)Wb)kcD~&WKwLKQE@cE*ANK zlR7h9rkr7lQ*_Z%di?I`qAyO#o+en~eJVn%;1R}j;H=>oRW`>45xT+KW9(thjym~J zhVfE%C=b2h#>KQG?F-YNXmU19OnOD1btgdr6io-Nz;`#6bPA?omyDMd*NZz^x+2D` z4$FUuV7Br&jiU*U_Jq4+6~x0`+Q|B^0r%XDqjHS_cX=#nN&-SI4vG)*sfuwG_@|WW zq5d=YGe$UT*eQ1Beez8k9vYc>PrS5QcmAGATj_7Io@HSYhs9?&dQXiyp?lgzg$3az zJ8a<@=?T>;=kd$Vffk{h83r=S5k>+ZXJ}7|4QB`B+>T zDLZrwfq&;3sQSoC?#AigHi-Z1*2Dp5(vwBXTsG#bk$))@p!i$NiQf^$PcEuNCv5`C zlt09Lttp?Xg{!S~1AKJ=MC?OAx^MjmwEW_aw7Lx>8`F$c#>DZmHU_^ut3MJ5_mwYz zt834kTp|JtUPh-%JdZM+6J>Y%Q|dTE-G;NU9xLadldy8zF6d-bx52fp{%R9}RaU8P z^G7zyhRu|Ni%)Y^b6D60-A|pD-^PTyuQQ6^BPSCIfHV=P?gcCm$H{tE5v!`fwZOjQ z8#H;Uklqf^h8t_La=NJ^((UjfhrYJmrBy6UNK3Yewx}oG85XkYmSh4ZE^B)YoDdiZ zlcTC=<_n2YlsQwPW#rVG0P?P|h)9U)p_7BF)BR;JIon#IS6^IRZz|B&7flw2J+dD^ zB1n3?>A)DcA{U3$NX~L(yb;`LAtM)mW6dQ+i_h{O&w1kU?-~eH+?tN@;^LVFd?eN^ z>|V6|<0eBUmwR{=ji4}q0M2xbaDPFVOXJxvpVrZYQ0PrHX@~xRsMf~h* zDRc8f&Ix;JDvngp6sKUyU8ALRGMGk9vZ!VQK9fl!F4Cob;96SQ+_#XX;*rLg6Xk+% zAZTuaUg6ybN^Yb#JRW|pbfLgPG4b5wS^h)4lP&HJOuf`ELgv_pR&Kv|TNvDSaJBOz z^v2nlP&NOz_;j(I&Vuq(N1HN0-ZfHu>i!+@B9s+d+VViy%_|&Hch2*yrqMQ|?n#p4 zm$Dx=GH*FUfKjxi-0UJhX~IPOevn|FbKJpr36k10FycY8tuoNEQ==(Zr}7JuM4rq# zW<&$ri8Gn_1~;Qr#uASd;&A~TpTYVzmBhUO*y(+RL${9U=;*tFIoack@KLerXBDJa z12D;AGTHs$#qfrt)(iLe6ShwTm)l(W)-Jd^7`7D8xY)_mExc&$j5@VAg<%Lgu%ZP@ zNFJ-_>6m`jKvSLgRbXLzzJJ}^*X}zq+1efCpx&~vJE=IkRag}zByBvxs6Mwm3sNCp8h7%!fYDO<0 zXbwK3j<3!6K2P6vm`~JP6t6S0jRxk#N|jtCR~G>7;cRS31P(bL>Gkm0o#V!@s}FgEpn68D%H~8%neOB1tPQktHajK7zgFKKE{w)*}yR%}8&nyn>R;{7D#+ zxKPUg!Z%?p?Sq@R5Rb2)LwYB|8K%iC9W0nSisnaF_hYJ?-Nxw2v7!oUeNIX0q?W|+skR;SnOt5kqv&-o^@Y^DK&~&8xPL}z-?zr`1)o25 z1erB_$GpbbrC{tA77cKSvVR5eYR@!w(qpkI*R!}}3}ROTuxeq}l~B4H)NYSSwu?VL zjuM_0ag%^~Qm63dxNYieM4%`^`xL@hY%4Rkvk>faR7iiwG#T5d8>qkBe=e3AK(3pP zCy3Ooav={LW&C(fGX~Aw8af&os(0}iEd%Q^CeZIhL^064XgAzwa&1ANb$e`9QRaam$;BUQfZ6?g{6@fW^BW%BE+5cl{5&@HSx~p{ zt*WfHYxo17H}2zpcu{6qzXI@`uIA|jJW!@i5P&5o0Vxb2;uBTWfK`Ua4Px)Tq5DXk zd73;_*DBHWOMKUTVZZJWLo~Y@34|e)REx~75ktCz)t-JIMLzGLNmq9rM)J^LXj@iW z{CMtqSg8(QwkQJat94%lVE?t9hN!4w$Yz;*<}T<+3icn+S~ab`Lq zwiUC_4OXLctB4UiL&#>RBEBDX$eJ~>zf;>)Q@NO&nU*ohz@xf(RuQs|c?xr?XL>6p zhvuh(##N*XS)XdSl2NI6zGg$B9SQF$P^L7H*0^Y=j8u7#h|+wz)?hw(e;>?CxOUVDcSR z&8}*I{AosG#qdR&a{F0i8o1C?9*C>cD#FahjiO7Ha~|5apArw(Qk5MOH?6DE9r1=Q zKK6=+yq(lUE08APh2E*YwQha5SD-iUwq0=eH(eetIKE)^wv(T1OQ;Y347JrRT10x2YZKRaj@4jl7U}2fv~nAwA6|8Y$pELw(?x75LzU1hJkHr7)aPu^e6_?UbMbRUI-ukGIo8+$pnHNR@rMP=94xK_9+b*_u+H)CDs! z`GsjbC3q*+TpQi}?U@z*@N3FIWOGC45uWcK5r7jtD~BDh9U}US-d5L+udyA|?|kP9 zAX@tbgjRR-Ck12L-n_z5|H%0PSA?%q%qRN1eXvPJ)rq9`;N=6BX=Dg9q@jAt^WCVf z^NzVM@vnVdiFVe`WhrYEX981*WkDkL%piToIFjHW?CL zD0TC9jzDdij+VAzibQqiQIl4as1^YU%pc0enk^^#}ot7Y&4;bralC(}O9o5r0k z`1W<&Y0y8rOMyN9vMi9^Q;v{;UQ$c37zya8pki~1`CC4Ug5_Mc_+2}Y97Q8#vC1&R zCZfR9-zZ6;MmK<_H81`%2j%~}&+D0NbOTlbf`=g+BXqd^Mn}+g!fpqz19ouap2z|} zD*2|mf)(U_^k{cd`i~cm$8CsZ08U&~c}qC5j9={1cXU_>FLQ06<*bch!pue`G(Aug zt6!GVT1E*bBhx~Ldx0p{?X--fvHjYAG4ge8Ax3S22RAr8_i= z<)y!^H{upw<3hUg-P~@s72byUD>U7Bmo;{}gLb4s6KnS_--ofHDI}qRjNh4CF1v6- z7{?`c$uK^$;sC5$w>WhD+*OkYb!EBG%oP+nltI*9LrPgW@ecJtMwGC0?I@oumF95XVhLjZ$6Latd zLz-DIYNnD78`q2_g?5!P8}U-13;Te$2H?tm20^% zMd5XS6R3dhlNGKy0(s7OAT)E~AduY5^i2|nSBO?@MgY1&E)Y;b^{TYw{rPeJ?+up{ ze=pGYm!@}|_5HoXfA(srF1J4mX7MKxuzway?N1V1{FiwDLC5{;>-}r&{cGm_yKCCN zrpCXf#=oY<|JJEdY8C7~y4U&5pH02Zah)tevpo^n)j?^-OBDlK8rQ!%C`T$50I20? ze1pcUR!Ug;dIaRx;Z*G_f%>h_yRb{k!#`P-hNaabVNn+fYbv;oH)0X#3EHU`pD+){ zaBJ?(9Apz^ zuh2r#{w3_e4ExKQU$6%C(SgK31A`+ZfGzZLuX`J7>6y@}9HtbTh`I!%okW1&*W>%~ z4i|8UF;?i#sZ0Q2eo?;AfhB})LuMl)a^$Y>h(-5#Ds#d0n~KltWs2?Jgk?J>E?tHD zDZ^66rWAgo6PE{S>!W2{(Wm)lC1D{wtg&aWW|o<5UvpK4f&|Yd)bw(63jE8M3wB45 zL*#_lrQ@icC{D3>e@_f^`xJq5vA(u9plp$VmDW1vJ~36Day~PyW+P!*9|%8?DKXW0 zLOu4JN2c)G++h4JyQ}6vUDtB|8 zRUKI1KL@WlpGoGWeqvBe(mM#roVIcAp|@{mveuvBVEgENp9w2U)@(=Xr(S#b!2Ks!MpCH5SZNiDQPlIXYRr-?;KVSAiJKNg4r_?$;Tfty*&dTfZJmhX`GrC_?Kb)p z!plkt7;@LMwc1x}4d>`B`?q;L*rVOPOzgldF9LMsskA~jzbupzI>&cgbXwh07O^J5 zE73?iTPmVatls|~3;VmHCVod{{@N*a|67l_cgL~6+VYnx0l8P0;q#WO9xj34qd)@% ze>^*=x(qYX$`rq=5I=D@%JBk7yMRx!){R+J+L>(hoA)8V|JMvHrNf7U#j`+nyb*(k zMfy&B7dksbI#+Z+zva`(xwI;|ESO<9bL%ytmxemNw%n!X ztC4Pomh&WlLu#UgHw{>@3}R|yv@N{!i2@G~0pSBmTSQ#bi-m5%do=h=3j$j#w76)3 zZ&CZi5f6a{t(7)r+hq4(LDxG_2p|lFnUw54;PTxAq&j;MO4++g*dH7j6aih4Kvbnh10ygX_YfyCoZkK z3m}5szC{4TP0J+nfqvUbDgZW#mKKNOgqKCH$;bImnxWO{)vSh-Ou{{1Y442c z>6i&SsyQjD^5gJ}MdxVDFsk!i2#{WF|FdlE>55cs3F!XPK1WWH%WyF#CiWFS?+W*} zc;Ao?p-0Z+^Gv$IcctAw1l$9|)M-ukAGCHcO?dFU0m}GG_uK{!sfm2Fw_X}n9`?kE z%M%{G0jqIuowF^AZVF7f#>emLhXzq3(w z-!O)Cgj#{AQ5X}mXWY#lAgRZUZRW640!$#u+9gs-<}EqJh1ajc5iv;0`rbW``G zAm)Dm<=IYT2ynIxJ}R9F*9zC{Bn9fuYVQ~tbor=iC?YKx<|QbqN&~6YF+uj7W*$+d zv*&^6q3vp9&Rw&QLk_fvZncb4>A01Ef6oJe1Oz2DFC%sK+?5ZRZY1K8L{GnLbDp~T_)eS zr*+?`e^yE5@~g~DR5}DF=%$0g9!mx&x^ukLTDLr>Cl&+mdCF8I>kC07j%W5AuW^hL zEGxQYJvAzb>}7{Ml@VO$c_Yn`EpQAjI+z?7KJ1k9ZF9T+Cz3fSyC}z_9HSkO%(`GC#}% z%Dur?THXjIWJN}~WoGbx*uWT?Wh@y42!QDk`Uq_vR@?-Ys=qm*w|7Rtq%I(do$$a2 zCiG=vlSZ%XVMd%cPe+t7HJgZ&8{kj_r;Z&XgD&k$0*|n%&T*5+!r(H#S`+!;V73&x;oH=V*lh;g%`&PJjh^#te>YR;aTkiAp z+bqfsEYW`9dD{bvUiu_3({Jpou}Tghau&pzx2v`;b!f2}>g%ub-kV_dYa2rM&U$Ka zV{lySiN-qPv^R4pc6(C8zpb{LvZZ{jnFTCJDU;9Z3~g2;nmk3FVlnrR{C04_UB{4`+Zp+o`R4(CMzDi`KLJOKf_7?YtI34vnqN3vm1Q~?rX)h zNdi)#oD)Lryp39C^2Ljj5ISc3mej1AON<26^n-8*fUD*Jv|DB?kOt@n%pw+F764bx z0E~x{AOST~mp{rrd|huMR*oIF6v-{fip3@p&_NraiR^pK<{6rrNQe}lOmvf4y!Bsf zv9I&!g^oB5lOLe=BXE4$O}wRm&lEbSaYikHS@X5n55wb9_*v)g?Re+psVo7>G;!3# zaUrn)9~k)US^nQkqWvRJQ>zKXszq-Fh$Yv0C7`7V4A&6z56VrxP@S|2L;kJgNq;AX zO4ZiUk_wrB{1AQrC`tOo$TWEtC7?glx~U#v zhCUPl85oalOujf$_-A15H*oGc6VsLe*lpmfrlJxgWukTB-xxGXhE7(0cZz^ZE}Wu- z*U@sT>(@pQFbDk85jC^)ivwTk1!u+cDSC@k%7_m`+zpC2BHtEf#tw*{$PB<4ZY&~# z?Twt@^{;8!Ap*vo1!cV-OMyrmG(%K6@Auytn}53t`ETL&TBdDca{%?91JK5=M)__- zmfQ(|NPl}6psv4scJNjJ&YXM$$Q>y_iqZ=T-bz3&8aXs;38)PX6{Y{E-5Rg7&52;+ zAN7L-bZDs@Pl;7xw4@CVWgJsa@O`x2t~w*{mPU!Lshr9Lk_UY+0fh0T{Spw%t^~uK z_0D1V_9%|rTRAAQH|e4FvHgCpsk?sggGa_D@OPksYQy-ktP_tRuY2{i5_U&jo!BalTwf zBf;4WT1m%i5h(EwYnHO=%g{}xMzKE`!jGJ4p%o>wAv>UC?%rhfImD|J4>w&puNSYc zWutMQqB|s>9j=*!%GF4!{=~A&OSOOHvaGn5vX*oFkvACxG*9+$*AmHm9CtiABeN!@{y` zX$HGnOR;;jNRwazft6RCF>hJnueKL;0?w6fCaVt^=9o3RD6O^qcu!#FZWnlDFade$ z7wlB{eX@sPWX+-vGA)bD_`leD^QflJeczwf-P($X6Hpm<1*Mg#DuV-(ss%(E(JEmG zkZlR|L&BPrl!0_)PEDr(I@}QQTA=(f9(-u5q{LD1GKav!WzK&7Y_k zaf{NA@sUGKhRpp-5WV#$XH2Ymxs&%Pb+(2fL@(b}oBF&0q8HN*%LTcjVj|PuXUDwmUVP_vST%<5bWw0(A|@lw%P0uTeYv^fxOt5i5YXGuuYK`E z^F7O{XL+={>-yrM?OeOr^YsZ|_YitTSQD-7fY@Duqr+UF=%599t zcFZFbH%)f>NQD^(dR&fA;yh;|2bLTY$bIQ=eQ(lRTAN{T$COi(gWef_1yMpG5N+H- zKLyOHizJ*qDoh?Eo2Sfn_*@Skgw|ZtAv3fM#oj3bJ>OZr3aav6MBUwWX<+PdWBq)f z5M&#C+r1vg4Bxfix*g<6_gscCQZ~Ng^{-8$1!xhP2)w=ycgD_Cq_E)yT}FB2V5g(H zEc3dra+=AwaxOSXy79>3E_rz7Sy;~53}!5=kcx>%$qB-<5vp(15AB79*z_KB@GJ#9 zp643-_7r80sopq8KlKMd#|SMf7)l-DGQ&7qq{X~^M_6dKpN|A5j3XM$-q^m)4n=Ih zV-5al=70-RKdRFl7Y6zTVdaQ%)xJcNI*f~+FRK2xjxxsBhrx75ryk|9EYeRVag`2aCTM7mWdtX@&dl>?Mwj zQed{-6PDs-Qv^9qNY#8oA9@d&GVd?xSE4-zjL!xllD?8|G)@ z7xrdI5X6zY03V4*5+7+#o{jj^i_zNEF#ejjTzvH**Xy)1RS7U={&s(Mi5+MCsMdWh zRRKinFDJX(DaosNBXUni-STcHGw$9ujns)~>vUR725`ISxL%&>Y^ZeS3bIV0?W?#| z16QPJR=sva`p=g~Mg@;{Z~I5ru(GusJ!D8Q)Nry>VEC%E2 zF^>&U08&L@D1`}aCHna1ErKrG?iS;9+ zs4e(Cn#?8%KHocTk377UMlZzRCSEUO!nQ1YG$0hI_qQshzc@E)j;~pLLr1^9+~+2>OpOUN>f2r=)&rUz zEy!{lcZsthe`C|F%y-RkMA&+DsJYwur-Sn_7HNZPCTnKct9zS}7Mry{uxHnBl0(7vLtLeI=BYf4En7(k zT!3|Y;DxlhakpB)NpUz65&Sd8^Kdsy5XN(TyYq?S`ZT{J8E=nWn6dExWzzs}O&u49 zHHIK;`KaJf7!1sn*aQbhkKjjnL9VbcE}ba3?my#@I~bt{j}p^szs*T2EnVgk9dgI@{FZ(z z6CON{&r-a%X=y{n#wVd0eKiB1Trs`nw39o16Qx4fXoZh@&rf`rVwnveUYnIOTA5*? zfN2q_trN!G7~sr8!jKO+-uD+a?*XI=wLszjlilbwYPXfS)g@vc=ESO!q+aS9Wgw~) zKP^(K?P)FCWK!Toj4|r1IFJ?3N@k+zn)g~!*RGK#Z0Wx|9Hd9hp=YYyj}6eX^9q!M z-|i`EYVNoPw0Gjwmj3Xjq}R5SgSj8d8e^kMW5fLR<-w6SVUL=tm6MhwzPg39d}5X_ zjHTh4lgbRTeTJYz^gIkf7qFTye@(JR&55%K@t}aZ0}6lW=a>IoC=6g$WQ4e%EXqnV zmk!}Na@{nyhUjp~b>^t)qQOw$s2V6j!#}=kb=Ft5hxpr~lYgH2EsWzxYLA$r#e=Np zMeD7?*O@>_=eEr}xdz(!lEbgJn36l?liEN`_3Ec&J>!DC zJx3>j47r0^l+v0J63+cjuJ%`IB`gr4%m!6zsG@q-l`=}WFnj04^1+9otXp!%9)=<9dBLZW%1#lzKAn?O0c@2EO zETw0^(tV1RKFFuVYK4CYLHo#qDM*aqt4Rl);2W)p!=6`LbCzb};n5xz8Eu8EX((bW zmSFEanWA)y-et8UnL*MglODby^i zC=4Zx^_k-WFJI{O8XeF)sX_9La9st=1+wn*V}D=hJ}~1N#u$ zb+^Cxx2>wQZFm>!neZXbvofMe*wcQ{U(B#SS)NzSJ#`PZn_-e4mzZYT`GrSOm<~N~uhO1$PNFs5=QU>kI zaE4SZ8g68e6u11xR;e{Of3m&f)hV7M@99xLbGdmqFS1dr9Ab%4__4!tE|@ZAdhW(i znZv?khGo?|yv3{c6dgs*OZio*%x#e@wh@aVvy1immpGTW=LXlrnj@f0T{g zNik{mh==(JRUKw2nDBIaq-p8lRc~04gWs1ZvT_d99R1y`kqMviZLgKwig%C$41;O` zR^Ev}1!zvcbU~$(NA{?rYvEBSE;2dOHE__#1DHB;b0uqA*G~Bz{^bdrQLZn?PGVzk zlqgQv7ItY4Wmn{AP%dX5=|urdcj_FMY#x;>vD>Vff7s^rSy_j{H2S{1nIZN;0I{3J z-T$uLG&Phz^sL*DN#RR>N6uz>pMHe}}%H^5nc zJC9k4j5nkW+9Pk-50L-RLVcORns!)VQ7yaec7dXIO+mfF{o**|H1cQ`+GM_<%0H>- zEW9$p`Nb6M6uS@~ICe}NDJD!4g=+~2paxh4544I``-9?>jjT73UzIVM{!*4EgQOEo z&`d3CO{BoFCVESeWnG%LmLuNqi)Wj>sz{#tGei`?!<|h%rV$ zF$2opp#Lr)@RNObS(^6$Oay$uaOtLT-!CNNzrfP`#|xac@p-Wr^xCTW?VYBt!NHI0 z$Hmq_rCuUR7u`-<@!*Pd<$$o3ztmpg661t;hdIzdR1{X_H~=6WK(cNkg{yVb6E>B! z)<*j#Shh&_fTFJ_A*f?Eq;dRqbD?39U02Hiht4`B2oT>rd2B1Qc9`lNS1;!MI#~;R zC$q;z)MtvV=&Ggmtc>z!h+XhE{!@xGi6bY)I z*}~;zS0D-|V={aN_6k=K%S4C>LhkjlpDg;zSjh3A{+4|whAQuF`h|F-gez&JpWmLr zh)>qnm&{nD4@181NbREe2lkdGBGIvEgRq5(`=sjp4DyS0 zUK`_DEz|;SSeJH)h;$lgbQ&PZQ3(1Cs@Ko=B1%4CoSar04vfIsU(Kh?zGN_4$KOU2 zM0?mf?On||R9F!P+hWSCuC4x@A~bWp*Wgm3PXFx>t29IY8)yqEVrV4xPLq8&#g)(cH~kNB+=|FuthWZa;2^v^tG(F}7CF zhfhzkJ{f5kV=(3(TH{Q(TmR+t7EqimK?P2g^LLa?GEYbP`}~48NA8{S>$gmyl9C*! zzG#^$&s|}iR0$K+U$P{5bx%BJS&whqsm?$BLn|bF3`mG%-{Vc&(T<@Vu{oE5N>E1;fab0-)t_e}`pjbrUJ&>?R2@Sr6RR#m z#3P$^2(>5HI-xMf>RoV-D{pP)zeYEKRx-8-AG8k#((g zoJ>7r-h`|SMK{MoFheL3K%N94IT7E04d6pD-~U7@{4an1Kdun`579TW;Y(0em}NxR zo(EX?KV2#!{^i1(QQcGd2~{0S;IB20w+&Ayc!)Pyb~JQ-#+V4X z#Qn_B`|+j0!!7*bH448j(d#p;W{Drll5I86bLVroU6*O$HzE9tiol_A<#15$?!`OA zY?_V7mASuXJ#f}@N?vvFMYm*ASINkfp*ffJ^Gl{h51_}Q_jPxm*>1${0J$T!o8wZe=qz+2#w zXBRO@;}h^cCsk9XL)aSo*C<{9<{@4mTK!J2JCpbA{&4AH&XM;SErmL>Hu8$oMVI^v zqEC=0h4K?%cqBebBLv7fGu zblq9rH&PRRK`%=@s(w=Lr#~6TCz-qP2*A3PqbO))z}X7KnRBD6F`N*iUNW4q=eN6p z1QKxBE{lQ}=)#Yda};qmnnrDJuT(TmYPzaa^t##NBXDfoD)3dKk5TyimNGWp1X&Pd zpMxYBE!rM!8=oJSh;QG_Cg03FCw&iNc!8c?&+^wS7)%(_7gWWTfy!Zmq_VvDV#BP9g92A< zN|zZ%7=EXZH*C#vk6$T2&g;VU$2yhlS`9@UF9+Elr4Yym^iZn(nPX-LuuA1wSG4YH zb>rmVvu-3PWG=6lho$n!>lX+ z+UrIAxHG7xsBD|Ma$j#2*W3Se$LPJ{Pr#PD?BAN(YltU`rmj(SoNi=Ls8+tBHhC3l zv~QyZe9k2pT4FZ%9!kvm>MI1vYqb1?***tYyFf+$9Jr>3Dxz|4j z<}{5EOAlb{qbj6m{Oe4s4qI783ji%{h49SX1_p$Q^RV*2pZ3zq zy4+k0(nd3bT^Fy6nse6MY#FC5@}=rp$PwDbxvk#Isz7p z4(2E5P}1j%mR;e#Xve;dFJO@MBtZZ%1bA_g!!*r~pnkOF&p#@{X|KXL`><|{u(0O9 zbFkA+Cu*B6(v&=;*9A_DiRlvXSMWr2*1X4Ela9Z~;lBqV2Kk{l6`g=(Imx`EcYZz7 zKG{XSVD>pS$>;n;>}m2%CLC@gb`dNI9CAX#MLA|CHzCLBr}S8HESsnxtl_qaXE)E; zZX@vG4-@wAxwe*Mf5*(zWPcHEq+_l5-iN$sLdT@|sz$QeuN-+#a1sD6{L{$xjgm5G zu#R{lZVVBSl+TB2{5_~dw0ezv{bkqRWF6Yi+ml&o>%#)$wg=t$+wa-)r88mPzN^+# z6(DFT9wHlEU?QYqfYYf_phbbb4@4!}NyIMs)Y5@)!{p-njqjhm>9@$PXGcIf+VC(m z9r0~l10&;$Y+TP42(%6UYs_Joig6NrcrITn41N<}@it+f+kaWP2y!jU=(DeF9&hfh z9T0>ZZ?XkcIBYF=;7tKp{x3|z2wpMVF8wSzJu+2)+vo~4vmi8U4;9F+*fc%fG7Y& zs>C@z1B?``<;V`(w1DTqzdJ*x+bRvTW2e)HJ)#&8^sUn08*RG6H?`-ZVA;e?R9h+f zG-07cuoW5T{@lM-<=MbU`usg<>S!Z;aQMQ@(>4rhGAyFB^-Uv4bJ{@e4GavcAP$M_ zgIk?_MJh`vzb{o|ig@=dBKZ8oOdA8*sh1IwU4ga@4OI@(1_7Xrr*fD}=u5V>XMG%* zgI6h-4PmPHs4E=T&fMe|3#TfsDgN z=g83)pMz}-W@qvkUcyf<;`z#^Huf8 zA?ZV8um@_f9y(w9oS>`s%5u_vxirFgR|A=ma64NZu32THNG%;>X{UPzu=<74#-x2Q z#3zt%;b%w0I$=hS%pbwL8k#YlNFyf|V6J8*ku+i-m;LZz_;daCk)NYNXk+%vhin}_ zyKSC>Os~xFz8T@N)@LR_OB~rs(7Z_7)mSuu+lrXx!>=kxq45||Df&R#J6s>CeU zq3*B4hFb+X<{l^dm8^ouS~Z_UjV#Ur)fouHC-1hSh@~SHk@`%7342~v9!fP>l{2#X zf+Zgf&b^(to^&ue^=tWD-%s!W<-K=;y=||Yo$fkj?dEI{ z`plz3v*H#Zj|d$t%-pifJ9EyqX34>qIQ5if$)28DchNISzq(Jvsq6YMWR zI+n~mCscPn^Y1dk4I2*(9wE8WSs1z12{nfwwDsYtFYu?{8EC$wkBPFrHS9>^^6iRS zjtlyYhl0H`fSsLrZjiqDm`p=UCmn*Pm91oZjtud@j^TaRDRq4r4yF6mt(s?Tjy69Y z;E3l>(0*m{fj~HEr>pU7gNV!OA{NF7jfA$z3z1IkEB7vy`iKqM?@{ScG$=rb%|Gpe zn5pAm|9uScZo4_Z*Z|}eKSuq?SsJ-?zxR-HOQycx(VSr-(ruWwiIhhnPB0&oW!}A4 z;;^jI=BzS~i`tGz^Sd`94xmpoMF>!g&p+4 zFun?otdJ3Y+L^k_@GvEogOw9sF&dT&eN;}nue?2O({0rTk^ox}t0YMtA+19|icCNi zHw8O9QP&x8a&4z<>Vy5vvk6?7aw0l8%Ru2rUep|t8Kj%Hys92WQEd{qz2neF!Rn!m zutvu@7?*Xd`Ejk#c4wf%eMQKPiNc_+}!Do$9$EGi>`))g?gYyxN*8Vv%zU34nUM6RK!^M$&;#GHw>jFce;$?TnSvTN%O5n$ z1s0LYTO6%8d%l%Z&bc)jGZp*r4=wi6e*fMm0}U6&hyPmx`bj!hhg_wCMkkzLlD(e8F_lNiMx2AL1&!{W=D1W3*`WU>d!xNKbUKVO5qW@UWfkqWdLwz3R~2TQwe^zV?$i zavSS1fU*oSEAnQg&QnUmYZtUlA=1IFQ9qnp$3!!0=Nqh-Ywo3#$$_HJL~>hwx3Dd5 z7M(V)v7zSG;J(iC=1^jdSbO@Sqao?Dh1L0bVL5&M-e{sGK1n&*#a>B@?utassQqi5j3nN`6 zx5ih|j>8*9DXoR&;G&z^y`%MW8gU|7FF-Va-PwuOUUJc9o?8!g(9^ZdPRPv;SVR~knj zIEI!4#5sG-BvqBq_DcDRQHO)0EA#D{tDQ>E%q6em8XEEaJFk7sA|NC{`QV1|J5->U}Ueu|@lXB=8%`d>X5gtGT23 z61&r1ftZEDJ2hW66nL?lc`3A_uMMKatk?qOfG{4e$Rkm7HXbjTK9W|?=yvkYKL11O zHPcy|*~+b~FIATY&!9h#+{YD)d7$!aEk12pSABD=_wZB`YITR zqT|tfNWn*P#JT-BuA#p38oQY-*fY&>{Ocm8Zoa`xU`GAVwf?lU^}wB;_n{ZIqAY%{ zvvppyDDvvCEL2N6=<|_VL1_Wm2FEv?9{iyd#XGB;8gy_xwnTB|FC3L!d3`!ESj5`r zSW+3eueYvcE#MgC*T!RwtKqzPE^Bo%Su;{{s8n%smJILqX_<6dLxi`d6fwUehWOcj z8S3Zcv>%U3Mv!e_%U!P?Hc)c5`7fytjG6&`61W8yZ@~3FFVb zAR9IU!&gAL@nA#T0!*WBwb_mFof*_DUEVwJ{N4*6GQOKMWQ@LPMp{i8ZVJRj| zf0Y0+tX@)Vq@DY_>rhfsyfP{R#yt1xXv%nRkhQepOi*e0((YYba z5R!cqq*?A)Zy!dR-H2dOAzdY@3AX9w3SH4^r8?Isjf}fS@yCL$exi^y0(*)l&LG6m z3-f^EL88aTGF{Xx>!{`@=Wq__8NLs!K^96$E2YaapPEeE-Lg7N{rlbC+J-JUY70=T z001Mcrab4%p(L9cT&%*O9Cv;c7b>z#75w=@AVS|yQM8SM!~FKYJQd}M%@V}BdVkRcyjE~^!n~mis~{@Sip@cHFkEGfkBN#cG79& z;I)C2gsc_ijESzBZ094;ifAu#U$|q!W-7WF``+r;kRt8)D(^L}Ylc6RWh=~sOMHIA z`g9eWjX6+uv@|KtV}q1kOuY$n25tODD#v0iSb1UCDc9^Z8*`E%MA+c?u4g}@&x)gj zJ)q|IXhURuOM6P?-S}Pon|UGGg&2_Llpv%~$<{f$RKn(d&?^|l#*HEZ2+zN9-a*Eu zZ#;d4a}5o(E`p?@B$g9)yELni6P)niK%pQk7aYp5aAk8X?(id}r|UtV{M&kSrpFc9 z!Lo}p)@Otn^|La|6#LhXi!S?%NAW#6T~^jKpIt-3@LVJ+>t1ay9pJb4K@`vdXA&&l zF!YfE_cEC}gew zqRyXS>ujp9Z~oaxGJJ;{;JH8hx4P7MXTZ=n@a<$X>eEeFb0a4<`((sM?f~h?NFHry zZBkK)>zVoKOoG*cVYI}-NC=uq4!9qux&DYI{PRcmucD1&U`IDK@@pM`XfZx?()+*i zU%f#MoqQla#Se@>cK>WY%M*$DdmNRoug`Ko`RxXJAF7Y&z$ifWIRS9WM~Htu}YQg-Ct zc>LuqBfl*o6U8f75reFz%|!8TX-jDZE}(eBb2Q)} z9UHH6G4en@?IZV;#V3G8dos{rt*-^$&~%BFD2Y#muTHj%x}UmN;-a`E8rq~4xezfT zQo7A$2j+wBqLsAT_P?FzJDewXiXskWmuF*=_|(D*)hSt3<=tuzlHZFfREslL8teYh zvW|<(K(}})Bi`}}(-$Vkn!SN@IWIKAK|nM$idchG+PU$`r#l- z))Kowl9v0YcWJE0=EmoWN4!$bnYjk?ijXjd@}iSk86FJ&lf!zT-ti~*`yME&f5#*o z>3ej09kKgcjd44DJk0U#l=Z9Hx_j|s?oo9l^C(}8ok|<&ZMFluM}8S}`KVb6`XW!% z#HQ+Hgf=CIP&30|G;lim&cZ-0Zo7lM?(o5$h~L@U0lC`DSx?W{?+U%xeY{@;_~%c8 ziJKoB(MgjgCv|h34nefSX>TD{bFo2*8CN_>`$THcnb8hPO*!R*E~Lk}z}-)tpllBd z#XhYb<%^q!b!Q&uJB~%}?H%pIz9K;lX##K1&d8?*xR%`|mnQ$r2_6M$CNpOEWDlr! z(5bO2*g!Wnos0tNF9|UDSY7@ldz~#&25I(>>Two=Bhn?m!1MaVh~4*WIwo?gKpgc% z?P}oiJVBe#5!B=g?d4S->Z{5*m-|X zxZ%jXeZr%%nULTBl66gu{lMGGBTcv*Q`|;6x#+TzbYgqP-X|A{^!Z`gH^W0=p)ifv z@lw#%lSI%-u+eh{cAHm<%B;p8TkYPLYI|GdimVF^j4ZYYMcA&g8Di3W6#uCBRc)W@ zEO2Q36+97KjOfnyPfkg8X-hQ8%0(M*+~xZ<7l(=IjGi2U7hMUUV8G4_JqUc65(;hy zIRJO9cx|;;CJEt)m@RCZx|`X-RD|7}^7@J9$l5w!Z4#B__QGkikCA8Gvu;!DR_s!q zM#LvM2ugJxe{C{xFFqL)5KpXv@lDyhwMGi4IlXYr`=$QZ5rz+z+ELUu`MJLKqjf1< zv3E#1M%uy&<@!jsJguQLM~;rJ1QSUz_K8kFKk)T$;R%ibom`ku)#o$O&KV z0A<(7h-u;W!F7mAzYEwpV1DglJ!h?d{f?lp0X6-P>Wk7&|1?h!`6!$;$>)If;1f{g z4a)aBDN?k0SAv;F`zBB(AgjtU|8lxS;jPgDDF81JwtIdX@0Va&Xm)*W<*M}1^jkk3@Ja(2Zwu;FVPtp#tq57Mk7i8G%rJ9s?&r)LCts7xfIdV*{fiBF+VBXy3l zHLn>H=B-Frt^(fA)7^|C`&fUgM4p!wq^f+-v5tUeRm5;4x6D9Y)=uZ^YnD&Y1_NHE z%lbN*M--3Deu+U{un7`1$5XS!HZC)V$oi+|GxaknV1H=&wYzS`skvLX0>Ch{9>p>3 ziN6!}pk4H)`lt1!D@93bU18kH%_qj`hIO|G1{&(Jw}xq$Cu%I`ZED{x zwd~|clH;~hyZoBJz)zWUlyL3z*92aco7Tbvhj5M?kFI8y=K~KQV%9D*=zz4ks{`2! z+y1+$`B+ST=Gm+jtA|mA{vP`0Scy^MF*`xzQiv?&f!V|ISBQY|f&h^67!xc(nt`g4 zKFDup9?G32JffETN%dX(7|v=Y-Dq%F7dkfSHPPJR&kd!tg`aBE0GsH=K&@@#-QQ5&ps`czV2^&m3)wiQcSf z{#LnvZ9~w>L6V;4a0PmDi=r@Y17)+Y))lqfh8JW1&=Oyhj`x%t)aWYGU8VUQ(bd9$ z2-;PU(Rr>za<8?|7W;c&}FdSSSN${Tpq8n zo~;(V_V$ScK*#oq@}CuOrF*|D5^6i^p+L_pqji_|dl=*YOo!_=BWw!&h^Q_PTFhx=RDA%A+m@) z*OR64tpBp`c1ViAGX{go@EMomy7v?OIL*={W2TP$ZJ(Pup)&Y++NtcRW8@RG*uLu<3)_N z8yEwd{{fip%J{U&^&ZXViUVdTcr!ls><^Y~#>8u#%z8mCvlS@ASE>pM<8cO42tl3m8u^ovYbU58iILX|e^W;p8$#_3p5 zh8c`QMV;*0zodUhkd%6rZG&-ZjN-|yACuei^8DaPROD8$h|mey4;SKW#!wnVf#PC- zG_bzZ>=6TcxbxYPAhhnU;uKfZWKhgd6K;SW_lH(Og(6ycddWw+NSYDuATLbe_e{Mz zUSEnfF1}K_5NQ9`dI|j~=L0f2GSm0x_ZL(l(sb1i=u{RDCY0vc7Pl9?X7p^8qd)^Z zu%f9Ll#+)>(*t*=s&4+KXgT}H{v7fOC!fz~BYL%G&JLniTR2<(%=6Nb=jy~OU{gOz z0}+ZJ*dc7@2X+0@{CIyRGU@WE5V<)h8GZF)w6U`ZCv+OzMAoUZR0bdtgeCfB$~&V$ zxIfjksfjv4 ze6;%0T6QJqCF(6yk^vWPY)8@m)?+GjP5xIko!@Wjzs5(A=P3NpK%q^JP0gm}touJ-{Ozy)_pknOL;m|t{P&#re`+aQ^5XX8rwQOm^P#*o zPToJ7cIp2EZ2|BWdeaaSAiz7KV9R_&JJanPE>=%r_ir!!fV?!zKQhrHu1LlZc9Gz7 z?F-edn@gGB`S~Di=c)|`Ym9r_N*6UpM$yel{x7Y>w%YCfaR~ng=T8=5U`# zy`DpMJ7^OnyWbs;C@45ycQRmW3~KI1rbjIvYN-cg39$Fw>d{OtxNqPPVti~s_q}3h zyz(sWG)tC?t)1=MXk5;~wm%5i%0VHMLqiM8`E#p)*WLp3DnaS~B~vm2b!${zVxUAk zF@8P}I{F`fW%YA`Yb4>{eLzgW?slu|N*B8g))~kL+&fZUHvOO9Ww66GZ9+4WuBN=q zT2ZjLs&}o2zE4{k7;J;jh1Qpxt) zddRpcQ19tK2Wa+>(T=qqih))Cd<8*PI{Kx6SW<7INb5x78Wg4_~M|BRU!8Wjdd;8zhM!S4L3>3Y+V5v!u zTn+y>u;K^(_Wx^t0!ALXm}B9cP8%&}79OsX4{xl1jMMhosgQzs-WtitnNv@yg}+Nc zaZ2MYbCJjcPF;>q)wn3VKYWFyuNqI82tp5$krdy{0$2$7i6aibu zavB^?(g8vuxsMnv(`lwjH-8a32Ld}bc5cO~sG{g^7{aa4wVP-F+*CFOfGFDU!I6Sz zie<85!Cet+@PHiC=0G?Y(G+=o_$j007S-38T6FHI{A7F%3aC7&-al<7Sj~T=ytX{H zXlh17o&5fpY)@rU?L{k3lVLWw!ZG&?&C0Q{OGCiE%EQYEvB+DO6tlAwAUO8A@``b; zaUAwyB zeBy7*L1qjFbL`~?;l$VFnW#EePa7*vD0W;zq!RQP-V|)~bG)%MEVr_%LUSl9u&uzq zy*0-^qau7Z#qrJ9$&*&NA>pynzqsU;HQD<~>|zoR#aekcOjsm?9Db<$Pd}iH^5BYR z!XE$FKmY5#_rHDh*(dYre1)Yw-#pHwM~ez?6O|W)zAeN-$9wK=zho2}y3q5OF#lnN zj%+7PQ6OC!_v2W@xRRGS{HU95z7C(wqDZm3XhRMCn+3623DE!JisgSSyATDj+mWd! zU9GMYwD8Wk2s{h3l*i~9@sbqHY1ew(5!ueiMy~mt^6%ge;>-IXLlna;B5s(~T}P&) zWmk9BT?>Hxf?2Ewq(vl3v-Y9;PQBO>xct+Tflu zs?9ISRm_)NnI z`eXvMH#h3T+j7=RwEAWSV~bz*`An4m0LKiK5<%kG7VIIb^$(^~du#9~afeg;9JlX0 zqva%B*cWLj$0QZagk#WLR$@PYlaQA8)B9ak^Q@mBEg#fzs$DX#Zu;;s^sX*Z))}Z5 zw1{x#(rhL_O5YVlY6}4k?DCskKG(c9H3ZZ12-8y0V1qY`xP}<6hUz!fSXtk5<6S5` zbbUSw9|%UQTl!V~eso6A$20|=>73i^Z>TtxRS=+gm9>%~NBy1~$LBEvS&5jz0K`v_ zNc2wJw;a4VG0E$DnzZ_Xey_Xi^3pYG-Vg@XnWSVc#S@lBhTdd}l+ATJX`?}AvtHcl z=i5G=k8tro*>Oe2DKvc=TzM(x$@#dxBRvy$R6sD|hnwd4;@GSI(oHnzrihgh7i6J$ zN>ZG&wzz*5;&?X6)L^CN0_hh$m)Do-+@}nugmhJ*HiafR#1^| zq_iyOv$GFZ&A#QV#KzK4O-85lp~b|S9pz=Mlr&QpqDYWY9mtCa_HAcPpw?g50Ju}q z*!Mi2c`kq1sk21S>>9^`Y~EdTUYbd&gdb7$M`Mg%QAttiH6X8u%tP_oukI`#4X7%R zK|0U8%&3Xna4{dRUmw&}Y`3S0{A~Wv5@^JB=`M%gnd&^Fw@pyLIPA!Q*2I6(S7-r5 z!g-0RA=3NM-}cm|O~(i`T=uY}u)>*m>aq>Qlj*&$gWqXoO~=^V$O(0b8UHah(-Q0X zZ9`MX1kSyR>mPJsoHu-AOkCu^lUorC1PIV^^UQ$NuY`vhgY@ofMXc(4_0hI3x$TL8 zc7KNh5<3t9#H@d`yV3g6*y&wqC;aF>XM_hBrjNZDKBc5RwXt6i6t${gbt?fW*{8xP z((B%xi6h;;CEdK-mIJny`))Tn)_7>{QNXBp%){79+UbP!Xh*wW2*GWPP|?WFb0wMU zXVLmtI<%E-PuR`p@vRH4b&f?Lnypd0Xsd9^iOCD))r?y&@jOE32#AsEwCieigShq# zkf|n@fLOaxjG5{Xh>-XA$VF8uW7*55>3(PUO_5Hdyi40MpWCc|!htE~5jZH`Cg9Zz z>RG*zMo150Cw8gpxkO}g(R4)K>61F zTUnbC+Sq2KoA`&;=Sh#txr@)1AV}dR?Zv2=m$iX^uOD<;iS9=Ebe4pKIO^|37=hgJ zvU6`Jk7l;=YGLXA8+G;+kqx2#0+{Si zC3q1+9}1~0`1RaP&=D1J?oo~Q}UP+v$$pkL}t`wYCTck ztTsZ7Q$3fjwTIi-=DWGR-}1VwwfOEs!n)pWCN--{5nt*T#9JhAH=T--nI$^aC2v=A z!niC)j^@~abfhPXiZtxap2@J%>{WYy-wbtgZD(X6;{XRio_~|-K2&FP>Nj1FvbH4i z8TaH^wW%C7j&rRJir^q*@@SK>Ef8XRKgt4p59l)mQDg%@&)|@q-Kzp398oPZ@1qI^ zegP3$Vb`}N-g=KjB?jTlJ>Ia!Y7mrRbf4)pw zr1`)$95-3qer*ruVXDvI8qdrLSQ<~S1qCL_e#QF0Uez&fRr9dEOR!C!{ZN-m<80oR zjbSpMiiwh2Rk0K$A{#r82DUBG%RjFcnxCVNzo8U>J~pp~J>QxH>J0?qpwh~GT15F9 zY&!J84aP3cMowUD|DS``R4g%o9gr>%z6fs_ja>7r^{+^9Kl9*_n+;g&42@)PSPL~8 z(9$Ne>nt{8G*j8Qi&cx8K6ke`vVb>rD8M4YCMHgv2Guyym)SGT;|@BPJNrC|9yHLB zEJC6e-0D}6?R|q_FlZ{C`c3!yvS+cZeY{0e-3i8dbjvn}+w1vuFKh5b9m*0ER1*@~ zAUkkwGN&|DjN0=DX-tCY&+%b)tY#uC6K!L7Zn^zj?Zk(60G1+<3d92ibWM!dtlK}T z5`Po)Nr4Ai=fo#rP-gxM??*C&3>zlxpO}9?Q5V~BS?-hWQ6w0X;Rd~qBQ_tOe#!Zs z@O#aU@vrJ#24`pwDDScMM8biHAC{u&?k2T-yI)u81_psrBrio6*-(RXlfEpNxx|2A zC=namZ+i64er~H57KU!8N3-cycV2V7`whSm;^tHSX8r^B#vP!U8A)Q0qukXMby&=X!S&qK7lP z>G(vaFU`KjVxl|~>5wnhtIG6h9US3D38=4u`>1D}(ddYF{q6IwD92N0Pk2htGtGer zpt0AwNUn4en>p#wKsE|cH|o><6nUa7o&KA9sFVmUba(}6@eQAEefxKS^HD*>S9kik zMuRk0mWZ&E&*d)TzZ{PtrRVW?V=$gZXFl{2f?(9ObGSem?~>G|a^_bmGTxl}YDFBy zcvtbmCKOWGa*Exe64D_(-u;ym_Az z!_tWyHA+X_w4H79ot>JudgE@kofkR+z<+;e>05brmY6yl%?P!FmSsqNZd^(59nPk1 zlJngy6sRNyqz~FX5ElRGoN1ceYriT%ib%0X8F&}?Qai?}iR{MvGhvF@oaA{}{K94V$<{~{ZJG1nTYS}_NGtbjs9TJ@lY8?jyRu$gDI zOoiPSBv$%l!&je~zGwf($8%KSBl|<^dqwWF4YJ2TA6j-aYu;h5xp6Dy6bEuVgipnW zj;Q6=ac2~=N^GC#t-y}6-|q9*UC~D>9@PZevVrv#UaJyns9g<%=QIEjeY zO!3!RKW6oC0{_)JlE^9}(Y>8ENiJe70~wEYQb)WpSY2wNT2mgq#Rtvf1yYeLA{jwO-^-;O<`__p+yHjT1V;0dg(wUKGG%0G}RrCffximmp-DBoLX~x@YYBo@9DE4 z&W6}>`=S+Ya3kKL5Dowhl!kZz3!jc+^on^1yZJmJ<_zrT8SzIkM8BhsC0rM?9OSRF zIc~YgOSH@6P38tj@2&jp>dyRd#2#lTFW%2`HM@EB3fxO?edg@Lhp>L!Y66`;6=obI1c{8d4qmb}*bIG2_<&ac7jW z2wB-HU41_Qs8|oEPvk{JMy>%SWBlUUAm^InH0^`WmvXJkL5U5IjB5}~Hs4ilaZgRd z_>vG2MX#MoSe+rE^CtHC$=dOQ6klH-7<&=;bt=rqLf`l$)ZF00AIm5`(rEUH#1I2$I0bfuWx2!z5 zgXcY&Zuo|d=q>0AA9IF##-X$=ZPL*sggZ_*Q_GB5J_Ff?tR#N?@`x~iqKe3ao*SzHQ z^)1wyQM==de@Rtss7uStI#5}So(ku&<@8uRAeTMifm8|_HWIqVsrNlq8&<+Pz9K5= zM0aMUnxzI9416nEzS7l$iyZ*&e`VAb<%AVF<*cKOB^ z`otGvyK+FyWdN0lpf9QbneU8_8!-;|QF>3$c!)8rs!SOnv8azZ>6kh&ZB5e|4GIgg z=o}{v&#U0xcCSA2&`oUgeVl78sj`2xDn1xdPKRy1%p_-iAtj#^&Pj>A|7u0Ec9N8Y z@j)hI)PZMEUVDc4b|VBQ#=+#T`eIBX9(3$s*_T*IPcV3zzHi+csa*K7KW#p^0(+{@ z>bi1kET)~ZQ*2TuX2NfwzwBG6d&hMQm$y+;52;SHkG@Qc$&1uZb1U-foRbX@>W%%M zm(`V{qhoZ%KBLH+OYpB6oe5IC0V_m+a!WP$aEitc0RC^(pUv2G43L}#JgM(SDA)(dcce5y z_Pb2sCdPMSH|}pWddbR4s7c$tQ0lD#{tLXH?C$kRJqLOJt1MDQ=P~i2QOs%@7qZP$ zybh3}a^dknLAvl19(9q%;D!iDi7BsTgo7dDT_WR!W-9C$ZfF6RS%Qt zJy9p! zap_aY#>E+k5^4njZ&l~S4<;sOIFZPd)R!xV_T;^*4Qa1j4yiVr@bv}CxW3e*Zy#jD zw8dWr>U6UwUYY5U^(^|05WKOh;Q?rq=?(Nbpnhr=bo#h*RwyS9Z7Fb01X<(n`#I<7 zH9_OxJmCLsW3UF#NuEmqH$7}VfRs!W3%e7oJdj+@?R=F=L4M(*yO;~|ftr4NF0^{9 z8-6g8f*8KEtWOSS^qaa|Q2reh@i}jtanEjJ>vSBi({0uBj=mV3=@BsS6#x?kNx?9r zK{kI0k6r&(y&wgxJv1XkMVuArWI_`2?ENx2N#rcwG`AD`vpg%yq~(~VUQkVG2=e7l zVy7}^Zi=E~)qvFz0vVOtJRMEWD-KHj)`b$O73RVs&ARTWh|v%#;i;{$su~`%8)*;C zQtx}?0<)d8I36v#s{`habHcKN5OUFMF6I7%aC5+s6-oDoTYft+gx{Wbf)CFn zQ6N$Sa<7%Wfay@<_;!J3!+_;}PgmcT|I(odXn@VpUEC0N!l!5Q?GHf^M@BNTTZg zY=-huESvZ6rSEf4Z#IVv> zn=smdJwLCV(?}&c>|PG++>J-a$$Q|HoE8EzP$M$Npolm)~#XtQnr7ddu!-c%8RP!7v~8T%VSBQh@ak`b;!EoMt$N;ZPP% zJxtb_UcRAzc`zO}d`PP@5PG_b!Icny|kDBmppkY!f3{H2~~%rJJB!WTM$aBvBUO zB|P^{?RejqKkvt1FyYPy5moOM8am-u-mVy66GM}o^c<@bo%_m1&Ktbrjc)8%A9$dw zj2aeXnd#lxxv5aI1Q?TDXfI8kS&AaWQNEqFTtPWsU@VYnUk=lJdM~CrRq6Md=tZU* z399J23^S~<^g_SY$(W5JPDfi{uijls)zd(^AJ+!v8iyi|mz9&V`PHrsJwLpO;f{8% z5+}FY>}E7NUMyrYr@xu9{NYiNV!TTl(qJt8*p0`m7+QG`Ng!SH<3_Z^M%W4&@n*CW z&F2~^kNb&AmoVHU{)9i<8xZkaT!D(mtNycfns%dycOU_qP=zN&l)iQK((dJl65ioh z)~5C%a=2UW@iCWh&*`>7w+IaJJo-SN)Kh3=*Hb)TWte5IIKs9W*KclO;Ql9F2lgFq1N7>tz1TQ}@p zZO-eQlhS0&-^E6*0>~|+P0nlKsM_vxg0_6Df2XzyD3NaOsWCA@xVR3L1NEt8G%prn z=kjFP$VrSCp{{3&m0@*Ni~$YIdT{<5^6-a=NNV9E&FjZmg>^8`Dm}yymAy=He<$A& zqu=<->g*fc`ksp3mXiY6f}iBXalSe;ph6|fxTmawSKbMd)<%xHJLsR}CD%#eNLZhq?F zpOJe_?B{^*w|k%1h&#LKt&L%CdEfW+D=`)qy-hbzcHg8I_l@b3FA^KDOX=;jf<|(9 z8KWok^i`mAQ>1JpUgA1Aw%crd^)B=oUpmex_f3!1amgAs%a{PwG!e5fzIr>}Ky+okjm*tG%+EDmr%K@|kC8Rpprde4h_n`i3Nh<-nFF?(({VO%weK%9{d3 z?l1F7JJwB~aUb;ZG&MHoQQVh9s$}CZxNIHXmg~G371yk3gmn{$NRI;ME(FY-r)ir< zp-D!+vU2h|WkdP!PEwbIV-{-xx(*~Y5;eC9jc$cHe`8lMJfjIo2og_J_unIt9@B#h z&x=S*RWqM|kpbC^z+-MxzUm(rM8wxL**MCgf*{`nES(whd{*_tC6AwIT7XIgYw}I{ z#j3jvty%yR_q|DRtLeF38X&T~*?H|n@vby!fsy<#uLI}1*GNt1h#0#!d#JD6LU~5+ z!Zj`WX@qR#_B$Lu1VDk5z`#<7X#tu1)u@|#Mz<)No!1V{SPbu{yU#l_?6VLTQ>0Cl z&^xTcx$wIyVbWe+a?IBkD_`~@cnOVKl)pxGpH4mnYW_+Cvwx|e=TEcU z045MdZKS>(IOr=40yq;_YT;Y|49ufyn#| zNq{N#)~rLSFQCKxNCO=USNRx!P$=D>N~RJw{c9ob1ugIxI=qz<>Fzi!1Ng zm$1A}KmZA3^wgZbYU3l$d?3c25o#8e1EW#qf@LZ(z}G4n7ww3*6c|aX8xxS?B_KU9 z?kiu6)kRK3I!y4~%%vOL|#=`;)j2~%5 zR}|l|0=gjLInWs(OAiOmV$qtYn?~EjB_kfn8w1__ZY^|r_E{&)bkyyHpoEva-6F39Eg?aj?Yzr&?BWGEdl{! z&2^9>Xru`oP%&9Ab;NK98iOhO;Y2v5 z;PZql5<}Wl(aKax-Vr+Amn2-Qr5FTv8AOK!;xnL_TRSIHzl%B zrkm{=k@yFjN^YGD+75pUiL}#RZV4b3W`%bz7ldiL)3U4RzotEc(&I6cshX zZCvo8%!iTQd{27|+6JI_fg!?Acr53ii6q0EsOH_|g24QwU)H7iuoWN5c@7`|#XpEq z7J5?R-q3WcZGxPjt(GyC5lNTWMEt<;=u0;ZSqU~2lM;GW3KechG%Ub7a69jw>97~3 z;vd>zzaZq8HL|iaZqf6C5Zuh5sj~g`KP{tYp4ez*y+m#y#0NvGS?#B1K&3|#dX@+A zIUiG-0hC!^`pPD=8*tOm4N|9(L7UxyS(L4f0;zjeVbQetN+{(7Epq$1;Y&Qg6xNtU zAhRQe31xeMF1Q2|XNEZYz0g|fj=AMEe=5%jB0TJIeynaTSS6?NHVAjQ4#7#MkPlu~J(P`y*5s0*lzOasgMuJ_Rl^JF`EkYP~A^vps^9=bW z)^$<*-tx=f&>-#V1WGO8tKUZHO)78p3>a3<_?{|Yv{}1)(zOT54P2e-2e^%?BZn#q zQ+@nv`9QxlmO~~*1g_ZpvPNYxm(YG!%})HZIOY)f=Cnc3xk?Fw*iM(2C=bWvk%HNh z9B9HLJD~)GVnJo&S?OJ#2U zfnjF7Ns*w0N^Ds{H|b^%4k8l(|<&BSm@B{RZc`@Lv0l@{rD;v64=JXW(qVsV-ghWB#V z-M7$WY#LB92Q(xbj=8wFv`_fX{vcqqvbDSf{mgM@H#aXRD4~`d=XE+@Px%Y9Pb`fj z&YpnzAo^FaPM=pp8^1DT=|$t%NM~YoeM(8fNOax6AuWvvQV^bF*Zn5WJD%F^ zF+sm#_v_inO+N&m?F8~5<|e^U5CFiKJ@q9`B~9&i=v1;T3!uPz|Mshl^IqYZ&CI`en83>L?-itReCkXd=B??fXMbeT1toxm z`maT>^yA>7|7w#{MxBB$fHrF4&YfBVUm&ai*c^LWo(Yh5s&JZUrvSi)6F{pKE?$KE zH+(V%WRTZOB+i%~MyXr_589keD0^Fa)ds*939+(L>f4z<;0H4RGSY25|K3MIC;a%}(-(G4m%d>v3VDD8R4zA(I4Jo(}$NJJw z-^PmlILv5yY7&1-NKkmtHdQAd@ws{YbqHjrI%!OLztXL_(OCC3(1-4=N)LvYxI9{&_^{Y3oz zFj#<<%B4zLy{c0WNoWM>_m(no?8O~6ESIXf>c$&}2tzIZ;+rl*Ra#2r)eyb%so&we+QPAm9GEF!K}3}@{AGNXTv*-bqj%8A?7hivEK+} zQ;aj+%CRt@iWX7$Ry~%VcC~aoeTBI)a*PBQ<*I@up*FK~szAZY8COxMB*QTS(R&46 z_jT?^>#o!-I^x>$p8EoXF`OW&WH3SjI%M_+tSDm6`8okV2}rOIOt%@ls32m>tf1cU z8*PwPzm55(<<#<)!{aMF-cX--^l>5UokrLm9(H|Z9w{f3l}^0s2w2}4VPdO=z;+=U zt3VQ=iz859u>>c*U!+kzuCCZ4L4+@-HTU-MCbR4*uPSntoEz`32gf_=GjqJ*WWfA^$6HD{ z8Xoq^&QL+cqS7T!+mgjz&TcO|4Lue>0Q(%WhLKNkZ>Pa^cZwCcYi}*g?s%KEQhT?e zwzw3PItt5nBUM*=xD^IyBM%akSH4f){j97uE+xPrfaKwIMROXg*KmEbeSK-sYT|C|^EU#t%tN2JD{f}A0pAIQP7HMEJ z0Yn^n5T3NF1JIc~dHr)>Z^tC(J_46g`TP9l50<9?Prgn@tSkPFcY63Ih+Sm+4dLWd zuf23r9`q!=ICt_CA`7`EzW?fw`M?!@+BYdxWo>9=9x=-LpL!N5u=fh`BTMi_8g)UP z_^}Sky$whIU`>0$3`pIBii&HPw?Lhy>^tz}kAKj#R`McpIkixpD%5B82D2dLUM-1A zr}V7J%JT_z^RBB)qs$~8d=KhSSlgsc&DJt>C0w7ea{ZwJS?zSC((?zL)Of#?6PV%=vE&P^2K=c?juKEy`_y4! zHZFgEKLIcOnoBzrOs(em&PgZCldtlrJQIiQS2)pEFoyew!gPOIIXi043n`ac${xx$ z#hCOguyxP9it*ci?Ly$0!!>ERlRa`?CV7d=+ol&3Zm#9jPI;?FZBsQN$f6PabEA~9E9;J;jqm*MA2rg$=V^+F~h*s4cHUm z3zVe?DrF6zVfUb_~S?F z+#0A%yuinfO8ha;OQaXxD^&VQ-Ya}hR0=eCg88vPyw?yXaYeCO(W*jNa`9XD;=Ae7 z>KKu?LoKQ`PY1SvHg6~nA<4=HWK>MB%u(dfFV%g3@;lyV#=8zn0Jb7#S~@J0N~Sl4 zT_u~Cht9HGpF+M$9~E6E$};7aO!TB(e}$Qn@&z9W+4W3A>M^7!7uY%hx%R~#X_ODe}h#HJR6`mdMvkB#NVfB9)y}e4rR$_OmSq%3o@EN&6HI^FU=~U z+o@kS?h`_}!NhWZFJq9d4T_v)3Udf~T#-Uv_iL{I!2B@!$#$g9$2jcLT>MfkS&*H>#kf=)bcoA{~C_NnV=6YDGwMHRR%?_1U7>?%@ou_LPv7rVU z-VrOdm}owulS;HmT0}BAqMNec(W-$BneDf^Mq5zP&O)8S0Dn0=$E@p`!_H;uZSTY@ z(IiBGdc87BOXM!-c-JWe72Kqo{Kc(}7UeU^Mj~q-%XtUBH9{4vm;B}IE*LgKcKyM@ z)r*X?bp3^(Lx!^z_o|3@&~rhzRljA(CIU@qXS&^LQm32w8Z!ZFJI!j6wGlwF!Cjgs z0}8!@f~r9T?fVRr?v91wpJQ)dkQdKVvLIV+ht|pTdmqCT!|u z$(TSkM)cc&%|7>_T=TqQd#=-&!y3MQQd*y$X=j5IuzBX&vk4Eo-KDc2cX<-?j4rXm zylzvL-)SWDtR5)w_%#tMXIg)B1o@x-x3Sdj`kt(~M?+6oJ~sy(w2;SK0+%gXFk7q+ zKVV4@P>~Q+l4$<)>s%=y$&UAHz@2mQmadR7jAaZ!bB_@4iz!fv^NFVFO_7*@crND- zZ-^Y!%%f<=oUQ4jXbx&?e&a9Q+4EZw;^>t7B;xlhs!H9=LfPAqWTfG z#uH|Y%`U-2F_*f=v-vNgjYA+4X&a=yze+#vDBmp;H)T2Bo#3U$Scvs#i|V+o;pAk0 zp)q0f9rHq!!+O0PjaLb@-MC<+Q3RV{d;bOySZ?R(wq|Qu*EXdaZ3)a*Og_olQxHdV zt{P?zMCLRF_y?!MUJXLjK%<^gZ5D$+@c4q4L+p4t z1Ky&^X`lhhKDEtjeiR-T1xX>kUCiMxB6M9O7O#hA?D{WPr$@)K8tctTUF00^-i3dETJYVu=(B+!;XxyFt;c zPd$yij+=7b6qxN!WiGIfI!0y%VHa6$n0PAL?#Ylidc;?lcyl4v*Fj$=ov&Z?H`-C%Q3xuHp! z=PJ|!8MwA;E_O3(1|1q@lcNkg%!D;bTJtb}mk9gyA?@Ud*a-sBzvS*u;ug_sS={E) z?1V-GXfq3{gfePl>J%{)Q5Pla&wSrG9~^{oyOXSawMt4IE+;S{qJ!dsQAm=elDJ2} zYI8S=rx=_M5O_=ryDm$V`a9LKy0_kC>!vQz=^?YRKBF!mCg#2^*IdRCS`3$nEksHTsfy$)6zR=4rT9VTD0Nd8 zBZ8Snvv_@n7#UW~_#7YwuEA*bz4R*zt}$e(PI9+TZ^R2CF?4P3cm<-`5n z`1OosWF$UoG&Jv5zKd^4Wd%Ro(8mZZ$UuE~sHp@4QSwf4JdGTu7yS~4? z&A*&Y>cSD-NVb|&=6v|UyzwC#5x3x{b0;^ZR|T5_@@|~h(J*l(4yD`1jK^65`A8+93bDP{m9bvj=FQ=;`Sh6C*9aH!eb$dZR{I8wl``gtS7}1*s3*cx0>1 zysR14gvbAW-QI|A?jfIZ+jf~<@5=9(w-CsG;TMK&pHqZrpaFncxr8-CVqtI%%GaoB zCqW0+u554`7S)OnIlj$|Y3lmmt))+Q4FbtN7!2k-AW=uP?KUW5`^gBpI53gW2n3(K(^tI|g2Qu1RPhPmYBo zoB7X{yE^0ssDGPTp>YNHhw_jdBQ{&=vKYVRoW%z!z;AzdzWyU~`u~N`{Ac45C^K*m z!V@aDJU3c$6{Blb`6fNId42dO^$g8X4wTwXLWR=`L^}ZYo&?Z3-|Z;=Z{%v z@V9G{7>!Yv`t~6feHa%7$OPRiUsCchxW(zrGsj%!b<$$$WxEd3(gK(?HBbl(^;Nd;wVcP4krWG@MEG7Fo0Gxg9eH{w@M%)by%wIJcjYQ*( zTFs+sA`}B~to8v?^!=3c%dez8wX?H&MiSyQee7D`tC;V^jQe*tU5qXjpHb}Q>Am&{ z)eAEQ)*fJslVk@!n1e|Ke!)1#-qqb!VZv*DPvyHS1+tQqGI2qx1&oI>ZPNUfc^m*X z%e|&|%5dayc-}-{(;G~ZI|OU$Z-mgvYIkkOZ7UDkb3S;a9xJM?TdzXtnIWAZMN6GQ?>h9XT95H-jW+Sa4ZyLA7gBLfM~v$ zaN+SljH7wpNOT?`w&eIE#2#Zf=hE?`FuhvyFz@A zCb^fyxEB2$|AyeYYWs&R+mnM#vtJBIGSQyHUOX}y&hPorn$cFE+Gx$ylR=%c=JePn zX3r3NnJ;?%hl3(@5tTJ*{2`qC%qU#BzJ6cDRJlP~QJr1{gn!O_0JuQ73>Bk^b3AH` zXYlZ7bAu^I*>FOrnlbv@Vytaj{skLPI?l^9{~C?lg=E)6XCphufz+PIZ9xZ2F#A-G zUL)`zR)nT22LM-3SXPypdCwR(Nr# z@@}WssyE0)P5qlQ&gqEFXA-B0mGtRG3FOa!KS*gU5&aVbQ~`XB}Pgo2=_MGC50 z-H|`VY^QN!FF9T7JK2Awxx**up$9*A>h8rlQXU_tgrC3}(wF@!{RZ!iqe@+&Z|aN? zF+1+HJaTiNTDcQ$E@ZpAVIGoxIYutT$}+E%F^-)1yJ9vUqq;bkKSI6=WBPwnGbgYJ zT>4)S{#9G%zuFb4%yyyK2W(8*MV5KTyae_;VMfbe0(4(j?h^dd|8zv-KfHf@#KtTw zlI)W78HeYRyZ4K|I-8r*l^Z&9v&sUfb4o-2?m0-y*dHPADqW~f?rcIMCP;;PtH#mV z>syadoe5~JIY@C)0A=k-Q0ar8OiT?{I|?0{-W+F6a8GV;W%~J5#Xi((vdJ=9xL*hCt_Vq5qoVfoC;TG?=+jfaXX?VQ)iTPr+guZ#k3VUK%EGXUCisP!kbSXpx)e z;4}c+&74(H5B81as6Y3P$B&Xy3e()&DzZ?66wzxBByE1*rS8XX33t#tq*3C;5ro!L za0c6CmIOcn7chusql$iw-YKwuin%FQ ze%v3LF@iYTTpdJW&fY<@!<5rWXKc3z)=q=v;|0egrVJ;Aho^$J_2lXS?R`Rb{04};E@WXd7oRwM9BaeF)omFOBMe_aemnl8jP9B3=@=g#<+(hx` z!wD~_`}xbRva+JJRY2^uAOW~UJC{yb{+!Nv`psQlg@F$719h^?C1*<&e>q~q;vnNT z%2r}8SLMW0iBKZ62E*Ol1VGJNn{E+REC(42HxXtC(5Im-a(Xo4IMCiwZg+Bj9TQXE z7v^c--t*ch-hgj#MY_Z6O~#)rn(@l(Vr<3tOlOtF$COM?7PNJ?jCo;~Zn9$658R5~@MeaE?%;U9K;L)eUR(nC2BI`A0ECW?in?MmyfU%Zpk5aP>eObsQ#P zVG@#8y$f~ioE1*bJneBRo~k`%UlvF)70HgC*!8hj6ERedb@{~!Ls*!TW|CH zLYo*az#SJmswHXl8WMu-Z#5o?963y_ZJ=}}XTj3EnYh{muH(SrfM@2%idiRU4l3Ae z#2>Av-xDW}RS-)#mc;twJXn8+dKqqiO`t`v@1(V!t6UA+f?e6GsS47ibkI;Wd#Mp5 z5wdW~%HX|191bZrEM)e{eU|RR2gTjX^G4D&?-kM}fs{C4RD%dDJs|ucZRG1l=aB-n z3vbPh0W%icn*BUPGX=TRC5+^ff(DkGOXd^IGNQ#z=w-}msSR{9wMx7HRm>3+!@@}? zjo8#fC%bywfw^Z0zb>b?b~XnG9(DYvU{s#8Hl8X70}uTJ|4*@l+acSctcZCt4bp$f zr98m>GnTYSpxfRn*Z`W!Pt-Z9T8YR|(1_=D^fsOafUuHt5$YPFIpVB;!>{3Ufh$)O zYro}VGnXh!vs!bk2?QDK`RJl#E6VSvVDhFZ*vyNo+wwQl8~2qwFB}vnI*>^olH_gL z^XkK+p$6-d$rEt4w49vPz%6& zxIh0}O7F3rhAsNXCLjk4x;trsDCEmDWGq@t?oiE2{Zt zzuLnK_|rf*9Hj@W+Wzhs`qz=2C_H2X%k?6hzR7-j3<{PC$}xz*EHy)_CXy9EI^4g-&)+1!{78q+Q6TpZU}1 z`gAO%4*w%V@a^aB`#0F3_x~oh}Po36l-K9dPC@XEn(|?R>R^ET8yR>RlKF|FJg) z8scqbpi_8p#L0c`4XgaQcGk>xmN!c`n@GYay7q>i{%-pMTDi|#*gCAIxRXB7H`+Vc zRbZ4}Wtu#huJ7`3Pql7h{iZ@90_thi4h z_u&aeNzHR3)|3mqN2z)TnABN>Ckxm6@Sd_6MZB4t+^Lfac|myJ9YmLG3ql z0SJYuHKVuoHE!dwp+|do@g+IHwA|xKao(j=RvsSl4>HL=W=uQztUnNSYVF29oa4)L zLUBQdJs$1GdPh2ON;}=@&ZhtpQ1qb!?>{)qi#tTA0@3XeDof+3CH&SEOifLs)UrJq zZZr~tl?5)LFQ*peIS*9dyT?4HuQ%%z7FNt447v{lOL&m`@l}p z)it!AVNbvMUSXaOy=Fz=Nk`>z`vD0VF!^)lA<~1I@xbqmf&~BoYgWD+E52l{D36bY zfa3^>BejdpZXu+g*N7W}fmn)?FgRo6ST##@Ye7<+_CRULEgRJOa*+*e9JzS$0#qu& z4#i0@;0Sx~1yyW}nTRO{2W|WNm{){C@QWhJdxcwt@~zmTz;X_54?qas65yD_PtGvM zu?eA`wr+If?#I0gl1L*%ekr1|WqnYC!!~L|*|st^eoPId$XG9`j~MHxJ!{12T`R$s zfehi~5PA&4TJDg2=>o0?AQ+Qz>L*Qr2wJ1mWb0O-a%Ch{oV55}VaErGjv{&+>NiwG zTmIo5t3&w6eOC?4^oa3|wL>bDU}!vb|LnqDSm%X4{OrO^9r9hUx7&~PwP}}A&mkLL zbha2k|NS4Ifc(clzEYb6cL8}`pdj}QN}9*6tjB=|uPgU^0sr_eq1;;wMk^`{7^x-) zBUD-3@CY@a9<#KL%rs-ymfITTt%ELQo5s2VvX;&0(oQ+&CG@Q^LB76Vxj+xovhgi^ zK@Xg~#nu!68tnb^@S)e7{^$D(*7Mi#r2Nrvn#=xs!ztFo4)Ra6AF<`%ZR@_zZT%=L zeD2qOFy?$7!yx>Bp09sg>ORM&e_$Q@9ES@&uQ?xYML(}OpVyo}?dktB>#M_#u}#kU z`}Ui@;>~NFoj05%=N_NEm=$o};dDtdGDp22$0By9=ilAG-hG6AZlf${wv8x59c6l0 zPpdn6!uaRgG1PgjReU`Y(KT>_x_*rn7DR+d=RTk zEsk{+eg$ysH6zZV(3v;@128F`;??gJbW2A$618kl<{Ff-@o1*SiO@G06CT4yHVThQK;p=pA+Oh+!-f43S@=|md!Cbw z)&knC&dO8nTV}k=P;oHq$yE9A)?|Omg=z<#fn~SGIyV=Ol&+dpZ_l}cQ``phfngNX zJ&SXSZuVYb%h0qtoszAS@$dv*H*q8gTYWpYXpXVe`;3*P;z>r&cFwyF0AV68xX@_y zkT~EW%j1F=+UkzSv)h{PHp%mA@$0iUo4+1~8~Y$w)WNpUTsZcMc4cpYyB^o+sO;{= zUSi&@-3|H#k_PM;tQ(%0Hw)~MeuV+;drvL9CSp~;nTMWUXBBLCc!cYn?hDvdZdLRY zZ=s$8nPKHo>a?q`JHhX!1`9Wl`YkkeT^DeT;`7p6Cs^~R)_U8W?p<`W4t$~4saR)J zKCV$%2M%2+i4y&$z{QW)TYs~z+!Z3!YML$I_gI>lAt(B<=kI|9?bw%mP2s-+B0N z;^|+Mfb;E67GVXHV!Xi9ogC#yNbp6_C7B-gzDy+wCwZ&Dn0fRDDp3PzY$Wd8!2ybu z>>hM%p>gedg-i1o-s06u?-j;5;=<(ukCuYAHVh+)vtnygD2{lquzPI+=H0d+kPptP zQ)U+02^hEJYs48Rg;^B2>Z@QTP<*l~IE3((T+m?`d6G|}ueEAKbQ1Hf?;poe#Zd>} z7;T6Ss2zQ;unQj$*nr3jXgJ0>etQBcM%hgzgM^(7D(Hb;(jdEj-{m+i6kLHSA2>0 zV|EErOGi!%yIaxw0~%GXS6pzTMQ(Ux^<2H3!-R#9IyVd;DDai6VRH63EmrXjHXglR z?6;%eYoj15o{Oq@8T@5Rsja_`H7PQX+>nds!sF1Z1xG-FnX4&)C)OM+d*oy*IqaCg z;hl&o&pQ_k&!NENi-&Ir@Jn|ZJ$y=}a6$N^M?qCn?%7XMf;@waHj1-$1kTdz&X3l~ zUhqissw2*9oN+BDn?FEEl_G_k;lhaInpl(b{ZeQE(6CUPa#gny+3X|sAnwoW!+a?X zph%r)KN*ajkTJ{kLG7;Vpu5@=OaeuPM5*neKv3`#wUnCN7t?yys>5c08H&8SKJV;A zs#9{k#yA33Kz5CB5kxD7*ZO7iN%g0b2%;5Ja=GV0J-eFouOzx;935H>^kfEnzB!D2eQO)--Z*a%x3u87yz58$%z|NXj$ z%Go)r0pRemlJaf{f2D4XIU;E2J|)y1Hl_mxALt?mfQ~8+7lV@T8?kE0k3b9ctW;r_PY!7+c}7`iEj%Yn6Xg z$=yfr$bn66&qyqni>$flkik);$*j3XgajJphUxen!Z)}cf!QFnTv5tOLEa01i7c;h z0w0qcw+?6=q9BKllEd3MjTsYO=y3UB*f(wCM{mpMZAw`V|C^5cC!vkMMFsu;$@{AS zfks8n0InH#v!y$sbEmDa3n^qYO$5!8L!ocrfS;KhAvuh@FWx6h1D115-LENhS+yuZ z&XNPJ9hOszQM>zIp=xxTmD1=MyW{tb&icGvV#wNKBRbHMujj++!vay-p98=12k zntMmZq&i=m3V5@rA{M9ax| z@X6!lR8A|)z6%&ct^K-4szojEs-jz+9qH^=sl98#EJs|fArTYwQi@>#s%EmWq|$MN z=JyKgk``9>-cvpx9_O+hTisoMzSMVRtTCO@=I%7s3|;nHAF;02VZnu9BqqM`JTuu2 z*_c21Btp{+sS__8M&`0{Wcu+JuXuT(tnArf`667X+k7|5-zvPJA_F%) ztAo48wv8x>Gn#+?|g~%pg=J3OtNauHpB|*8aHx2tddk{4NMg4s3 z8@sDgUpn4$%f8ZQt6|)s6~p=!=vCnVLEV@TAbV@ZjtFiTsWqj+dHRvE@%sbeb4nlBf6jm{{t?MqL2q<6m89-j%By^H$Smaq<9J_U;^j@l z!(ETtHtv=hPF_;zR9UV2=nKkI!eQ}ki4AJ?2!-?|VFbTga`sdL8{wVD%l>9$XV=q^ z?Mas8?W0yQy$qETVLxUb_B36b^Xs{cdS||NGX8h9mo$vwW`cy}~Gc`|Iv;>@au~Q=$tN?-ihW0GZzcWPnUI%hi3ASGHTJ zyjQ>ga9$f=(_R+GKwKo#({K6P4FsIN*6a zH>>~TLQrlpMd`i5D-iuM9lPb$_U0zb-b>UuWuF$z{2W!j${EC~17Ml_sKrJf-z>;j zeAM1ge&cW4EQp;S_s^ehmh{Pm{-K*a_(|_T1bqiEuefgrk`?H(4j>_Z+G6R!h^N$* z&D6O^AZ&L$oM^}BqnG4I-qj)HXjecC+y@x7zPcZ^xbTyqHo*DhBH*zbioxhS8XRLbm*MXf z6iPuq|5or(tFa#qwZA*Z`mcLKv!K!7Z18_+ie37&-JkR!0!+b=r|Lg8#hgEC_a{^A z^QiqiYX4IU!{<@^Io5uTwg0xA`5bHixPkk;L-?=Vgny2;pJVOkSo=BFem;%;j|Dtl z2G5L|XN+!11%7^<_#dm+xPPB`>gKTV+9lG7E)E|L5VJ2TOvzN1cSpKEGqZwF3fa)U zyQ%7~Z_=4Hw-6OYaS7`;D4(*6D*gDdje8`H_yAXNj}dJMT+h`Kqf@zT{AB=FZ4SOw zRe8ZdYs16RZ@VGxFR!-Jf8AlX<-u%WY(aL;puJl9j|arRareN^;?}OAZPl(AnJr>T ztg+vNoM*4i&VM-Px%b}^YX8i?|Jg*{zxZxENcBm(c4yo5RJ}>$>*fHRN^#%BaUVsY zed`HwDi1el)CoWAF2;7aCo~D>Ab*+3b^Ynxtc{*H*|Dysvd`B)xG1F}?`54!9sH1+ zH=;+1BL~Hh!|CUhd!qEX8dMf^#HMrW-g5Qv zkS8YK7}YP;XSwKnxU`JyF(DalkwCh&8>k(i=JBWmy*Dt4jAg^C28DJ%t(Ye*cZ#6- z4q+Iq{BpcVdWxgH{tjHta>t|=qs#JjXf}LP45B=6Q^y;`b+3%(1{_Eb)>qegnh(#tbfOL;4M(72 z71fJiVCV=1YLY@=GAMIMJrQ!RqV@jJVQnx#&X%2MHyW# zbBxZ6adtfgo`sj{${ta4q_&h-S_7t8^caf~*`~QUAR@b~EI=p@cdn`#m=R4;uZL+E z@uh97dE+W+&9$USwyEng^tyaMpbF|?MbpZma?cN2^6uN|s|v z{W)K+U$;JeD#mOCx^}vc+8tR!i}meuuf}Z2nmgV}l1Ela$*|Njk=Xtfy%A&zmAI5F z-yZWlL#JmkR`G&>ABTTEWs$3Sm&-meRhSyED;3s2xAE|Zjt{UoFGq4=%egqzyeL~t zdr)wjqT5p<_%(l<0G_&A&^+Z@dp7pL%Awo=GvA!ct!NKVy;x4(gnYK^d^LZ+-UwDj zJ^Ez_jRAW_p#@9yHC&X{O-B; z*?;ilfvk6tcV(^he%JT=8H#t6rwQwJ$xct6N^>8vBAyWP*e_=i^PEm_1G3M2JxF2d zg#Cm%zhr%`Rax$M&{>i>iG@tHE^|WKcII3dq{FmsFKcL&cpm80`T98*$8>wqbD)9J zSE0;%@N0a1#f&DESc9>_ocDt|M)hy3E{T#e-8Yw^!sD(5>-JrMajHnqYf^_?h^rWL z6yUyUDdhtet*Snt@(mMseNtAOWWVW(;G68(Zd<>7bQV!_(fUVIIukL&lJ+h=8MdJC z)^Mp*8v0W}D%M-EOA0Dv6=KXgoi-vgV}B;t*Q|JdI5xL>u3yK770A7IpxR=dJ>sKL z{XBe$3wP+mB8-y+B3onFDg{-l106XBtfWW#{Q+6obcA&b)=> zo%hjXL_O-CZ_GwJ`2m_DZGJ7Hmldiln#C4AdaKEuowj z&Xw&VmZ9Q}}$B2)|D7L=q=JmU7@}nz+2{ggN zULRSk{^s6??v*k9fhQ2>x*JVS6Zf++!qyd3V{)jVKxx8)SJP z+QMTHkAB-g7Ga4>eyc}r^3j=HAm~Mtrh0qn9it;_@?z%+uT=ZU2fdZEmdG(e5MCHpjr^px z4BwYa;O;RtwgssccE^F0ISP#`S(t;iEh}QcpwnCLNYh9J_q0bG6S>B7$#WCIsXBd; z9KZzqDqs;9owz!e{}SceO(`<)wdNc{lMPvQbf>_Y%O}hH;Ks&kDMZlCMnJ??rD?om z4Dxjf@Y?1)>45AcU{bM>_XetG(re(pA#I1Kjg5pisq;ZeHXsxS&CSNjC6%* zO>`V&C}_m071}ADRJ&8wf?oUB#)X0 zFn2og+lHumtgifE^p^H((TQP`qjc!C+Bditcs$^vYBU1a4>G2o9XGt%qj9jQ)-ep1 zh~&I}@G@HK#%~+^;TxiNcCMeL?JD|wEK=R9ee!gpiZ?16Igr}z(}_aVKlT0QNDG@G zw&hhPY*J>^)g^N8q$R)%eD{c^&G3siu|Z|Emtnr4=lW;?87xgyy@Fwm{i5#rgTh~z zl(sJ^NfC87djtWWUe5faP}@`OX=tPEctA=?;F41-IkW%rDu8qetOIIF2P?1hl zCi*Cq^dko^nyA7hHIOpL(d`6uB0ccIWkkXO=|?=1;R5#&ve&1xI}uvAz(uxoy7nnD z2-}U$jqzU!t3x~}e~Fz9l^cu$HXJIA25DxBb&0T0<7@bK7}8H`)Xa(@Hn#4W7lF|E zP(&k+Or_9|%I(Hw{*&8NgmIzwD3Y;6#gQUyiRi{hC{+^i0BLyd-j*Pr`g{pclQ*Ck6`hDGfH9u@1b+je8V z1b_RAq|^2Sd9=Hv3hs7@K4SLO(jn}Xg!$bd?_2t9*4S63Un5KVO1g64+lk_|+u?>+yFe0rikSTYV z@qD>DoYUEF@LCrq`a?#>+D)Su}4 z8q7|}#e`d}N2Od{VGWu5)KELhU2GC@&){DcJt*E9QLN@W)5MB>b}Nw(hiq_N#~ieqw#qQ}Tk(u5Zr|d_HB5 zbd~AHmyXL$YmMG-L1}U>CY|YgTFTAL^@sJ74dnN6v~zXsI4vI5h$b;AsS0_}8}WMX z>O6`#<@>?c46A|yPtfgE- ztBw#s?vZN97~xHiJ0%48@-@;AmcMPF+CBeJ{ND8sIRNs@;Q@d!F$Io#m+)FU*k2Fz zA8^)Iin5v>KDoad5L4hwY;*Y{UjfnyV@#9as{rG(>Yan- z@47nw{;gCV%r~73)y*|HpBHotwEKVsb^c-@nY?eFOg- zmw%4Se`Lk}bH4n~nJ)&0D-5feY*)pZ&0KQS@2C(v=TkRC0-~dTQnYgyIE)qxgaHp> zU6eiwCHo3(e#$%YdTem!{C?Sfw2DPI@E8{yOBkuQ)OvM_Z&g%@JMzhr(%pX;I;!jB*KKtpb#MHuKWA{z`uK zkQH(%c0RqzX`5`;T?#6%4-Y=<7SZZ5d8Rooad9*#u?gU7e8A=XZ~}3JK9(mrKyN5T z^lW{`3IAsG9Oq}>K)3N{yDa@9H~2iLB8#at-%PkXfalpvIO0|jqkoQ(^4FmAkKHET z>*#lw)87gEzI$9pTC3jN+q$B^s6hrWszA_qZ|smXxJ!%onIr-q8=7h?PpgjmBDh(n zLbSmUO8ps6Rh_nZm(?fba!k7?_e4@=d9%E3=3oKzw5fICE$)H^`1i0!1h-vWgTv* z@jB_qM zdr+Dhnwy(57s8f|2%x3CZ8J)KF1Xe5q|8fzR^!4Zd_setwNT^+I%z3J9Q1D+?ERPa z(wl$VFb=DR@Qf0q13*3(GL{4=U_=KFE;IIcgjbw8OmBc*T3kIMPN%r*b5+aN`Fi#T z{lgzb=Rk%?#=1dDd{jk^M??ecHFIk>hd7BAf_NNFHx+sztj=H3lX%r$x8!kI=(T?J z+Z%UunRez})#ri%`%*brvsH4dJ;7+SVkA=CppV+jS;+`SgL@QW(S7j0>2VCWyVNu+A*8{l-ZLPqR^U#KLEQ z+bEloz5PYP=(e~hKrNJM;9VlCc)aeFjzWpaA$}oZ5%exWE#>pZNWokW}45`K({$o zn!MPxtMrC1VSrM;^~}J__+4+<%eQ<*!+tRVBL_a~m5Z{E?kl!gz+bPbV>UHn%_TYQ zyd5LXqJnZOjM^tREv@^8O^({~GOV)zlh+k+Mu6GHRLNKx7FsbO)4DXe%#8o8r;Aws z_BBe=sopA2Y=zos-uILi)gFtww2;f_hYL%e`-LM*z6=-(2UduQv zD-eq@`xLFz92Ksc_!*8RY#I-NL>1_Er(+~SE6Ojpn(Q?qmYlqR!zhboyW>`Dw>!r) zCOj)vKX`rmG2|-~jBbe1WAvcJ^xD>y1o|FtTB09)+l;eM%k%38FH zK~39m+u*!43%0T=469 zEk%&I*$+WLiK_2ij@IlRJA*{4_Y&iaCeaI;*1lSEOSbdlL1_7#sIl>)j~>y7usZjA z`IjfSaOjJZyF15HtL?K*5PM6v_R$(wn;OFvo{$wA@Ot1cHcda3kx3udcYTB;pTC3x zf#!}?lYU79RFvMCw*8J^^GCn$2DbHA`oQV$G62#(`Fqm?^51*d{hpq3)%)a(4ZB!6 zd4l}~!NvDF4tcLJ1-Sa3ALDOd%0EZrzkP0$&z#83$U#r@VVrU2Ut;hUCP1#&%}K-F|mS%Fr*aatJ0Nv+Y}t#2hFuc2c;&EpP*aK7j2 zg!vu2w;53J2lj?oFZo-(!c(6O#2~yJl@g$Z>;$iv-zxzw+EDqgy;$MiYYlzTb~8Qg z^%oyj{GzQ4!|$`P87)?YADT3`CH-7NP3S`!ykO)cl(m>7=)N!IH8Hx%9C&}n`Aq0%ia|1 zqmlzv7)>sd`h(FQT$`;X_bBIV>a?_UX+&i>UoDAe{k9>p6(RaZxBDknwr^ILFLA9e?j3C4Fv2{s{@Evhx>r1JR@m}w>9U`r(L z(Y(%GT->e=#qUq^-}NwxEX27#YW1*7cX_5(-u)t=8I;9%X&o-s1g4SPwm`p_g3G0c zafa>Uzxd!Ebkhnf3t)BpaOC~<^1BcI-?X}H%z%HxElZBzgT7Iz zZ6%23Ra#{y{z4!<^1jv3@5(y=-`KxKSEu`g5u+Lb@hhwzM`$PAeM(`mmTI!0jbZGD zBg)}LivAqy-;=!X+b?%ZVEJuU-i?694eWw>L{|E3k#A3_sA}bQ^Mz~qpo_}#RSq@# z@~?66yVnYw6q-?cM3J`yJDD4fdE-0dV2qbefoSDc0D6-Aym;LJc-@LqO%hvwl<(-@ z`Ieok0J0;0eDlD6agLjP}`i=cbf;EJN_Uc z-#DkzZYly6NQw2uh8=T`&*6!N7WCb z!-bo(tr)vxxYJUzyN14r8TEm>BaYuGS&Og0Fu+2Hw=9`XY}1K8brOJuU6aW>V}lbT zJk2|8wKY;qgGPx~FTSJZZEB^0(#t^;-dC?FXpAWWP#9;NtiF-?vf|c$nLkbfbM@!ZS+_vY8Sz9;Y^g#3)UlO`@6+(qJIpxt095676sJhf6#<8qA z8lh|C;3&m5Ia5Mjp-=49bk`EC;lESxt>cyf_Mn&gjOswzWyatnedOZl=I4~NV#A=P zLFwf;N=&C38c%8QE+kx3ifYklWr*Xx*j&M(%L+1(H^wCzFCBeRpPckImt=Gx8zQ}fdj3t3K~kxuWy7P)6<9L9as zKQJAG(*h6-wYFa?@2L*REPC3Dj!GTtQ9`0|*E{nlw06%uXy_o@tiZagmH3p?VKJZm zO(1pgw+%5Z6tjgD0 zIltDn9Q*>SD@n@3?q5OewvkLqS3|FUzoDIf2()j4Z-t`C zklCIWg@;m>&Bt$bCzwfVi|ayFN2DK}7@P2#%p zqpU@$<0H{e9X>l|uCNvsdOm0!G?cvh{>^$|TK-4jr?5JG*={R6jO$`F4+Y}*b;u21 z?!j^sU#}e;D_h9hRKD1(G6r@wO@0(>``kWwOG*va!O(VPH5&nL>VF8cPA85O(0c`q zp^gLl(p%aB)1e~79DP3^De|?%>-Ja&#H@J5^izk9_=5f{RcL99kzIa%TW}U<0k>QI zXKaroSjgzw<@AC5$w?<|pV@V<&O#e>)0@h&QwTy_ZNb5};jRZ1eVi+SIT+>=wu+3G zjICryyGx|u0>#eRb$qJ*p-Hs(n~eGC_Uh6%)l(49$ZZ7dk34KzlP-fk0f-U-p7RwX z{;dS{p*i}FP~?LuPV`~>*3CxE`EA{dKqQP|sajgKuuv^wc-+-C#4GF}`9bWL60ekq z_&E3HVSpg*mac zVfJ@L5jpOL**0$sv)%U=xdzSj^^JDQTvtG{@RzV4KKMCFi9Xv*rp7g2$?RM&fzI5SH~1kj4jQ$c~vZTCa%$G}k{)r^Pn;c~9X9UU@c^<`sNn16c0 z5{PHG$EkjcqQEuDX?6*CG7guG+O6mU(sdLYdWSwbF=$0e9xP6vqf9dGsNs%Zu^_z zuNlkF1Vj->IZkN1tNg)UN26olw%#F0>CV=^6Ni3WCRzvK7u@z!Lso6&9OTH=-!=pn zCsSi(>Q^LvAIY@c10+lC_CwaU+-L#pzO@{T(u%uugu(+^4@Z0b$~M1YN|^*T zG*=9k4~nlU_;G}ma0&A9>R`yxfd@AUn+va%hTab|`tf9_{rL{VuZ*$d8OO%Lu}sQT zms!XiiP!i})oDEdQx<>*^b>Y3drF6e)1CUs{N2l+ERdN4o3Y31p_+}X&m}?x`!c6M zX&^uCj6N|YLRB%R02efBtGr$R6Rhz|kJsiA4quGJPL=w;-qE{~QfllUDvB@CFD#gk zX~6PhDA>Nyei?HmNERvG*hO(PV7)zVX06llL7UsM&O=nB#v!E_n>d+!rI|z8v;=Av zrpddC?V_hwGGw%>=rN)0^i7jUCKLTJlJTM`L9anFfw4J(&wfPWBU*!e_h)+@ZCb+} zsmtTr%nSo&iE+YbIVkCHH`Oame3hi1tW81&3*M0K1jr+d{e!PrS$eMK;;28 z?&4k=P3`G$+#|6mjw~zOjuC{SjG*J1SR?f@3@!HBe(E$bPWQN z7XdV4l$ei^QNk*&XMR+0K~^F}r{pNC1|1Du1#4Ja0nGuj3U3}Xtu>2;rd5V%T7TK} zVrUPGFIpV2)RZ%!CtN#qk(2Pb;A=uM6|O!@#V3cAW($A)#2__<<(zeY$qCRm=M<%K zqx&Nut8--%PAB_pn*YO)^&d-I8*{wKR9M_O4+=SpVojo`ITc#jF{Xa`@hPJt^({?9 z?Txlq(mjbduMma2|M;qj|M29GAA={%_~z) zgA|UAjhjw~K-~P70?^#wHk5Rb0m=%KVtQ%0c)UZf(ju63_zblbiG-@3o%LyKsUHa= z&Wpps46{8__BUrZ=Q$lMi+@U_h&Yt6!xJ}(8_{WWe<0pZYM<_b1U@USWAnMbF?btX zK=#VVuQ=+Dlfz@K4vHseZD4gD{eQDf$6|hH(|fJ2Mz=%@7Suzl%imnHds2^%fxXIN z0#7!#z9BcjZmq8?{d?68Wp2VgnU*hQlw|4z9j-MED_Y#+FLiv#sy6t`rretx8stw2 z;}?{<2f^onIM%53C4;!C!e91&Qrt-CGKH>OlbsZ%>y5Z$Fq5Xo$qSpm;@c=8$c4Fa zA|At2F8TITq>V~{PlZxC+F zq@b-U(Q-Si110nExr!vg;g`&OH0cAAMVe2)(ZbQ68v`5qf!%Ql6;?%>KRLyJwB;rM zKW?wnl${R?Xj64bGb$^{eZ&?~2-`YTM=83%mmHC3r08qc>|9A@C4Bku@|u(26RCA^ zvT49AXL|NyQV19s@8?4GQ9Wa{QM0`WUTfDgl?3@hyW{3TtcG3{!8D}-=6${xOxMF^ zieP6rWjP{$N;qz-qHQ#%M4>N9N|ZdqC9@z$%x{Uv%_O{Mc4)xTfnQqkAud5tCPVHm zlwi?=sp7+c(4Nb4duRD2#kaS-Wrv))W6GafZ6djuMyzS}T!+WMsVz^xzvKufIP|RI z=7^C_8x;G6P8+b%vTy-k6L4Y(iB9vBc^zz>Q-yoVyGBqd_?Jp?H5r4`WUm`P=5M7R zedz#Rx-e)pO=@d>R6lHHlsywlfW}#8p>pXxXljG|QamtpmY}7V4(jylMPDM#Wt9fQ zPgn||0EQ{Ke|3LNYYkIRqN1e zMje!Iyj=UbtDCjBifg2yqI){{TcOx6$Ir(c_lIhPKV&1dv{7xzn1~^0Xr<;T8wh@` z_NyBEgI3*9%6`ql{A8GJo3r!ukh_7CV?g?Wc_3MA&uR@T*x0E6 z*^s3glc0fHD1}F5bnwZch>%F*q`^|3+IbPdeFrZ3DaRj_DN9(gc7$t2(*%8xEw>>^ z_&yoUyN#B1mP#*}OKtkT>d6qy0#Nz`1;PQEQFdi!@hN@B9sE8_Tu`QeK=D&Q;RB) zoIf|(zfqX^BmI;V-*F-WOt_?{GlI1?s&*iG&t4<5LMB5E2*<_)jzAF)QNR&qDv$#+ zbo#)Gqrd&d8>;smy6c1fOCW#Y(wvxK>8h*jUr!qIDN{O1v`_T zGn(p*hJ5{Lv}}*0q+=`>FgtCk(T3mJ?0wi@n$zW^anW1SC$-I3_H^osYSU|=Qu9%z z1f2rh38xdzu(kkPWC2>*K-~FNGh?&Uu8vv+wR4s6W|4wFK<>#`TnGzUQ7}4ILpJX0*4MR< zDR3}2I}}0G>Riz~4EXvUGi(bbPG?Mk-LNh8g}lqD{Vh$^W_)~P2&LNr{?%AyfrG6Z z-e+Wi{vCDK_cPj!YSF%=<$~+BdA!SU`1vY|6hIvS5@_}zjUnE7*Jr77>#aQEcJiS+ z0nO=kZKK=y2N|PlaUgF*9iD;;>5`ms&h+*4tXn=l-`&=>KRCvKz3LIVf~r`>&8#Rm zxK;(c!{u(d=$Wd0jd7&_kph6K zi39i>hwflaB^TYKXA8^NGb;_3FLUmaTM?n=qGowOR?jxOZ_&pK&`%j={8jJB*h{!g zR>w+i9>w6A^mn-nJC(0+(q#j;e*JaEGLDYMFW|_R#nW@l5{J5^?;518DG^zMUbFK0 zIOv`2b=yk%D^LWlUT*}Js(<4$Z0J6=Su?;FDH zZlxo2tkM^O=3u9l>U^nr7qEl!uwGet`vs9`uZV?s9& zhqs@H{wg%7F@fwRcs2G+QSJ_QKkaY9Vf$yS!QaUlq_N#7!LC-vL$rC-B3}XQ>m;4eGtpLqM@jD>re)cyw{tIQU=YF&i z|D|^aZQxTV)`^7j@797O+z%IWw}|mtteGZ&)=SCvN1L_5Qu}OO~E*Az{Uu`q<}(&Q%dp2 z3MY-kx1JDnC$C0&_M(mlSM%A7M_dwz>WsVtaYpAR%;MA!B{?C`sd}GM7|C8z%hO|+ zVMKa3E*KqSWt)SmZBBA82}*KN8KvjZOwAjD;*uzGdxfLas(5omyq3H4BMcRvR7*x7 zrrnEJUSt!C%f{}aGlM<)3kIBN%NV2{KbJrcUMZBBmnlAzX50!bL0m*=`?cKX%;u@$ zyIMz@Q~}v8&q=Jg=l$gSXrAjL`GSNmc90sh6V4`=zd-(Eosp*AMXd3TMG$9$^0mFS z+O>>@QqPU zR1xJ(YtSk5X)E-e*M0}-BQogjRq`LJ1pZDY4|-@;mh=(-(`T44QT{+4PDmQEE%({a@)^ z0&i29Jr`qv=;D)IY*w*GvLdlry9V#pRCF{r@We;ElqfFGsf$|9v3$-#|}Mk6tK4vtUb65SWHt(FXZ za@&I03`>-p-10B*mfZ!l$Q3>9p^6c*M{~{9Z(&1TmMGBfF5B+H6CWoG$J~$3@fxBQ zUYI_e^({!Xgkw6xSMtGyKfH}SQdYR!R)SRTuj##pvZ%@G*9r3m%S*K>ZXf;$cIE%X zHe27*=LK2u*EYXHtQx97VSkrq{fk%o-#pv@ldu0p3D|cCJ_SV*-1!C<2nV2MTB^fm z8@vRJUC||7Y~7K)O8yG@=OI`qhr!N=2zNhJ?czoEtON|US-b(SlVagNTEjn$cJdmf zJI87FQDLkKM57Sxog}d$eM(wG&`U3wf#6ceG-&v>2fcC8O%0|!Iodr&qiB-Zl$;TFou!q5 zbru!_@)5VCMPy)EkO#WdgTIzp$5i<;t2?%OZnP;ucNPbOz?s+A(iBjP*#dQRCHS^e z@IaF414$B50Ne7I;GCQsd6UnEL{pXw=ugFtQA%DgQ#m*41+XU@;q42F_=_;v#mCxb z)2>RDw_Nq(PWV-BUanYl}*dsg(mppPvVZ=L0DvnHMn zrGsoz8%>`#2Km&OCD? zboQqZzm0=$Uo6v(TZzAdKF&PB@>;HW{-GbwE6*`FQA`9XK`2{WrB${^ zG)|lknZ*f0WYu0W9m()1iE9cShI$yHzxA#1`mCp1gjo=3>xueTdOQ=O$6u%`M zW5Ys?uA=zub)Wdja?VRXTHo|DK~z$X^M~>80!1wA3^6}ANF}}w)CuS6CBm;p9fb?F|U zK`{!;g?Q);7#$}no{am4YTBT4qr{;$bzIJZqoFcY@yVoqY80fq7A8Roqzy#$2!oq-)PFDb1%;fO3v66eNReqn!d#e7dRqaz$y9lW*hJ2m6I z+B`v2-8`p!!8}aN=(?oZqBtX~Mhh~>2YPs)%W@t@6ZG$B7lcsDZ$k5>3}2++q~SE4rj=(9FR3g6Sd8W}W! zOS@KS;<#OAd3q3QT*8*cK2^7CUsa1KfgUMrvPcZn(3e94a#ZVO*Ywq27w=0r2!x#) zs@>SL#lnetxNNLvCD!cz%S3V`G|mG&qHRM?UQM|Eq`pVq9kiC`wF;s2n+y4!>RVQB zP;$B*&Bsg>$E%}JG6XD^7h)d+|N1-c6$-efUJ`V21imSR=4C{p49hi$ho3NUz@7yXvjX&FMR@6303Oz1G&v zOd=I{ntM{9P-7oO^A*2Ta@hsue*~TSzss!rpZeRM%;@YE@a$ywYhF9)Dvp*w-1QX) zp6qWLbBfx_5-yiPoeK2L#(~n?lia!3+&d#RW~hWg>$(CR*J-F@ufc%SILdm;(OPD= zxlq#B9TjWceg7eH>Mmw?NGS>rxZt}q0T>&Rq0~s8>Ugv0Z2mI5^ipa4rIy|&) zhJZ_)DTQ=uZ%ouVI|}ZSk3SB{X)d<-*3oOn^;NbfTagDJlf(o^UvS zBXc#u!)>sxe-_wXD{Jp3xd~$~*yc<*2W_AX(NA@qc+1NI3&7l!7shmpa$ z&oWCU;GY+U&F{@3Sfhw!Jb?AKlhp|*V+!|4j@bM03BY4C(gEW^0>nBj)^#0SQ8r$U zN)~P{JS$A655>W)K&n4jzP;MCGOy`gl783nNPW*!$7P54d%gzgWuA-M9^_56;YIAf$8O50<8(Hj`mTOlT(DZe82oJmw$Y8hGzgVZP}RJ{67pO4 zr&j&JXC**@erMVJ!63ak()2TT)240%?!lt6ZN;x?2|y0j{&z8cRrYWjZCH(-=rij0`{RgX% zngyFlRUGcsiGMcy=kWY!iL;C1@&MSYH z>-pP0w;l7i?c>1&{nu$FuQhkPAs~7UnP?qLvp0M5sX!P#I`~(<pS@^JPR8U|+Kj^SXfC3RIdW^l*y-!B8mwzYf!GE| zVy|VN5{Uzut_Cj5B^-(9r(0{(a^>MHdus@FWC<4C-gALMYa(ZRpzG(!6TD#V8n$VI zq09}C>2%;(dY?aM4V)Z%yRgGN#jMucy!Hyg&SeoK&jRh%geErD@x*yrBBpUA-cPM*G0omZfd9FW8s_M$W|nJ$|jZwocZ6oENY;vnE2y$#)4 z-nP}(Kwd*UMP>1}S^mLlmoK}M3z+HNK&4dG?}fgmE2i7-u#p9TvY_&(qf1-T>Bs9P zUK7d~sS)uw=va`Xzve^$^=i21K8&A-=~D`RaXv`SX(91tGox`2pEz(_CCy)D%ZK^YSggq?6rhr)GAIfW3E+ShQ5Zh6<&+KH@kxmb@q>^ ziPiPz{j-HZfz3G>E+8a1&Q+XHeJ+DWA}eGmHCC<2XUQE=7?$H@{5C$jpmb?(0-AuT z@1e;U3suKA-dzu?SAc{C`ddO85#Ai0bzu7EH?X}%Sp|q5ZGa16i6sm444>1_W2C1S zg_Cj@yY7*iS3?UmZ)s_nW@A_5a)W&H_Dpg`F=+>Y=C=JU$l*4Ve!X1GOYV1XgI$Ls z%d({f^X(0%td92a=K!L+^R2u00aG}gEv2xLP|wS1v_gM0JJq`_l><4hCG&+&)p~;@+lz6NTr<@sk6lgq zzRU{i=3Il$&;jiR;HiK?=KFaUM3U>fTEvnq zyLwE4e!zUWAUmzOhI`aDurfdfPNo%dP89G7dP1bG=_IGkuZ@XFTc&?`1*_>opQ{)rcEkHm}xX)p;wUbo}E3?Q;*q3>6$m{ zOs*l=5MEbY=>)X+P#?zt%Un8AEACfVUEgwa1~Q!<;k11Q;*fOz;X$IP-ZsZylZ8Is z_HG>Lj5zrSwKzQjX3z-x>V8&k|WcUEudwD$TLA1SZjNq<$Ne--$0rN z_>XyCA{jf7D^!%S_>TQa0jIiGhZ(}&Ye>yxlEJFInp{wnatc7{JV2;dL?_Y@Kagd5 zNy2eK%4*M|U;IQ1%`#d&_Q}FFuHm~#={#f5=Qh@CS17(touS5>1`g=lF^^P=4*l-v2R4bM19!k#5vr(DPlYJzs zcYvLDSLl~B5LgfUYS2VIlZ=mI1)^J3L*!^c3&~!i2B4rxF)3DQzTdG(w0N6ocY@Ej zKb=icvu;}ug25p4d=q<#*{|b=7ci3Et|=@lA6vYk-X0_o;-@;ZCagPutY14yLhdVw zYUn+S35SdMiKrs+C_ z^!@xA$UocN&_}fbpkh(qMr(F>waTD21u;nVXDtF+b)`{L(P%9}rzYHlo*40J*ZxPD`sC3h}g_kV1uVor8K_! zNdeR;*-^cr1<8fz`bL`aAl-xQL_;7S4k$*cq$NOcEEL?D{qxQJfVHCLe>BSbx9v{? z4pn;LxG@`I>=VEYEz>S@T{K$vmuk$PO|o=m{#djL76106P`PX7X|5E=(O9ABvp5B4rA=_{YX}c8uiMhX6BX zZzDn7+YR2}%l$fC-qIGtIrK{vp!;ex6QY5Y)ODe^L{FYC>iA0Pm|MeZrdQq~Swa#bWQn{S%Y-qM-o>B3tsZE6IO8p%m0$_0~xHlKjxh&WtI zDM+QgPWWllZb5?GXrz~x0y2U%2eP|n)dKYPkQ2LNy9CcwP)>_sw(|xX7TOeSxmYzH zgs7GbOd8}$SZT_4#eix*+0CVIP=2CkNS(W29dI*Wr}ny+Z+j%~KeHPe-i^4@P8u zJmYB5H1zF=|A;MY!iD0wt0=(HZ#2Wp%vs(yfg{883Z$G)t)gvX_8{A2t8^##MmnLL z74PZ?jU>3u6Ew43Lsok9nW|w+vr#{A+6VI^-p{n*JtdP*uwO2Op~CJ8f4ZQe4Jl9>Qb600IkN+*&2*u|v83Af+RFf@iEAM0(W0F^Q zk00Hz?#>A2j*` zZ?#>eJ%U1+*8#_=)U(pQpPFQX#g>32ibbP2bNS^XJ*6|Ys}_K_=4Pg(Qxb2x9B=!) z_W{hPvC=B-fHE<_XHvX2hYQY2?$({p z&lqs0fTw;`l2hCQPDAQlG4jKET(Q5g!QQ?8_wdFxI~OZrfS7E|x^hOx*q!ifqUK2n z6TTL#8jz1Vy_m;M3(e&KkYW7Xhvz+3?26Z9fb1PmZMqbnF|A4fqDzL8GIztlT?~J~ zC$RVqV_^WO?YsUTz3!4N_z|Do|G(Nh_pqe$weL5FnkjP}v$QmGW=cDF+Ihe!O*@R5 znVA)+m{NHbkxUUSmFXasrcN0;>Zp~b$wBh~C@4-ICy~qpm>`mvW*P{Zh;ryzJ=eSU zw)cHzx3B&5Uf1(n*Y*zuS!=OatabnH^}E0K=ZlP9-6^3(l;i~XxTqnOVo8Jd@F)bSt+|3PJ9z%a|Wy-hF!0GjL+ z?QFR$Zg?#ZHY@J@d%gb?hyPa|m(K#`)DB{&=>kYo&@^;Y>E zB`gGIjR)Vt(8);@}ot1K|xwR+9JD|2#cfzUq zKy1yXO}NA+o^Kw*A28;Q;}i;hFIf{p2j);AlN|%0JX^pz0*dQ@e@%MpDa+rHGJoh#MF*nK1}O`B{jX zN!e@BBoXp{e>vH!O-9c4=;@$n|QhpmL32t}uOy&*4r@YTsPWzY2Ok{W4Q-`WO_jlFU9+T_gk# zpzI4FffpWASPn%&Y4zYm^zQcefXP3b@cI-AMlfJp@JmvaDh8%nMFRUR!zC^QGydPV zlK)$uf3)!N0k~p$9QuGqzC8E-R~~x{yA{%@{E@{_L@DC^k{@S#|9AKDqipb% z%j$FZWy;GBF0-%qdIk7P0V=niqmg8; z2;GtD?;@a8tpbh7{G8f~NuklV?^c|J@BJ;qm?XG^?>p2`Fp<>_vy8E0E#L<%D>ioy ztoMGZ-p($OP_L2q*&md)t444tdSGDEL)%sli}Ww)i6W{;p#T`KPA{-A*LAq@-lKxd zNTuzVzx%29RlfnVElCI6f~vzUw3GcvyxeZsD1R1k8-Muno50s=OkeQ>`j6Mj5z(3t z6!IwXD+o0Uv5;ocs5o&Rx`75TF-jM6->u*hA8^DdOYGchT5<#CLBL}nxVFBIgJQ9^ ziO-3i!RI~YW!KI5ozv?( zjc9X{%_6=!PA_ilnRx97f$+tO7roY$-hzXBY6jPaWOR!oXgU6mS^v zc4*t2UUxkNGh&JWsQ8TOD8;~zn6Gxv`)`=*NwaWi7{lmu{obT3}@ z>3DpPZw!>;8v?4NF|bT%BY>V8(*TgO}|fh^`U zE(lR<8C{6fl$&%cC0Pj|e(9ux;vou4<`btDgVcrqh_sm=0KYrwxDsg7X&%+?b1+I6 z?O7KtvJQxV6g!C+mOG9gbX?zyOj#NZ*R?KtKozyP`waUdHo~k;=YV7m z{vhnntqP8>N-Dng7OuIeE*jTTCl`6uk(ND*i?nukB&hv%p__nqmF`qj4mH^^=? zYi-p_9vL4jaZNpW2%#No_?NIY@m|dHF@+;$x`|1Yf%_J=fIwOMW3 z&Ji5yhe5q&r!>QjxO!?Xg72Bz3UzL~c+)x@bioBX_L z62o1o>F8~0QexD3>3S|10cQv@PN^plrW_r6+&>Fw@7Y}&O6tcBba41)7A4~f6KZ4 zq45Y1xFelj8Ugt7(|}a}J^FRI)5xD?`k(CoXE^g`?fzhU{otwrv~$tQ9wgkCr?+Nk z#9ws|*EfLY@C;vCSqyz2qBi7t!!a7>39{m~IWb%+)=_&38k%O#4arz^p%zRzGyB36sHl2gj^ozrYj z*IFaI_UoJp$$b#LbF>6`4j}#SNrnL+e1lS`!@mA1kBt1cPk7_6qf3I z%ozC|05t~61_15HQyEuAt@bg9w$BHa{A3rvKgg;+o>Bdib<#6(YP&xAN}}zdZ+XL$Rl#YXOkKnpq!!0!l z5A6_0QdkzgYqG!G#V5}-ugE-rDlwAXk~;maiW=te-5g^5$_AttWLB4#o{So;f33EYj1*Lc zTbp+jW7Z5G+94)76`qi1kCE(0fqJpCkPoK3UNXMMz1Heb(*Z;VRmsg{+ZRCNj;DXr z*eSVAG{U;pq=0Z5+Dfm%WqPXCxK#Pe`I6f`W2u*4N{T9yY;!VVF2Nn|ZQezUdVvkj zBk|DA+2;tCLTVhfwB4U`aE_flPPIlN`>NLg-^hVhT#aOP3oZV>uA|s}~_*=(~@FLA#v%0W(g5Lugb}@?M;xk4@ zS|W2Vu(o07$lM8qz}#mLbFK%QJF;CrGyU%Kcvp|cXMi6p9O|0-RvtOZjuhjZI0+H- zpp09-elsV{F~O_nv%Z>U=t_IqFgo#qgN$BmDm-)#mSr{yfn*{c;cr;rO0pJMsvi2B zEhP0jT9(MRQoj8$<9m+!yF_zNqdDqaPN-)sr4hS`+EW=adzP9RCd0EPHb5W>JdbL{ z34SF?a>4@r0NPwR^>=iDp)@KLRYK%$m)$|{W+5K(9}w-dPyZka_O^=hZ=UT@;n_4* zu4Q8&aP+?o8*!8u3Xyul5$A{6!_gE2+rF2S?59}a#J^UP`kI2LWL-%I1GtSQP1x1GNJ9hu6+HuvJVlU7rjW4k#=2a`G?J_9n$@b?4KGuK;n^> z#gEhSTqet-!ovuC*tNu!$V&K@4cL&`d3Pg`>GeuV=R)E*;D?P|3Z!xwZz{i38ut#4 z*$CD#dgD<;HY+7Ozq)kU$w;bZG(Czx5;Adifg)1thiPTkX>pUsev(>tcxuN(&W<_B z4=cS#Sq3eT!Q2z(xrLbW$K%$Ro*&gAAz=Su?g+}g!!vN>g&pH3DL~geU;<5u8l^*YGZbGdtt79mH6lhuKSs!)=o|CbQIB8xn_URk zJ5QMmst(SIP4-KAuXb8UhKd>5%2j;~ort3EIPX?aZF_X(<*>h0-V;W?Y7yX7~YO}kg zd1`Q>&z0t))?M_z0L0&G?^aN<(v_}+srBYF2?r zTM2a8X>7w-fEmz*Y0QwKrO30b;L=+fo5=WXSXBb8X}XnIqRz}w1U|=i(YL~cA^fLP zzXPPb*6`7dsIM&@zXt%YGU{HNe8Wb43Y8p*C}?yF&9oF~apS|h|};6Up-`(Km-yEsCu`AN}7ds?eUKGa^%r8bw5;63R`W%kDV-k z3Qy1BG8+KJUfmO`8Zk1L$y0*4$z8>D3)dNT@|>|_gD)jX4|-%sUdS_|35~+3%WG_h zPyx0hID7*-vEP-9ws1sru?21l3&6w1Z?rI(=`YpI1#-hDV6C`B2n@Q%8k(!duT)uqo}C#IeunyO(l?nd^Q7w!$1 z7vROKFN&{^Y?pgIoH`Mvcl9W(A%$eQwZ0CU#H6EhdI8d90$i5RAY??!XA~RcNTEvA zK`k4mVwKK2O?-t}-PqCQrfmsEb_#c35*ysMFguLeyBh}xt=DE<^%QAHKWhv&N-J)= z;700<;CS^I6Rlh~dn}arK%LCJRcs>Am^;_zv~J$Oj{OolJ8L_O;B>W&@Zx}p%dJ)AXk#0V<^!SAV2oq188_Y z^CtkmDrW#GWQoM-3aTmYVCfJq_Xe!wk7S=g^_Oo5^1&mAS48{fGp+`b9^eHBUXjS% z=xEJ1pEC_sxr|kA49(9L?s^J-u#;Iap1f9GeaSLKw?`BQxG`^rcFOjiHe;@}Gn?vn z-5mtsGCUW&XIuvO_QI;PGw=dXFL4It(l{`5Kl}LX3u@ZnlSPPgYHR`ExReJ-_ulS{ z`CWetJs~D0A&7;FeMHPYrj)2T{ z;X?sU^TvlCYi%+?HAj(VnQjH>(;?#J0x z4hJqFjfNt(H}hzH0(C;wLSA#4%VI&qV6?O8&w%Lb_vIEvP(1ZR)9}9y$jfZhP|LyL z|5;1_oRq4(D#A?XzgsbB8-ah0`NMHg-B1P0Z#?{<)DE;b{p-&e7%ApDxoWjCVo>P} z0Jxr}miYK!!e4iFGiAQ{NULDjj7uOOR7RuR?DT$dEWur@qj32!1tMpUxiZB zjB47g-Ee@3SOlI$#x!~V;ktV!)?4F+k?s_YuWq_O{RrAhnyhu+Ty@6p&1~nkbBFop z#8x@(&I0r$GTA&n`xsT4S!IkZFiF1`q4j{*s^carD8Pp07(Y5#8Nnj?z;aQUP$l-4I(wI8(BBD^5}Se+jTQgHKfZ16E66 zj~X%sKio`{G%!r#$K0+$>5Q6ZTLxQc*Fey$+aVCzoW3>9P%tGU_Nqj@0+h#?XU3p< zy%g16UvRMg5v7S*m+g6ZB43*0k^yzO<;_gVMK4j*Gb4*RnQmIY$^eRA9bkNJE_tc8 zD_U|l_s@o4uXfeRYH%BDLJHqyo3ny6t*5=KbwF`l4){H0%#mY@7LE*#F69xN&DTfB`2Y(y39TJ?fnSpj_z| z8TacgPTvd}kLUhMQiwRF(Tbh|1uH8TIsc%mv-=`$!}y4SwDX42bBsU&jc8gJ8hMgy zXbt(EyUVgg+SZ7ht9Y%JBS=nxG*o!3;zh-k;d#{{HKl(N)F_n4#POVFqFycSd1lcw_=zoA6@!(U(%{O=U=a00WS! zIIW4e*|Xl=H2sh77aJQ+7vG{H0Lq_uM);u1B5+``syKOr{8jr!S8St}3BfQ$e1jP9 zq^xs$fho)^r7ZYX9^*taaL-F+&gs8J8p-L9K3j~|r2dz>HSoPp3U4D0p@&{zjR6Be z?}?B__9?9yuQL<&U8e0pa!Gh|^xmII>_uL8c04ZfyCBBr5klqvD$z=a%l&x`FOLJIx9X)W_FFKcP;Hbh8va zxMUdJ{Z`hmGN`Fe3$%_qgRSEqMAsx#p7GsOx2T+sLoS7GmzpC6K011QY;^hS>;G0e z{Iheae_ie?JNS!f15cf4{gK7sdbaspay=HJsg#dKRX>1JuVRrKWM~w2=ISL zTrzVsWFQW5-9jL@!|kHlQjy(s%0&zT`u0t}RoKM2>R0ORa#y&Jb!@_&g@me^jrp4@ z>rnNOy1Wt|6?>wqJZu=xq|q#O;vs@|x3ct#(DXNY5<&aRM+T)C)E$vobpMs>Plsw% zC{6pwa7!JiM|8XntP#8Px_2=4f=l(TMT{k1)s-X?wEHY|{!X&?ZD`)^$06BFE9M8p zHq7v|x}sPjX(F+=HfeHjoKI=8H@TU=r_^-wstO6(m0hVkI0uVxLlPU&)=*a~;(5O7 z_?w0slo?+=CQQidHqWMb-AK3#uaL+qPgIj;%l7en2mX zUL-uL342nKe;{xbuZ6-1FdIO<%|X+fDOk?oU#oI-w|;-3+2V~_suUdsA{iimu*2i1cPIk4D$u(cqfjBMCf8)(?9&@8Q>K_5Ag8 z+3LC{*|D7I1+cO$*o*+PeD@8$+iv+% z6IpYnrituH`T8>_CvG{E8;oMG7tztI{j8556qt1*g+Wc@I6+j;IUMgW(q2}!5QBW7+Uu11V)?-%RK=8g>1z1DZ%=kk~x za;-)byokf$Qo+Ht953gN^H9&0ecN63+IOOoY@ZfC^uUvc`v`96=xqoC8-XQjmMRee zAa#b71Xq>mVEr)6yN{(=s5IFG<>P`h)KABeLpk~?m%Dqab^Eu=sRCK%q4uXe14Cf8LnM?7nFae3ZPpig*D_^}o zp?@VN49O1|a_mEcR#1N^cy23#p7RqhUo2LM=sC2d9raJ0T8+MWlel$D#wuLW!vkj@__@IibGg=Q k=>;)vKZq7f`Sb}Ovm`vapSJ(6L&1Od?-}ml&?`+s(FxYFYJ?CD{ zoa>w4T;k8-G0?s%7S7dJ?ty{MI`U_m&0KeOIZrk?FH`{ja*s*=* zp51%)NbZ)Dl-he>pVZ#{dnG0J$?e;JP)1f(c8|3DAvu{t2V`Vrer-fzD^TZ~ZM(K@ z+a)6{!OvJ(Q7}T*OqU-+4{|n zU-gpM5)9m1_kXkP*xBs|F4^z6<9G15&cmHD-zUAU_^|7Q?hUr=-3Pt9>RXM+^ z_M2w^Yl=PkFKPB?#r~of4YX&g1kia~_k+wp!s}a4WS*0e?yFIbGdohZpKWYOq?#aE zI|I~ho|*hCX{mx!57Y3&=tfFjOEpR@defJvmwWf#Kw#47#EgY$>Iza|;=Ud(2IW0M zvu#Sy%NLZ{oW$WJQS2@;=!4d4G3evkJo@ewdR6MC7z9e+I2`T+?My5@*L*5ceL6yN zUC7(T()8@VTFHC3tB#a(H*P^8aI!8uYjH8z3hXHQrh)vh=3N49N2asGq3gb|BjXow zX8RvEjjJAFooU|->9?r(#fx1@J*kF`7nZ-4&Vu>L4nRyk-rGz0857t!r1kt&Uh?xq z7m&*tU7L$LLtpuUk+=ms4@DldK=}@~xy;|YB^<;c(IqZ)t&6g%Kma4fpfAdSV$f~67<3793#f7Hpp9AA?&5Uelt>Lk zGzW`8SHz$qC~r-92D;iI1}VLgyak4?;x;|RphSC7su*;Y!kGmT{vRlLL-%&Ev3rUz z6@i!VjZHZ05JDYVBv@bECS(6U{03WRO#aeSl~4C;rW zmr6lZ<_>&ip+UN6vy}THD{7Onejrds%n!!`JRAqOdclyL2<}C@& zSnRslCPG`dW_4e9D%FlZAmI~>g{xbYSIG03C!=T0m!sGm3~lEc;5)uTeiplw@)`CoV-)XoZ? zVWUV>;Vp$BB<;vrN2||^L9=4e=!z+l>p*@SGx;pq%Y6Jq*G<=m&grHH{4@T_Svl%F zO|S84G3Z%!_C)TsXg-dmyw@;A4BC?6?26=95PnqXex!}hG8p(alkV8(h!kAK51V`( z3TCXxPbSNWL7wIgxE{OV;PpQCbTiP*cRbzZ9GO1BzS`0aBgAx!mRF^Mew-L#rK zWXZUndwrm22Jl5fdK)b8p>IA44BQC(X1?7oqW0YV=y@lS7}P5T-6EnXSDegXFY9sa zV$5;TH#TSf1W_&L4&T1k-Ze@8NOzI*h|lUsLQ%$@4G}K<>Gf9*cEgS=og_RiT1yOC zz3dp%^sZdlFbDedwcwK>BE16k6z?IMFT!jipnTebSWIW$VPCo-}gJ1?fJrr5NJVMd6{Ef%a zp_E$IToCiDE0W<7Jio?n{jk|8kg;Sf#UKTmdCZW=j%)A2AzD)nXEd?HUR7Z42@e`Y z-H;(#ar`RF9(GVF*LgY9&UxZWR1-n)y*Xj2&K3uzW+OK4aLgLGxqA^NA)9!)NZFbS zTk}B0#xEAt^Q6?QV8)!A;tqjK+}IDBJ^ks;hQ}1_G?CD&HLm_kBwxQJqAZKT21CA| zXms1cgN$X%4a|%lA(MV-bN|z$LuNm|E&FZ^RZ1ZvFx*J!I{Y}-E^TGJDQBPE=hF4j zE-bnG=xXqFVWmL}({MrAi#dgsTcECd)=1oCa?}J8POv9KBlk~Q)7=ov*S$CQ>)%Yi zAPscWqZGJ->9@iczWSOv%SXR&m=(2nYhk|}uvF7Um6cIKLve6nGSG{AI@qnE<~cLr z0m8;GmgnC(3K+acp}ed{^7%gPfT#0yw)Z#h1!Gnsa=t(ONDQi)@nG?t&L!H64oe8!dOeY<-?86x*Vfp6hHVIz37G&y^ zeMNf10Ba{|F~$iGZi|rR{{Rds2_CSj-);H54269~`LD`^{tTFloj=D2d675t(>W7g zBXYVvtjqGOv2a>(Ba^1s#M~Fq)paVy1g?^-#Y~^7$w+k-ra9enz%kOkS`s3!2@dnF zQWWYg^PNH*&OJ$AvDI&mUL3kU+O2}-_G>Y@&KTy{>?(VJ1BPn{_1su}Fp19Sj=%BX z`Q6CZG+wu5J^Qw)hE0>`L-Knccu?yk-zkdhmIH@V0ZHX? z!z^T$PuUA%I!Tn#s#+@s@%5Lgp@ds$O!hT+?oR!tTy}C%KA=-223^KK5y=nclij|; zbti}ocD$QL%JWs@8{^CR7NOcmDw#|3s39bqBe}sE==D$JxNqNEp2E6hdo100@C4m> zh}gB#Gu?D+o%1|@-EJtApFd+`MG1&F1LwGp>^`0M(_i1`(T74tJ3b`3%M1?Dq8O|3 zN94i+DcO2ij{XKrbC;=AT!LKKXs+x+Vw8!aKm6oEvU%a61C5TZcMt%*tO^~U7B-<* zv*FKqwqnpO>CbgLzHBhraT|s?!JmVc_>NfyOuVlclx7@^0Ia*FDVI$hc1jtn(`Tt` z%YF^Yto~)n#`xvRiBXd4*TyQ(YD~9iE4od+*KA<|F#0OEi*LeTf+=Fqi-;kiKT^0I z3w9NQHg+)3Lk7S?-cd;j5@1r9BEe1V7c<{yr{qdys<``vD3iM#~1n+TS$c>Qv zf%55C>l3i*4{-gK&qGVPp+0oSDSj7tL3H|Sx=T~oeZ%DrKb4&CD`{O{@L9o#mFkUf zz$OrG%eVYclkNh3yTYo0D9zdw#a&OJ-Xn~V@`aOi^eoXvnX*(RF&0mc3t7x#^Wl<5 zg1X*V(0$1rqN+B&=>laP(D@m5BcfkD*NS$v|MENXy5DVN6Y{y+9-fU^42u(4k9v=kR{?Zw_RD+V%QLy#gds07Kyj7iht%sF-onw+6l@)36T!)Z=+;@Hw2 z>sPMba)n6G+DJlp<;247ew~eYg+^bJ7Db8k(xLm6)@N)s_8kl~&k|EbbFi z9bUv*?)JI2@WK6823JfJ)2vjX^+%FwF)hsbN-Tnx)l_TwIN(_z#Q!F~7yr>kKW%f} zBl=KhvIguOeYc@*snE%Bx{i5tn#^8m5&F(bDTGTd+`QC#kJH+y{toY?4=3I&tMoaD z?>O=u#_~w|ejFox3Dpk&gmMf)LD{F)~*AP~RIomC29AY~0ezT~M zBgeOn&-ZpN7|{F@eV&3ZaF$0d9VXA>x=54#y!baLTlB#h<-Ly0!O8Gk1=NM{nb$)B zbIgvNWW+;d16iO8GofX9>4^bwT#UbboBgz~ufpqFnV>w2{1$b` zFt}sf_AZNQ!KOwvm7Pc!q?)01iWv;mrev+9dVVHX43cv#5FH+vNb6g&kH<1x`aTHW zO+kv)6@8%@l+nVw&1^9!E&wf))jgkmg?XK?(lR>nF(o+t6xS9pl)*D|WJi)%aCvrp zZiZ@6_IfBz<7Qj*;L(mrdM8DD$#nhm`eNcVpF~Dpef`j(Q3!0mhMdh--EY#oSpk+w z_66g&h|ofvzdUd}!MQT1p1ZfOau~v#P5y)_j5|_L0YM-tLvw@lp&1!jlk|5%)eB`m zEvi4}z2T&P$o&Q%F6G#=x$1VbKTz^ZKX$P=*21F3!xOv#t`<~Oj4-u3AP#Y|Yt@M! za>XW&M#$(oi*7`Njc zng*d$#`JoW7C4tDEFYUnIl~-tbcXKqY7%Uy{R?|4+D_OSZ}%|2<$xTY>m#)b;(1Zn zko1Y%#+PJHApY%b!I2nic0H?`#HFofQtRtG>6AF3+@P6Y7vMa1PrS;+wPyP_q4yJ( z_6TmXTdJ$c->@f(DpV;M@X3Zjv{!`#J#hV*f_ch=wq-FW2=^kveO0%~Wi2sk^{9>W zK>4n!0i<37mi5 zuxsUv?8Y!>s5!{=*q|8XEx~bY&BQ$xfb&Cq2SRQ(rqvFl{b)T6BNZ$osHsR3dbIaF z2gb61qRn0zJ77ZH3yj>fS|7pe zAYg?LSYvr9x2ULa#OshuyN51UQ$qUsjH9De;QYkZLb^g{DV^x+#MN4$N4l&@e@3<# zaLH#l(^0Q77pEc&q)caF^Cn)WV)ZOxa(1JxBR2fc_flY$+hEs;YR4@`W5U)b|i-eta* z7D`)VP}Y+~T^R)lo97mW9rTfB(}iALN4$K*An80&U-ugu{xN}lgbU!=UgxRbgqI*x zV8^T@8&9sqXMi(;eQBK-`efdFZ^1d?8&d_;?V<5r)EU;UHQVpK6~>PnLB8zn#c@J` zVG3-R9kD}==v^X}(GIrpO+1<=`5{LyQyFwy(N6NKT=n{9_Vc7+j5<)Z2!FKn0?L}*haB#Q0u%UIon85y=mOg!-w)$Fr$A_)HypB`+0SrYh_&#DPqRx6 zTG&93xyJ2_Xl5lI`RXQ3VGDDrN;FeHkx;d6S&9~&UNvXya-3+CGjv7eti)ftiO35r z3%Lv#LVH9wbhEBuS=~>KC{Oe2Ii~3)@MO18N2o_uCk!#1fp9s0I6ofjK2pXE_npPC z8(XXWLbi@GpW@N6Zr@TP4Xh`a^J(nPoNqHHK2;*Fm?w|g$RQ|Re6uB!1&TUazP%Lp z;(Q`iZ3=aEB0{~zFl@3qOSQu$4_ciaa*B{C>r4gztZs#w(i)bZf^Us2gihYcroay@^F~tqn4`U7#Csfx$!E9VbHK<0l_on)3V5OgXn0@}H(@z9a9L2_kJEoX& zh%RhB816Z=yI(IWQ-4JfZp9=q-nj;MxhN}{1Q+tQvOZz4mnJ%LWq9x_k1r)!^{z@E zg*d@8ue#k1cNrWshetU3%BER(0#+^yKweO*1<(_}VJhXNHK+$+ij@gQawDU-&3|dFc`_cTD>L`h#wOa`cZr zo5-i+6SD?b_W~Z&t|npPCgc*Xz}vMA(j}?A+0{jyR1Tjir%V{eB3PXCmR`;&mS@!% zV4^}SEh;)VF)TYAxEQ@T~6}pdPim=rPHdpDOV|B29S0piAK@Xx;|xQ>0O51 zy%Bgp@*4S8&9hU)1PIZl(qp9ba}ql0-A0tpK$?ubIViNd*)Gv-8&6d$Mfv$!OP}hX z0=Odn{-T!Kcs>}mrR;3Q1~`4ClxDV|$ew75ROmCgRKzRl<62M&Apw%ECGJU&eGV9W zUdRYqOt1xD948DneX#D-dfyqJx%u4IW$Os8?2ZtX)mEhJ!jtw zUkf5&%Fz_>c)KRb0fF^+cig7kGc7$}RrHq2q&B~SsJfU{Z5a5_?=R4sq-vIE}%_8n2JLjObn=fQv-ls9(XMh~46gVudO6`35Qtf<#| z9F7gzCD43R1xt7P9^cxQ5a=|Ol zd0EH~A_9I*?FWO1?BZFu%_r(ATE&wN9*h;ONybWKfY4HOpg?8odXcL_ z|7Kd$bc`=PeV{O5Y~)kF#sgiw_{p*d7T72TmyE2#kS1~P0kZ*ZVGy9Bg{IeOn)&g` z(4}XQr$)S$42-rBqXTV2TJIcPt%M;YO(O3lt6@|UC@WoL1obO;oP3h^3_b>ZR-3w3 z61zElov)M;m_K5-?#euS&m$Thm@=7Zu#y2_FLYAyXx{0Jqu|75*Ee;Lyst#leS>=U zs4AH%q=8J?>zsx@1USLk5LJ=8I8@;HGArAXWtKScuq?&4x9$W+*H)v-leIVj0D2_w zb&5u~uPoPhgA5j}g%J$)tI^S|UDLtMtB`ABTYXZJzA7>Doz9-i-*Y?Gn8F|Gi4~HI53e=L#`nXm1}4S=IDn{&?*`_ z2z7)NlV(o|EO>HW#dkxn4vFsj__YH_n3=?9WFKTe1A5@gC=0?;NC7X>WYO5vWeIa=OnAmKp`?i0ARMYQC7Ey`y;nuGcx$PJg+bK0c$MGt3Z6lvB_XE}D zZJwr#{dMO<-z3`kY#6W4@+j<;abw;o(ZTMWv*a^IG4#k*VOB{SY~4?JSkOhk2cg$B z0l13`xpF%IO|qEpDO*tieVVJxBVYH!tJC^6dVJHY3ZC&(lMNi8A22=b9GUMvmj?PB z8HtO8s1iSZD)Tt~ph++lv;I+-i=A{n&a-EIWVk-4$H}oPAjgNS#Pk z1Fn)J3+Y-pa&PFDueoof-`h%p@dH>BiGA+~NuG%d3aRs)MJTEE{7DaF{ux{zW%#H$ zN`%b(lNx{LyiI~UEE|V|3hE0MJhs* zaM)lGS;yooxnbE}W5o-uuTZu<`pQR8))iFL8RU65|Mb$Aq@qN36bY3o(xfy)?=Nh9 zE2G>{(d4J3E_>LbP%V<{W{$2a8P7jj(49hYM-C?f__>7Xx2Wrbb=IaCn(5Dcli{lU z)3C!}j|_4|d(Fl@7gpr?$Y$n27EcU9b6m$-wgY0V@*Fr zERv5oLpp-tIHx?kU)#I7x9u3ENpCt_i-{hWo#dcog;j|$X5(AXN4?LCbEJpOh$dr0 zW?p;}s{2$IgUBOu`QXhh)I6iTL#Z-uFkf%ZC zpNm1eQHM+DO!$z(bEEjt;!=|Nk!9`C7Am4t77UKSbk-U3HAJ6t4)&DkK~AuOQj2HO z_F?M@_^_8*cj6s=O2i;bT%W|HbF=5xux~ggb+XUj%zQpj*tKM_S*fHT?as1`F#}Ox z7X5?2-8te&Kyg+Q3}}@tZ+*%Qn3c6eL%#r`i!xaRMx(a$W1d4EkwX<_ngP7 zHl5pWD0I8Qebbs|nnoP8qs|nb<|^fbcDtO$Wg9*En<&;^*A}H6m7J8%9s1O@D8}=6 z75U7-))c+)bhy@%h0C-a?%{ga$z0jg#8?3s0asc`YJD^A=xRZb zi!PvsL6meMFDf75YWygLe= z%HtfALUeag-jycTus+((3Ji_P6o* z(cxL%Uy}8ab)}tkTVs9m$C%P{`-VTDU(FDEoR83&{k@LF?chI5pA>_B8eEW($Ud3q zt~o-~cK01RczHeaq_?BLxv&MfC}^E5Syh&rF_kC4At$a@^v|M)@tMIQo!3tpKiXFZEaW=< zXd0PLnV-wWBk0p;3UQLb#kjk+Q4rkt%MTjejgP}nI9DXT?)tFqjmgO|0fYuz|(FVRelGL zKXs{JkDdl8aMDC+?aLgE&v;NQq|x$z zI=hCH{3`l3Z4lyURWUGGg;-x^&@FYpXO7^LAJ$S7?aw&pAT~(CzK5d0M!h~ zqptJpMMe#Z@D=KtE{ez!ZRRfkG>z^mH1k&fq(tqtbj0hKps>JKXW(ADFYT`QB;9JE(M*=iv@k8^xR+DsW0p8a0ZDLbzNuzX>=(aw(LqAANicuc&Vy?tUvXR zvquGyuP1--r+$qPu$h5boX-^D>%R?vcijg}dU{rw&%>j=U4V}|V8-S6P-P?>`4&T>NPr(kiL+df#|ebM zW&kaRwZAB9_U9TDnjB?iODMIx~|?c)xtlb-Qp=vg!gh#YzU@h#vP z)dQz~?mF1sr}3hV6wJp$qP|o6tLNT-`R3HpGykBM|MJ<;e{TFAkBk3**7#-UY+1v> zW|+n%m2}(+U~Dx(I3_eFbDs@dj0bn zfHU%Mwp00^+0OrwYvgYnU)gqnIVYAU{eepx8Ws(i7`qb*3}Sbt#r{?$G|kgxN<*Zh zIsL@s7`8mk0fBUgvC8Q!+9;dy^8T_>Eugy?h!$sqw)V5VKa`(h?MPfisYc|%jIJf? zX9qin%F$@nf~RTch60kUPY%*B@6a(k%t>^dp;2&Swqujwrt)y0`WaD+-si(D z$}m{U3g$*GICId=iTgryCK?@#5)xn2XI)4TUwNbc%kbR|B^~^!m?eXy#r5L$=FYHD z^TV=V!k8^wcEd=#oekkG`3T~{E)8VeSKoWi1`#<1z;<~waY_tAE)Iwe@~HB$IQ&ye z)nMNl{26_$1?&{7<39POB{ve8aZj|gS$qE8k=D{*Wj#BNP8brMVQW9t?||-U6BZVP z8t$-y<3?>@DsVl2b;N9Ml>u`-Xvls{``HVW`Jn!Ze3xeeW6u|5VQ*is!lNMz3gx4* zWu&a&Q3U>-lfTkKbD0~be{~T5*{y~J(4@zUl({VQSACySCP48wofEwyiXLA)5|y|K zC{z9b{k6J$ss^sS)&=m@0T8hl0qMH+6VURD-)MCkNEW&Yqlk{>W^N38c~*DSAMPbv z06(EIb8?9YFnAdqN8-4Y>FfyW+n-X#5GO3z3+pj5wwm!Px2*$CMs(?3>+GvC)M=5(8Q?K=@dfpes*rR$w8*x%b$4myI69~~OGsPP7VQiPT6In|1QVAv-1}Ae z`huj0N}9<+LIh>bh-el*^~RUHDKG10u{TaX|T9>W&t0L zF%G#GCHtsRkICU29*IOy7(f7LI$E&5AjGlZY=~#eNc{10U*i9iN<>xa>T03KY;RA+ zv&|Z5`a&#i#D|#dLbvwtArhCbRK8~i!D<-<%@thx}l!aC)ySQw;EXO z_z1mmb|zTa=Pf=>WUaZNIMv>&2#|O6<)6Cz0K5og&XP1c5OVV}TiBKTJhO46RljSJ zWcQ`)$Bm3z4iI1zZ7Dat$V;3s6uloH*km8KHCTeAH1-d>(yS_V)vc9jauzAPf<&Pk zvz8fF54UGeCcMGUD3pzhh70jn-}cX79m@*hUI6U$xXh+oM7Ov1-aw!1c0hO@vF;l$ zNVWiAlEoyl%fXAG^@%MP?(rt99`nC%b?jZc;9{%STs-4wEmgblqNO9^)Z!GHA!x@4 z7brow1|hAtua1Vz*} zPGM^do+!WkRqwsKhpRD~=7H?E@9PK4^;OgD39(Z% z+P(pE@R<|%n(QmNI#xqGqS~TpotdTIKPOVC;3zn0vPhEZA91f~e*Dj(1zXmRcsIc% z&S)HhrAWeBT9OECaz4`i!LvKZ4f3iExdx%SM=Hzagi9GN__d{{Ihsx;P?n~et zhM^lSZABvuUHVJOvHga>DDD?7rB5PBD~{nMDE(f7joChzt|Ki+AIut)-d4H?B$fG) z&?Hfzx-NugIKH$GZskdg{wW29#SZ7|!EJhl~mQCzNG) zlt$+MQ@y_UIjJ4y6=^$&r0Sp=62r5~YSeRbdA*FH-OkhzP;>n`UQpux8HIh{8N?NQ z{@5O1T>k^}8he+VfpCRB=J%(A1 zKKNKlXlmF^0{U^S+?(T8DYIez!g!5S2m_Io)WFU{u;;e|`U9rn=tgaS-S>UxVmQ9! z+UYog@Pv5|-#(oU#g6mJNE%4QEkIu?V`RS1yeK5M{-OtBv_#QX8;o9Z+A<~$a%fdVh z=<>Q%nb~>`f8g`Ree91f%FOCk0KU`JTpfT1%G3-1u;fG_g&|0!T1oX?Ww_oT_S_r1 zkJOx}$wIZv6Rf_(b>0{BX$>-jv#a4i7*av0$mAL^s4GzU>6Hlbc~>?138x_>HxdkO z&1{Vu%lS#K-g9s%=Tm}&qs6mtutZw-D_%Latsf^9f2^HJGB{Z17~)e znGTC-LGN>hRV&;oVg$|*vKUH;D?_%Kv-&o7YC5Yc7Lzhk(*B#zI^TA!0K~o^F(}j}w*+;nhoETu)Q+|=m13U)w&{S; zjGt<$jAx#gUldTZE~q0L$TC&z$ZXZZU(y}7IJ>Z%mrlWo(k%F$odbr7dzMd&PKMB! zjl@Xpnsf(NGOl(Jo=rqB6}6 zulM3(k8se#URk&TX%t-Ond)8Z(t*4Cdth%{2Znyr=^BuXkV>?xrCib8$4xkkx;&9> zc)On6JoP%CIVZ1Rl3AYp#?rH=@nGw5%w@eucNE145y2cSvpqGT{It$_>}Yta5g1#h z;Q8|%$s*!XQIb1t=y@krIEd61GMLSfLgEL+E)9ZJQ5|*daG)aiZO$h5da`R`)po`g z3{fp!a4B*PQpXOYL=QO*)EVYC`=i4V+c2=?k%*kWJIagHtVu+n`$ptaXDPy5$j%YWDD6ukzhMlLY!*&Vuc6*6N5mWj1mt(oYxaE^jO7#dj%MQ0fvp zVTOi3GYzKr??h^ABfGylv!WAvO%aG}ZU|gM^L@g6vBGENume_uM9-1iCp6-!twwb^ z-Z=q?);>O=#U=8SoB?fbZsCYe_&fz-7)k}?8Y(N;gB1RPH96|hJ59wB=iu*&bs$kwoj!5WMy7p8kll94&6<-==4a# zLBWMmXCH?!)TYr$X)C%&SbH8dX+DW+=9ADatG9Q5cY5l@gB<4x<;Oc(dU(eC%iw7B z`0ato=t}2MKey(0hhGm*0lSgBL|)e7;5}#+cQ^pxTgt76@oB`^d$h`k^x)43nzc%@ zjBwhxZpMtnB`<^fy7 zLk}4W{ZoFWBn9i=0Gd|2_|G;d|6hClCNy|^E=-@=6t`zzzKu#e7wnkkvyv+j*&{l@ zOJVK8BaSUjkini$AeSbw4Y<)|5mG3L2*{N~e@P2z6}=zI-BiV}D8opR4KW(MFee6C zd%lwHQTQ#%31`U5#R2qb+NdHsCk_=5ZwXCC>)v!~0=5y2N;3-j$d@0RlN^6M7Od+& z`AXUNb#!qpHWmoFe6^qevPKkBhiLzZP@zrW!8Y@0;gswPsnt?HvT7nsBmATM7m1uXFVG z9&~U#JL4$t67_bV+>;oqQyl|uFYWX79wwsNDO$`d1SkwF+W-{#W|26jhHFN)1bpBY-{xvoJH8uWaHU6%x z|DQ26k|55(TT@G+zuS8Ik8z$@zXJ>O6&)N$69nrumy!E5Md$(aAGygmf~f{@%7O+b zQTPyNwrbJp|EaaV?|K;(s7vNf%O7tTV_^A1B12g6Rq=x3X46}-hjs!?XWV#MhZrQus~wy|?>Y5E|9rnl&Lr}L zXfUpgf9mr+vNO?_L3*&bk&a{&A{@=-w|+5J{)X2+&BDd#pAl#W7TS8T=y4sb{i_{L z_XkJq&O0FS>iSJx$14|s%-ODYkqcp#PfX;?okVT5>CuaN!+Apmy=yjs709)Cm3Xr> z3R@F+e}rSLz4`kM=m+Rd6K6K%N`(3kaDE5!u(z@oxxVK9$c#cQZ6U!ES^s5)i0qoI z#NI=l|c^lgNt<_2XY+a z6hQiU5Qd%qc~?=iwOP=pM{D-U!420ANw+tAW0=c&y7Du+Q?CUu<_1o-FGpxe8Le}E zxa2f)SyBguUB;kD6q4J!LYRZg$=vO^#m*n@c=+a>LoDPwO~j*XODRFP-X8%ZsNg?4 zZs@oF|8}y3JhsWG9rPj!>RXd^HU=}Lf zA^iGj@(nmO{eht4-{@tsoaj{kHKxEwh^2T-NjmqLgil;h8nl(>Yqd6p*|uQ!W|8a| zh8@Zs(!hC0qQODY!jDi%GlbA)eCRz^ZprzbXoO=ID}Z91XDUNl9Yo#L71hKf=ge7{ zp&neJO<^NE*cU-?*t)6{%ez8fuN^Qon5Aq(>cKNijEw{qS&d<9X^Ex6()JB^0?g1- zFWdnhG*U)N2WWU$=x)&{S(nUijYj!lQJU5Lq9ZKgE`0QK~%?6Pk3~cuN<>u znXYG{yl=T7syFo;aZf^@}G&D%`46`ndtVesoPg8 z+6a+QAcz>wC$K2{DBifIwNXAx3H3cfM9b#b@)mku+!annJ;3F_L6;MnW0k*9hDj%H8v;}HJMa_dGO&wJIX zZYyTUx>&8hr62LDGVBu4rdOAEr{#)qC@uEVQ$6i1Gh#NEVq)L|R2{AFNaJ z_=qbb6s?EgQ!>&g9gvItlP{EEy->wDG_h;C5xp06xr)Pmgk6O`HI*5_3(p61ARsOc zYdCO0^$Um&E*;3afE`Ys1Op*C+^fhAG$8QG6d+1=II&f)sEcfAm+B_b+5j(M7#94ZVLQL$8R6=Q(>ab&;=rehifpF$R6p>O> z>j1?^q{%_b-fqNk3n0w-K|>(151|d5*5Sa5!GyraJ0p+rLfI?VA%;yZb+wgr2EK#1 zpzJ}UwU1t@Tpa1Bjgwd-$S&`QH}mCpotiLjeIzg&&pxpHX^?rXHUYXnB>95Uv{S;e zeo9bImE~mHBnCwVg{_=t4LLIA%!@<{o0$_yC51?{QSFd+Us)S)uCR zVyvgNmsD01xrpWR$^4hnB_+6@P1Pf*IhsA>;~`6pKwhUv>C(g@Qzh72AGgyaGL_s& z`~;*xA%;4TI(DkgMD%;~Xmi$jJ0~Wg4(Txtj5{E#zEI~x6iHQao){_=m1>|$JUyYR z>F~&vAS&2_-^nzf=Dm;bH%IA?GifU@CFt4b;u z89HFmH7V!>Flk2sR6xCo=r>0}#}esPEea!}5|lyFA=j^?azlb!EP)h z95IDE`$EE}GcR7|vv=WvSv!$3#Hd6A9wdv1H?qE>%HSF^ELZ7fN$(L0| zuWIy#CTnkM*5bM*ONpWj%M8wrg)WxzE?#g`kPD^pa|%m4y3W3k7k;BhlVGPq^Po7u zVc10j+$b2Cnu?|Y{S;>&68s*TXnLF%KS5FEDYoPssAi|fp^gvw+a8U6y*>wz3ooHr z(axI?Hqw0uH{c^a|2F}rb(GdzJhLG;T4A^sWk1m+Z$-_A^f}x5)q4M$Q{x!K=82wG3aZtH=4_#@G{NDYa+J-T#;)%V$jkAnxl`Z+&WT%lL8XT$9d8K0&EOy zLn-lMP*X3k7BsSzMX^CK0BcwgIA$R~?P7T9KW|d+!AI!_d3A;&#h5WOq0EA`NMtAm z9kdh}O20>Mo}noV1xV4!1ZRoGTaM9UP~(q+9RLoU4bYUCEkN2Npum3+rQ_QR?Xheo z&sX^;;BdH`cr!kaDX>-k8{N*|+wsoc4UqJ2<^EeqG=Ac$sW+nk=2PgQ|AQ!#FO(+D zLy-Soa?(FKCdGuMAjZ}A~1{8P@--wO%+AK)>40^FTMMfBzraFCYwU|^yk z1ctb@c_06Qs)(`Eeo0Xi&H&x+@#c(%8MWGV6mx6*zQ={8?y6$7hIP^i2(Cvj6sLTY#Yw$Sh6X1lQ zc_4#e0m&Kx@@jGF%5Oi#-x-%uXtm}9y(tJuzcjQ|cexFkrIqUv$)?MrVe&-Jsi?Fiyv!A{9^L&TTkgXpPQpoaX zRjpg@${gv}w`P#&?IdCQWRJC066P#WD_E*YA9ABAG-u?9wlEbBNTcX(^&qA2NR($y zYe#&QMOz1%l)Z{9lE(BIA8P=)Oe+GEg(p&;C-syqSN($85nC^N7a-BW^D%q-1z~jh zdCX{ap524a#=5$$ohx#0v@j`rc}}6V#Lnm{`^;Z?PGWs;oq6OZ0Cwt#d3(U=6)P{G zFWwm1KW`>4g<2Of4TTh-X%t2zM^oK`!|NB=*P4t-QG1yH*CO`|=1{;Gez6QxAZRS) zNjp4(OOZX9{w{f_mw2~5rziEuPZ>W*T1jv5OB(6LhsRi_vfz(H*&|&4LBhdKj1CJ{ zfPaP7S9d!Z$ikDwD@$IJB>nqJV|-M}n2op-pBkT{m4AexSJzorM}gQ9tLkjq)5_Nh zLsl{Gszrqb&*gWzjh@nqpKy7LHxnO_4jKpM_fD4DHFqd9JH`_xb&uBHI6FK@Fvx1P zWi!{zeI6>kRGGGfkGq_-hN;JivaD`hU4XCQOR^gUM(at)YmSVBY>&sC^%!$AGFQQr zCQQCDG?la#@F6+EH3y4AvMyt;k0QNp)&iDT6ba6m?utK1aXX~may|xlW(+r*Tk<0H z7j9=Jl=d^QF&JQNp6aJ-R%Jedd@=6s4GqqC_aJ9J0%QHJ+1D?C4?EPW{k33uqy?AyF+Z$XGRU>p*1jiW*Q!8-~X#k=32K2qqtr~QrPbYM|O zeTYuUf6gv!tL0Sv+Dnf@c>0Av)S@EF90zu`KgQsP`c*SL3g1TCxH=`Hdt|3@l1%e9 z7kdTfDR}sKHV`Vb6Hd{7&-JUN-jcw*UXQtw#N_mrqXP5!jB~QP^gR_t137U;8a8>U z59teKA}8r|LwHZwQv~@Q`XE$xY_e~wwdgVN{tET-vA$5`v8gG~BgF=>Zbul0Tsbtz z6hY|$gUB0On0ha1KR3(=@@(x+&J@txwSg+zT$J^fPS+){<$OZYbJ-E^2`3A;?rR*Oe#YnOx=PfP>Lr??#B%cf zI&YtC2UV&O<208Wf-Pd?ivafLaklSVVE>^e6|VqaHcXALmSqDvFsa24_0S60%uGB~ z_k0wy4<9WD6RT}7`f(R&=r!(p)-~#RNm8ZvnB$q+q!DUN^-{vo-sVXA!8%GXWU2{B zdWIr9+VL+Gdn4A~OWph==|rNe`Le{ROz<$cyC~*tGt4toU19W%cW}08AcKTn5lItN zR+ASh@HQiLy)4F;bmFIDg5e- zbs1!+FCa0~)}pyVSkF9PL8wozcQJqkE5=#Rupz5#7YK(Ez4)t1MQy~*%`(C1)5sQ1 zt;o`^7y_yTR04@EnZskEGNffxFQC0q|JOV4k*PKgFU$C8IBsD>L-FiUGa1qJLtg@m zOGc_sdmtL1fk80na-nlm6Y=38VYcVUii(5LWqhY+Xt$QtUIQeaHYc%i=9b4hFL1j-y2AUdXLqpZ8 z!ynhy3#|jl6E=zzLX*D*FF}K}I;-HIg@Mr8K)=H8Lm_OQy#$kR%cP=GamC5M|DD?$ znTz0A8bzeddavz|wVDg$b%ADeUSjlc}%7~Xx%NfD{3VqYYJSaU6}(EKYAJs zNl|Y^*EcpbTV#>I3Z!V+w)jmW!(i#()?>fPAmhfSNkmKW<5i*+BCI_pk-It*F+oF0 ztNpi)Rg{d@HJcw>@di)QNA*K{lZw2=w5()&)c$MDBr^QI9SUS(E1)`iu{xxMf;VnNoX6ySUmVjXnp0f%IqDhrJ`U!8DYB)8h;p2L_)UYBon82vYc| z0Lk{UjIWyWHZlnl<90`dfjnw%D^Y#=JH5EeVTC7**GzMNhKY_N>YO=jpr*3v%&t_A zuA)aak%Cnzg8QA`hbp+X^7Rs%E6ucJp7WisGF-bfZe`N`tQ0ae>TVnz_CdIK{UMC0 z9nYtY{Lz`YMcMS1S@WIsQAUy-_jmjP1+|%Mi;XlAa)*oHld%xen6d!bYS!24UR)KH zg9OlE(ZIyx$LJwvO)7%V!;)E%wI~O&c)GljKp4uVzF&#+#=TNpId*Qh=v1s(eF_Kw z6!hahUMC%E`qq6Yi$+a%rwwGq^+~YqQYUf4otlMByRF}DPD%HjZ%^y!#Gl)QCQVcq z14U_Xc>-B+iafTdKWPc#aQ)XyrN_l9zdV*f#=OR(Z7E&$Z&oZ;IMN%|D?l3wGx4EB zJi^>F=M-5%D-?^C?FB~r7W^a0lIS!YI?tpTHIfzL2_7DnpUQH2;$o$6tI+#5-P9 zI7GqrItEw0*Mq%i7MmjC6wKf&<0OLr>Y`3#4ug9m%waYbT2%we95+B*Hh4I)(R-B+~_l}z4MdRO(=yYW?*wgx2S8SYdh<(>PnI=OR-l$SV%53*2YlNg#>=O(BJ z1z25yY3UQ98~-8u5L2(O_B^G)yBZ5Q4~3)UP=vwAfwc|ob#=bNWWhnJTeN*aTIwh+ z%Le@z{N^beh*9%{vQK7Nwpj{>=O@#O zAQ~AY9O)sECUl%vw!j5i&6Qr2C*aR>suT>>;>ilD%bi~nU1a)iu{(N8znnZQvEIDf zZk16!4foMu2aI&Z#?RIXu_2+k!Bc~DJ5WtwIeq>ft)(8%5gg8~;K@$$)miP6SK1`! zd}X&iF9|$pfu&`;3gP+zK=BiSw1mbUBM!W}bYM+86-Ysu7kuuF>k~deyiiycmd#r5 z_c~_hooyQiD*_9xKLsW`ELD~kb5;L7N3gr(U;7NuhsOAM?k%_>`MYGj749ycS)U1fl{~;$GrzE-^sv$@?4&TjPR>cO#2g8Cg;M!xa5&qAjC|c>7DtD3v~Hk z>I%RI;p>Xqoj?p`x5s~8DVW%-=_!+&mLC0c+f&ibbL;wBX6L<#w2ikp9K8}+OG zc>R+P$2lix?MUXs(hV(HEptqD+2g=Kj*>!8Hn14LNZ6~egHiiY`(<$P3Zql~n+;F29%@c*9qe8XQy-43!lFavWP65+Emh!8*lI)~Ru1_+D%Lw`{A;KmRP+r| zzg3LMQtm)rt@4l1gLQlDj*R^*4l-`70~T#hc70<*2)57@pxwR5WAT$MUdinTJ?-mw6rU#DcX1{5U`hu4_hk2jVFS^R|mEx8xZ!hY8o5oB|fw8!Bfa zmUTM|r8=$X-KA@TigSSh*2QyR4qEFP7Mx%OQCb=@X)k);__YRVASyR%6QiZ(#z*3V za4DFU)aIyuJMKUef?dx^jF!XSpH~`H_s!g=LUhjHnKgZnn~R-brXi8Ka~oV^h&|S6 zZxXUEadNVxdq2~5{&$<=U9raycYkdDCLopNrwvV19IAVpy!)Zz$0dA?EK{)NHD-NP zEMyhM)CRFRTxj3nvbe!KcY|27*7utLHD+$L$Q@Uol%*^DsAH(p)sczL*Ca=j#y4%5 zYv&t}pNk$1^0ebo>jqxdQ4$e^RM^{Jhn4nVn~}L)+yLO*q>|p#Kw9hbsC;+*Ui_i- zFgA0w9=lqn)zJcZh~jy+x>~kI%qpx0|J-H|p33t` z#T@*|C7EbOUjxij6y}YdP3h;u%F9L>4LD{oGB70X3uTW{;@n!1O0=1v!4E1l^y04R zN3A$dmTZ*#2pWnXz=;1b&hfth;|TfR>v(Y2=6cLbFHhY@RNpzX?{M|&J?SB8)7%`@ z#@gk&ljmHQ%us`Lb8Bhgt&?S0C5Dd$Mj1Dm>ABaeZ4d0TyZuQ)@V;X&3FKH5UJ5rH z>ojz3C;K6l=xfol@~YOSqXhOBUee%5-Po^zl}1&UL%`u#1mn`>ws7OOL)$(1DcDP! zQs1^o+tV&Hkt-{QvMuW2ByWhVxBsHm<5kPqdAe?-% z?P<^4IRgwgkYxJ#g)=k2z9lwQV3axP&j=fpxn0RV{lk_QU-5my8dQ!En|?58X^rnC zf>1_>Cp&zK3G*uS5DDSF8yco$X!^6}R?=!P(iC?s>s>UCffQkyVeaw~C65X#XeBeH z{%M8wxt05@f2SV@Aus%=8`qM3ZliQfGr9tQ;I7MA!a~LJ9xl6YSD}RY=QfKOtRIyW zrr$bwP+*jnAGu|J$GCO)?P3-?`tmH zp|GS0(LOel?tx{HyhFHTP@ua-f&r|qmt{yGkIQwlx!qU1NYkE}mFdDO&K;baFV}ut z=ZSEvBkw=kE|G>CdqC(Yc_D1roo}XTI_k>GPYjw1Wi} z>mImM-&lYZ>gfXAYU#wg`;h5_6p-^y4dA7Th*}aX!+&lPYf`eV!%FI^hb2VclON(*%Z>bis|q|CPmgdIa(Hx!-#ZzsU#_Y1iybx zjNqW%w*%Nv-~}toWN5oM7;ZTIcW%sHae2gpmZ&9&D~y>-e5d3o$~cy zf|z$K(DPj!$$|c<#3Z2bYK0xNDp$5BS$CZRBPF zBpVqQQ1c6>vz^soB1tc+mL*(4ob#M8h6{gSorp#);NoIbbtj!iwQ&l@Y8{d3A7o8t zp?ne>7AN&I=L@Py>G?>@(i)S5Ket8uimLJiTxi0?R8unMxRsE{jgk)8fh(q`Ag9%4 zJ&hAK7(H3sKyaqWgBodBa_Y6(&+g?&QxkbwUQqjP_vPotql6=L$%!WM@L0?FjSedh z;%;|Ss&iCMKVn#p4lkk%M$E2$;O0;#wil&aksez6IE1B{mfnF2qf1%kn{u0@WJ`K=aeqOr>|gi-DvDl6dJtMrV+b_i+5dpWwd8 zLRPQBi`-7eh7|js2+0^!4GrL1U65QN16ox22`%H2#BNAKRdivTq^Tp1MwCOamDzB} zGuUwvieT^nG-juX@roD)U1hK`!|`MzB#@dsOqCPCvI0^CjV5MiJ{ zg2G=}dphTcJS&baM+!QQCuUgvLt;wnh)=FfDlUMW%Cd_D#aBwKx$7{o2((q-)H&(P zd}1ES!aI3kV&&$|yX;TO!|@&Y7WT*7P2ePdFHtnC0JkzZY;J)xb~S+=MTZ6Qj{VP? z#c@DAmwvb}1L*DV?Msca{BC=G8hI15<-mEhsS$j-%dCl9yz3gH4Go(%YY_65Cr?^; zd5<7MWBA{7*WSS<<8&AgsO|po2B{ zqlOv*`X0IT@GvsoYd2Fs)Pi!j=MIIHJ zvWj%+2>2zN;7!p?4r=6kV6C5*R29;6pNYo{o%Yj6lUCv0-s4t! zwwyI{w47{#owt{Qao5z5R9M>MNL#|}-azP&1cP6P^jdD1r_O~UsV%YDeMA05UA?Sz z0!Cr{Vt>S@^BX$};NN!Xk>+M&lW_%!h}=Twqo^UlnmORAJhv8TeSQ|!)f4yG2Gila zzt`XD;YdTANs7NSg#?Wf65}eeV{VjguSQvveII!Eo!79=t2Mk2K){O!4oC4Gx?8vg5xX=rzl9cCu45f~J+No>Kb6u`=mGplLR=+ zyt5;2CvKEE`6t;!#+}UG1O>_>pxtbebo1_$n!Z^k+~l|`Ewt4d9(AG@HJMoin#w40fOVva3TH7lB|mFvE<9Me=I7X$tPN*?o8XVPh^ z-TtmsoR;R4XJwaNzN5L;LzBeJF;7qt-Q{dRA`5RJ;Sjwm(a6F|$*cT7TS$%ik>B;&uA$Sy2CMZj;E1 zc$VjEt!P4xG?5iofU;wfj`J>d)VI(iOf)vxKEr1jve-zNV#FUBq;uB}ph#iB2!n>L zo4lFYv3&9I1l8v=gZszobMc5==nb-BfM>Ue;twQUan{oJr?p-Y<6b9Sg6-pQt#V>_ z_BQq%{9~m5_*uFc-W{0JS+6K3Y=?ensFZv!cH`gwi8}pd`pQsuZZGXeot-0U(IB_Y z`>NkptLUX@(B|zBWS;E$O?+OhCo7Fmo~h(?BP>zQ)jU60*taJje(!h0uadON{PH`}|-Ae%Y^2Dn5m4BS{ z(#YWZj@sh!!c$$1ep65xjRimXD9yWkO!*8oWUXjifnx{g(^1dGysp#)9wfJ+-kRR} zRn8?v=rZKnzJu?I4KgVWj?6JYj0W^}2#n8xxr2wrS3VXXJ86T~vvcrC{~7X3oRJ@| zcNYIHYHz=W8$~gOX^;cOIs8^%nw5zx?eXH4{oNWivqpX$wji$8dD_c6?slxHKWd~w z1XKqG>G5PA+C^~GRikW&?SSqVz5XnC_!HvKtz?gPga?UDZpXiv2=<-L4bHi-qM9*B zZ~#HlSLU>0?rBT)dCL%K2YB*T4y>_g7W~w`85Nj|VG06McTj8?u2O)Y4+WY4=ZfLewkqmEtMQl^F)^O=34u!$ZKa{>+aXiBCluHX>~|n$fGa*x>vG zAul=c*d)fq>K}8ChBMElkm;`rKSdoiTjGg-g>v7>@=i=#z5T-9nJdSU&FalfoV-9i zU3g{7D}xkiO||7OffEEW!fTT5TxzGbKMaPuTh2p=tYx&&&}ZHtS|X)jKbLp*4gRV; z$#1;X6H(1;2_OwfR{1{6A%;Y*rmAe`q#{G41D`W6{FPy{yXdZENruY zwqd{z?osQ&Z(`DcCdNx)%Qx##ghx>{pL%^KK;$5RA=$CVng0ED3Ec zGE9PB8h8B0IkN={)p%c>t`YxibvBovY>jwBL$>p_bj6e9k>~@Xai_kvjN@93{nSeS zwf}RwrxtMiWBg)o^b#UWn{YWbddJWv`XFe9t|vhdI{~<$WUX^FGNuo4=p}0`dDQ0m z%Ryc|D7Gc4jz{%K$hU0GiJD8SqEgNF$#{5&&cMM#D<@`?-Ko zfIyvCU0VmX;CbSG|A32&Xk&QC_IS%61OcvHZ8;lI5^cJ@7NA!eG>e~lerZ&$ScxZ9dd zN&zj@n{%D8DvXk1cFJx=UGl18T-zjX1?b?Dw8>}l+zmyR(1dvqy);|Kk#<2OT_#N? z5d`Dt+}~2?IpkQ(bML5twp;Smq0XQ$iv*z!Rnz+UKjjPAHJl@@joByI2fix4%8i+~ z$&W@_($l=@X=wjE4&U`jrOfAiR@c_WQCm3ulK{WPeP>#AQkbpvcw@|vmrUl3hhB3Fb!V;vD1WwbMRA|B6m z6?HXEO$`ZT^O19DepQXD-cXP)mBLbPs941Kikd$xl0FqXvR!&^N0sg`3lKldeq3*H z)531ySkJp33CLreJaaelD{r-^7=khq)2}GvtG25D(7n(se)ncwfnzqOrt=#PM@&HH zRkJZI@HPx3XvxLr#EsIA#JJ9_oZF(=@!4ULZ%xYx+nf;r8ef=J&o*fET)yg%SD|N< zy%aYu#}BJ5ph43zjBDwu&iO5m(?5f}2p$T_XJm(l$uZo9)l8gX@?g$>Sy^PQOhUCs z+jzRzq?~N1q&%gK&a~$bM$D{MD^A(5yj6k~^}%sb2JPdbE+;E70SE@g8e{i)*wUYo zvDKv9kTCDu0vvJ$$^hPoG2$fy7k-gm_0pTw9=>l{bp2D>NlT1u=9Z3^>oT z$va<9w?z%P#NW85*x%*c%FC#??FfTojk|A}Ee7e1#C@?r*9wAm)xGWyp@gi4Z`Is*oV)&^C?Dni#@V^^_x1<*B#Lrs*iKr4g zc`H#t;$xcM^+up~$(A4zZ6S%NEMi-gR}*s&ztQoy=Lhj*s#tP7BcihK(uO z*KL}^ILY%80_HOok2s9bkq~S@NmoywJm42bDkdl2Fn6t|cS-RRU#chws)}p@$TDiQn<3 zvnt-JKbECD!Gtgcc-ZG~R_UQ}<7a-3Wg7_zieMPGVLlWjAAlGeh|HlQ8Ty1HUNB9l zjO;y8kMe^HYNM6Gb#Czn=pSAZ!VO~T>aQy~Xc%rDxd|84eLZ_G*2p8u=pb4%0vL0` z4kUT-rT4y^^yw)?AEx0T6NTS5XY1!UctCBj-hLrS0WQ{xU9LEb9tWd4$fu-B3^QVZ zy<}k-KG3~lsnG@w+`QVVBXxQgFj)-lDoCO!${(3)J)hqwFRfHK$uiE3P|pYu&fR+5 z{+gv7e2&oY^bhj~4&R=REI_s921D|JnaUhQG4b9|Tl_9#9SeUhtXB^Q=>q4#wD?a& zmg}zx@xUcqJ>2X$+qY>OILMPuGGO)>e|jD*Q$WcQw`{6=y+5ANQ2rz!=xkH!oSzAp z9qB~Mq;h|X+RKB_Q+3nVIq@(XpGhO}k6PQ7f)1(b;*9qC`FR}9;d4^@H#9owl$=3Q z$rqD}(&8!woeNP5&B{KlK#u|okU8A;WVZ+Czr9vokY(*A&%mDn?;EV0X1mwjnig2& zx$6TA%Zbird$tLi6FgPd*dWBWk$vzZ-xG5mO5r>6Y{@w%Ss6_J0whFk6z)UdEl zNu}EoA5!8*@EH6Qr2rksB*8}_fFEeOLhTq0voYLSiPs0n+7G(+B?CMkcCEOKv%T7g5ocvq$#s z`(az5{+W&mf(7Ye$dByQs@1yw*g3?SBT0Q2b~NeH?$XTeHK>B=Dpjv}-^d+AjQHYs z|NQgffI%-;`+f(&P>drnB3py<|2O|T0lGf_%fS?Wiis52Lh~SHKeQhaefkD;cpw&Y z|Dp}?FO>Djm;dL~lfNUP8#qNb`c-rCh@L*B(q8!)q^jc9R|=|9%aN5K=OcP)5-lwA8RyZZA3MEPR-}sQj8$0otuuHf?e8@a8doj@yV&!hFdD z(P~ z{3{SSBO!W?TiSLUn)yEW@&(DR8%beE+&kZSm0*y*(RU4u1!4X%OqOEXK)^?Cmg<(y zT+UnUPQnCYeHdeitJ&I|`Dm_I;mF`1UfrCi@Lt&$v6b43-(6{nZ7I2Xv?XTR#%#=$ zAT5B3%qA8=ZYnu4YWvdm_jQcSO;lnl!EAO#Y0Rl`%k_i6GLcIGfg)1Os_EAHjN>A} z9)g#BW`7^$Q0Q~PrJ8-tAPX69k_ig10IF4VQ(o4|6ZzDST)y5ot7Ev2WvqH=!`!}X z8W?m6>O0$_HVrNIqf6NHsLQ}ly`;D1(83R!7VN^t>!aj>=5t5#Tb++W%XMy<6m?CfDeO0LY|5OxtBX7&J4h$ck}oh2+d9gQTRwq3(K#Tdo=|01MXsZE-NHnctY1+WQYNdGl9pWl2NyUlY?62zw{;%mHy<|BWm&-EG z&rZapI$zGS<>&|b_kAevH@CrqGPliR_&zdwb=qk_f!O}^&utPLOeax{RPDTJq}I3! zSPAgzitK-!ER%VwHGwUK7a&QzxaRzUf`C`*1H1Xs43Koc#>h$*YAb)=9j?~K+`+uR zJPiEiq2P#s`vA`L34pBE0G7ksRwMP{_NcGtKQ8`rTN>;wZ|n1QXQd9PQhwlE?4-Y- zuYdC%BeMjDyx&lgdGgl(TZ6N0ejL$`&VX+QI`ne9NL)&POImOsZ-o%}uYTYEclLU% zA7no2ou8SzDBRWk77qrzGOJ)kYPtMVqOjq+L4=ynCI|Gw3%!U zeE?-E@yaS?lC}@8bFk-@?;k08mmJfK07d0i)uO!U1>oNzosOW+OOB+Wzhy5T<3+KW zlJX`M`+M2>ewMgEMD_$>vLB10bA_QWT(qbF@1pEBTvMC~l-*XmW&PrOA9tk)lF;rH z*1X>>js?nfxMkt7CVccNicY1_JAw*e`bo$hLlYTPVh)*XIqkHn;1e1;jBbkn((?;g zlp{S4w-^`ZT3y8YPFc$SHUwAlqLrY5r^B8Z@FHkfCU9N(0W)xlMz6)!lYHj#r zXcI2O#)k{OIn-R(*Arv~k^s*qQJRM*)UT-^1%N~HWzc?(mNbFjWAU8=enwK1q4Yt< z+W~#C5~>}cRp9Bgda_VQy>yJs2zZqx>FcDK%I+K8jYFKZ4C1#WlX8TXPJ#mjofGrf zI@l`6pWFP}U4TFlTOJ+&hM99efo6kyj@gZL(tgwb7Nn6SsjFRK?8nlF23dOb*9Ha} z>vQEya^gc3cj8<(YzN9+V%=?M(=I;J$q}U`Y-M!$wS0kobFia~ZLPC9?R7+I!MkY- zpllq9YNE36tHIRB8SCtz{okmWY(YV6wQ#s4#_o9pa-E~gWQ=)|+#)w)B zk)+=-x>q?23m7X3$VK&xPA@vQ025&fgWFC!kpC8QpH%jD<@fEQD04xUK7*;{IXdYz z-qK<3ku5kcs>EGO*i1q7X4@ip_TttF1&JNS({#M}SUt8c%74c%BBG^;>W8d@c{MC- z+Lzqm)JHYJ{CUyPtYmGUIWlnBmDaX=r`Wf;Io-*ERA99<0ms=IS=S*ADe7VNH^cdhgKwNb%zEQpsDxNse zoxkwVYo_MzGAfo&|BM41Gonf+=6Z6J9t~d=Ukgc}_DF~3;NmbA*XW5NACC&zT9=dY zxKme^EbE5}B(>m~!nR;N)_s=D&5vpo(t#MM1k(uyQPPVO4E^#gu~T2Plb2-0yVNW> za<3|#8{A&pBClN2Isc|4G|vkcYdtUuHkkbot>v+|2QT-iKa=e@N=F-UlTKY4x!OK% zYi(fjZM{8Y4GQ4LH_VgsZLlWS1{eyuwW{Rg&jx*5mfC(8#Qq)!Px+3KPG9zfzzI&H zt2!kI>0O6bR6L~y;%Wr_3+l;uKGq1rA|Z}rKnf1&9M1%8dYA@=)i~#p%%cUxn!|h{>S1!DbW?5Cgsb zDp{=J`7^W#gGvc8as+6C=qJH=Ta`Dnc$OLT z))_+zuUF+Lbp^=Jl5oD+Yn`zRz!BV~xtTOh;_3`}P54K=p^x|xpos1mqOvs<5*B5; z$El_w)BM=&f9KHlH-feDb^|E#BUPW&O*{tPuHHNa0nqkGKP~}r-2Y zpK|->FMpd0|NYTl6Y}3P@!!wH|F2$!^Iq(}!pv!C>U=0?l~wSUiQ0dSE|2Y7=YxN4 z3pbTbNPPG^vaD^-)x1gCxApE4{QL~pbi7Acm6nLviHFX$zpHsBUCjR8&j)TbSEI|Q zHRx?CUr?KlAX`%XUzrQ7wx&Ix_{hn)SXTzAfIjU%huSXodf~jwRs$#6_5NsNQPI)* z;{ozfPh(dCC3@jNYlCQI5-gnIJ?hDKZks5W5FH=TeWTPf8PtYO(k1yxbu*otO-tCM z_B#RcJOn%~G_<&qJGTPF_ghtS+@LJ~vNsY8Nj{<~(^bG88oU??9r@Rf%zr6ziNgGE zCx{E!)oy-S;bgr@cX0lo-49=@c>C|arTei}=D2z&OGSK@vn->tmG7S&_%U;FfYAn> z3$1h5eWzLmb88-r1I+B>9?d!Zr@jBIsbC-nn8o=K^;HjpQHKb(6au|uI;a_$z}4d zt!aY()J4ToE|8bVp3gq6$pbpFx)!t7{zFB}h+lyx$v?NN{|rp}|FQr5SCRMwhVD5T zCqX;yH(MRv-CHLd+FVu(rngp4gcQwlR`K?ZtOk4?^!-f)t33HgJ{-Q^(+1~(aM3Sr z&6E?*j?Jqcq4dk`$dV)7Ital8l}Z|0=jhwsKJkxdumXRNuuzVIPUV!`a(Y}rOCELl z(vxpwkfd!Q*~AMxHG1qF(jU$(>3oZ~?AB~V(GP=Ka8f6Oy=O&BSy+ep=v&DQK z*&KD5`Gi^)PV#jmm7IREc04%`@d0D%eM@@NeEuWF#ih{&LnAWc_>WH|yC0|4oiq11 z$ebWl+2wzsUN##&KL}#G0`wXt9v*&PHZw!i4@&a-X_$KY@j^*8{&5g5pt>d67@BOk zvd^GS=o2x^EtsmiKEUfDWqs?EUO|f#_w~>Z$P|6fTGsWsH3$%eaEdOzJmHI$VU|W%Y`?EY7Y*oZey4}?1@#E$w z;4MD#52u2PW*a||bzI7UcysT@ag#JX!9?iV-!CEb3!t*6ydMAfFF&xr{?i|yy%Mj^ z;Uw}4=}{UbmS21gr#Q>=ZN)L{Zn(AG#g^(hQ3_}ff2K@RvI8P35-*PVu`D2L(W^Xe z^i@}1+fQc@`1qaV!Nz`RQT(=>p8v61Z^XyH{)d5P-M@QtLu9yaJDRq5rKWzmm2=u@ z+M_r}VMf6wi_+C6T^dlPk{u89T?#wbzK7mPuIzUnBtTo1S4-S7T$x+5-wXs~}8+Y}D1-gmRWtPeNr~4N7O6SIQ}Z-Ch&>Y6K%L zq4!mJf_T6LCb0QvVce%sWGDH2uo;7#m<-OU2!_5PS4ak|<-NqO>mw>lH#F`uegLEFcH)xCZn+KRPa zwyjUVh%bHB=QCb;37R-qj*Ax3t-u(Hq--;rz!1-tkDv!A9)eRtnt)ot~Ix~;mSE%eBknYG&&HBSgZW& zhJ8qE(BH^1G|e%;*I!R|BBv-oJ)EN^(N7?G_@D(lfJIEtJMl)XQnipF>o%d16{DW>k zNm#{(Pl6wxk9&9~q0h8u{JL_mj^l@V%kf1`to)m^Y|vFc9h(qjB71_DowhK!eahMH zRI2-|iaRwd$3$R#jBYv~#fSV`~CAk-@(0^l`-cOH05jO&$G_<1^3ZF4=dM zX&YT+*%FMqOU{V1@sFXV%Kq3ygJBXrTD3X|d_4*foc0SlDn|mUizLpS4zIAaQJYT2 zV-4#JE!kFkCTQ*axoui4w9axm^xjbODWz?k^u-}NmS=79_P$~hvjIv$igFOY5&CF% zUFO?39@c3$T?{D}B$JjbojqvYdpo$D<`$Gh8_P9JJxt&~Izc;<nZDsfG`{sXpcS-F5L_U}Sa5VAV4WT(no2B+IIQ?~s6x2^UL$ zXv+gj%Wc;y9jhE9`!K-aJm4f+iP}M@FxJlcE+)8*8pO1@)Pjmk87SB5#~CRP zfDS;9&l~eQbCYuqY)8^! z*9#H<;5qL}>p0oQ*Z&w5&f?-M@Dv#uRJDSUe26?sbr zWLlv;>Pc1!2~ch2R6Wg1<}q`a%_iPeUaVr0=q!IO`z2k<0M7;bIjlt`6@DYm^doIP zkTrkz@LK#ghXV0&2NK<4$g)J#K$l_!2$NKPGdd^DdeD~hB%&g-`%y)kKGMK)s2c}5 zov9Bh*$Yn>o#8ww`Q?a^le3B5&~!|P^g-i$$*~v6j~Fcv?+A2B z&aRX+OnS;jg2n0?uQ)g(wB)o0Q^oro;_AC8T>KuPyyB~jY8|K91}dHI{FJQ6nyAjd zxKCN(;vJR~J^d^flX>g3L*^RK0QJMOVMq61{=*$-aNRVMwqJ-&FUg-F4RjVcsD2aR z4TQq4lI*+6bZrz5&9;#HIQDmtEsCDTcJjnwm}O`v*RcPYpLS6})VB2>%pXu?ZEriA zct1UBm$()ncURFuah`E?%?(0Ik?J9@2cqZ(?rVa;P(ZMJmIp_l`Jv@CkaJ{?ic2-x0Qi!Q-yOY3hAU zirU}KJK^2DKPgyp`G;olay$KA{J|sV$WTs}ZNB16$b;W!LjgH$UXBh`8(i3t=yHA1 zXn)+MME6dd%BHXGjqJBT8wV_@v*rBLzWD?NhbA&gXaPtc{Vqd`@rD+aK5UHSv2?LL z!qTeH)n!BO^~1J>_a7|i8mt8wIn}ac0LkDiVAxXok~CVGW=+}bN?sTn5OCFI1LC2c z91>iwH&=i)SMO1I{Mh2@>e5clh9{7KfJ@<3lG|Xt{)tCg?iFpR#sasrc$MKAWDMm} z6BNmUN!DTyj>q2BKKREFbQLeEfH1V^{(fJi$;t6(F#XaI=jweY&WJLs)_ zpI@dMK|4C8<^f1?JE>ZTwXI?tN?$+aQZEe%#YC@}SJP!> zd`oNh#tAE>TlI??p7CkY*c7n{cpAOt^?Y|QPr}Of!Q6%z321GTSa+7{QCmn8 zsB_f|QlA((OA;_uk2v(_wxpDl$&^tIGAh z0t7V-g>N2d{aEodp1zl}V5l`tJ%eoBVzGNYX7|t;$LbMB26#fu%{J$cQLYmEOhlZ@ zYZL{KV3Ke(ZUsdo7`eGM)uVIj zGb`<|Fl0L=c7k$Y$3>TzZ&9vKo7yIQ0;pp)xIicMsn6RGXP|>&SU=Pd07l7a!cbY! z%;atA$+t6tD=b93=PBVu_BUhm?gy&E0*KRUZ8=Vr`<-EdON`W68)bCW?xAL1rcJHM zIN;6L77Deiv%Q`%hPctwq}PDTsBM64vO~H6(GB9!jG1E|;xjbkKevT6^;(pyDeQ$t z_L^kS2?JQ6hAcl>0Y68x|LSfMF_P`cT!x!`%jH^Jdk)k`l+MBnJN#T?L5nSikJ-Ux zvzO3cjm6=!3b?xx6Fu}DKKM#GC#ve^P=ON8d9l-D!S5t^vN36RSr|=yUv)|9>0I1; zVxm>aqd50`XISZy98lI`lxC@PL59XYdCiX0^}lknO0QcQ)#%1|>hT{CJ#KMQfy zYNpM1=FPnMl$+63LFf>(trf5j%{@BH3?20aJdL0w30#MrP!@chCDlrGi~*`)1&JtL z5BUZt$=^C{n7QV)PkB=eOSeJjdKdYUI>tyT6HT`Tqe)!ddHF+JqJ|-#q99EWv2nnW zNuR{B=xHdyo6eR-3CWkg$n~RBR+~uM;y(8=uLGfU^_^OFBtgp2IR93DBjL<-fIr_vq?)RvgZmx1DQglBb_wIUfz-l9ED)RBM+}4l>E( zq(1)av^8s=^=GfUVoj^=*9KZmfaoitPRRos)02^aA*F@2CZ_W^*w~CV=Q1`wiBVN< zrmdnkr=mBwtHOHcoyva3Rs z`s??2zBx9M|Ri(}cARU%Rjpn`%5ippfkR3Z?ELO?`BB}fobii{B=$`BwBhXND?3J?$|L1mN> zB0~}&kSL=VkeNtAh|FU`!Vm%(yvshP`&Yy5u0DOb&%M9<_(vX}CvNu6Ucs}QS z?OoyaX7VhsfY%(?oneIv;)OB8)}i<<)8X5Mp@TNIqD?#X>QZs}EN^;3-Hy~vZMRfs zp$d9m^?uhEbln{}xMkIXj{?7u%v&02tDB8ek?-<{Y;}3fdqBV%RU|cpYaue2FJ89v z3OE`oCpFPDL9lw7lXocxUo!C*KcUr$Lcn)YFKdJdzN`j1k=k$tDGf8{Tn(BK316`` z2z&AORSIP!s{mC`2Cv*s*^XLz9pY~cy)(pF<)7iW&oGN&rkaR&Je+tc31LY(ja#A! zKxT60(YduLp70GOHjsx4K0);Zdphw+=5Ze!8D2`(Fx@oc!ZOQN{S3-ElruG?=4|x- z)#(JV19ck?vuN-xv%<1rC$G*m?;Ejnb$7c6A6!mL!_vgJcsq`J)y#FeY)W|% zTriUbtLoCxhD}SZGWwmc`hI0$6McY4Y``r)w>AnTFF=-bidR4LN+;C!Dvf0X1qZJI z^;z6pBEq)ZvQ_=t=B0GgBJ6w^hm5V_jn_VruY<)UVBASQK{@Yj@|ZL~|9L$Ntr+!) zK8m}$8G;tT3`Ny1V@8(}s|MX%kL0zjPjq5CGzgMF8r8IHKBrqC_nm;IK# zTd4Zik4Uu%3pEN3tDD2O3gH(!!gr%m1aq}c2&Y&z3sN*xgjctF3yQwh-lVS?#zaZA z2z-kevD={Jpd*20P;rb;o|jCV&v@6*MgJkNN=C0b|gTRLWy$w}92VbrlMpT3a_ zUh)$>gJDy`a?$Qix@%n2!n7|`WA+*9g$y>&n?-nyM2w=a~I9A^;crQQ3TVEKAW z2R5u16jT*b>*OO>dMGbGOxs+r;%gbXC=)6kKRJ&^?4^Dre2XD!AIqEF~+UqdZ$cV_YETg5Jt37(!jg|cA{@)u(AMgE+a(D@&D?b>ltG{z0+R4P>)@G}~dgb%7! zu|h0V+TMlM3wqwPmuZn_BtA{&sJKr!R#OMz%bq~aw>bV>m&q;p`Y=o*WtUK|P{@Ga z5C5`jp<-sxI#Aj`i94b=);Rnw0i6-7p5U0{Oq&BW5|xKNn+q$7!b8y-LbqY$-6i-} z)wX!CX0M5lmwa8>;L%v+vmkR(b5O)dw5hTjOR#rMH=JYGuyxq!HS>Z)ytVgDR*Cjc znkn(Wyer+&`ymO+%1MFFeV+=1kg?Ktz0TB`Axix7n1|w4Ug}ITe-r(f&~fl_xn_*K z9BTYzmp}D>FZV6ZL-OR-xTck~yFGI;gtcS-1|+veMxZls&@ii`Be_ z96v&LFR6%z2t}@c2u!e+l7X^^6fVFV+n@b7?!r97_HL$el-0eV{X)NBooK3pawCR^ zOKlc;gEAZe=4dl(RDx}hEG3bJltnib{)0ok>I1MxRtk`4zEUFHzlWN*d{(XX*&*{@7z%-!7{@G__;CY~pu8zQa=qouQ_+YL0ynnvBa zKAe%1fnG?B{+)18op{FpVP`3y;?KSpZp^cd0txT$yV;j|$`PaBIN>?7(N~#cBTc7( zA3b2$ixiFJ^4p?JoREX;2bl^byv*zuPcWCIz2)7wbV%7YM_gYr#b@B^vKBdz-mPzc zS^j5n@CVzC)3DSA*1jgo3XOVA*U&4W$xdFqUjZzlkK_wQYNv9SaPjLWl(J&O)%&OU zsGtkHeaWg(85SN%G!i+*J;Cwx!4&6`LU9qMrn9CKOx-B4jnJmhb}&}fy0q8iIG&73 zubm1dXXGMczO$zUs|DCINwW?+i_yw_IUKbe$6zOY<-Gho`#&))5|W(N1T!i~Qh>&NZLIL(EplPIcUZE~lH1&?7>ZauNU zF{4A%zwc4np83uyCpH^8*JJNUac5wGehS?DleGd-R4d)vWw!sUzWj!%1Se-1ky!g*>Isbh{QdCDMmlO9rx=r-99%)Q_SgHtI3kC_cjnt`H~}S?l3`!J-ZRjS-Ya@rN;xEmsr&Uu4R(qDc zX?t+t_oN;05_TP);jJ8uw!h-%g=3yN&vxoeyF#{{m*QeHwPI$bm^Yb$T!Z;<0dvv_ zrIo_u%R1YMkCaD}!;0LLEo(c?0=uZ*ZGURwA*~{}@998W1IbFh#cxz@m(7IvC9Jom?HQB3~SI;D}m9 z@cAr-Uvjq-sKNcBQp*7VGFiv z;_u^eoEdZvbIjs;nQsE7kEN$8DFb+)hIkWX5E*4qfFwwQT=*BiA&z#9cyfOD2@`0m z9Rw;!s%Y@*6D!*BQT{PDn$~4ewp~R-m$YX%!yB8{dmkz+qy`|2lU-@FP1!1WPqx@y zZp@FFUJAkAqkK1Iyn?d5OkW_kemT(U)_En)rc|p_PZKDi^GbU(=mzof;>+D8XV4pm zY);fc+h?xEX)2>&mg>-i^dmu*g}_&VTjo&JarOgxaJX%iFs9LL553y@N;Zoz_06R5 z*%vvo5%vj4?UBS&cV05ltwkLqo_Nt?FsLpp=p>&WVbFTI_F^^VB@i*ViW!XNj(M_N z0f*4uLE1-e@4ir>Vm^FyClU}L6*yv_{0Rpa^-iuoeg?}l)wd9k10B;XN9+TgryBYk zgD`|k;fK1!&U{_-j@(`oofJdaA(q+DPHkfIjgcr_WxAC!M2QGdIJK*_Z^;p&e zjQrwzv)#htZNyZ)q<#3!*8#$t8j0 z8xR@ousEc8NNgnZ8?rI9m5Dw1Jvr{EG6&sn5xj>Rcl1Z(em#phU?C6d0y=}UnQ5O+GQWL-F)_;$%XKrPO7gg#|ci| zxpw|dLTM4^AgI^{Tl%VKnPu;~S1@Ts)~2!U1^G?h0@%BZ{HAqN=V74uU;nTnhXPyn zE0v5w;gWT@lY_RKQTJ+9bmJX)0@4ewxyxR2ms-^gL&8bAkCpql*q<15J~+$+%2(Z^ zyr78k8Z&E22txH6uch<-&TmT3UUjnK&>HvXpG?{s!WFZi!mbaoj^!P<^1TGc zcWF1@=I%}qXX#4+^xk{EZH-v3#|LfxVK24a(VwFWDv?)HdpUiN_=F==hBbh=6GVKA|iAqR`s(m(+XK~>h9{S9Nd&JGo+ z?Qxs{H(?zI@oY4%C)=&UQQ6;*3r)I-ZFsVM0M| z;lqPqNlrvTYFApbW|~y)1i0zL-cD?0gvEZ8=?%`j91U`h69!FUEnX`zDci}b10~uNvs^wh1Q(gcJAX9nK)I+@r*q8o zv}d+G0-D&6Ou{fvfxHVgjW>I*`dB&~&?h%gZ{y2Fcv(b(b%FQOlOy_wu`d%1p{!j= z2V3U^u$5x)Bbx@cbs9hv#HL6&uT1&Mt*K^OiBibM=S>1s%27_Mo)!7#8?3Sg3CSvs zYRLSOFYA@{4SHARH&s0bs+*=`3oZ#G(9}}%^$&}1>Y%4Q?HvO-A7uc^vxA`nMBGKk zJ@OqJ+`PuDj8fNeoLd?mgS0p}pq>VM3(S2pWkg6^Sz%7qMLIE6#kdn6V~o}6Eveho ztcEF@BTixJAyFZ$5vro%jcauX&$7sj9G zt7I3UkhP2j<2ZtsyGaZ-)EZ~Z(-oIhMK0pyL2&3134CKHO6r?6f@1ZVTBaUY60y0*s<{qk+JyPYh{%CfmAWIqJL0ZueL^Q2{E(mI#H=&ZnN4;m`!&Ojs zb+-%ihMeR#dc!>KFVLJkUE8zE5^PiDQ}@=?UDnjsBm;MK-B!sczTlHhmA3c<&m%@k zEATW7IaQ{egmP!wlx0{EN)oM`756!+QK}Mxe4xy^CLr=c8p8&Clvyvf#th<>`(h8m z32@X=t1{AjApVKpmYcx|TFos^m+%WQe}A;y!PuKwqCpw zWMRa}VJYhZUC3pi3(F9iK0{=x2xXspr7ErpU^*KNJAtZ=WOQ4`^_!ToB}Dh91%&!1 zl-+rhvVC%}S-WAE4`oW2_}d`@o)1-~zzVxbw7qkk=}{ z;5D!>eQHc#YSzoOEo;{AwD5B;)%&D_`;JAcqWt{4@BlDfWQ>o%U&Os!O%Gchb7@$%c0W%mi{5o;U;m{;S?q%&0oE-4nWF|)a?O8Gld-$( z(I5e6ZyjbG`U-Z^W^^i0$)=~ZC(Sl;9hmY&93Y&P5*~-<{G%Tr3g!5E^?#dh!G43{cD`N`pkc zFz1UvDCd<_Ul{z_rhco6a#zO1?tty~<~e?ajV(sL+LciAe#7>)1$Hi0gH*=mvp~pi zlPq2i6;-zqUZatkGnI}j@?)}yoo}gH4?D^ZmgzpmRu!Ud36eVlniqoId2RRoTU9T0 zE1xOpEgt%j(F_zM5X!+f^>ii`l7qB&JPkEas1qL(rn&K?VJb}`BRhHk&c$j^*Fw!< zeXqPZx_@ldDSLbS#xeKVvpjk|OU;GX%@}304Q3z^kwo%67dx-LMQ_91!dgkf)G?@= zPxmUO-KDMJhuig;KxdzYv?Y{P#^yy1g;w+)QBw{&Typ)e7=-t*b&rJ^mT`?vV?Ec+ zA6*FEbk_F*EzX8%sOS622LK?m=e}%JNZ9(o|H0w|eH6jzKmdw<>%DfZK2Xr{oF^-s zhwKpEwA9JB!JI+)*k8-Eua3Wdu$ADL5vbt!Xz}TW=;-XcslfD_pB+-Ro*Q7CCTC|d z%oB~kdY+1N0U-4Lzy0}dd({&Fwu&N6qtopRE`O+-G z=Y`{BO5hU7k3C?qE{7ftPaU`1I+o-u27V7dgiwj5Gz~M{;~jnM-`m#zj{f_6&;L_1 z=X1;d&$j$u=%W7;3*DP=1(0OBOHo4Q*DL;1XV9CPciF{!2Yae7J2vekrmP|OR1)u~ zeNV(|pIfIRo&`X?Ocbt`v{owGbddPEpzd)g8OvJSX~wiKtthL$qvNBa=9zofzQ0sW zuBfQyb!+gPv^8tPMnimLkAvv>+%Rw~xUf2+w_2`9yer9)zaxb5;KjkIUc>WF8j-l` ztn7-4I2I8otrlVT%?jnhHoDx(MX%;5#umc76*G6nb5atu+N})gyMoW_vfMVz7_D3I z-ll4ZMhZQEg8DdO}R5O=iiM*kX%I zsYLN!(@7Ok-j8O`nh-VDTxO6YWd08JSHxgHfE`%(pu8x6c4@%r1f z-xd=vg(eW!h>d2*(#Iq2328*ma$rQU(+BD+r>;TO^)^7n)XBWdz$x~$yxi&bCa?9h zAia}nKJhKc>Qa!wXkqh|R*baRmO$DV6d%ZaUS zn>}0uyJ5*_KYNdhxw0k`z>i&4DFevY*oi4{-(5o4!L|1n zW_M1ctPr0R6La%Xal_D5M^agdlVi4*I`Uh*{L1$+d)^dwNM?oWaNdq+r7$Cbe|yn5 zFXfp^b$aR4)~vomOu|rU|Cx?f^;>Atx+%(+ez!!<2l#tk= zqkOJ1aodNoxYn_VjL~#YSNBUJzCi~GA<0`V-u~pJ_qRUwPQTDH@$UI$%JTLRSs)_= z>gq;7MjJ?#-slAqGt$at;IhXcr@sKVRQO|V_8)P#|9|%X|B`^R#qLbdy4=ScyMf!P z=E043_~Wl#_V?&>AZL1V)5p*Gq#*Z3bl>PV?7gnl`b}(UVM91FgAii+_et=7qahWb zA7!M+m*9Wa10wudgZ$}+6HSs^82S;&f|J9o%Lef)W58bVCT0T2LKhwbG1L;&q8yWR zAq=vK5RM|mZT1c$EA}aoMD3w;tVzjm^LKQus7Rnp`z%u~^#UB}HoYQ_uzUrGL7Wj1 z0Cj4kw4E($wjFivq7vqwzvEy}wUx1&%_u;KljnLI(^9_bOZ5h6AIY_69$pT&v~9~85_ zW1WJQ+`Ty%PpX`?2ox~Cwpy6ImwB)oj}w1A*m}g5TE=mo6OS3j+~86IBD$Bvtqz0V&lPr#n(kaCg%z$Y)VU!ar> z0rE+Elg^8RzR42H0a$a8ac$ASy?}5#A&{;Q?ZCJL*JA1`yqpjwU9ny7Y>U8n zO@dH^`WSV;GW39xekgUo9hW69K2##90x$LyqF7|Mmp!%BWh?LKjN2lSd!ygHEW-qv0x_DmxUgZ7C#E@1eEFkHiM!~N;ijCo-Yd)xS;GA~ zSdjyYSs$**hsNYiz;d4?PAOpou2w`;ea1fMhSqrO+DM%mqB_m-oKlB|Cg7y;HzFpkjzXjUoF} zq|^FXvIq#Wa;F%QQpnX=x$s8n*VPC3kU?KUk*CXHkZ48(&ocOFc*h=UFiGfiQv&QC z9g`n*&qaN^g2iAExiG7Mpj!%0%B`!c#&6umO1q3sFCzq{l9!9BcYwUc5!XSWUQ{1~ zj59b=RID?amUF2`oe=M@t?U}KauO5DrPHVcqv%B>gDto#`8izCyDGWy!JzIIRH!X~ zU$&R06rN_#a?@(pGWCIL)b&u3kC#%VJX1{o<2B9D_@Hc_beFrht=gz)I!0Gu%3<2> z#MK8Wg2k4*oZ14750cz^{_W~T`h^~?1;i1Z+2W_AgoojChzE+_(IsQv`mN{N9LwXT zYPrhOUTbLvl}!uV zh-<;Z7qr2|vmonw2J@yzWR7{?ATNDbjqh09o9FZ>3ao`{|6{5=pOCbF)8~k83}nvQ z#Py@h4~RC&rXFA<2qwZ_W*tPZ_)#XO%z=@mh#5&iHA!F5vsU(Ehp2r>sWmAQ!vSEW zmHSiqcHpLS`>@#j*4~tRFZy3G-R|~UsUc6MiY9v~|ts2pM`zha=xcJsz^2dQhEYyTwJ z*bg$WClTrgOSLjia2=Delg9IHkuF>53t>(TAx#gIZEP$qS4R%dFfNx`t=HUHeFNXR z2kVQ}4PxO>KD#sjQKpfjQJ<<}N~}rL-Qt}o8*_%aH|rk3wse3o5FE8Mt5jN!CHb<` zh{?#53a7Ssn-8;!!m~C4yv%1od)Iv_9&KBme0m<;LvW;l1;FvK+U?f=E zXVN>@fKM9>3tG-HN?Rh?-7qN|a)=3t*=gGZWqT$zK}(ZrW3&Ys0e(tvx?B!}q>Lp^ zNnAwL9e^&A?BpBRTTn)p^t`)zFHY*lE;=0`_?*?u!1^2>Fk~qKrQDt%G*+zXQlbGC zC|jSfon2E~Z?Y}&n*8*a&U9L8U7OdY87+~*#-s;9{JTR@sf@Ej-oj2gZ1PTpQ5g%Y z6V|~gi1|XkN8McrbGOpkm&lbj$m!&UclfrpKOQ+)IS%1U7*o?( zJou0PZL3i3v=bdT2HmD?eaWd#{n+_$-QC^yrlqBUTu_bknm=z%K5b9_@bk+pE$&y_ z-G8i_Z}kntP#f5qlx(Px=Q!}yK<2TbY%qQ^b2{8V#4Jrd4MpUaN2|?4J?(=m*84S* zgTlt}1kb!Dw}o|r_mX>?hf^b~wIQ`kh#X3{fw3=#E{nP%T7Uj~+Pp6U<@hj0{YI&n zI#7gXs0zLnW(})GtH=pEc+3WvZUja9k{8cuir;cgB-c&bn$or-)5+Jlh|fucqyhaM zQDXFw$Euky_{_I_TD@vHsV?gwm*-!B2@D6bh_aVPw?ugk_cCeWM)JeD` zz-oD(WpZ11^n=EXDc=cf?Kpm08zDPT%u{@SQi>kN`*P#F#ZCJ8lcStC-My}JuE6xz z?Gw#_g)hW-%F9dvLhRS5@z+(~1J(1)%IyA0LFWKj>wtPfGBhFCkvK0t*fY81e1$5| z{i^$NdZkIc`J;@jP!&6+w40xoYC;znKibqHuC$cKD9V6sPdAjX`9=eILL-43 zU92Gb3Pt)bO38gr8>Vh*p$9QCT8-X6WP8S@E@i^g2v;I7qgewixSySc-s++FEz(z; z#HS<=tL1rFF1^>Uo45^rfB5~had0p$W!OLC5!c>5Ho0=p(b1mbLw0%xV&D5z!6V7j z3$@bCLMGp+ZeF$B8f2@U^8eVX83(p0h;`R_R_!-z66-U(dYe$3e1nSuC;rV#Jnw^!>9h2nJU4MWOmdnbR*fWu8Y0>Kt5+8q9E z8YXI#wS=YbU)^RBz`xn`TDqr%C&`N~6lT>MK{+V>8kO%DmH=*A?$m!+h$W8##dhzS z5143}YP`OuuFt-dMu)2OhN6JImwbmR<0aR$RWQB#$7M4$Z}VA^)DN@KjMSLxTd3e5n7 zqBOV2uv2u~^d8(WRFh~J;98lC8P69P91D{R9+Jq~lx5D%!@4*{AJTAR6`51|aYv#C zV$(84kpeFrvGX&pmrPnv(WKyIzs2}Tb-G<)v+7%;1KRVtEmR|sHT;mxAY4h=po+|M z?lvT4YW6Hz5hr`AVkPNsOC(vexKne|>s94hhAiO3*Df^TVW5%YgI9&vI;{@RD&LAS zi-NixF$n$Cx4j~umOkLZA&0}c9Y2`T8?qFuO$Rk45ZjbFP1fnz^Mp>u+fL5`M6iZW zNqGXd9}AlvhRau09w?qH(oV>!&RgCbr84VbR0`0Up8fsVbxHtjer$ zQC2{Zk;vczpm%IFIWnZW)+<~+eMNgy-8nC{I{{wVcha&Gvac+aJfXR)`hbj5(YiZr zo9MIod}L4fp4T{C<_{!NvJa+gz%VU4YRM0{k^%(w6~(6~A(&TI4@2HY?`RziyJ~Z@ z>rD6c+9o$dffF}<^2wD7QU(_*ha1D{^elUpc=SCTMddp{K2+%Xpm#p4d*KM1TzMF1 z$Y;SEF$JWbfV*_OB>8$F{rDeC8T}fQ#@hc9u$CFo`kP8g-k=xS)+GY9z^3@f$YBK7Ice z`}b|GO_Z;qrKc2nQRn0c0GhNC7qb4GKlqm>eKhX zI==sN*{X}()Dh(Xkk!gswy%2e0`tajT3QL2!}|Y6iVU7oRa|A@NC8UL23WLZwQv~0 z-wY-2x?$@XIVpaPq?nN=4RNKGrl^&s0XWitFb*bAd0V~Q>L%(|3iG)zNcxR!gy1Fc zX52ky9Kn@n`&MJY^Psq~on+=HgEqJ5U3~sXe9?Y3A zghl2KZYxjI+37Qj<50SZK?ANjoQ|x;1kQ3Y1(qKmg0Xpo5iX*SOb0tlRlRx`!$_u;#E7OydPrSdPk{gZ}# zF4UGGNQ~Ks;j94pMAG?_JG_ZgVC{L!)Aq!#N&=w z`{|@H^`XAcdFi5w_1>n`e45*>9a_Lp$x98H;mv_cMtmv*31P8zSKsK&?kbIGtw`G) zFX}YxIIo5+_~CVO)kYW9fTzvFHtR$Usdfzb)E6sU1Lu=AJ{B|3m(A)3xf+mxrJeGm z-wK;eqxRvDGK_xIh5GQFrA`JNL-cZB4ywN&(tU~5Fv9greXbb{vgeKV7zVavM^iDd z>)0Fd-#w1~U-0(|`t0Q|LBG+}LCE_zP5txT0m?qQwv`m}6YV07t5;s;DMrcj*p`+s z8T{c+P-Hn91siG+C=nz>ul0_roqWb7rA$A>hL1~f#nTjiP^7kr#~?7$S!%Q|?a>-5 zd-Bw$^sABcI0D(o;6sQ`{E)3YQ+)ho$D*ymw1Yr;Fyb1eBh|*KPfbb#am-N=aVZf0 zmU@u8>>w%3Sz8K}iL)Zn0R^`Clg2+LvR{Amgj1}&kMM~s+hU)#rIfoIv|+K2{s47S zWFb{#M^Fh60;C)>xVZ+vxkR%z0aYjk0Joz6(+>#qkUD8ksE;KOYm;xZfxSngE4u=m zEgC!C>qcmEwXchJ8hl9N$OYV>c%&ldxQs$_&NAR1%dbP96h76Tl^33p(>s&Z80exB zf2r4le(f5o>Qvk|dspBu+M>1$jIat{s#@5=cL}M(I|ejk?D;#XOb4rKo>^Lf+}>ID z$nVCIE+!iKbd(ZjQ+-F7l@21RcE~^)U%64L#fMf5D0k>*WXXAk&;W%w<)Qo4I|n89XwwZp(H$I87;*Up3>T{_-t9hf%i;6=HX%% zj;w!4KBu5abHS!(-T;O04>XqM~JQ^ogusnbaxWe$?; z4Su!Q7YEhDH;Od|WxLDh)ppsQsqx#=v=?`tP!8Vy^{zt>KUltf*>>jMklbV=Qop1}KrZOikz;x!*-62^hbxT@Gd^Y zKbtW0b+zqK*4E1thKB(cI=()&*-s@FxywF)WFH&uZhvXXaEwt%FuDu5hFL8(gKVaj zs&}`ej{}d_aU12ZxFcs;I$+Y-B|mOO8j(1gh6PO>G0Yp5Mz4*a@&Z7ke`0LPHVrmO zw#dt3K1_kMX-qx`ur&{h>O_$3A7#t{H|REX&V(ou=gH4^)7XTD4XPbk%Nv{pM z;92!3aL#+>x-9WKZhZ0*WocGzjydK-4tIWWMYIj&ae_B~S05ZBgk=qxn|rDcZsbHb!trYkaiob)fMeEiFt35%wIAUHHvoCw|vSU4rh9GHGFhlB|axWwH)~trmQFXN^Mr z?|$*`|Flz3{?Gnt2gmDo1L0_h=KbIOyx)(Z{}|a(f^a=48ZehvVMYQM95()LsAd1Y zy9xPX+N}JH*h9RWD*fW4%p1-`vD)u1`|VVUNBnkFFgq^)Zogpl_x*1p_~+U8dG>t{o-#w9gXb?c zo1cT{=dH`XaqAL*9c15^Zsvzg_f44X?;w!tAGQcq0-wp%N8i4EsqoufZjU_jEEn?% zkCPAV(YV`iA?D{vtGzw>QVVnQEzqd#7i@sBiw|CM({>a(l?RdxyO{8mk*v44_D?1* zYrp01s|~XrY>U3man-xrKk0C;Vbh|!r{l$NcZ!ZAxtxiOk6qDvfmE~b*jb+{Pm;iv z4_nM*UZp`P!a|}TP|(?`bvwTc5|Y1_x#N}S+(bM64ul59sv)EI1OW&i52Rt-T2J)0jFiwnDF#gjBVRMBCUH48Mg7(9*5DC({DO; zY137wid0q6JKHI{ynmVsO7afz&E1K;R5Oqiwznt=R_5z@%}oRtl|&u(0Tsy2&BRpNMZ#UAu>o8_!YC-EzM`)8rW~lCs~i6a@ve$onLSISBx=gi z?hBT0S*;Tn)d9$|255TXPd~qH(kuF48>y?64o*PjTEJb0P)v9nAgkFJ%^esUOv{@a zfmh)wBiZD$)xzyhS)NsH4`GpHH=V5U+vPxMh=UjpGD#D0H+^%^`n^m_8x4)b4<)JxquaYY>8 zoM_Y7QKRrIZG^q)G2nLlXZ+!yH+)X+6QE!VpzmLSzA~D_s*Iy`42@{OH%I_infBmh{U!v=I*aO>~Pjj z+*UFel9YSD{$d^@G21?`+^gWI{)#@V3v(@Nt?Xp4tx|MHXj zO9r9RZSg0786w;QpaGmEK$3@>i%pO4wnj=8n1SG!gK$>x4|4!KD6`=uwYO6CJIKUg zymWoma@GM%-Np&{f+jfI2~8>BSh^q7U8=QHCmg`Jix(OOG`Adj%FnM9t4hO~xW^^b zpE2!jaPfA?yxIBRU1Mx6%G}52652LTx+pm*_UL|ixs{&qo*CRq zYOVN5$~6GM9@Ztk5M(9C`@=_6nafR*FYUn#0Psa`R9AS_R8|@hrV7qdB(~UGl~*6K zRkCgcB9%u{h0!YxK$Ez$S}mJ~ML6)h(ywbPHQ&0ybcH6G#~x;LKG{jW!e0GnZ%&Lh z!veN{mY?V+402p`VY>tHjIM0@IT})UWSN8@%{Hi7%)SgzNHEeW5&?AziYkh z-}fuD!+NR$v%I8QxmFJ<=YIV>hX3o~@t=-hpXcj8wyu4SO`q4C&+F?ywRQU(oBn~FG*ZeZq=9cWY;%*cW`^rOtxIZ!+j zd-Ll4^LdkJf8K0`b7CV|H7d8Qoj%W;Qp%$aWjwKlkvc~@1HQJ;;fZbh1b*7xnV>M&QTb*q7O2@RhkLKo z-d?9klkOFJK;#UR5eRV9i?Jt=D@bJ|YFBU1v&12H<_^Tz85}uC^j&e1HJ|fiit&YM zbM%1%cv+D#w9M~M4(7>%VedC@`=;m0!bYEND2;CFuC~tcBoT?QNUWP%aNubsv;VTz z%`@y7tRbwVD15J%QOJ0V|74DFFt3GoGBg0)(X@8>){8Y(RZJ4K-8i^a(IMJf_Jpn=(ir7*p&|JRseDOi3S9qW|sM1n-`| zma+fpQ2r|>R!1+zMDi3evS_jv3>XoHvltd>OmE66Un|Z`y$xumE=UQbcs*Q9{wT8r zqc@GZC9J@&@SQ%&SbB70Hhh#BImT;~+|$M^oTtl+&U!1cqF&p6leGbrQdbKb!6D^} z^9xwGpW)RNv9R5Gnp)>AbF&={iMTP+WP$ zP{{2eTUF8R;sZw$vg1rU(H$TS8xRiTqQmfgXd|9s*w|w$Y8bj~~ z-7UiGv-}^^TL;n#tcNT0ooI5_4ZSZXW4uYNzKoE`hh(5&5TuDI@&c8MI3lIDL%Ip2 zS{LZY^fHre*Pr$*epVAFYpS!&&Ud%{C7{C`f7rm@9$Eb?92P9J_z=mbiuY-M;H;?z z9eSKS^FExp1b~rZCYdXyhURWvJqU?escDKelM}KVjREMvf-&n~4EMAHCE@kPcX2-+ zsw=WsZ-{YaO3RvKeNEbG#7;zRanq1PyJhL%GE>AfNOuOU(C;}1N5tKtV+xhZ4L+d`W{=Sjo4{x@5F zS)GTukN^bG7l*t?evW|du zE==X0JjKrflsr_ZW6v)B-jp2nH#4+;Jxj)pKsmijx)MFagtU#s<@gZt{f5~JXc%uS zVyH%=k-?36Dx4oG3UNs?%(ylBc;P!AUu{7xuF~OM;((WtkEjJF)`WXdBNl?_8L%)= zzgOlZSpCAF8sjO_;SAWOk%CgmV|`0O_L{abLJ!-{h1kRNx+)t=TK1W;$&_>N8(+SE zL%!R*KGaWHXKXoDHMr1sEijbzT@2HduDdJ!s@p(vp*~P$Yv=5hE@S{UtK7NBND&%z zN)hlL)&yrEj9Q6Q3(a~1+rHl3dBOAcaGgGb$PDewgn<01bLQ@f!{<_LwO!Lj;WfnX z1Y#z|9-v4;%n3KZzx!jV@P9foe8PeMZ~3gvM7_l*2%Rn&kH!2+s@^#gd!?$~KG`i6 z0B)Y!sr#V76o!EQI!}0)Q9CqSPZ}D&KuLXvxC9vpd-q#*ogv=PGIJ&I4LjzIL>E`pdO^OL9j6x&Y_|wY=No+TZ3I|l zM{RA+mFQtF`*ibIOz@r;%horjwa;tJfOScW*%4IGdA?#YpvE$f=&BR=(4^o)@lwaS zowenOVG0PER2#MRVkJMS-lLMYvnkf80d__)L{9u|?Q~vMZAbY{{fD?OmLKlxr><y@$%u9XyN5WzoOS5Y2f|1fM5bQp zW62+4%IW@2K1yB&M<496EXpoR98z>+>bEzxl)PaZj;hYkvHz*F@<)Y{J23UJK`!m$ z@7e}qznQLDtdH?QtTubIC-BXDo^JExjYa2oHj384m#V zuPy53SJw6cfV~hSHFZfC1p)j^RgR`ZZ;_s_2$bV{siyvjg zAvJ0=GB9Z50jeqh-W%+Gb-#+6naJNh(n7pf^&7~9DhMNYD?!#^F~lM@=%b7&n5I+> zP+^q)>-*gpHDUkanIH&>L*j*}L0`6;wE^{)TcF*h7UFegrJT79?fHZ`@COvEcg*k*iI8N zb$M1^vJ4U1|Kd(DQ3rs0(QlsBjrnV*XnyJxhn!z@`!DXZ^|vG7)02KRgdG0<5c*F_;YXkmqx&+M}8gwpGUyw81R?C{5b}EUIjj{0-yH*pZ5Wu zcY^vd$RINLwc)<421tu{sBgy3%2{u7gpx^ z@i6!hJjsk21xi02W9Sbslm2bM*&&*1cIHJM051&b(n^gvY``yphQGR`{>3fzzw_@G zIB+Pr$E8kZU?vQ_f2F8j@R$33kq78GyO{VDaL#*+>#?zx9w)&U!J#YJ>FE~~m@qO> z%4W;SORPHq%2IQJl8a9EJYgv@uU!>WNU%LmwJOy!tCWN=Q>P71BM>WUO8!& z(HY0lX#q;V3LjDb?A6AMdWr*E06T zWxTM>afTy^hS=ciOql5`37qC&5d|u^gj41yx%~R+C$p!*UwYYq%4;md&2d>(c|B%_;eC#%^ldYoyx!dvWj33> z*be0$t)Y6jk3YeUeaZYNV=^)7I%XOj&Hs}B?%A^`Ad}IlrR`2fifvU=W&2ydC9wqh zX?Y2b8Qz6hbO~pI0025!3m2Xap1*F|+1inCX;#10tarbQZAseRSYk72447=`9poZY zvUXL*KbzbC2EQhLm`L&8>EGm)I~79AGRxYr)sWvNm)u^N=ISOHZz-bkbgK#3!|-S` zO`gp_Mm`*OKZGBgGOmYz zwJ5PTT|q3T#Gg+8K&#BLx0BTizMdP7Hs>`#RYq z&mPLO&KLj?b{1(vhNv;<)6(sPP9OdI#f#UvN7Ahu9yP6Fu)eZSwkH{ma;Df3JBrXv zBs&*_D}0aoaCHK&_esv=iW~L*q9R4)$CR1i`HL1n5K zpdyf}l|iI9F@}&TgFp;XnIVv>1r$UC3KS@yG6^9dkO2}%lu?E-50a3CfS?Q^2~z?Y zykF1sd(MB)`Om%Qzw6$0S6#~meEGgN-}LVN?!BM=JWN~>`@*AVVYNX>3c__FsH}W0 z$2HMsgj zbCh@TnUYEOj|yc`#xk|i(Lr^Z^j;Nh{{C+|TFb`GDeI8lhmbYnvwya37!_loaYq64 z2}ThF(8eDAs8Bp;UHUvxzt7t4Uo&r?)q#H))t=%XD7|Nh7pfLmS78HP9d3A+Fl(@m zx+C7p=AGr`72WLN4i-90vNa(zH@8>WlYt;Rui`R#QVzFZ^gT`1O$-XB{h8rxpmbNZ zRPDLikZ7X+hB?I|km6QjY((GH6B-9rzr@!YvQP*{^WXKJs0ZoEc^6I%FDA+W$8t~)C8+^?7I@4qiGnt6k`>cT^BC;JX}JJH_h6t5Iv6q;vI@I6?rPZq3$%jk2L;v#$M^aBa*hgi|X> zZA&^R4r)ID^+fG%KHOTh;TbkpU4hVbFPN6#Yz^-JvhoPH%$Lb^9?~StisiKcZ!?%4MN`5QEjWxEh#x`d=tY zY=-_rvUS{FD;k?W^;hMeHaUO08C@*|$(@$DBw5lR<~P?>`kU**{DBjb>$` z?Xoo`9%$hOAkI}Pw3YZ6(<029W9{-fJ^6iCg#I~l%0fqAw`zo1mp`!AG~eU@l>2hE znk?AnAto9gH+q+&GRU78y0&DuKTUVMd<;&3Dee2LnIqNfBPc&ml^@SJX@k|QW!S5K zy%ePV1K_Th%bGai3vuwoNG_M|*($=U%P(`jnIWZ}d+6Qt63HZzWh2-P^Cl(j z@8sog+ree#WPFYN0q3s`zlZ$qI z{W`poliM}U#8B$7wH<}JkNs4?F@#UZQ!8xC(jzg5@W^u8iegqnXAKi(fKbO{j9Cmo z|0-st99WfX3h}=g`gW0E<$Z}VJbiB_dOkN!ZTI+DdAa|Xb6Aa($}x=ca@-FNwllb; z&9yZ5K+-FVCpClJqI0p(th$DLt5?i7H8$l~^)_sMED~E!Yjm+h2pEw4Fb8eK#zLuXLgVT7=|)&D`{ZnB@+Av^!;Pi zl&+@Q8+la1-Ad^(!q6g~*<}TH?hDmFb+9B!W_H!x52!HY-wASYY4-h@$IUhU0KRlP zKlHZX#(5}gi{_z^3ReXyIp6K^I2~Hj^W6SmYDSU!jzlKOB%UQAp@+SjJ}P{%lCm}L zi?^PfhxxZo-{@rbthk=5f3V_u21RTuis%)T}7?`~Fzb!jKzj>&E zD0(->FKyTjyjLBKw1_BsA{Wp4>PnP6==C}D%^TlUPQab^XpNOe8RXVg`W$xKuWj%Vss@vK} z?+zGwAaOqwq3=C?vQpn&45|<(eJk8M7nPqIISIS7(Ka`c>!PZtS$p&`=t%7eV>S!W zMH=QcsC#Y?bllq+K}{6|d-OM0RlP>O_>k$UjvF_JjD<12=N@T; z&|v6|bN}GNh|LHO25*Etyty_qVDO}-a+tM5oMBzan#g}%usx(e(S43M64i4*isa!^ zcHq=Lr`Q0!}uwjvc#oCOl#(2ic%nDROYF&8}MPt_P??{nDZvA>w? z>l|Q?2}fmrRKTcL2Vri=T&&cZ$TwX*bgQ=}7#UOPs7i)u%Lyo5j`Q*2f#jV8zT3VZ zF7TK%>i7e&V}a`->}_ z01#yTurr2hFCPG{SVjPAw$HSI$#iUk(JVi`Mq8z0R@4ceoGU5I0MEeZHwf|?09x?W z10>c3y+Z>tZoAW3D$sYsY=Voo0h8|_P@VbvO|GL?91m=xy1EUP%peJ74C_+R`z-k% znq@v%pt6XE2rfY0hygu~eFZ$=tIv{tq6zjuvnhi*u%Ida;+7rmn z3x=ko0PyYAFPMr9%m>v3`L=|0&>>7yx~vHEAxr@D=-L9&^VPrK;Jl`!%Ygs+lCJ&D z6=DAK>Hp8JXoM3)l(-X?CnXnQY^Pr==`=Reg*dqJD<&T<-mAQtRwer!uq&^>1{e>f zfC}Md3_eA+lMSE-UfBVQmF+42X$$og)8^khr?(pZOxyq24FMj@e|<-`FwFW_fmR$# znm`vce;B^8`592%ye7A82daNXK!6F_&};yf?mq%gdr1ZKpEt$aB_`uzzK8T0OSdI# zbhygz`eP=3!K4a-%)(I!!&aVv+J>3X!ARq5|L$-256D6!=-+u9zhC-)o(8`^^?z;x zet+SApX_V4ajcG&?TOUJQU>L%$naS~Vg(Z>6f zhUm%<3>LerSY!X;!R((Nai6|DA9?)gXJ54+cz@tPI{Yjgh(@ddVqB8^zRTXl#M`f5 z+uL`3&zNH#s@grNXH)#erQ*$|o#O#*MvPN11o`ZX=ZSpli=XBj{#gmIHk3I!h<7}* zXh`*KY)Dt)gdd||(+4dvx0PQj!ZqM$Nap(7cOyXu6GRB@nA{#;cq3`x$5V6qpbMio zQ@aVK0V4Qtg}D=AW=q;P?<5snGEN2nZ1EXsRoLfx3iN1Y)-u%mr%}2^g+)kI;U>_c zG&8jjo;wGHN@}K?u zw+ooRh0cRzRU$_P*OveaJf=z(43}-GcS@8WZ4_kvq&4rjlzy?EOsc+zNd1KyP~s+S z1L_FjK~jC}C;MaP8Cwb_ky`T(Lp<-s!Nv5hoNcl5h6<5eTX2vT57!?<5wPGEvyF`y zihUaUQXo;Z8|buX(qv{r08aBIl4SU7OTj*Z%)FHsUK$wg5udWNac8<|L)4k}YwGx} z9f_km*xnZ^=%QE{r`anhK!9l)v54W7zjd$-uX$~-zS9l z&}xDVd97}-b=J{*ht?qvQ)-=?^+1J2kK1?o9v$_kS)_bS!*D6sZu(^Ml}RH~zyxUj zTtHY-bZJDE#B5mlB&>Pug*THVrn;-wgA&P{hf;TeXws`#m4GV!umNjr`xzWo>$2=( zRXzYNopIL)n`SP_IF01S8y;fQ30}B!{SY4^}?K~@vnaL7w^c`)C8I{DhK&Ea7E;4dXulfq+fAZ#TV`jRb5ymIF(+Nhd?05`ny4ur?|sea(#$uENVmN3*El3KDsx(VJ;YGdZ-29R@N zr4Yj?pb(nE$Ipg3ooMpt4kLc;>A1Fj;d3_1(47ja0T-BYvfjy$-GpqBYHz8_DfgEI z?U~{|RAcB$EC7tGm8ufkgCv0>NFh6l-GZ(#%Bt_=8**u?VvN8E@>a(yXDeceQo9e9c*SQ5nre zy-L=2wPADjFz>l1g6#8+Nqg`KrU5eypV~sNcBeJsO@R)IDQhP10_ima1g3MBy}qI~ z3f%@OH99YmLA-4T$4I2bVGQHN3QavCpE+sa9+4=IF&cP(C1;?g$MLvV+Z-sjV<{+u zw2fFY;|^iK2%`>y8?ToU_ub?jhgMNpmoH~s`C?=i8lF?lUmAW0GEyh8T(Aup@RH_aH!R=u8i z>c$C9y~_V`_=lmDD#d^ii@5r3*WiuKO{S^^qDL+>53?Ti|4lrdpx#6S2N%sugJHI7n z50ZkWH7<>g5n6y|7rmb>W1>~o8lRml^RiWbq8si%R>-1sAuA|Z;R4dW;9ck>^Si4s zRI}EOhG8(1?)q(=z%aS;ml3u`x`)K7jl(twtT7&&L9uGOIidkxRjuW=Ymr4_^KWaZ z_4Q;b>E%8BhgB*rkveWeC+t+R!*B4Be(%LOgoT^9m9B4)$b$|D|J~k+UyKGmf59ln zzP;@la8ea0sQkw0{4WVK9OHblDk=XPAkbfr|0lFhM=Q(?EfN)a?oGIVcPQI45_9ZH z1vAcGR*I6%LimFip(*lba}P7)r%$&RFv316u+!x-^N$L9ZP)T>A|SCya|Ar#aAVSk zBX6nqemsT=0AMFkPLdUl)DiUQ{w{0APYbVaY*#4zkDnhy!6wN`~PEa zQxtIc?TBw9dZ`prP0}~ZMm}z%;S%~)uFx+oa%NqfTJX)o9chu_ z$4Qxm;#?(>|L3BA^v)uH-owyY3nQ(oJ@}0K)-FoRfj<))f_xkxBN_K|@MWKaoo|8*jM%t&)6@Bwr~I&CnVn06mq_kBy~H`)35P!<0`;&M!^3&2CC|2I2UxmZ3eu;l z?VuzG-d+6cADr-S7OJ+A0M$5EXJ}+>$R*dg>m=^i+1^tvTXP1Gd|9Cij{o3L9ksPA zN21~bKfh#v2qAIIMuL;iM-9{e>81V)_HL-wUa7*^pdz&`DbsJ=*(FcI)*w*G3%OQj zA{+d6*QfUNf6W@7ti0w>L<+ccBh@`q7joj5Y$`>(OFGx{U%eow1A;d%D4I5;(iLOI zw2@uk0s+!LatW6`JWoe|PDkkkpW%erG4JF&?p*7UgUkjZDzEiezHxUCUiDD;^bw|f z`l>(gln-8d)PVrI9>;=j_SN~ifFc7NSue=YD`1B^^NrWW&A*nf9r^4Z4p2U7Ay2;+ zl^L#{{HvN{^2?4}SOAC>_?}@XA6!G3K_EdN74#!mq>`|Wi<|!8DA;JVzDvN&P`LUJ z-n;*%dA08*{Pedf|NorEndrOGyAxG{XV4YrYj`R#Hp9=Z8lMmSBc|kEgKPXrn#j$< z{85M_#$^cKc*?bA$W-Zg1`3CJ@2*4gV-b9uC8g6rkTKlce}SrOIJ@>lR?(m&`T7G< zlz)VrIe)o9c};V#lXEmBnbx{&QeG-?VI6@&|fyghz%Bwch7Ha(#ur1Rb~8O zje>a<+sWBb0Sea2&mb$dw(@-+6=dzu=>2QjIeN1Mm307G`sYtnK)?rHz{I%MA;w%D zo=j+=DOIgFTIC{Ovm50cljvUbKrF&+S!2n+f9Y0~x$2q3FR%@AeNl@CQL) z&>jyu)+69$Yp8)@bO=`C0=BO5?bLcWrpr{$i@oo<3G~`&%UJ=C;NedCKeN<~Q5q(+ z<*k*`A~jpm>`m5l8+x;gpU4)?(fGkl@9EJ~dM%i-IZ4Ycae)^>xN=EqNfpN7YG$$5 ziQ!>)&T2rhw#%lJm}i)gpj04q@F=K#-;$DKL25zjT@Zd3wMgO8UF|ZLVjNrqLlzgJ zVkXybCI$7sVTYI)q#4tfjat{KfuebU`XFCpSb9#i9#hyJ`JyD__OqZoh8_K9x`yK> z8)%oWU%YWck+f7o_vAFEj8R@ewym}fH=HW+Y{qC-SCeuiSIAVe%;F4c@8{$8hnlsf z;hA3-PQ%ZKZ=%P7Be0Xsqird>BH&G4;G`xGuZ#&_p`OJB^E8LxwcH15i|LDPUFwT+ zXHcBvwDg?7Hzm%}dot!aAgb+}B-aXLo2GnA94y4c%n%OI&a=*X?5|k7_g1HRIPIN- z*KU0zg-yXAUV~ES$uB~S`wc_tc%$E^IqELiO_5TeA*mI2On{?ENMx4?F{_-wsWoq| zierdJ&{o{{Bx>R|177U#3s2ZNZ&^&cZWT{+D%p1OeY03^ zbk%rYgi{-b@lcc18|P|2=-P?TncELu-`}vVcJmY_vfdpU#;3rY0%%*L8pUG~hXzK^ zITHn!BcpaW-aZkN9NRfUiM=6vzv$yDo8dQ7_~_v`VyCt9;%gHHdp>Ner9Wjgzxa@B zzGh$G)>xPDz5DeD-^c9zy8ctinKoJ;l^jZYC*^zpG=M?}1CpYIvAF{KMfnlvx17Bp zh$L)6N;8pyFS*AGQgy&^{ZeCHmQ?_ycV~D;fQW;qwM5glqmK*TnVZuD>4==H2RB2l zFwNv0;iFDk86hK*XN1%SLI5qSn_lmVK*9l8dHZM5F9jGtU`}OcUQzBKlgn&q7h*h)BycdvM=) z`ysm4Opaq`g_<%+Bh~}m|2b)4Y_eB9qMf@iD4usBSC1O3>SGV*EHy8fz@df8x?G&d zJhQtv{20+(-$ifX^hX8Ves0Q!beOg>8|sxkp`75QC5Ca|;2}E#IAe@Jd!1IxS=}^y z9pr$$PTC2}$MoUsZ0tip$^usK{uP-{#XSqyVCUUNgoti6gC4Hh5Zw1?=gKrIFPw5X zp-BmovX1UUcL=;M3U!O83ht7pNe^;u1~A-RqkKQX;=>}w>wrVAUqIJWD-i1C>&&bf zc(!{ijFKB}D^LaiQ>`4(JV#k-8+ij4NL3f2yBYDhdI{QuL40n&!8AJ`0@Y<%cWKPq zW!XH)#Scb7e86JHre2xd7N10d-77e+2kp(04p$b2YB1N+4AN}eeY#UA#}*$ucD|Jm zT$>@apk~w-ahvj<0BcZm%$b46$_gG`Hk9dd#gWpT#_lijzveeS-mWg2!8X`^jk&J8 zZkDnGGdcrgM@k*a!xv5g_pfn+h7-xNJY7nE!uY}ytT&G!SCNWNBP;Vs(9(wZFQrc9 zrsZ$CyEz|tcdjwqf8jAnGqd&+v@s~Rw^`=bd{w}UCzLw+o^sku78hKNzUt@a_ff$X z5&u&%pa;)t7*a*MOU%)ZBbXbUyXyRSgB|H})Z+usMlF>!G?@f_O59*c9A-YR8A>Nf zYE5H~OB5o^KnVq3SUPd-^rC+WaoRuc5lA;?KRFxHzlahMM_-J|dm+jps{-|A?l0Q} z;It0zOj=|SVoPdR2O)b|m%kA}nE@E1$`K`kg1dP5=g`1gA6#@hObyS!cO|Fvra6jS z_e7@KFP}h;w-ATdZ3Q^@zcC;8KD{YwbvmU*qu#L48DTc`thzco ztZGqY8$wb7npG696pJ$^AzA?i%Avd{eMNu=0}oQ43p2FNR%RHi$WjT9ZqZ5Iuo0fd z3H{jRa5^Or=r3u1rK~WYrDa`D6%48E6)!S0AiGJ7!&hSVUBcfs?&MGyI8h;EY@I4H zz4Lxz^gjQ2pLfBz&0r;qDZ9ZlqNsF6{oo^!}xlV`G)P<&x!2Ns&1xp zRu_x{4UHE4WW7i1`YNl>>qFF~8ghbNZJ%~JZ?Bz8+hDiE5^~CXeA#E6x*W8gjm;Hi zEzkw?lUtL5(6+b_cRT~l<2RvG=gcoM=CqtP3JB2lW6ifo{)#i$+pTJ%B|LXn>YGw4UvAyv%~0dR*8-@+QQb9Xb@IKDOLAwpApjXJ6>3k4 zTJv<7Q>Dov1!{~jf5=W#|B>G2!GK*IYad+NdM5q-a{0&PKxUQa>xb;_BDk_FVUn+< zX2Q`#zU5h*{~j3J3vV2=BPwplEQ_bMK_n9&pNpTU1VGkqIFSo+6?0!pabyLe;V&W| zE%;veE8V&gwU(X5O9LNog5Dv10Q%N;AS8{n@R7&P7Lia#psTpzwMK!{FO zonyYfJv_ug7}v zm{h(ni1q!n);60)Gm)31YSc{7ahmZ z=+iI!3`Lp5kdy}&qiwXeJyWfS5UR##sPRsg{o&Rh_t@_1uSn89JA+Y-M3Ssj$ikct z1pb%4gn;841ePT$nwZ-si)mUP?lW>LT(Ky=7JHLnw zi!owHa!LLxnQn!TfC6Z&(3Q*+tMHn5@5+%~X2G*{EGl5h%J zRRitm!#`=CdBdRq<}3m5XFEArK^|rEklKW1UU2hBln!Ix$OutAZ>0K0u=fiVl*Jey zx7W$qbN2RuB!gJ*^O6ieV3ia!ex-gB#E+vE(cTKI8{>K(NqeNo_d|VMCs7Z1bka+ zc2R0J@GnW7E7g`9lb#e9)uOQe9Y^jj?u3%P`=Elt|}FB1C{7OT9^{tZ&kN?k+#Th^9V+;cig2SH}IpJv0A7Rzfm$% zn~M_pq)~eW@CBUUJy%|Ptvn*E*<-MzCa?4!3++^HRes{c2}~iO`HamBDl3SMxXf7# z6Gx*!_ntW^HC=g~Wy10Q+(XiebleZHy5yvGYpn=V@+SE;tr&1yPVfXwJrfKga0)w-D?Jx$SviBDdws~2h1R$T%XLsUIh*95 z2i%bvxypJBR?k)p;VLO|e+8n{3j9_0RyyK#fKjf_K*!9vlffqVN8JnVMd`soz4UWk z)>nk`)z&hB(C3k9^Dt^+}(R0JqK$`~FpwM#Qw`4H+n36!2<&7-I=3oOiswfprIGirN?Gy6Ads)VhB~d3PkE z{OHrKL$P1Pej%Q3Rwt%<7X_GE$u!IHOU01s%dUN) z?+Uh4zeLD2g?}K69+{rf40lbiC-hhFzp__H>FQ<_rP;)yJJipyNb*Qr6w7SuLvcA0 zF$rqJ7HnaPvn~sI?iYRNn{!nvw5x^zAceF;a$yFs4GR*;W&+Vv17UdcFL>oI@FUcd z!mz5CaMirfNb@n3n_TDABC>BXEutKdrk0x=01N9$NapqzzvtuVz%RpvsF+}?3}XY_=P3Q#4f#T zDKty$D#3r&IS<0K*cseBqe!r#qxqF|7jl27+Av{D!uNwghH_E%t!h%2Ue~k~E252I zbGEjE_D3X48OMiwz=2;_n!gYSI=V#rSB*>|D*D?7Y{9o{-drUPUu3#N!#lf(WV_hU z1Blj|)q^2byCYK0E=4$$&E;zfH{Sz)J;yNUI>5ei=NhuJvxA~U z$RgZ|Jub@!+Rc7~H+$&by&bzF_Gp6VaOgumW^fv1bPkbnLR>W7&s)bNEcR{Ddt_}9 zv1}G)19Uli7(ZOFymGB@?v_1z5ZbHRS@C53PUNYEdX?N@qzjB@G}n)M6h*yTId2vf zlxkF6#))VRIl>(>^-~qp2F#niLm4*IiHup!s7cfs0^k#ikLdU7t1_4#4INvJZuHO$ zblTN%TLT_bu1R&kV3*d)D~I96L5%*SiTEU~nw(ttTrev+27RVy$(FE~Ik>=d0;b$B zZ=Lv2AzDm}PYKy+-b}-T9`IMK_64$)i7y9@8crL>RW5!tY&m;yNPF4ZH>C^Kx0n*d zW75XtYp~Kbr(I&`L`mvo-YEydFwdv1o^%`$H8cRH-(Ks`e*9(FD|WCMCnFX{TdbU0 zc_H8Kf7|vC*MV=nX?-NU^`Ybqz~DA#lbQor#ms$Fh~O9lUCM#=%-*pqn`rosR^Q2# zJ+s=)X)wp@YouM8HBQWI`5>mJoDwb8u zZfY_2aEi-v6TJWFIZ4iJ%)Pj@=!Vw^7u-ncLLDzm&d3cm;&aR8da}aMj$~PM{*4+x+*=|eMR5uIeWv8KNGI*lS-uif?qiNhW6hkZefLYh&R|+Gr*kX8RLeL9~FX8CT(menoh{E8xD_| z>mCFcN>SU`9cA!{T$g%*r3LAsX|l1yy%?xtyp^hkdjkPcg}+b(Lw93|obXNx3Ef3` z5463?2QYiBcZ*{|iXU?D>ahhkA`iba%OV@*4P}ayNFj-%yYPveW3mysy3>x>A zD1Gt^eyFh$ERZqWlWtW}5a{Mp=aSx?UHBNnwk7rWEsa>w^i3mrd8Y($2l3%4%`JmA zLnTAt#^AH0QA4RDDY^_P?a)bn7JwUjmWw#UX-Jt4EauV}(UZ1u;5=o=m!n@HG*vZ{ zpl+2bOlZnVbTT`%AzA0K4318V&$FvTK;!M=F;8m?G{D7Lgm*>9Xe!7eMC95jT5#%JLe?O zGu(g$$s|>So)g;v!J%z3$x)cRyFdx`Uiev(d@pFXy=L391GZ5fw(12&oN%EDKQ%jR zdYl=Hq~y*`p71nOW})^F6{_#3?J-DzHX5O7 z;n24%TX1I0vZUS_Dnc%iI@v$_vyb;RI7F7Y&#L$2m3svQ$K-FYJLgUo;nNFP-Q^q# zDue1G=P95ro%115N-C-3<20iDEh0FAjQN%3?|=4YfMvttSbau#NXJv9Kz+!iAzVm| z#oUH`PG3XcjS;XIq~kUfw=c-g4|1&haY;ExjI|jAZygOk*gF)?TY{54->FXQsC>{9`?Ef40MTM1ZtA$w#E;+5cBR|CjXu{=Lraf0M=@0ONhOj+*FwI3Vw&smYC{m-BoInq!X$ZIke) zUYwB7c!;A#0 z+blII>?cQP0#@7yp+^&W((~+$(zoY8S)gBV?VaWOVm8B%A1H0-?4_QR+RE~uiCejf zP8wZP?qDko+abUr?L~wUXQs}jtqGAn&=g(M$ByM3e^{3&c5S-C_<%U66;ey$ZJV+} z$oC=&64iU+1cA#}YqMPA_vLIs+Hf9P_mtzbrkB;k0C#lWc z_da1aQloFe^F`8(>0SiMSxo2gDnL)bZV&I7cP-hER;b;fd2FG4dt;Ii?ehYCa05)*>- zaNV>V-8vf8*xszqAG3e=E1k>ozS!7MV1VvbWI{hG_%nh~8v#0*f{zMJ6T%44_}VXY zDG-=*OA;nh_C}DI7yGPLW;dn*=8G7J<74(PwO2yH&Uei)-^qEjVD&|01rxi-q{Vsd zX%y-fPHtOKJZeg}y&!L+(2mQO5SmPC4_(HXHU*+mZ!RR!yKmH0pqlcc@}aj@cIiXz zoA?pP>McO+Q61=0w9!|dtc zn;LLz2$B#x59FxAtZ2sF!w&_bsqmJn$2nQAa9c1BO(L$y_aL?5>Y3O=24kI!&os}! z5Da9`YuW}3ZgP(C7z0Q+g~UeNOMsP18)V-CAvyTFfl%Ykv+O-YL0bX0gO@8)c~%F+ z5GT#i*y)aGp4KX2a_hpw3C;sAD}gV&`1yw=v0ihI3OfZ9pPc`8v^!1pvEA7D^;mei z)KWTBwL4#1P%st#Na`$9n^;T5&A*Jezu@(?5EDN$>e8cP(W2^NV%pfmgR@AEY%dlQ zlUOEI7H11xIL)Bif|Q2SLcNPZFl3jhS%0YGmbtc(Q_jiH>im6@3m^Kpt2`SA!f}>V ze`)ls2k13sllTru@pMUq_xIHaP3+tKml>aZS-o87KXW{8agGi|O{!Cu-O!o>zl`T% zS6hN$^=I|)xJA`v8=~{6o4(sSwIFN64n5iG`AKVye(P_f=ZEDW=`nF0q_zG!O<0w( zumU^#S`pz#T+77003#8~i_+;-?-bD^`m#a6A|X5n-b#C*uFTtn(tCu4SGtRXU4%u9 zok>P1h>|(4{S72!7s^}Ph7(8tR1J#K!@F@yyvjt>+pJ3?azhUUqfvCPaFAzdtNc)B z7>wDD%O|}6Uy1Bzj9;jU1QLvtA~z8LJLj0wy;ul|xzyyPIG$}u6xgJwC3%FAEq)=% z-sf|oez2Z1}2fgXAk{ak@`|nK_zsMnw|D~)-qXQxvZkd*VWNge&>hH&9o_rBOuVs9IK8bo~V$ysD&?7|6L>->2l z#>^U4161Eq_1~`Q?=ExmWXPT);AUniJ;aza+k#1#1wPzRlY=q_8E+6*tL_wS{imHg&?~UBJ5pPgicor;iQS)}iX##dLuUb8 zi!BTh?gbL678j3;FOT$`$^+TfQBPa2#!qO21DR07Z(#3mA-ow?B~EWAF{Kd0q+luoBzTDV(kTj%cl8ld?at&Al+8uYd>f2l%Hef zqFLY8*lL{5eYgLdVONa)l<_0#-u$lb#ks*##@jjK0%-1O0#Xl7dgvG5>Rrn;=bK{+ z&%A13YY z(qp5zzJUf?->&?NS^mVwa@_H9X4$(tF+^rEB*wz;=~UJ%-nrh8PMiR&UI6y_pIIikmX1J& zmi5;oz!Jo?U?*eFeBp&o+V!`;=#B3=f~)~DR{rXX&SxGOcWwq+ME;nl_+Q|s{2S$z za)9(~Ntp}?Rso|v9WMXtXrD-&g<(ty<&XThn-2BLEdlDvN`z-gsqlW2 zhjrua!a4Oi0+p2Zjo&hI#%(G<#of7tIbFquPgYg}d-Faj%;$_IISEFg!m1>ywi~^y zt1#lm)dqrlp}d!M_#FbVldtQCET3Bs zk8P5(fP_JSN5d&k$2-lJM)CXlL(68!!nnu9sBk1gCzIpNLew|(QkK?{^%%*q4uN@s zafRjSXSn?PRkQ3v@I5e3FSutjI4rpYpMJt)vumE8MAoW72f$N0;|J+WrJ!)s})5@&KElHEJ*@TizRd|gobIu0IVJ$yf`YpGa z)7)%661!J`qdjbZOlmZmfp&H%;Ytcnw`ZVpUcu-ATTnNS)Wx6!7g=eS9y}Z4?MCRs z=hDw_pboY5sMgh`Bh0hZD}h3)EGM5md5RekbOu5(6l>^kWrK#OgT$wx{Et^Vp}g8p>iq3TUcJf)T7 zKQm5is)@{{qR5>bc46kh?wF(J(76G{;^&J~E_I|nLX5?v<&*o%0TdcV94b96AkK|C3HQ1MM2+St$1m_J(Dg?9X#{io851c7%=AtP7h*V9 z?`%E3!txelTg|L@_WKQ0ks1QK)K7oDp5~}+HvHIBuIsmsXtK0_7V(@vnK9kkzV;|Q zxy-$znBzVhuBHX6d@DNEU2AZ>ozlImdQPs3%DppMigwSjI>vuZZvD`cQZnywc*>j= z=M(SmolP5a*Lj?@_4rx2s(BI(F9#ZCznDo_t1|A9zXqyNcQFJsjUO2c&jPs20BNAl zE2o;MN2`U8lXez+H){L&v$u_QrR`TC;_ny>YDit4mDmYQYe1NW43lopJc>Ml*lWKz z2}Q1BPFK!Xk+CE>Ky|Ao!PB-Cr(C57Gpl{uQpymHhjWct+1~MA`eb9q0<_Z-x_+?Q z@@ZE0e>-HM!=jxqyNC&EDdKEgIUB42l{uKP089$oW1#A&P_;@gF4mNYyInZ1#z3<* z9}oKthKyBabxXcWCNvO`G{eiDufhvk$v)u&9uBvV#n9Gj=lbMAcU^yJtJ@fD1WjYf zc{fj#V^72kSN+=H&W&8f>~_EfTI|F|UsF!PEoN-~m40Z9m|xyqUDe$+V-QW0jle}^ z`D#E` z&bh4D9Kp|q0PN;kOaZR|0feniA|elXE|2>sQ#AZzmHJ#gMo^jUj(*PWIcBJADxU=# zp4_l0jEt0=ghg0JL<9Zusd3-FlMgU|J_dSAwsYEhgi)ae6-Tpiw;owb(C(KVFf!FxDg)} zB4b6OvQUWn2eR!0JblZO7$_?zX}dE`NPMD_4~_CdfRfX$i2KbX*EI9Ye!Pi&C@3osCp9r7m-8f@0E*|kA&u{aiV;`N3AG9) zX5@5ZK??=YL@?OhSD0c(*vuRX0Xt8zK4cf8>wV6_oS z-i(Mw%pG+`5J6qKZ^I1+Y-f-%29ZY-(L)P%Hk_(pL}a~Aci2*Bou}0HvcIvXz_gsv ztaEGqHZRS7zqg;e+4>`#oPOYLcl18%ulpA*oo{pT)MP?ebSOD(5TS2?l)_no)m;9r z0Qr%DYeRTS3jQs<-a89n#H;A!M9Yc;g@9mb1jwBx&Yk4%2~e$R#RY_IAbZlm)iv#r zgdkN!45y`{qMXA*65eO!m^A`88Kv1}@r^6@ddkz*5+CFA4t-DJ2G4NTh|r|6NZbZZBk!L zMob#&*;P}Mngrss5kI2Ds&#;X^{C1wnx^`sK~4otC)3sfPj32U<4nU;3q-{zjejZO z%*&dzUdJa)4k8M88{ghMm@ALOBK=ac#8enoHdio_nHy6QarhY}Z8fz@7vaXg`HtYv zWi6Gr>!jIL%XpOBFvcr^VSGWffQU3(8qvrLZVq?7?1;3&!ocQox)t+1MfMiZ?tW2p4=nrrerQ#Ve^x{!QMOB7taO4RjJQqbLWGzZLrs1Je;Z*he zO+?I2w1)4UQ8M15W)9jwNG^=?us70S(B`IOQ#M%n051CQ(LbC?Jo4h-W*@(|EVl%f zOqEEF)znBpTP_^5O2>Nru~PG|W?vqH_FHaeNIu+e-8ed&k?mN(EL`4X%=QeLm(v5# zXKIE+L7zpW4W9kwbfsqiX}4P^G>93WoM{FqED}e1ny0dwDaQ9)!oo2-S1)m4)+cU& z)hnOSd}_&Y8lqq{o#=GZ8PiC19O~0r=fV@&JB4k9R`GWyWAxn{DN4rjl`B#K-9~GLN6z9AYhrr4YRdbRX=k1Yx*nk^|9n7F2cXRa1j-4NXz<#eC zR%*Ix`qg&*CTV#2T)wfG@Inli?}52Cu@Cfx99d0jekLW6?%3-jDj+M^9u+sHoyWU} zUAYBqw)$M0ngl2lxHJkk&pco>(tjN>|GL5qcMA)?V?!x+W>j>R)67LAXE){8=M5*I zvZs8oO#gCyV)J+SX!tI+zrGm>XP8I*3aD=t%#Do!?_e9w+47X+EJDj70WRSVWNm9S^wJFu2#!XWYh$V68+cz;_WYcKD1xF76Ul+y_lwhuV1vk2%^*_2jv{Ke1mSp9?17h5@iEf+(5FG`4s7avE}~Ch;r1<0QK1P8|tYQAkd2G5^|}Z%BQ`zQCkWW-jz&I3aGzc>gM8A*Q_% zewr%uoyp=x5S$bvZ2TuiPz*^&ou99}j>OU$x^-#foZomC1!8zfsK(n{=|9Ro)<6^# zH}d18rY{%_IKUhfurF^a`yf)-JbkC*eUPsoJZt{-4^|tA>q#(i@svJjuLer-i$}re z)9K%l9dBF?P(3Hv{-(NJlDTvZ@Tvg5-jyGjOe$}XJtgQvnX(@uuImZ33M!R8pK_6e zZ@}~38yK1}*Q3!H*~E_u6)Y*#|E1RZbCC(n$`8n*7UNwJzc@Z)29S5Ama%6qv6E)f z5D%aJVoT`)d_HeStcWcKO;$OT{iT2G?|=Sx^povKf{2%aIIt3NWys7W-88S>J=Hdo zw$v*t5S3%9#{n707PtriIMh7-!}F|#aLBSyO|}l?8DB^=d!^EIK#wyJW+|LM8rvTH zQ30|W!zqw=ru^SM?nMypf3)}IVNG4@`n0yTia09@0zIOr3{quML1^_LQbe2*NDxv* zkPt(PK*A7+mLeq>P$)s55M>fcLIe^51c=HcLqujt2!S9VfrJDE0vWuUbAG2+&(xlC z`+J^ypXcxo@a(nM-g~XJzxD0!d*AoF#Aaj+3O8pD`ue}dg9a#6r76SVl>|2~G*OIUU_32zyCZL}Sq&sOPz>`=63g)B57%=h9#Qb9( z@7NvBXXG70Cs-g5E*AXo{QM z0OZAXP^B8Y-|(wdj^^*R{op1w{&U{noxJ~y%imwr|Ms|~w#t)+%j1MmUj?|YZ@*e( zJmq!VwJQ0=IBEfUvLBMB5{drtyS_H~nR-PBR%c-oyqr4EqRHhol^HGx@$A9m6F@m9 zL{9!cdTVZM#KYb<8@cb>nveGW=e+-ChW}5F%f~h2%J2bPpEWh2xrUy8O`>_~wjHn~ z5^+<2)&;bCnKu5>dDrDgBTaSPZ4@keUwr9Lf!1I2;Juw?dxk-h2a%ZKuycabDWIAS zO+|2>77IAHl(tl!?k;IHXlVt^D+YmE1@p|gaVRNHt l>c!^|+LsBDu0*f%kaX*2 zTL4c0#AI3zrAa6VAc9s?nS=svV?6t?+{tGI8PVh&2kueY2sJ&?r8))i-<7!Tu0%F& zw=o0)a+wk(6=JS98ioeQeHoxEHKV}WBfyp1RP5ds=L)M;F~54)VQ8jys-&y;5a_?CQN1c-~94 z2I@zm6xUqMF>AwJ{(j=hK?~!>$nr7^EQ=jqE~eVDZywpDUKR691{0N+Kt&uX7uK}P zPdGSHtb=U5*LLkeRvN+tTdV9V?D$IG6^PDOS*HhSbyvM$KJqT8%{GiA_YYqsP9V90 zG2TY?G$N{pr+(417*H)JE5E1S+yyYHgs4q$D3ZZw`1vk$(6L;!JrAyTg^=UF5a?Hr zFM5L85~;O0_@nws!RipN$l(U&&E|@-m*Qat^GDCB#dtl|iH_niEkHHBw~~OGza}Lz zF(a&a=u$6g+f%)o*{^pYUdQq(>V85}4UGW-x=LaV8m}>f+Y;R#p`Bn=b8g1V6I#|a z_b_&jpx-#*L+{QwTBP$e^ST5CLYbvqr}_r~+;z*NSDj_VbgRJ!dTe~)gX-#;Y%71L zRXpgjO3+df8$CZQq@B#UVaIZpf0i5YU4TBAOo1~Mpha3~0^T;d`WtSSa5Si=S z4}eTefZ8X024#&gp~Ay7-nm|Fr;47HjG$pgC2VSuW)nCIK~DorFR~F|fmz4$W;klE zNRzsMf2~>*QS{X3Xg#@5ggHWK9F0L{gxfW5`Hnmz@xhD!iP+TdfNvyU0&oTj*wPq# zNefz3ZZNZsvFrwZmFTTBOV5EU1&b@W?=<>$0Z4*hB-xhZ%j-jOs#ySC8owXw zFcebym^fqR2C0s7Ek_>Jce&S-8B%(p1R%{=rAQR+_xhDU#s(HwtKDH`cJ+Kmz5O55 z&w4`->$rHb_kr@_f^9ek_iDYzP?^)9LfE;(Nrii9SxPsu(=cjPKm+^J-G zGR^O?>sUYv=;qnyJDNH-ySG=>6g;>p-Fiu?kPOJgZp#otAO)d=arAbn8;7?TW%j1FkgD!hiT!=;sjQ}XQEc9rq}sXQfB;| zgFygTZQ9LnFq%A#C`dlM3%(uNKE*~`0RcX?IZ&Z&M^ndJ_ZDv2kJH-ExZ?>)388zd zL$4M$BTx^h3R6R5AF-V6tDsN~I13ou7X3rArub?@iqfNd?E{QM9EsCePNA|~D-c@{ z;kjxN{dNlvK%h|FfJkhSKaFH}7RXUDInt`pxv1-dUV$E8)OsSlg}YN7GadNO*5p7C zvh^anL3|$cQVA0lP6bvUD}kerT(ri_FIC^+O_Vy1nWujTU~8(uTvjF`s+Ye|cxG&^ zkEqIIoV58OA-GX(#gN(Drd1!Q+M0zh`q^}DWeFkXc$Eva2oN3^Zv`?XJS7M-%qGma zx;3h|R>(eGWFp!>bqY1z)H>QYYxPV$i{cNoI(-qWKkI1a@nwthNmr5PujM%YuUmeT zRkdg;_i0i5#?^S+Lf`ZU&bjck=@Zmhv+(KqNwOGIKd38pCA?kot)(!vp|SijWaiXk zF;M4EX`7yk!J(`BzJ7_-6ea-?F~htt``NneU0qzi>Tc?O>-K_0oM>#88LbFDsAP-5 zCGmDmotESot00HZl`e&0na{3O)$OWk%nT38A9T?XL-KC9)?QU=js}Pk2MNG_d;fj8 zp5J26|J63XX8(VS=qR~-QXRP*c<}_RR4%elIS<4!0PDBqt1XhZ3A!w@ZEM0wzp64n z1bmbLJqbxeTtf{Hu+3XR#n^p00?oOKPSdNdE-2m)#FC0C?w7TG?&Cal-!~mR0Eloj z2Z&3U0csIv-@nTJH42ASZ+dks;9I$lgyK8HF&+)q5>^B%c)6P8}Z+ zKX*TAc=GcpdaG`fb*BRh1z+6+;D(=oi@qM>d@O*bK;n*fUFy98NU|30&bvHZTZ-gE zJp_Y`>ViXW^&7KwU)-=I=l8je`ug6=A_%)`Um? zI)3+&+DJB+GLq5^a~xosS$IkfZdIQR(rYXm=9{=@Sk831@V1Tl!X)H_D(#JI_f<*Kd$AGjUAX7>NB<4>ugXQ;bHIe$#l2H0JpuHVc$%CzHjw#ruG zJ1wpI%*t(_U-2qE#!=dq7x${SLTAPH+fBzAs5Mx_ZhHG>PdZ0Wl7FxXxg!Nffg3@# z7i+6X^U+?6=?kFQO)cSZPdp2!>}*4adC$koWhY2MySVk*4iSLt1nRJ-gpO32sW}l* zMm@9J%a_Yeo=+POIJbbe0XRh;$fEtLHuSf&|2^#Nvd?;|d((guLeteUVRJyGpC32p zhS3v_`8PtRMv_Jpa|$QT42u*)c&~8*$5}Icuc(@O%o)nd=cSEx#x4Ee`wzD}2_5l! zbG`>sdB3cezjdkq{;^;8XqS8X-@E$AL6^(QhtLdY&J7D7fACIY;-T8sX&$Bm5Y|*93C@M|S}tmM}k{mfbN1 zi(o!k;rs7c65dfi`Lh<&`I9(HzFOMn~f7@%CTN!(0 z_DmpG0PQdUrFJ|#eY@mfl7$_#1~ud$HWcD+F7YdwRcU9Wlg{eY+*YGE{kZtt<=Un= zA`LCa46j=)XP=Yqxh<*Qci+EHN0LBo4KHxH%09L38px;8Ll@n%Sc|&P6Gex~Cky)p zVR3GrCZx{?>-37pGM_F5LylLb*Fv_WVr*I$*i}7F4Dlo3=#=k@xs+%2`?{2<;ToHZ zoqnDKd-axzTW9$mxFh+pJxx$d9grbTXs9A1yWYC2VywCq?n~HSPE$B-Qz8I-(NK03 zzwwdONXWhF=V}A4V8mg3O?Hr$HWR#AGS={JQq#H>&BI5}Eg{Tw#^!sYjvWGd@Sm<} zweT#LBp5Y-=Q&rf`a9D~sK!xA**HH3E(X?FttVTd*51lq`KU;aIawx4<6gTT`^^Ls zj4dM7jKdGw2a(O*F`tRu$YGeLI^zlUUtquNv(wFi_;*Z#nJ%GPGknVJlBR$` zZ-C#NaS~t>3CuYQwa}7cbpuJuy!y12p7z&|rHPs04@CO=qJF+Hjxy7O zZr0sd${Z|!s#`Wq>aXj=nvIZtR2#`^W217l*7m*#*2;i9qfmG7E`c_BnK&6tXDUdT zu>k7YuR{&E2rsEy4E8ux`IX$08U__Ln7$xgMTTq5h=MP2Ue}>^bcjKjZN?Pp0*D#t zOKENAF5$G`YD0z$bwg5ZE99@8bb`>`#Cs(^8N9PB7mvW56^`7N;wu9z+$c!_B8MvW z!@`=Ls=w%{j*{8d51LnW50~LPr>|Kf@|%%+vb5^Id{zr1jRCj`X5cF_0K~}+R~Q@@ zCasnCV1e+cPyL{&?2qUJ%|aj4rM9zna9&=Xn2I)vmr-Q}4vqU#@iCPROwJ?#PL|jRKWe{E(Lajm0YZ7c*XCNS^DW0Q41V6l=6wo=IJ#w0nJkP%-8Qcy37Gl?K?3#OBqBhmH zaG$g}$*7`tPKl=jIjYxZa$G}vHY|32x|qkU-VcNb#Zb_vv768S4Hoz>nBY&h5QI0Q zjN-3nAo(@1v$2V~q=zXJ^v{9cVx&^3qG20%pcV&5_+ z{TYtlM6in-!twSSYzq5k_Dr`W`*Znf!Np!XlQo=C5z&F`zQGHbBw1|LHN^|wHlY|% z{TElsU2x`Inv2ef6d`?CR}@}pg&M&1kGnvbK2^rkvyg4OM^ZB26RG1HyjAom?^Ulqoy<9^pOMW5a@ zxqdR{d6mcO&i%v4m(7=3_1`GV6#%#iBT+K=SVOs1KGvwm?rZ8hjX*@Une24!C9mW4 z`nxI<5tW9q=(^R5L5=Y`5>%=?8Otk7+qdU#SkipHj2)+Rk)?pDUeZ63jtmp(@?U=s zVbiDkDPVt}SX>WnIxi`NpfsB)9tG_0s&^Vl18sSddyl2qrH<=>pVqC@_Wq(PM~Zdt z5Q;wGt1@l4hKqSA=&dK!7s*)0bT0r?U5K?K#ak~-gpwiHuPYTN7`~A^7oAjkX7f!Y zpj7k;c0Grc)pHXN-n-TH>`T#lZ|rf|LcBOI1a4^?W`#rryyDq}MYr#O)f(*Y&kRXm zjs`@((bpTM@h}a2f>qHw!-~W0Gz&$8deWuIRe7(R!#@=h8b`s5>aq&nA3%IH0C>bG zdic>D0^QIWd-H8!2RjcTFR|p##bFOZKi4@a8LEm0TWu(1wy>-H)bpdTj4Ctj`2*6Q zk;3hEE<%INVN0_f(#bH8MCly0BgJ!q!pqpk-8QnwG-lr8WnNc_9_2;XM{n_5&1L#hKFmf>bsa$$O;8v1=d3)uZrIEJiw*zC~5Q z7(Q9M5neUX;r4T%wizSsI)KiDC;R#i2+Y%%axz_&>HweKfJMp78nbB4Lkbttl%SmR zz{52ZNrhX4=&`8NSh84<6d<{X<)xBqQ|7&-GhxiSBPt?GXtsaWQ7V+5{$pu1 zLHlf1s^=BVXRZ87NP_8I`Zb*#Ib31(hP@UW1Q)iFwuaFfGkP-%$_WL|!pK*}TCwDV zWxJ3&&H@#EVHqd<@+G$y08c3Ie$M|aeErW{@87}YzS9XLoop2su%;wo(^GTDritB2q@P|l$IQS?o?TWLAq8QF5={jJ!(D;*a*n+)1wul5FcYJ z^>%}mB(An_MOcf&7Ka~Sr+6mzP;#?fJgy`j%A7v&m>vzKDaalGL3jMe=-nb~6uDVz z^flr&kD3}=Q!eH#pz8p(bI)6MrEOdjAVb4b)eKLU0)jM=m$WbPvLxVv1Ljlxkc`T zyKE84Xq@ErsHaMUnd*;O?&=j@0s!^sG=%$PGGZ#$xr}MPk?nLxSwd5;>UWnF|8z$~ z2*_BjZ1Qvg)yA}YBJN3qy01PZTCX8AUHr1$YEM1rXYx1a)6Tajx63Zi!&aM-xZ>*b z!b6OtsgMc?aa$myry%k2b1DN%lF$YTkE$>Yu+2~UtdY5pK@lPRQJJIBgVCp`Ewq#w zO2XO+n{blJK>(-sCSv@CShG^mv#OPM|A5s=haWqX#_JP57YkdLvyE*!ceWXT1UFvnN=At!pxy?hoGHgQ@oEC zix~(&oY7UE%7cer&Qr>_TX#T@ubO>ryRp~pAU}FRTEbWN$pXn+y|p^tHg@UswznY7 zcH{dt+J+Yy9009zMJ6pNsu7a z2MozycH@uUXXuCu@`0=#kGAysokk1w&kQBcb&H>Ifw-B~JB^nCJ)rkWt5MJVl2!i= zmDm4yTOcwg1TxzQ?@jU^nj8-Za%HVM@%XX2t1C4CxA{oN|bVw}umpM>#zPzjY8WuZp&yj`@cUA2#O8@MoU8KsDA zY}Ry`+9vTq{DYGGZ^w?3cHYLz09-MQfZb`8spWaP)7*+2rSBTp@wUVK?yjJdWIIy| z2DQ2lv93lX7%qac#ac&;9|;@iCEriF(I3x-Ad)>*)|1HEH|;=X!s{5@Tyf$0No-S9 zhqo0K3+baL85qTcSh^Y=;`UmpyWtPio$!wr2`HN-2Dh!Hh?ky@pvp-?&Uru)Jehm6 zhT0=mKSDkHqT0`+k8_y?R9xz~6s%Y4xP8lTFcvIpOj*gmMETx7N{b8q{2IexR~t2S z%<^i8ExShy$yM-})}qWtk}gfTPdJ$nWE~rH2^ook8XqG(Op zI(eej(NL`;CXs~_OOEjH<;=DbgtDcjgV|eB?Iyc=Mp{2;^|+#xnFi`=Ap1YwKRn+N ztf;{3pDOQXnjqC19Aoay-Nqj@X`DmFtIhD{n?zNCnRx^L(Oc1R=bQ%}D_F^gdR>y1 z^}xZ4m_R->Sb44*X)Zh}rmwe))+=>sH9z3Fz@vai7DzS=nFjj>lP;-A_vVaETB~hF zk~YanP3}-+wa&#O_9TRFXs~6rkLw=pPCwo@E#@s)wSa0PY_HEc|5WEz8p6L^2bte} z-r1;x;$#AK+L_AJw8VvG23wB-%BF7@9or&^%ab)|CLb^K5D}q%+1%X>!TgmYscn63 zRy{0wX2}|})k;Aqpy_4KUxWi;_QZ&vpj+b#;o~PdeQUey1-pr~aZbpUiP5|-i^Z&% zXn+}`=Ud=CoHa^cIQhodV=1OzSGq&P3L%(DCh~IRq_dN;+LfG@}$(H zzOe~+yHz8o?Q)J~4HlkpA$oJq_UyexckMgJXTz^oIv56W*~Tiqpuy#*i%pL@VD?HA zSv&r1l2(|$E7(?(zsaK}#Wm5*8e9v}Jk@Gwfz8JO-@B-^F{?L&4l*<~FF?@c6XP}AMyOC(*N-1X- z$(OIkavgVKj`5fXB(U+Sin`^~kwQJ2p#nzGRSW`A*yI=8Z-?0V8d-nKAk&yvD-jPw zbO7%w6u(Gny3;&FSW$n(icwP&D^7~Jk{%e+9B6ofj}{n{PYR|36#_A`csV$|=SS=o zAuXOud=!en-G76g@ARCIlH);nFn-}>o;x7U%9eBi1eSr)z0YYH*E)MoFMv*H4Jz{d8WFUHqGPb% z+`=COhy}RnT0Mu8%HCvu z`)0p9?6y*_FL-)Pl{_;GVt4P+*3O7k;=5k((8b~+;0FK;M;TMtYeK^5<{KuoHIMcX zYcRXTFO$8&PP~-uD@J)G5FEF-7r+0rRl|Z?*4YN##i1sAxAQXn{G*Rc6sIX~#%!bd z4**>aCk>96L7*=)fvc-Un@w*TzmUY1I(D{x4hEP z*5K2wbdx6bHDqM9rDY5PE6FA{M6GOm7uiH5N{)Z6^c)B)jrp* z{mOipozVQPjk9jxnSKLw)#|+!FAUDh-SY!_y^ZJ@La7luTx+f?xA@I#!e!8m8pcxm zq~NjfC38tPp?+n-(WjYID7%SbkJH^$OyFScFW+C;6qyH zha^d4FL~@(ZFjN&5+?DZC;EB}S0%Of2X8_SmYNgU4a_9gDM04P5n+|euYE*AKC3`bp z)Jbsn==`FGO6)sB?&O6Hj7cVU2K#|%$C`7E(k%26ceW0LQ+;y65YCBXQr;WwXX^e8N5QUyv2Pt(7jO{* z3`*J~*e2h|Yjk@!U?m<)eFX2cBt{Toia4u$xvjG%CMIi(S`d_+gxf=O(hm`*Z--zK zMN5b*hclTVL}flWSt2~iwT9QiGDlIvyag~5>pN@$+Yxs4%u8B+hOVvr)IkQz__?+! zt3}YVys&rJ+_v93;cZ>fV^o_lZ`2)9miG-Z%l(@D)$~e?36{rJWdxQ&5b#KHxE1CjH))Y5?wHz&3})LF3MrAgway z#~k>_1bh(TLNj1qwnNCU2Jh9l4YgTj!ecd#<}<-mEhV!%=s+Uvk`h<1d@R|Md6o Kv6-xR*Z*Iw>s_({ literal 0 HcmV?d00001 diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 7f01b38ddc8..faa02c8f1a9 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -327,6 +327,7 @@ jobs: tests/unit/llms/sagemaker tests/unit/llms/sail tests/unit/llms/sambanova + tests/unit/llms/scaledown tests/unit/llms/sap tests/unit/llms/scaleway tests/unit/llms/searchapi diff --git a/README.md b/README.md index 141e8232de8..9987f13d127 100644 --- a/README.md +++ b/README.md @@ -407,6 +407,7 @@ Set `LITELLM_PROXY_API_BASE` and `LITELLM_PROXY_API_KEY` and every model call th | [Sagemaker Chat (`sagemaker_chat`)](https://docs.litellm.ai/docs/providers/aws_sagemaker) | ✅ | ✅ | ✅ | | | | | | | | | [Sail (`sail`)](https://docs.litellm.ai/docs/providers/sail) | ✅ | ✅ | ✅ | | | | | | | | | [Sambanova (`sambanova`)](https://docs.litellm.ai/docs/providers/sambanova) | ✅ | ✅ | ✅ | | | | | | | | +| [ScaleDown (`scaledown`)](https://docs.litellm.ai/docs/providers/scaledown) | ✅ | | | | | | | | | | | [Snowflake (`snowflake`)](https://docs.litellm.ai/docs/providers/snowflake) | ✅ | ✅ | ✅ | | | | | | | | | [Strands Decider (`strands_decider`)](https://docs.litellm.ai/docs/providers) | | | | | | | | | | | | [Text Completion Codestral (`text-completion-codestral`)](https://docs.litellm.ai/docs/providers/codestral) | ✅ | ✅ | ✅ | | | | | | | | diff --git a/litellm/__init__.py b/litellm/__init__.py index a6bbd132cbf..554b37f11d4 100644 --- a/litellm/__init__.py +++ b/litellm/__init__.py @@ -674,6 +674,7 @@ nvidia_nim_models: Set = set() nvidia_riva_models: Set = set() soniox_models: Set = set() sambanova_models: Set = set() +scaledown_models: Final[Set[str]] = set() # Price reloads update this registry through existing references. sambanova_embedding_models: Set = set() novita_models: Set = set() assemblyai_models: Set = set() @@ -756,7 +757,7 @@ def is_openai_finetune_model(key: str) -> bool: return key.startswith("ft:") and not key.count(":") > 1 -def _populate_provider_model_sets(model_cost_map: Dict) -> None: +def _populate_provider_model_sets(model_cost_map: Mapping[str, Mapping[str, object]]) -> None: for key, value in model_cost_map.items(): if value.get("litellm_provider") == "openai" and not is_openai_finetune_model(key): open_ai_chat_completion_models.add(key) @@ -918,6 +919,8 @@ def _populate_provider_model_sets(model_cost_map: Dict) -> None: soniox_models.add(key) elif value.get("litellm_provider") == "sambanova": sambanova_models.add(key) + elif value.get("litellm_provider") == "scaledown": + scaledown_models.add(key) elif value.get("litellm_provider") == "sambanova-embedding-models": sambanova_embedding_models.add(key) elif value.get("litellm_provider") == "novita": @@ -1106,6 +1109,7 @@ model_list = list( | nvidia_riva_models | soniox_models | sambanova_models + | scaledown_models | azure_text_models | novita_models | assemblyai_models @@ -1215,6 +1219,7 @@ def _build_models_by_provider() -> dict: "nvidia_riva": nvidia_riva_models, "soniox": soniox_models, "sambanova": sambanova_models | sambanova_embedding_models, + "scaledown": scaledown_models, "novita": novita_models, "nebius": nebius_models | nebius_embedding_models, "aiml": aiml_models, @@ -2054,6 +2059,9 @@ if TYPE_CHECKING: from .llms.featherless_ai.chat.transformation import ( FeatherlessAIConfig as FeatherlessAIConfig, ) + from .llms.scaledown.chat.transformation import ( + ScaleDownChatConfig as ScaleDownChatConfig, + ) from .llms.cerebras.chat import CerebrasConfig as CerebrasConfig from .llms.nadir.chat.transformation import NadirConfig as NadirConfig from .llms.baseten.chat import BasetenConfig as BasetenConfig diff --git a/litellm/_lazy_imports_registry.py b/litellm/_lazy_imports_registry.py index 9caba6a060a..ce69de3b240 100644 --- a/litellm/_lazy_imports_registry.py +++ b/litellm/_lazy_imports_registry.py @@ -277,6 +277,7 @@ LLM_CONFIG_NAMES: Final = ( "NvidiaNimConfig", "NvidiaNimEmbeddingConfig", "FeatherlessAIConfig", + "ScaleDownChatConfig", "CerebrasConfig", "NadirConfig", "BasetenConfig", @@ -1103,6 +1104,10 @@ _LLM_CONFIGS_IMPORT_MAP: Final = { ".llms.featherless_ai.chat.transformation", "FeatherlessAIConfig", ), + "ScaleDownChatConfig": ( + ".llms.scaledown.chat.transformation", + "ScaleDownChatConfig", + ), "CerebrasConfig": (".llms.cerebras.chat", "CerebrasConfig"), "NadirConfig": (".llms.nadir.chat.transformation", "NadirConfig"), "BasetenConfig": (".llms.baseten.chat", "BasetenConfig"), diff --git a/litellm/constants.py b/litellm/constants.py index a339fb76976..f9c55f59f42 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -825,6 +825,7 @@ LITELLM_CHAT_PROVIDERS: Final = [ "lemonade", "docker_model_runner", "amazon_nova", + "scaledown", ] # Resolving these providers runs an OAuth device flow (their provider info IS the login), so any diff --git a/litellm/llms/scaledown/__init__.py b/litellm/llms/scaledown/__init__.py new file mode 100644 index 00000000000..d571093869e --- /dev/null +++ b/litellm/llms/scaledown/__init__.py @@ -0,0 +1,3 @@ +from .chat.transformation import ScaleDownChatConfig, ScaleDownError + +__all__ = ["ScaleDownChatConfig", "ScaleDownError"] diff --git a/litellm/llms/scaledown/chat/__init__.py b/litellm/llms/scaledown/chat/__init__.py new file mode 100644 index 00000000000..e6ac554eed1 --- /dev/null +++ b/litellm/llms/scaledown/chat/__init__.py @@ -0,0 +1,3 @@ +from .transformation import ScaleDownChatConfig, ScaleDownError + +__all__ = ["ScaleDownChatConfig", "ScaleDownError"] diff --git a/litellm/llms/scaledown/chat/transformation.py b/litellm/llms/scaledown/chat/transformation.py new file mode 100644 index 00000000000..44eb4785615 --- /dev/null +++ b/litellm/llms/scaledown/chat/transformation.py @@ -0,0 +1,648 @@ +import json +import time +import uuid +from collections.abc import Mapping, Sequence +from functools import reduce +from typing import TYPE_CHECKING, Final, NoReturn +from urllib.parse import unquote + +import httpx +from pydantic import JsonValue, TypeAdapter + +from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException +from litellm.secret_managers.main import get_secret_str +from litellm.types.llms.openai import AllMessageValues +from litellm.types.utils import Choices, Message, ModelResponse, Usage + +if TYPE_CHECKING: + from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj + from litellm.litellm_core_utils.tokenizer import Encoding as Tokenizer + +JSON_OBJECT: Final = TypeAdapter(dict[str, JsonValue]) +JSON_MESSAGES: Final = TypeAdapter(list[dict[str, JsonValue]]) + +DEFAULT_API_BASE: Final = "https://api.scaledown.xyz" + +DOMAIN_MODELS: Final = frozenset({"classify", "extract", "summarize", "compress"}) + +DECISIONS_MODELS: Final = frozenset({"decisions"}) + +DECISIONS_UPSTREAM_MODEL: Final = "classify-1" + +NATIVE_PATHS: Final[Mapping[str, str]] = { + "classify": "/classify", + "extract": "/extract", + "summarize": "/summarization/abstractive", + "compress": "/compress/raw/", +} + +DECISION_QUESTION_TYPES: Final = frozenset({"choice", "noul", "score"}) + +EXTRA_BODY_KEYS: Final[Mapping[str, frozenset[str]]] = { + "extract": frozenset({"threshold", "top_n"}), + "summarize": frozenset(), + "compress": frozenset({"compression_rate"}), + "classify": frozenset({"labels"}), + "decisions": frozenset({"questions"}), +} + +# Set by the LiteLLM router and proxy on every call; handled by LiteLLM, never sent upstream. +ROUTER_PARAMS: Final = frozenset({"max_retries", "stream_options"}) + +SCORE_MIN_LEVELS: Final = 2 +SCORE_MAX_LEVELS: Final = 10 + +MAX_ENTITIES: Final = 1000 +MAX_REF_CHAIN: Final = 32 +MAX_SCHEMA_DEPTH: Final = 32 + + +class ScaleDownError(BaseLLMException): + pass + + +def _reject(message: str, status_code: int = 400) -> NoReturn: + """The single place this adapter raises its own errors.""" + raise ScaleDownError(status_code=status_code, message=message) + + +class ScaleDownChatConfig(BaseConfig): + @property + def custom_llm_provider(self) -> str | None: + return "scaledown" + + def validate_environment( + self, + headers: dict[str, str], # mutable-ok: BaseConfig signature + model: str, + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + api_key: str | None = None, + api_base: str | None = None, + ) -> dict[str, JsonValue]: # mutable-ok: BaseConfig signature + resolved_key: Final = api_key or get_secret_str("SCALEDOWN_API_KEY") + if resolved_key is None: + _reject("Missing ScaleDown API key. Set SCALEDOWN_API_KEY or pass api_key to the call.", 401) + if not api_key and api_base is not None and _root(api_base) != _trusted_root(): + _reject( + "A custom api_base needs its own api_key: the SCALEDOWN_API_KEY from the environment is only " + "sent to the default host or the host in SCALEDOWN_API_BASE." + ) + return {**headers, "x-api-key": resolved_key, "content-type": "application/json"} + + def get_complete_url( + self, + api_base: str | None, + api_key: str | None, + model: str, + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + stream: bool | None = None, + ) -> str: + base: Final = _root(api_base or _trusted_root()) + operation: Final = _operation(model) + if operation in DECISIONS_MODELS: + return f"{base}/v1/scaledown" + return f"{base}{NATIVE_PATHS[operation]}" + + def get_supported_openai_params(self, model: str) -> list[str]: # mutable-ok: BaseConfig signature + operation: Final = _operation(model) + if operation == "extract": + return ["response_format", "stream"] + if operation == "summarize": + return ["max_tokens", "max_completion_tokens", "stream"] + return ["stream"] + + def map_openai_params( + self, + non_default_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + model: str, + drop_params: bool, + ) -> dict[str, JsonValue]: # mutable-ok: BaseConfig signature + supported: Final = frozenset(self.get_supported_openai_params(model)) + unsupported: Final = sorted(key for key in non_default_params if key not in supported | ROUTER_PARAMS) + if unsupported and not drop_params: + _reject( + f"ScaleDown model '{model}' does not support {unsupported}. " + f"Supported: {sorted(supported)}. Set litellm.drop_params=True to ignore." + ) + mapped: Final = { + ("max_tokens" if key == "max_completion_tokens" else key): value + for key, value in non_default_params.items() + if key in supported + } + return {**optional_params, **mapped} + + def should_fake_stream( + self, model: str | None, stream: bool | None, custom_llm_provider: str | None = None + ) -> bool: + return bool(stream) + + def transform_extra_body( + self, + extra_body: Mapping[str, object], + request: Mapping[str, object], + model: str, + litellm_params: Mapping[str, object], + ) -> Mapping[str, JsonValue]: + # extra_body is merged after guardrails ran on the messages, so it may only add options. + # Anything that carries prompt text or picks the model is refused. + options: Final = JSON_OBJECT.validate_python(extra_body) + operation: Final = _operation(model) + allowed: Final = EXTRA_BODY_KEYS[operation] + unexpected: Final = sorted(set(options) - allowed) + if unexpected: + allowance: Final = f"may only set {sorted(allowed)}" if allowed else "is not accepted" + _reject( + f"extra_body for scaledown/{operation} {allowance}, got {unexpected}. " + "Text and instructions must be passed as messages." + ) + if operation == "compress": + return {"scaledown": {"rate": options["compression_rate"]}} if "compression_rate" in options else {} + if "questions" in options: + _validate_questions(options["questions"]) + if "labels" in options: + _validate_labels(options["labels"]) + return dict(options) + + def sign_request( + self, + headers: dict[str, str], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + request_data: dict[str, JsonValue], # mutable-ok: BaseConfig signature + api_base: str, + api_key: str | None = None, + model: str | None = None, + stream: bool | None = None, + fake_stream: bool | None = None, + ) -> tuple[dict[str, str], bytes | None]: # mutable-ok: BaseConfig signature + # Runs after extra_body is merged, so this is where a decisions body is known to be complete. + if model is not None and _operation(model) in DECISIONS_MODELS: + _require_complete_decisions(request_data) + if model is not None and _operation(model) == "classify": + _validate_labels(request_data.get("labels")) + return headers, None + + def transform_request( + self, + model: str, + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + headers: dict[str, str], # mutable-ok: BaseConfig signature + ) -> dict[str, JsonValue]: # mutable-ok: BaseConfig signature + checked_messages: Final = JSON_MESSAGES.validate_python(messages) + operation: Final = _operation(model) + _validate_text_input(operation, checked_messages, optional_params) + if operation == "classify": + return { + "text": _last_user_text(operation, checked_messages), + **({"labels": optional_params["labels"]} if "labels" in optional_params else {}), + } + if operation in DECISIONS_MODELS: + return dict(_decisions_request(checked_messages, optional_params)) + if operation == "extract": + return dict(_extract_request(checked_messages, optional_params)) + if operation == "summarize": + return dict(_summarize_request(checked_messages, optional_params)) + return dict(_compress_request(checked_messages, optional_params)) + + def transform_response( + self, + model: str, + raw_response: httpx.Response, + model_response: ModelResponse, + logging_obj: "LiteLLMLoggingObj", + request_data: dict[str, JsonValue], # mutable-ok: BaseConfig signature + messages: list[AllMessageValues], # mutable-ok: BaseConfig signature + optional_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + litellm_params: dict[str, JsonValue], # mutable-ok: BaseConfig signature + encoding: "Tokenizer | None", + api_key: str | None = None, + json_mode: bool | None = None, + ) -> ModelResponse: + try: + raw: Final = JSON_OBJECT.validate_json(raw_response.content) + except ValueError: + _reject( + f"ScaleDown returned a non-JSON response: {raw_response.text[:500]}", + raw_response.status_code if raw_response.is_error else 502, + ) + + operation: Final = _operation(model) + if operation in DECISIONS_MODELS: + return _decisions_response(operation, raw, model_response) + return _native_response(operation, raw, model_response, request_data.get("entities")) + + def get_error_class( + self, + error_message: str, + status_code: int, + headers: dict[str, str] | httpx.Headers, # mutable-ok: BaseConfig signature + ) -> BaseLLMException: + return ScaleDownError(status_code=status_code, message=error_message, headers=headers) + + +def _root(api_base: str) -> str: + return api_base.rstrip("/").removesuffix("/v1") + + +def _trusted_root() -> str: + return _root(get_secret_str("SCALEDOWN_API_BASE") or DEFAULT_API_BASE) + + +def _operation(model: str) -> str: + operation: Final = model.split("/", 1)[1] if "/" in model else model + if operation not in DOMAIN_MODELS | DECISIONS_MODELS: + _reject(f"Unknown ScaleDown model '{model}'. Choose from {sorted(DOMAIN_MODELS | DECISIONS_MODELS)}.") + return operation + + +def _text_of(message: Mapping[str, JsonValue]) -> str | None: + content: Final = message.get("content") + if isinstance(content, str): + return content + if isinstance(content, list): + parts: Final = [ + str(part.get("text", "")) for part in content if isinstance(part, dict) and part.get("type") == "text" + ] + return "\n".join(parts) if parts else None + return None + + +def _validate_text_input( + operation: str, messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> None: + if any(key in optional_params for key in ("state", "document", "document_mime_type", "text")): + _reject("ScaleDown accepts text through messages only. Native state and document inputs are not supported.") + for message in messages: + if operation in {"classify", "decisions", "extract"} and message.get("role") in {"system", "developer"}: + _reject( + f"ScaleDown {operation} does not support system messages. Put instructions in the field definitions." + ) + if _has_non_text_content(message): + _reject("ScaleDown currently supports text only. Image and document inputs are not supported.") + + +def _has_non_text_content(message: Mapping[str, JsonValue]) -> bool: + content: Final = message.get("content") + return isinstance(content, list) and any( + not isinstance(part, dict) or part.get("type") != "text" for part in content + ) + + +def _last_user_text(operation: str, messages: Sequence[Mapping[str, JsonValue]]) -> str: + message: Final = next((message for message in reversed(messages) if message.get("role") == "user"), None) + text: Final = _text_of(message) if message is not None else None + if not text: + _reject(f"ScaleDown {operation} requires text in the last user message.") + return text + + +def _validate_labels(labels: JsonValue) -> None: + if not isinstance(labels, list) or not labels: + _reject("ScaleDown classify requires a non-empty 'labels' list of objects with 'name' and 'rubric' strings.") + for label in labels: + if not isinstance(label, Mapping) or not all( + isinstance(label.get(key), str) and label[key] for key in ("name", "rubric") + ): + _reject("Each classify label needs non-empty 'name' and 'rubric' strings.") + + +def _child(node: JsonValue, key: str) -> JsonValue: + return node.get(key.replace("~1", "/").replace("~0", "~")) if isinstance(node, Mapping) else None + + +def _extend_ref_path(path: tuple[str, ...], ref: str) -> tuple[str, ...]: + return (*path, ref) + + +def _nullable_branch(schema: Mapping[str, JsonValue]) -> Mapping[str, JsonValue] | None: + if "allOf" in schema: + _reject("ScaleDown extraction does not support allOf. Supply the object's properties directly.") + variants: Final = schema.get("anyOf", schema.get("oneOf")) + if variants is None: + return None + if any(key in schema for key in ("properties", "items", "$ref", "type")) or ( + "anyOf" in schema and "oneOf" in schema + ): + _reject("ScaleDown extraction cannot combine nullable branches with sibling schema definitions.") + if not isinstance(variants, list) or len(variants) != 2: + _reject("ScaleDown extraction supports anyOf/oneOf only for one schema plus null.") + branches: Final = tuple(branch for branch in variants if isinstance(branch, dict)) + non_null: Final = tuple(branch for branch in branches if branch.get("type") != "null") + if len(branches) != 2 or len(non_null) != 1: + _reject("ScaleDown extraction supports anyOf/oneOf only for one schema plus null.") + return { + **non_null[0], + **{key: value for key, value in schema.items() if key not in {"anyOf", "oneOf"}}, + } + + +def _deref( + prop: Mapping[str, JsonValue], root: Mapping[str, JsonValue], path: tuple[str, ...] +) -> tuple[Mapping[str, JsonValue], tuple[str, ...], bool]: + """Follow nullable branches and local `$ref`s (`#/$defs/X`, `#/definitions/X`). + + Returns the schema, the refs followed so far, and whether a ref repeated (a cycle). + The walk is a bounded loop, so a long chain of distinct refs cannot exhaust the stack. + """ + current: Mapping[str, JsonValue] = prop # rebind-ok: bounded walk along a ref chain + followed: tuple[str, ...] = path # rebind-ok: bounded walk along a ref chain + for _ in range(MAX_REF_CHAIN): + nullable: Mapping[str, JsonValue] | None = _nullable_branch(current) + ref: JsonValue = current.get("$ref") + if nullable is not None: + current = nullable + continue + if "$ref" not in current: + return current, followed, False + if not isinstance(ref, str) or not ref.startswith("#"): + _reject("ScaleDown extraction supports only local JSON Pointer $refs.") + pointer: str = unquote(ref[1:]) + if not pointer.startswith("/"): + _reject("ScaleDown extraction supports only local JSON Pointer $refs beginning with #/.") + if any(key in current for key in ("properties", "items", "type")): + _reject("ScaleDown extraction cannot combine $ref with sibling schema definitions.") + if ref in followed: + return current, followed, True + target: JsonValue = reduce(_child, pointer[1:].split("/"), dict(root)) + if not isinstance(target, Mapping): + _reject(f"ScaleDown extraction cannot resolve $ref '{ref}' to a schema object.") + current, followed = target, _extend_ref_path(followed, ref) + _reject(f"The response_format schema follows more than {MAX_REF_CHAIN} chained $refs or nullable branches.") + + +def _build_properties( + properties: Mapping[str, JsonValue], + root: Mapping[str, JsonValue], + path: tuple[str, ...], + budget: int, + depth: int = 0, +) -> tuple[dict[str, JsonValue], int]: # mutable-ok: serialized JSON entity map + """Turn JSON-schema properties into the /extract entity map, spending one budget unit per entity. + + Each property becomes an entity whose value is its description (or its name when there is + none). Nested objects stay nested, arrays of objects become a one-element list holding the + item's entity map, local `$ref`s are followed, and a ref that points back into itself is cut + off at that point instead of expanded. + """ + if depth > MAX_SCHEMA_DEPTH: + _reject(f"The response_format schema nests deeper than {MAX_SCHEMA_DEPTH} levels.") + built: Final[dict[str, JsonValue]] = {} # mutable-ok: filled once while threading the budget through + remaining = budget # rebind-ok: the budget is threaded through the loop + for name, prop in properties.items(): + if remaining <= 0: + _reject( + f"The response_format schema expands to more than {MAX_ENTITIES} entities; " + "remove recursive or heavily repeated $refs." + ) + built[name], remaining = _build_entity(name, prop, root, path, remaining - 1, depth) + return built, remaining + + +def _build_entity( + name: str, prop: JsonValue, root: Mapping[str, JsonValue], path: tuple[str, ...], budget: int, depth: int +) -> tuple[JsonValue, int]: + schema, followed, cyclic = _deref(prop if isinstance(prop, Mapping) else {}, root, path) + leaf: Final = schema.get("description") or name + nested: Final = schema.get("properties") + if cyclic: + return leaf, budget + if isinstance(nested, Mapping): + return _build_properties(nested, root, followed, budget, depth + 1) + raw_items: Final = schema.get("items") + schema_type: Final = schema.get("type") + is_array: Final = schema_type == "array" or (isinstance(schema_type, list) and "array" in schema_type) + if is_array and isinstance(raw_items, Mapping): + items, item_path, item_cyclic = _deref(raw_items, root, followed) + item_properties: Final = items.get("properties") + if not item_cyclic and isinstance(item_properties, Mapping): + built, remaining = _build_properties(item_properties, root, item_path, budget, depth + 1) + return [built], remaining + return leaf, budget + + +def _extract_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + response_format: Final = optional_params.get("response_format") + json_schema: Final = response_format.get("json_schema") if isinstance(response_format, Mapping) else None + if isinstance(json_schema, Mapping) and json_schema.get("strict") is True: + _reject( + "ScaleDown extract does not support strict JSON schemas. Use strict=false and validate the returned fields." + ) + schema: Final = json_schema.get("schema") if isinstance(json_schema, Mapping) else None + properties: Final = _deref(schema, schema, ())[0].get("properties") if isinstance(schema, Mapping) else None + if not isinstance(schema, dict) or not isinstance(properties, dict) or not properties: + _reject( + "ScaleDown extract needs response_format={'type': 'json_schema', ...} with a non-empty " + "'properties' map; each property becomes an entity and its description the extraction hint." + ) + return { + "text": _last_user_text("extract", messages), + "entities": _build_properties(properties, schema, (), MAX_ENTITIES)[0], + **{key: optional_params[key] for key in ("threshold", "top_n") if key in optional_params}, + } + + +def _summarize_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + instructions: Final = [ + text for message in messages if message.get("role") in {"system", "developer"} and (text := _text_of(message)) + ] + return { + "text": _last_user_text("summarize", messages), + **({"instructions": "\n".join(instructions)} if instructions else {}), + **({"max_tokens": optional_params["max_tokens"]} if "max_tokens" in optional_params else {}), + } + + +def _compress_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + prompt: Final = _last_user_text("compress", messages) + last_user_index: Final = max(i for i, m in enumerate(messages) if m.get("role") == "user" and _text_of(m)) + context: Final = "\n\n".join(text for i, m in enumerate(messages) if i != last_user_index and (text := _text_of(m))) + return { + "context": context, + "prompt": prompt, + "scaledown": {"rate": optional_params.get("compression_rate", "auto")}, + } + + +def _decisions_request( + messages: Sequence[Mapping[str, JsonValue]], optional_params: Mapping[str, JsonValue] +) -> Mapping[str, JsonValue]: + questions: Final = optional_params.get("questions") + if questions is not None: + _validate_questions(questions) + return { + "model": DECISIONS_UPSTREAM_MODEL, + "state": {"text": _last_user_text("decisions", messages)}, + **({"questions": questions} if questions is not None else {}), + } + + +def _require_complete_decisions(body: Mapping[str, JsonValue]) -> None: + if not body.get("questions"): + _reject( + "ScaleDown decisions requires a non-empty 'questions' map, passed as an extra parameter, " + 'e.g. {"questions": {"category": {"type": "choice", "criteria": {"billing": "..."}}}}. ' + "Question types are 'choice', 'noul', and 'score'." + ) + + +def _validate_questions(questions: JsonValue) -> None: + if not isinstance(questions, Mapping): + _reject(f"'questions' must be a map of name to question, got {type(questions).__name__}.") + for name, question in questions.items(): + if not isinstance(question, Mapping): + _reject(f"Question '{name}' must be an object.") + question_type = question.get("type") + if not isinstance(question_type, str) or question_type not in DECISION_QUESTION_TYPES: + _reject(f"Question '{name}' has type {question_type!r}; expected one of {sorted(DECISION_QUESTION_TYPES)}.") + criteria = question.get("criteria") + if question_type == "choice" and (not isinstance(criteria, Mapping) or not criteria): + _reject(f"Choice question '{name}' needs a non-empty 'criteria' map of option key to description.") + if question_type == "score" and ( + not isinstance(criteria, list) or not SCORE_MIN_LEVELS <= len(criteria) <= SCORE_MAX_LEVELS + ): + _reject( + f"Score question '{name}' needs 'criteria' as an ordered list of " + f"{SCORE_MIN_LEVELS} to {SCORE_MAX_LEVELS} level descriptions, lowest to highest." + ) + + +def _unwrap(value: JsonValue) -> JsonValue: + wrapper_keys: Final = {"_value", "_span_anchor"} + if isinstance(value, Mapping) and "_value" in value and set(value) <= wrapper_keys: + return value["_value"] + return value + + +def _clean_extraction(value: JsonValue, requested: JsonValue) -> JsonValue: + """Reduce an extraction result to the fields the caller's schema asked for. + + The request's entity map says which keys are real fields, so provider-added keys such as + `_span_anchor` and `{"_value": ..., "_span_anchor": ...}` wrappers are dropped without + touching a requested field that happens to share one of those names. The untouched payload + stays on `_hidden_params["scaledown_response"]`. + """ + if isinstance(requested, Mapping) and isinstance(value, Mapping): + return {key: _clean_extraction(value[key], requested[key]) for key in requested if key in value} + if isinstance(requested, list) and requested and isinstance(value, list): + return [_clean_extraction(item, requested[0]) for item in value] + return _unwrap(value) + + +def _extracted_fields(raw: Mapping[str, JsonValue], requested: JsonValue) -> JsonValue: + entities: Final = raw.get("entities") + matches: Final = ( + tuple(entity for entity in entities if isinstance(entity, Mapping)) if isinstance(entities, list) else () + ) + scalar_names: Final = ( + tuple(name for name, hint in requested.items() if isinstance(hint, str)) + if isinstance(requested, Mapping) + else () + ) + scalars: Final = { + name: next(entity["text"] for entity in matches if entity.get("type") == name and "text" in entity) + for name in scalar_names + if any(entity.get("type") == name and "text" in entity for entity in matches) + } + structured: Final = raw.get("structured_result") + return _clean_extraction({**scalars, **(structured if isinstance(structured, Mapping) else {})}, requested) + + +def _native_response( + operation: str, raw: JsonValue, model_response: ModelResponse, requested: JsonValue = None +) -> ModelResponse: + """Wrap a native /extract, /summarization/abstractive or /compress/raw/ payload. + + Summarize and compress return the upstream payload as JSON on choices[0].message.content. + Extract returns the fields requested by response_format without enforcing schema types. + The native APIs report input tokens only; they return no output token + count, so completion_tokens stays 0 because it is unmeasured, not because nothing was generated. + """ + if not isinstance(raw, Mapping) or not raw: + _reject( + f"ScaleDown '{operation}' returned an empty or malformed response: {json.dumps(raw)[:500]}", + 500, + ) + + results: Final = raw.get("results") + input_tokens: Final = raw.get("input_tokens") or ( + (results.get("original_prompt_tokens") if isinstance(results, Mapping) else None) + or raw.get("original_prompt_tokens") + if operation == "compress" + else None + ) + content: Final = _extracted_fields(raw, requested) if operation == "extract" else raw + + model_response.id = f"chatcmpl-{uuid.uuid4().hex}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.created = int(time.time()) # rebind-ok: LiteLLM fills the response object it passes in + model_response.model = f"scaledown/{operation}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.object = "chat.completion" # rebind-ok: LiteLLM fills the response object it passes in + model_response.choices = [ # rebind-ok: LiteLLM fills the response object it passes in + Choices( + index=0, + message=Message(role="assistant", content=json.dumps(content, separators=(",", ":"))), + finish_reason="stop", + ) + ] + _set_usage(model_response, {"prompt_tokens": input_tokens or 0}) + model_response.hidden_params["scaledown_response"] = raw # pyright: ignore[reportUnknownMemberType] # Inherited hidden_params has untyped values. # rebind-ok: supplied response + return model_response + + +def _decisions_response(operation: str, raw: JsonValue, model_response: ModelResponse) -> ModelResponse: + answers: Final = raw.get("answers") if isinstance(raw, Mapping) else None + if answers is None: + _reject(f"ScaleDown decisions response contained no answers: {json.dumps(raw)[:500]}", 500) + + model_response.id = f"chatcmpl-{uuid.uuid4().hex}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.created = int(time.time()) # rebind-ok: LiteLLM fills the response object it passes in + model_response.model = f"scaledown/{operation}" # rebind-ok: LiteLLM fills the response object it passes in + model_response.object = "chat.completion" # rebind-ok: LiteLLM fills the response object it passes in + model_response.choices = [ # rebind-ok: LiteLLM fills the response object it passes in + Choices( + index=0, + message=Message(role="assistant", content=json.dumps(answers, separators=(",", ":"))), + finish_reason="stop", + ) + ] + + usage: Final = raw.get("usage") if isinstance(raw, dict) else None + counts: Final = usage if isinstance(usage, dict) else {} + _set_usage( + model_response, + { + "prompt_tokens": counts.get("input_tokens", 0), + "completion_tokens": counts.get("output_tokens", 0), + }, + ) + model_response.hidden_params["scaledown_response"] = raw # pyright: ignore[reportUnknownMemberType] # Inherited hidden_params has untyped values. # rebind-ok: supplied response + return model_response + + +def _set_usage(model_response: ModelResponse, usage: Mapping[str, JsonValue]) -> None: + prompt_tokens: Final = _token_count(usage.get("prompt_tokens")) + completion_tokens: Final = _token_count(usage.get("completion_tokens")) + total_tokens: Final = usage.get("total_tokens") + model_response.usage = Usage( # pyright: ignore[reportAttributeAccessIssue] # ModelResponse stores usage as a dynamic Pydantic field. # rebind-ok: fills the supplied response + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=_token_count(total_tokens) if total_tokens is not None else prompt_tokens + completion_tokens, + ) + + +def _token_count(value: JsonValue) -> int: + if value is None: + return 0 + if isinstance(value, int) and value >= 0: + return value + _reject("ScaleDown returned an invalid token count.", 500) diff --git a/litellm/main.py b/litellm/main.py index 88c1257ee2c..c2c6d5affe8 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -1938,6 +1938,39 @@ def _complete_ragflow(ctx: CompletionDispatchContext) -> _CompletionDispatchResu return response +def _complete_scaledown(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: + try: + return base_llm_http_handler.completion( + model=ctx.model, + messages=ctx.messages, + headers=ctx.headers, + model_response=ctx.model_response, + api_key=ctx.api_key, + api_base=ctx.api_base, + acompletion=ctx.acompletion, + logging_obj=ctx.logging, + optional_params=ctx.optional_params, + litellm_params=ctx.litellm_params, + shared_session=ctx.shared_session, + timeout=cast( # cast-ok: CompletionTimeout.resolve normalizes this value + float | httpx.Timeout, ctx.timeout + ), + client=_dispatch_client_http(ctx), + custom_llm_provider=ctx.custom_llm_provider, + encoding=_get_encoding(), + stream=ctx.stream, + provider_config=ctx.provider_config, + ) + except Exception as exc: + ctx.logging.post_call( + input=ctx.messages, + api_key=ctx.api_key, + original_response=str(exc), + additional_args={"headers": ctx.headers}, + ) + raise + + def _complete_xai(ctx: CompletionDispatchContext) -> _CompletionDispatchResult: acompletion: Final = ctx.acompletion api_base: Final = ctx.api_base @@ -5859,6 +5892,8 @@ def completion( elif custom_llm_provider == "ragflow": ## COMPLETION CALL - RAGFlow uses HTTP handler to support custom URL paths response = _complete_ragflow(_dispatch_ctx) + elif custom_llm_provider == "scaledown": + response = _complete_scaledown(_dispatch_ctx) # rebind-ok: shared provider return elif custom_llm_provider == "xai": ## COMPLETION CALL response = _complete_xai(_dispatch_ctx) diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index cb910ae12ea..20558be3422 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -71842,5 +71842,51 @@ "image" ], "supports_embedding_image_input": true + }, + "scaledown/extract": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/summarize": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/compress": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_function_calling": false, + "supports_vision": false + }, + "scaledown/classify": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/decisions": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false } } diff --git a/litellm/provider_endpoints_support_backup.json b/litellm/provider_endpoints_support_backup.json index d9a6feb625d..a0e75b1f02d 100644 --- a/litellm/provider_endpoints_support_backup.json +++ b/litellm/provider_endpoints_support_backup.json @@ -2064,6 +2064,24 @@ "search": true } }, + "scaledown": { + "display_name": "ScaleDown (`scaledown`)", + "url": "https://docs.litellm.ai/docs/providers/scaledown", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "sambanova": { "display_name": "Sambanova (`sambanova`)", "url": "https://docs.litellm.ai/docs/providers/sambanova", diff --git a/litellm/proxy/public_endpoints/provider_create_fields.json b/litellm/proxy/public_endpoints/provider_create_fields.json index f20369e279a..633e9edf71c 100644 --- a/litellm/proxy/public_endpoints/provider_create_fields.json +++ b/litellm/proxy/public_endpoints/provider_create_fields.json @@ -1,4 +1,22 @@ [ + { + "provider": "SCALEDOWN", + "provider_display_name": "ScaleDown", + "litellm_provider": "scaledown", + "credential_fields": [ + { + "key": "api_key", + "label": "API Key", + "placeholder": null, + "tooltip": null, + "required": true, + "field_type": "password", + "options": null, + "default_value": null + } + ], + "default_model_placeholder": "scaledown/summarize" + }, { "provider": "AIML", "provider_display_name": "AI/ML API", diff --git a/litellm/types/utils.py b/litellm/types/utils.py index d6751610e7a..13fd86b74a7 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -4224,6 +4224,7 @@ class LlmProviders(str, Enum): FRIENDLIAI = "friendliai" FEATHERLESS_AI = "featherless_ai" WATSONX = "watsonx" + SCALEDOWN = "scaledown" WATSONX_TEXT = "watsonx_text" TRITON = "triton" PREDIBASE = "predibase" diff --git a/litellm/utils.py b/litellm/utils.py index e7a171f56c6..6774771dbf9 100644 --- a/litellm/utils.py +++ b/litellm/utils.py @@ -8655,6 +8655,7 @@ class ProviderConfigManager: False, ), LlmProviders.FEATHERLESS_AI: (lambda: litellm.FeatherlessAIConfig(), False), + LlmProviders.SCALEDOWN: (lambda: litellm.ScaleDownChatConfig(), False), LlmProviders.NOVITA: (lambda: litellm.NovitaConfig(), False), LlmProviders.NEBIUS: (lambda: litellm.NebiusConfig(), False), LlmProviders.WANDB: (lambda: litellm.WandbConfig(), False), diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index cb910ae12ea..20558be3422 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -71842,5 +71842,51 @@ "image" ], "supports_embedding_image_input": true + }, + "scaledown/extract": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_response_schema": false, + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/summarize": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/compress": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": true, + "supports_function_calling": false, + "supports_vision": false + }, + "scaledown/classify": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false + }, + "scaledown/decisions": { + "input_cost_per_token": 5e-08, + "output_cost_per_token": 0.0, + "litellm_provider": "scaledown", + "mode": "chat", + "supports_system_messages": false, + "supports_vision": false, + "supports_function_calling": false } } diff --git a/provider_endpoints_support.json b/provider_endpoints_support.json index 33ac4e4e60b..ea617cf8462 100644 --- a/provider_endpoints_support.json +++ b/provider_endpoints_support.json @@ -2349,6 +2349,24 @@ "search": true } }, + "scaledown": { + "display_name": "ScaleDown (`scaledown`)", + "url": "https://docs.litellm.ai/docs/providers/scaledown", + "endpoints": { + "chat_completions": true, + "messages": false, + "responses": false, + "embeddings": false, + "image_generations": false, + "audio_transcriptions": false, + "audio_speech": false, + "moderations": false, + "batches": false, + "rerank": false, + "a2a": false, + "interactions": false + } + }, "sambanova": { "display_name": "Sambanova (`sambanova`)", "url": "https://docs.litellm.ai/docs/providers/sambanova", diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index 7166891bb1a..cea2b4154a7 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -67,6 +67,7 @@ IGNORE_FUNCTIONS = [ "strict_json_schema", # harness: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising ValueError at the cap. "toml_value", # harness/codex: max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by raising OptionsMismatch at the cap. "with_json_string_leaves", # transitively bounded: only runs on a tree json_string_leaves already walked under the cap. + "_clean_extraction", # ScaleDown: traverses the entity map built under MAX_SCHEMA_DEPTH=32 and MAX_ENTITIES=1000. "json_unrewritable_labels", # max depth set (MAX_STRUCTURED_CONTENT_SCAN_DEPTH); returns the None sentinel at the cap so the caller blocks. "_flatten_form_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). "_flatten_form_data_field", # bounded by the nesting depth of the already-parsed request body (a finite JSON tree, no cycles possible). diff --git a/tests/unit/llms/scaledown/__init__.py b/tests/unit/llms/scaledown/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/scaledown/chat/__init__.py b/tests/unit/llms/scaledown/chat/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py b/tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py new file mode 100644 index 00000000000..82f03a2cd80 --- /dev/null +++ b/tests/unit/llms/scaledown/chat/test_scaledown_chat_transformation.py @@ -0,0 +1,1213 @@ +import json +from collections.abc import Mapping +from queue import SimpleQueue +from typing import Final, Literal + +import httpx +import pytest +import respx +from pydantic import JsonValue + +import litellm +from litellm.llms.scaledown.chat.transformation import ( + DECISIONS_UPSTREAM_MODEL, + ScaleDownChatConfig, + ScaleDownError, +) +from litellm.types.utils import ModelResponse + +BASE = "https://api.scaledown.xyz" + + +@pytest.fixture +def config() -> ScaleDownChatConfig: + return ScaleDownChatConfig() + + +@pytest.fixture(autouse=True) +def scaledown_api_key(monkeypatch): + monkeypatch.setenv("SCALEDOWN_API_KEY", "sk-scaledown-test") + monkeypatch.delenv("SCALEDOWN_API_BASE", raising=False) + + +def _transform_response( + config: ScaleDownChatConfig, model: str, payload: dict, request_data: dict | None = None +) -> ModelResponse: + return config.transform_response( + model=model, + raw_response=httpx.Response( + status_code=200, + json=payload, + request=httpx.Request("POST", f"{BASE}/extract"), + ), + model_response=ModelResponse(), + logging_obj=None, + request_data=request_data or {}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + +def _build_decisions(config: ScaleDownChatConfig, messages: list, optional_params: dict) -> dict: + body = config.transform_request( + model="scaledown/decisions", messages=messages, optional_params=optional_params, litellm_params={}, headers={} + ) + config.sign_request( + headers={}, + optional_params=optional_params, + request_data=body, + api_base=f"{BASE}/v1/scaledown", + model="scaledown/decisions", + ) + return body + + +def test_auth_uses_x_api_key_not_bearer(config): + headers = config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + api_key="sk-explicit", + ) + + assert headers["x-api-key"] == "sk-explicit" + assert "Authorization" not in headers + + +def test_missing_api_key_is_rejected_before_any_request(config, monkeypatch): + monkeypatch.delenv("SCALEDOWN_API_KEY", raising=False) + + with pytest.raises(ScaleDownError) as exc: + config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + ) + + assert exc.value.status_code == 401 + + +@pytest.mark.parametrize( + "model, path", + [ + ("scaledown/extract", "/extract"), + ("scaledown/summarize", "/summarization/abstractive"), + ("scaledown/compress", "/compress/raw/"), + ("scaledown/classify", "/classify"), + ("scaledown/decisions", "/v1/scaledown"), + ], +) +def test_each_model_targets_its_native_endpoint(config, model, path): + url = config.get_complete_url(api_base=None, api_key="k", model=model, optional_params={}, litellm_params={}) + + assert url == f"{BASE}{path}" + + +@pytest.mark.parametrize( + "api_base", + ["https://staging.scaledown.xyz", "https://staging.scaledown.xyz/", "https://staging.scaledown.xyz/v1"], +) +def test_api_base_with_or_without_v1_resolves_to_the_same_host(config, api_base): + extract = config.get_complete_url( + api_base=api_base, api_key="k", model="scaledown/extract", optional_params={}, litellm_params={} + ) + decisions = config.get_complete_url( + api_base=api_base, api_key="k", model="scaledown/decisions", optional_params={}, litellm_params={} + ) + + assert extract == "https://staging.scaledown.xyz/extract" + assert decisions == "https://staging.scaledown.xyz/v1/scaledown" + + +def test_decisions_request_carries_state_and_questions_without_chat_keys(config): + questions = { + "category": { + "type": "choice", + "instructions": "Which category?", + "criteria": {"billing": "A charge or refund.", "technical": "A bug."}, + } + } + + body = config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "I was charged twice."}], + optional_params={"questions": questions}, + litellm_params={}, + headers={}, + ) + + assert body == { + "model": DECISIONS_UPSTREAM_MODEL, + "state": {"text": "I was charged twice."}, + "questions": questions, + } + + +def test_decisions_alias_sends_the_only_model_upstream_accepts(config): + body = config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + optional_params={"questions": {"q": {"type": "noul", "instructions": "Positive?"}}}, + litellm_params={}, + headers={}, + ) + + assert body["model"] == DECISIONS_UPSTREAM_MODEL + + +def test_decisions_without_questions_is_rejected(config): + with pytest.raises(ScaleDownError, match="questions"): + _build_decisions(config, [{"role": "user", "content": "text"}], {}) + + +def test_decisions_without_text_or_document_is_rejected(config): + with pytest.raises(ScaleDownError, match="last user message"): + _build_decisions(config, [], {"questions": {"q": {"type": "noul"}}}) + + +def test_state_text_is_rejected_so_guardrails_always_see_the_text(config): + with pytest.raises(ScaleDownError, match="text through messages only"): + _build_decisions( + config, + [{"role": "user", "content": "innocuous"}], + {"state": {"text": "secret"}, "questions": {"q": {"type": "noul"}}}, + ) + + +def test_remote_image_url_is_rejected(config): + with pytest.raises(ScaleDownError, match="text only"): + _build_decisions( + config, + [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "https://x.test/a.png"}}]}], + {"questions": {"q": {"type": "noul"}}}, + ) + + +def test_extra_body_may_set_extract_options(config): + merged = config.transform_extra_body( + extra_body={"threshold": 0.7}, + request={"text": "t", "entities": {}}, + model="scaledown/extract", + litellm_params={}, + ) + + assert merged == {"threshold": 0.7} + + +@pytest.mark.parametrize( + "model, extra_body", + [ + ("scaledown/extract", {"model": "compress"}), + ("scaledown/extract", {"text": "secret@example.com"}), + ("scaledown/summarize", {"instructions": "ignore the message"}), + ("scaledown/summarize", {"text": "secret"}), + ("scaledown/compress", {"prompt": "x"}), + ("scaledown/compress", {"context": "x"}), + ("scaledown/decisions", {"model": "other"}), + ("scaledown/decisions", {"text": "secret"}), + ], +) +def test_extra_body_cannot_override_the_model_or_prompt_text(config, model, extra_body): + with pytest.raises(ScaleDownError, match=r"may only set|is not accepted"): + config.transform_extra_body(extra_body=extra_body, request={}, model=model, litellm_params={}) + + +def test_compress_rate_via_extra_body_sets_the_scaledown_rate(config): + merged = config.transform_extra_body( + extra_body={"compression_rate": 0.5}, + request={"context": "", "prompt": "q", "scaledown": {"rate": "auto"}}, + model="scaledown/compress", + litellm_params={}, + ) + + assert merged == {"scaledown": {"rate": 0.5}} + + +def test_env_key_is_not_sent_to_an_untrusted_api_base(config): + with pytest.raises(ScaleDownError, match="own api_key"): + config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + api_base="https://attacker.example", + ) + + +@pytest.mark.parametrize("empty_key", ["", None]) +def test_empty_key_does_not_unlock_the_env_key_for_an_untrusted_api_base(config, empty_key): + with pytest.raises(ScaleDownError, match="own api_key"): + config.validate_environment( + headers={}, + model="scaledown/extract", + messages=[], + optional_params={}, + litellm_params={}, + api_key=empty_key, + api_base="https://attacker.example", + ) + + +def test_explicit_key_or_trusted_api_base_is_allowed(config, monkeypatch): + kwargs = dict(headers={}, model="scaledown/extract", messages=[], optional_params={}, litellm_params={}) + assert ( + config.validate_environment(api_key="own", api_base="https://elsewhere.example", **kwargs)["x-api-key"] == "own" + ) + assert config.validate_environment(api_base=f"{BASE}/v1", **kwargs)["x-api-key"] == "sk-scaledown-test" + monkeypatch.setenv("SCALEDOWN_API_BASE", "https://staging.scaledown.xyz") + assert ( + config.validate_environment(api_base="https://staging.scaledown.xyz/", **kwargs)["x-api-key"] + == "sk-scaledown-test" + ) + + +def test_max_completion_tokens_maps_to_max_tokens(config): + assert config.map_openai_params( + non_default_params={"max_completion_tokens": 50}, + optional_params={}, + model="scaledown/summarize", + drop_params=False, + ) == {"max_tokens": 50} + + +def test_router_params_are_accepted_and_not_forwarded(config): + assert ( + config.map_openai_params( + non_default_params={"max_retries": 2}, optional_params={}, model="scaledown/decisions", drop_params=False + ) + == {} + ) + + +def test_every_model_fakes_a_stream(config): + assert config.should_fake_stream(model="scaledown/compress", stream=True) + assert not config.should_fake_stream(model="scaledown/compress", stream=False) + + +def test_unknown_question_type_is_rejected(config): + with pytest.raises(ScaleDownError, match="expected one of"): + config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "ranking"}}}, + litellm_params={}, + headers={}, + ) + + +def test_choice_question_without_criteria_is_rejected(config): + with pytest.raises(ScaleDownError, match="non-empty 'criteria' map"): + config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "choice", "criteria": {}}}}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("levels", [1, 11]) +def test_score_criteria_outside_two_to_ten_is_rejected(config, levels): + with pytest.raises(ScaleDownError, match="ordered list"): + config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "score", "criteria": ["l"] * levels}}}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("levels", [2, 10]) +def test_score_criteria_within_two_to_ten_is_accepted(config, levels): + criteria = [f"level {index}" for index in range(levels)] + + body = config.transform_request( + model="scaledown/decisions", + messages=[{"role": "user", "content": "t"}], + optional_params={"questions": {"q": {"type": "score", "criteria": criteria}}}, + litellm_params={}, + headers={}, + ) + + assert body["questions"]["q"]["criteria"] == criteria + + +def test_extract_content_is_the_clean_fields_and_raw_payload_is_kept(config): + payload = { + "entities": [], + "structured_result": { + "invoice": { + "vendor": "Northwind", + "vendor_span_anchor": "Invoice from Northwind", + "customer": {"_value": "Ada Lovelace", "_span_anchor": "to Ada Lovelace"}, + "amount": 500, + } + }, + "input_tokens": 171, + } + + requested = {"invoice": {"vendor": "v", "customer": "c", "amount": "a"}} + response = _transform_response(config, "scaledown/extract", payload, {"entities": requested}) + + assert json.loads(response.choices[0].message.content) == { + "invoice": {"vendor": "Northwind", "customer": "Ada Lovelace", "amount": 500} + } + assert response.model == "scaledown/extract" + assert response.usage.prompt_tokens == 171 + assert response.usage.completion_tokens == 0 + assert response._hidden_params["scaledown_response"] == payload + + +def test_extract_cleaning_keeps_requested_fields_named_like_provider_keys(config): + payload = {"structured_result": {"_value": "x", "vendor": "y", "note_span_anchor": "z", "note": "n"}} + requested = {"_value": "v", "vendor": "v", "note_span_anchor": "s", "note": "n"} + + response = _transform_response(config, "scaledown/extract", payload, {"entities": requested}) + + assert json.loads(response.choices[0].message.content) == payload["structured_result"] + + +def test_extract_cleaning_handles_lists_of_objects(config): + payload = {"structured_result": {"items": [{"sku": "a", "sku_span_anchor": "s"}, {"sku": {"_value": "b"}}]}} + + response = _transform_response(config, "scaledown/extract", payload, {"entities": {"items": [{"sku": "code"}]}}) + + assert json.loads(response.choices[0].message.content) == {"items": [{"sku": "a"}, {"sku": "b"}]} + + +def test_long_ref_chain_is_rejected_cleanly_not_with_a_recursion_error(config): + defs = {f"D{i}": {"$ref": f"#/$defs/D{i + 1}"} for i in range(5000)} + defs["D5000"] = {"type": "string"} + schema = {"type": "object", "$defs": defs, "properties": {"f": {"$ref": "#/$defs/D0"}}} + + with pytest.raises(ScaleDownError, match="chained"): + _extract_body(config, schema) + + +def test_deeply_nested_schema_is_rejected_cleanly(config): + nested: dict = {"type": "string"} + for _ in range(200): + nested = {"type": "object", "properties": {"x": nested}} + schema = {"type": "object", "properties": {"root": nested}} + + with pytest.raises(ScaleDownError, match="nests deeper"): + _extract_body(config, schema) + + +def test_more_than_one_image_is_rejected(config): + image = {"type": "image_url", "image_url": {"url": "data:image/png;base64,aGVsbG8="}} + + with pytest.raises(ScaleDownError, match="text only"): + _build_decisions(config, [{"role": "user", "content": [image, image]}], {"questions": {"q": {"type": "noul"}}}) + + +def test_stream_options_is_accepted_and_not_forwarded(config): + assert ( + config.map_openai_params( + non_default_params={"stream_options": {"include_usage": True}}, + optional_params={}, + model="scaledown/summarize", + drop_params=False, + ) + == {} + ) + + +@pytest.mark.parametrize("wrapper", [None, "anyOf", "oneOf"]) +def test_extract_schema_follows_local_refs(config: ScaleDownChatConfig, wrapper: str | None) -> None: + address: Final = {"$ref": "#/$defs/Address"} + schema: Final = { + "type": "object", + "$defs": { + "Address": {"type": "object", "properties": {"city": {"type": "string", "description": "city name"}}} + }, + "properties": { + "address": {wrapper: [address, {"type": "null"}]} if wrapper else address, + "name": {"type": "string"}, + }, + } + + body = config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "text"}], + optional_params={"response_format": {"type": "json_schema", "json_schema": {"name": "n", "schema": schema}}}, + litellm_params={}, + headers={}, + ) + + assert body["entities"] == {"address": {"city": "city name"}, "name": "name"} + + +@pytest.mark.parametrize( + ("definition", "reference"), + [ + ("Invoice/Details", "#/$defs/Invoice~1Details"), + ("Invoice~Details", "#/$defs/Invoice~0Details"), + ("Invoice~1 Details", "#/%24defs/Invoice~01%20Details"), + ], +) +def test_extract_preserves_fields_from_escaped_refs( + config: ScaleDownChatConfig, definition: str, reference: str +) -> None: + schema: Final = { + "type": "object", + "$defs": {definition: {"type": "object", "properties": {"amount": {"type": "number"}}}}, + "properties": {"invoice": {"$ref": reference}}, + } + + assert _extract_body(config, schema)["entities"] == {"invoice": {"amount": "amount"}} + + +@pytest.mark.parametrize("reference", ["#/$defs/Missing", "https://example.com/schema", "#named-anchor", None, []]) +def test_extract_rejects_unresolvable_refs(config: ScaleDownChatConfig, reference: JsonValue) -> None: + schema: Final = {"type": "object", "properties": {"invoice": {"$ref": reference}}} + + with pytest.raises(ScaleDownError, match=r"\$ref") as exc: + _extract_body(config, schema) + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("field_type", ["array", ["array", "null"]]) +def test_extract_nullable_array_keeps_item_fields(config: ScaleDownChatConfig, field_type: str | list[str]) -> None: + schema: Final = { + "type": "object", + "properties": { + "invoices": { + "type": field_type, + "items": {"type": "object", "properties": {"amount": {"type": "number"}}}, + } + }, + } + + assert _extract_body(config, schema)["entities"] == {"invoices": [{"amount": "amount"}]} + + +@pytest.mark.parametrize("wrapper", ["anyOf", "oneOf", "$ref"]) +def test_extract_rejects_sibling_fields_instead_of_dropping_them(config: ScaleDownChatConfig, wrapper: str) -> None: + address: Final = {"type": "object", "properties": {"city": {"type": "string"}}} + reference: Final = {"$ref": "#/$defs/Address"} + field: Final = { + **(reference if wrapper == "$ref" else {wrapper: [address, {"type": "null"}]}), + "properties": {"postal_code": {"type": "string"}}, + } + schema: Final = {"$defs": {"Address": address}, "type": "object", "properties": {"address": field}} + + with pytest.raises(ScaleDownError, match="sibling") as exc: + _extract_body(config, schema) + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize( + "field", + [ + {"anyOf": [{"type": "object", "properties": {"x": {"type": "string"}}}, {"type": "string"}]}, + {"oneOf": [{"type": "string"}]}, + {"anyOf": [False, {"type": "string"}]}, + {"allOf": [{"type": "object", "properties": {"x": {"type": "string"}}}]}, + ], +) +def test_extract_rejects_unsupported_composition(config: ScaleDownChatConfig, field: dict[str, JsonValue]) -> None: + with pytest.raises(ScaleDownError) as exc: + _extract_body(config, {"type": "object", "properties": {"field": field}}) + + assert exc.value.status_code == 400 + + +@respx.mock +@pytest.mark.parametrize("question_type", [[], {}]) +def test_malformed_question_types_are_request_errors(question_type: list[JsonValue] | dict[str, JsonValue]) -> None: + with pytest.raises(litellm.BadRequestError, match="Question 'q'"): + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": question_type}}, + ) + + assert not respx.calls + + +@respx.mock +def test_unknown_scaledown_model_is_a_request_error() -> None: + with pytest.raises(litellm.BadRequestError, match="Unknown ScaleDown model"): + litellm.completion(model="scaledown/unknown", messages=[{"role": "user", "content": "text"}]) + + assert not respx.calls + + +def _extract_body(config: ScaleDownChatConfig, schema: dict) -> dict: + return config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "text"}], + optional_params={"response_format": {"type": "json_schema", "json_schema": {"name": "n", "schema": schema}}}, + litellm_params={}, + headers={}, + ) + + +def test_recursive_schema_is_cut_at_the_cycle_instead_of_expanded(config): + children = {name: {"$ref": "#/$defs/Node"} for name in ("a", "b", "c", "d")} + schema = { + "type": "object", + "$defs": {"Node": {"type": "object", "properties": children}}, + "properties": {"root": {"$ref": "#/$defs/Node"}}, + } + + entities = _extract_body(config, schema)["entities"] + + assert entities == {"root": {"a": "a", "b": "b", "c": "c", "d": "d"}} + + +def test_heavily_repeated_acyclic_refs_are_rejected_by_the_entity_budget(config): + defs = { + f"L{level}": {"type": "object", "properties": {name: {"$ref": f"#/$defs/L{level + 1}"} for name in "abcd"}} + for level in range(10) + } + defs["L10"] = {"type": "string", "description": "leaf"} + schema = {"type": "object", "$defs": defs, "properties": {"root": {"$ref": "#/$defs/L0"}}} + + with pytest.raises(ScaleDownError, match="expands to more than"): + _extract_body(config, schema) + + +def test_summarize_response_is_the_native_payload(config): + payload = {"summary": "terse summary", "input_chars": 400, "output_chars": 13, "input_tokens": 78} + + response = _transform_response(config, "scaledown/summarize", payload) + + assert json.loads(response.choices[0].message.content) == payload + assert response.usage.prompt_tokens == 78 + + +def test_compress_response_reports_top_level_input_tokens(config): + payload = { + "results": {"success": True, "compressed_prompt": "short", "original_prompt_tokens": 52}, + "input_tokens": 52, + } + + response = _transform_response(config, "scaledown/compress", payload) + + assert json.loads(response.choices[0].message.content) == payload + assert response.usage.prompt_tokens == 52 + + +def test_compress_response_falls_back_to_original_prompt_tokens(config): + payload = {"results": {"compressed_prompt": "short", "original_prompt_tokens": 900}} + + assert _transform_response(config, "scaledown/compress", payload).usage.prompt_tokens == 900 + + +def test_extract_request_maps_schema_to_entities_including_nesting(config): + schema = { + "type": "object", + "properties": { + "vendor": {"type": "string", "description": "company that issued the invoice"}, + "notes": {"type": "string"}, + "address": {"type": "object", "properties": {"city": {"type": "string", "description": "city name"}}}, + "line_items": { + "type": "array", + "items": {"type": "object", "properties": {"sku": {"type": "string", "description": "item code"}}}, + }, + }, + } + + body = config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind."}], + optional_params={"response_format": {"type": "json_schema", "json_schema": {"name": "i", "schema": schema}}}, + litellm_params={}, + headers={}, + ) + + assert body == { + "text": "Invoice from Northwind.", + "entities": { + "vendor": "company that issued the invoice", + "notes": "notes", + "address": {"city": "city name"}, + "line_items": [{"sku": "item code"}], + }, + } + + +def test_extract_without_a_schema_is_rejected(config): + with pytest.raises(ScaleDownError, match="response_format"): + config.transform_request( + model="scaledown/extract", + messages=[{"role": "user", "content": "text"}], + optional_params={}, + litellm_params={}, + headers={}, + ) + + +@pytest.mark.parametrize("role", ["system", "developer"]) +def test_summarize_request_splits_instructions_text_and_max_tokens( + config: ScaleDownChatConfig, role: Literal["system", "developer"] +) -> None: + body: Final = config.transform_request( + model="scaledown/summarize", + messages=[ + {"role": role, "content": "Be terse."}, + {"role": "user", "content": "a long document"}, + ], + optional_params={"max_tokens": 40}, + litellm_params={}, + headers={}, + ) + + assert body == {"text": "a long document", "instructions": "Be terse.", "max_tokens": 40} + + +def test_compress_request_puts_earlier_messages_in_context_and_defaults_rate_to_auto(config): + body = config.transform_request( + model="scaledown/compress", + messages=[ + {"role": "system", "content": "background"}, + {"role": "user", "content": "the question"}, + ], + optional_params={}, + litellm_params={}, + headers={}, + ) + + assert body == {"context": "background", "prompt": "the question", "scaledown": {"rate": "auto"}} + + +def test_decisions_response_ignores_the_upstream_cost_field(config): + payload = { + "model": "classify-1", + "answers": { + "category": { + "type": "choice", + "choice": "billing", + "probabilities": {"billing": 0.93, "technical": 0.07}, + "confidence": 0.93, + } + }, + "usage": {"input_tokens": 62, "output_tokens": 1, "cost": 0.00543}, + } + + response = _transform_response(config, "scaledown/decisions", payload) + + assert json.loads(response.choices[0].message.content) == payload["answers"] + assert response.usage.prompt_tokens == 62 + assert response.usage.completion_tokens == 1 + assert "response_cost" not in response._hidden_params + assert response._hidden_params["scaledown_response"] == payload + + +def test_score_answer_survives_the_envelope_intact(config): + answer = { + "type": "score", + "score": 1.43, + "confidence": 0.57, + "legend": {"0": "Cosmetic", "1": "Workaround", "2": "Blocking"}, + "probabilities": {"0": 0.0, "1": 0.57, "2": 0.43}, + } + payload = { + "model": "classify-1", + "answers": {"bug_severity": answer}, + "usage": {"input_tokens": 68, "output_tokens": 1, "cost": 0.0000029}, + } + + response = _transform_response(config, "scaledown/decisions", payload) + + assert json.loads(response.choices[0].message.content)["bug_severity"] == answer + + +def test_decisions_response_without_answers_is_an_error(config): + with pytest.raises(ScaleDownError, match="no answers"): + _transform_response(config, "scaledown/decisions", {"model": "classify-1"}) + + +def test_empty_native_response_is_an_error(config): + with pytest.raises(ScaleDownError, match="empty or malformed"): + _transform_response(config, "scaledown/extract", {}) + + +@pytest.mark.parametrize("upstream_status", [200, 502]) +def test_non_json_response_is_an_upstream_error(config: ScaleDownChatConfig, upstream_status: int) -> None: + raw_response: Final = httpx.Response( + status_code=upstream_status, + text="bad gateway", + request=httpx.Request("POST", f"{BASE}/v1/scaledown"), + ) + + with pytest.raises(ScaleDownError) as exc: + config.transform_response( + model="scaledown/decisions", + raw_response=raw_response, + model_response=ModelResponse(), + logging_obj=None, + request_data={}, + messages=[], + optional_params={}, + litellm_params={}, + encoding=None, + ) + + assert exc.value.status_code == 502 + + +def test_decisions_takes_no_openai_sampling_params(config): + assert config.get_supported_openai_params("scaledown/decisions") == ["stream"] + + +def test_unsupported_param_is_rejected_unless_dropped(config): + with pytest.raises(ScaleDownError, match="does not support"): + config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="scaledown/decisions", + drop_params=False, + ) + + assert ( + config.map_openai_params( + non_default_params={"temperature": 0.7}, + optional_params={}, + model="scaledown/decisions", + drop_params=True, + ) + == {} + ) + + +@respx.mock +def test_completion_sends_x_api_key_to_the_native_summarize_endpoint(): + route = respx.post(f"{BASE}/summarization/abstractive").mock( + return_value=httpx.Response(200, json={"summary": "s", "input_tokens": 120}) + ) + + response = litellm.completion( + model="scaledown/summarize", + messages=[ + {"role": "system", "content": "Be terse."}, + {"role": "user", "content": "a long document"}, + ], + ) + + request = route.calls[0].request + assert request.headers["x-api-key"] == "sk-scaledown-test" + assert "authorization" not in request.headers + assert json.loads(request.content) == {"text": "a long document", "instructions": "Be terse."} + assert json.loads(response.choices[0].message.content)["summary"] == "s" + + +@respx.mock +def test_completion_routes_decisions_to_the_decisions_endpoint(): + route = respx.post(f"{BASE}/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": { + "category": { + "type": "choice", + "choice": "billing", + "probabilities": {"billing": 0.93, "technical": 0.07}, + "confidence": 0.93, + } + }, + "usage": {"input_tokens": 62, "output_tokens": 1, "cost": 0.00543}, + }, + ) + ) + + response = litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "I was charged twice."}], + questions={ + "category": { + "type": "choice", + "criteria": {"billing": "A charge or refund.", "technical": "A bug."}, + } + }, + ) + + sent = json.loads(route.calls[0].request.content) + assert sent["model"] == "classify-1" + assert sent["state"] == {"text": "I was charged twice."} + assert "messages" not in sent + assert json.loads(response.choices[0].message.content)["category"]["choice"] == "billing" + + +@respx.mock +def test_completion_extract_converts_response_format_to_entities(): + schema = { + "type": "json_schema", + "json_schema": { + "name": "invoice", + "schema": { + "type": "object", + "properties": {"vendor": {"type": "string", "description": "company name"}}, + }, + }, + } + extract_result = {"entities": [], "structured_result": {"vendor": "Acme Corp"}, "input_tokens": 20} + route = respx.post(f"{BASE}/extract").mock(return_value=httpx.Response(200, json=extract_result)) + + response = litellm.completion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Acme Corp invoiced $500."}], + response_format=schema, + ) + + assert json.loads(route.calls[0].request.content) == { + "text": "Acme Corp invoiced $500.", + "entities": {"vendor": "company name"}, + } + assert json.loads(response.choices[0].message.content) == {"vendor": "Acme Corp"} + assert response._hidden_params["scaledown_response"] == extract_result + + +@respx.mock +def test_completion_cost_comes_from_input_tokens_not_upstream_usage_cost(): + respx.post(f"{BASE}/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": {"q": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 1000, "output_tokens": 1, "cost": 99.0}, + }, + ) + ) + + response = litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + ) + + cost = litellm.completion_cost(completion_response=response, model="scaledown/decisions") + assert cost == pytest.approx(1000 * litellm.model_cost["scaledown/decisions"]["input_cost_per_token"]) + + +@respx.mock +def test_completion_honors_scaledown_api_base(monkeypatch): + monkeypatch.setenv("SCALEDOWN_API_BASE", "https://staging.scaledown.xyz/v1") + route = respx.post("https://staging.scaledown.xyz/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": {"q": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 10, "output_tokens": 1, "cost": 0.0}, + }, + ) + ) + + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + ) + + assert route.called + + +@respx.mock +def test_upstream_payment_error_surfaces_the_detail(): + respx.post(f"{BASE}/v1/scaledown").mock(return_value=httpx.Response(402, json={"detail": "Insufficient credits"})) + + with pytest.raises(litellm.exceptions.BadRequestError) as exc: + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + ) + + assert "Insufficient credits" in str(exc.value) + + +@respx.mock +def test_failed_completion_invokes_post_api_logger() -> None: + events: Final = SimpleQueue[str]() + + def logger_fn(details: Mapping[str, object]) -> None: + if details.get("log_event_type") == "post_api_call": + events.put(str(details.get("original_response"))) + + respx.post(f"{BASE}/v1/scaledown").respond(402, json={"detail": "Insufficient credits"}) + + with pytest.raises(litellm.BadRequestError): + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + questions={"q": {"type": "noul", "instructions": "Positive?"}}, + logger_fn=logger_fn, + ) + + assert not events.empty() + assert "Insufficient credits" in events.get_nowait() + + +@respx.mock +def test_malformed_decisions_request_never_reaches_the_network(): + route = respx.post(f"{BASE}/v1/scaledown") + + with pytest.raises(Exception, match="questions"): + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + ) + + assert not route.called + + +@respx.mock +def test_questions_passed_through_extra_body_reach_scaledown(): + route = respx.post(f"{BASE}/v1/scaledown").mock( + return_value=httpx.Response( + 200, + json={ + "model": "classify-1", + "answers": {"q": {"type": "noul", "noul": 0.8}}, + "usage": {"input_tokens": 10, "output_tokens": 1}, + }, + ) + ) + questions = {"q": {"type": "noul", "instructions": "Positive?"}} + + litellm.completion( + model="scaledown/decisions", + messages=[{"role": "user", "content": "text"}], + extra_body={"questions": questions}, + ) + + sent = json.loads(route.calls[0].request.content) + assert sent == {"model": DECISIONS_UPSTREAM_MODEL, "state": {"text": "text"}, "questions": questions} + + +@respx.mock +def test_streaming_request_is_served_as_a_single_chunk(): + respx.post(f"{BASE}/compress/raw/").mock( + return_value=httpx.Response(200, json={"compressed_prompt": "c", "original_prompt_tokens": 9}) + ) + + chunks = list( + litellm.completion( + model="scaledown/compress", + messages=[{"role": "user", "content": "the question"}], + stream=True, + ) + ) + + assert "compressed_prompt" in "".join(chunk.choices[0].delta.content or "" for chunk in chunks) + + +@pytest.mark.parametrize( + "structured,expected", + [ + (None, {"vendor": "Northwind"}), + ( + {"invoice": {"amount": 500, "amount_span_anchor": "$500"}}, + {"vendor": "Northwind", "invoice": {"amount": 500}}, + ), + ], +) +def test_extract_merges_first_scalar_match_with_nested_fields(config, structured, expected): + payload = { + "entities": [ + {"type": "vendor", "text": "Northwind", "confidence": 1.0}, + {"type": "vendor", "text": "Contoso", "confidence": 0.8}, + {"type": "unrequested", "text": "ignore"}, + ], + "structured_result": structured, + "input_tokens": 161, + } + response = _transform_response( + config, + "scaledown/extract", + payload, + {"entities": {"vendor": "company name", "missing": "absent field", "invoice": {"amount": "total"}}}, + ) + + assert json.loads(response.choices[0].message.content) == expected + assert response._hidden_params["scaledown_response"] == payload + assert response.usage.prompt_tokens == 161 + + +@respx.mock +def test_strict_extraction_is_rejected_before_network(): + route = respx.post(f"{BASE}/extract") + with pytest.raises(litellm.BadRequestError, match="strict JSON schemas"): + litellm.completion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind. Total: $500."}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "invoice", + "strict": True, + "schema": {"type": "object", "properties": {"amount": {"type": "string"}}, "required": ["amount"]}, + }, + }, + ) + assert not route.called + + +@pytest.mark.parametrize("model", ["classify", "decisions", "extract", "summarize", "compress"]) +@pytest.mark.parametrize( + "content", + [ + [{"type": "image_url", "image_url": {"url": "data:image/png;base64,aGVsbG8="}}], + [ + {"type": "text", "text": "invoice"}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,aGVsbG8="}}, + ], + ], +) +def test_multimodal_inputs_are_rejected_instead_of_dropped(config, model, content): + with pytest.raises(ScaleDownError, match="text only"): + config.transform_request(f"scaledown/{model}", [{"role": "user", "content": content}], {}, {}, {}) + + +@pytest.mark.parametrize("params", [{"document": "QQ=="}, {"state": {"document": "QQ=="}}, {"text": "override"}]) +def test_native_input_overrides_are_rejected(config, params): + with pytest.raises(ScaleDownError, match="text through messages only"): + config.transform_request("scaledown/decisions", [{"role": "user", "content": "text"}], params, {}, {}) + + +@pytest.mark.parametrize("model", ["classify", "decisions", "extract"]) +def test_unsupported_system_instructions_are_rejected(config, model): + with pytest.raises(ScaleDownError, match="does not support system messages"): + config.transform_request( + f"scaledown/{model}", + [{"role": "system", "content": "instructions"}, {"role": "user", "content": "text"}], + {}, + {}, + {}, + ) + + +@respx.mock +@pytest.mark.parametrize("nested", [False, True]) +def test_classify_uses_native_labels_and_preserves_scores(nested): + payload = {"top_label": "urgent", "scores": {"urgent": 0.9, "routine": 0.1}, "input_tokens": 521} + route = respx.post(f"{BASE}/classify").mock(return_value=httpx.Response(200, json=payload)) + labels = [ + {"name": "urgent", "rubric": "Service is unavailable."}, + {"name": "routine", "rubric": "A cosmetic issue."}, + ] + options = {"extra_body": {"labels": labels}} if nested else {"labels": labels} + + response = litellm.completion( + model="scaledown/classify", + messages=[{"role": "user", "content": [{"type": "text", "text": "Server is down."}]}], + **options, + ) + + assert json.loads(route.calls[0].request.content) == {"text": "Server is down.", "labels": labels} + assert json.loads(response.choices[0].message.content) == payload + assert response.usage.prompt_tokens == 521 + assert litellm.completion_cost(completion_response=response) == pytest.approx( + 521 * litellm.model_cost["scaledown/classify"]["input_cost_per_token"] + ) + + +@respx.mock +@pytest.mark.parametrize("labels", [None, [], ["urgent"], [{"name": "urgent"}]]) +def test_classify_requires_labels_before_network(labels): + route = respx.post(f"{BASE}/classify") + with pytest.raises(litellm.BadRequestError, match=r"labels|rubric"): + litellm.completion( + model="scaledown/classify", messages=[{"role": "user", "content": "text"}], extra_body={"labels": labels} + ) + assert not route.called + + +@pytest.mark.asyncio +@respx.mock +async def test_async_extraction_keeps_flat_fields_and_usage(): + from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler + + route = respx.post(f"{BASE}/extract").mock( + return_value=httpx.Response( + 200, + json={ + "entities": [{"type": "vendor", "text": "Northwind"}], + "structured_result": None, + "input_tokens": 161, + }, + ) + ) + client = AsyncHTTPHandler(transport=httpx.AsyncHTTPTransport()) + try: + response = await litellm.acompletion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind."}], + response_format={ + "type": "json_schema", + "json_schema": { + "name": "invoice", + "strict": False, + "schema": {"properties": {"vendor": {"type": "string"}}}, + }, + }, + client=client, + ) + finally: + await client.close() + assert json.loads(response.choices[0].message.content) == {"vendor": "Northwind"} + assert response.usage.prompt_tokens == 161 + assert json.loads(route.calls[0].request.content) == { + "text": "Invoice from Northwind.", + "entities": {"vendor": "vendor"}, + } + + +@respx.mock +def test_router_stream_includes_usage_without_sending_stream_options_upstream(): + route = respx.post(f"{BASE}/summarization/abstractive").mock( + return_value=httpx.Response(200, json={"summary": "Fixed billing.", "input_tokens": 90}) + ) + router = litellm.Router( + model_list=[ + {"model_name": "summary", "litellm_params": {"model": "scaledown/summarize", "api_key": "test-key"}} + ], + num_retries=0, + ) + chunks = list( + router.completion( + model="summary", + messages=[{"role": "user", "content": "Billing was fixed on Monday."}], + stream=True, + stream_options={"include_usage": True}, + ) + ) + assert ( + json.loads("".join(chunk.choices[0].delta.content or "" for chunk in chunks if chunk.choices))["summary"] + == "Fixed billing." + ) + assert chunks[-1].usage.prompt_tokens == 90 + assert json.loads(route.calls[0].request.content) == {"text": "Billing was fixed on Monday."} + + +@respx.mock +def test_pydantic_response_format_does_not_silently_claim_strict_support(): + from pydantic import BaseModel, ConfigDict + + class Invoice(BaseModel): + model_config = ConfigDict(frozen=True) + vendor: str + + route = respx.post(f"{BASE}/extract") + with pytest.raises(litellm.BadRequestError, match="strict JSON schemas"): + litellm.completion( + model="scaledown/extract", + messages=[{"role": "user", "content": "Invoice from Northwind."}], + response_format=Invoice, + ) + assert not route.called diff --git a/tests/unit/test_scaledown_model_prices.py b/tests/unit/test_scaledown_model_prices.py new file mode 100644 index 00000000000..02c7983f753 --- /dev/null +++ b/tests/unit/test_scaledown_model_prices.py @@ -0,0 +1,46 @@ +from typing import Final + +import pytest + +import litellm +from litellm.types.utils import Usage + +SCALEDOWN_MODELS: Final = ( + "scaledown/classify", + "scaledown/decisions", + "scaledown/extract", + "scaledown/summarize", + "scaledown/compress", +) + +# Source: ScaleDown's standard rate of $0.05 per million input tokens with output unbilled, +# as shown in the ScaleDown usage dashboard (checked 2026-10-09) and its billing service. +INPUT_COST_PER_MILLION_TOKENS: Final = 0.05 + + +@pytest.mark.parametrize("model", SCALEDOWN_MODELS) +def test_scaledown_model_is_priced_on_input_tokens_only(model: str) -> None: + entry: Final = litellm.model_cost[model] + + assert entry["litellm_provider"] == "scaledown" + assert entry["mode"] == "chat" + assert entry["input_cost_per_token"] * 1_000_000 == pytest.approx(INPUT_COST_PER_MILLION_TOKENS) + assert entry["output_cost_per_token"] == 0.0 + + +@pytest.mark.parametrize("model", SCALEDOWN_MODELS) +def test_scaledown_cost_ignores_output_tokens(model: str) -> None: + prompt_cost, completion_cost = litellm.cost_per_token( + model=model, + custom_llm_provider="scaledown", + usage_object=Usage(prompt_tokens=1_000_000, completion_tokens=500_000), + ) + + assert prompt_cost == pytest.approx(INPUT_COST_PER_MILLION_TOKENS) + assert completion_cost == 0.0 + + +def test_every_scaledown_entry_in_the_cost_map_is_covered_here() -> None: + in_map: Final = {key for key in litellm.model_cost if key.startswith("scaledown/")} + + assert in_map == set(SCALEDOWN_MODELS) From b6037990c258380ea95572ded12927eb34f8deba Mon Sep 17 00:00:00 2001 From: Soham Chatterjee <96soham96@gmail.com> Date: Sat, 10 Oct 2026 23:02:03 +0200 Subject: [PATCH 06/20] feat(models): register ScaleDown text pricing (#44167) * Add ScaleDown model pricing Registers the five ScaleDown models in the pricing map. ScaleDown bills on input tokens only, so output_cost_per_token is zero for every entry. The classify and decisions models bill at $0.04 per million input tokens, and extract, summarize, and compress bill at $0.05 per million. Co-Authored-By: Claude Opus 5 * Price all ScaleDown models at $0.04 per million and drop unverified context limits Every ScaleDown model now bills $0.04 per million input tokens. The 32000-token max_input_tokens/max_output_tokens values were not confirmed against the service, so they are removed rather than registered. Co-Authored-By: Claude Sonnet 5.5 * Price all ScaleDown models at $0.05 per million input tokens This is the standard rate in ScaleDown's billing and matches the usage dashboard. The Decisions usage.cost field is not used for pricing. Co-Authored-By: Claude Sonnet 5.5 * Add regression tests for the ScaleDown pricing entries Co-Authored-By: Claude Sonnet 5.5 * Cite the source of the ScaleDown rate in the pricing tests and type them Co-Authored-By: Claude Sonnet 5.5 * fix(scaledown): type the pricing model registry * fix(scaledown): address pricing review annotations --------- Co-authored-by: Claude Opus 5 Co-authored-by: moe-berri From 9e7e5c24e4315ca8b5d87d2c30de64b5fb816aa1 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 14:02:16 -0700 Subject: [PATCH 07/20] refactor(types): move UserAPIKeyAuth, its token view and hash_token into litellm.types.proxy.auth.user_api_key_auth (#45800) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): move is_jwt into litellm.types.proxy.auth.user_api_key_auth Part of LIT-9423 * refactor(auth): alias is_jwt import so JWTHandler.is_jwt is not read as recursive * refactor(types): type UserAPIKeyAuth, its token view and hash_token in place Part of LIT-9423 * refactor(types): move UserAPIKeyAuth, its token view and hash_token into litellm.types.proxy.auth.user_api_key_auth Part of LIT-9423 * fix(types): keep agent, budget snapshot and is_jwt names importable from proxy._types Part of LIT-9423 * fix(types): restore main's _types.py content alongside the user_api_key_auth move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * style(types): drop blank lines between proxy._types re-export imports Part of LIT-9423 * refactor(proxy): keep only user_api_key_auth re-exports in _types.py Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): carry main's UserAPIKeyAuth field into moved file Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(types): drop blank lines between proxy._types re-export imports Part of LIT-9423 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_types.py | 335 ++---------------- litellm/types/proxy/auth/user_api_key_auth.py | 331 ++++++++++++++++- .../test_key_management_endpoints.py | 6 +- 3 files changed, 361 insertions(+), 311 deletions(-) diff --git a/litellm/proxy/_types.py b/litellm/proxy/_types.py index 4abb93454c7..7b2a63974d8 100644 --- a/litellm/proxy/_types.py +++ b/litellm/proxy/_types.py @@ -16,7 +16,6 @@ from pydantic import ( Json, JsonValue, PositiveInt, - PrivateAttr, TypeAdapter, field_validator, model_validator, @@ -39,7 +38,12 @@ from litellm.litellm_core_utils.initialize_dynamic_callback_params import ( validate_no_callback_env_reference, ) from litellm.proxy._experimental.mcp_server.stdio_gate import MCP_STDIO_DISABLED_MESSAGE, is_mcp_stdio_enabled -from litellm.types.agents import AgentCaller, AgentResponse +from litellm.types.agents import ( + AgentCaller as AgentCaller, # noqa: PLC0414 # public re-export +) +from litellm.types.agents import ( + AgentResponse as AgentResponse, # noqa: PLC0414 # public re-export +) from litellm.types.integrations.compression_interception import ( CompressionSavingsMetadata, ) @@ -68,11 +72,17 @@ from litellm.types.proxy.agent_identity import ManagedAgentContext from litellm.types.proxy.auth.special_headers import ( SpecialHeaders as SpecialHeaders, # noqa: PLC0414 # public re-export ) -from litellm.types.proxy.auth.user_api_key_auth import is_jwt +from litellm.types.proxy.auth.user_api_key_auth import ( + is_jwt as is_jwt, # noqa: PLC0414 # public re-export +) from litellm.types.proxy.carried_budget_state import ( - OrgBudgetSnapshot, - TeamBudgetSnapshot, - UserBudgetSnapshot, + OrgBudgetSnapshot as OrgBudgetSnapshot, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.carried_budget_state import ( + TeamBudgetSnapshot as TeamBudgetSnapshot, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.carried_budget_state import ( + UserBudgetSnapshot as UserBudgetSnapshot, # noqa: PLC0414 # public re-export ) from litellm.types.proxy.control_plane_endpoints import WorkerRegistryEntry from litellm.types.proxy.spend_capture_rate import SpendCaptureRateCheckSettings @@ -198,17 +208,9 @@ class LitellmTableNames(str, enum.Enum): from litellm.types.integrations.slack_alerting import ( Litellm_EntityType as Litellm_EntityType, # noqa: E402, PLC0414 # public re-export ) - - -def hash_token(token: str) -> str: - import hashlib - - # Hash the string using SHA-256 - hashed_token: Final = hashlib.sha256(token.encode()).hexdigest() - - return hashed_token - - +from litellm.types.proxy.auth.user_api_key_auth import ( # noqa: E402 # re-export after the definitions above + hash_token as hash_token, # noqa: PLC0414 # public re-export +) from litellm.types.proxy.auth.user_roles import ( # noqa: E402 # re-export after the definitions above KeyManagementRoutes as KeyManagementRoutes, # noqa: PLC0414 # public re-export ) @@ -3058,296 +3060,15 @@ from litellm.models.verification_token import ( # noqa: E402 from litellm.models.verification_token import ( # noqa: E402 LiteLLM_VerificationToken as LiteLLM_VerificationToken, ) - -_UntypedDict: TypeAlias = dict[Any, Any] - - -def _is_str_keyed_dict(value: object) -> TypeIs[dict[str, object]]: # guard-ok: pydantic input and JSON keys are str - return isinstance(value, dict) - - -class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): - """ - Combined view of litellm verification token + litellm team table (select values) - """ - - team_spend: float | None = None - team_alias: str | None = None - team_tpm_limit: int | None = None - team_rpm_limit: int | None = None - team_tpd_limit: int | None = None - team_max_budget: float | None = None - team_soft_budget: float | None = None - team_model_max_budget: dict[str, object] | None = None - team_models: list[Any] = [] - team_blocked: bool = False - soft_budget: float | None = None - team_model_aliases: _UntypedDict | None = None - team_member: Member | None = None - team_metadata: _UntypedDict | None = None - team_object_permission_id: str | None = None - - # Team Member Specific Params - team_member_spend: float | None = None - team_member_tpm_limit: int | None = None - team_member_rpm_limit: int | None = None - - # End User Params - end_user_id: str | None = None - end_user_tpm_limit: int | None = None - end_user_rpm_limit: int | None = None - end_user_tpd_limit: int | None = None - end_user_max_budget: float | None = None - end_user_model_max_budget: _UntypedDict | None = None - - # Organization Params - organization_alias: str | None = None - organization_max_budget: float | None = None - organization_tpm_limit: int | None = None - organization_rpm_limit: int | None = None - organization_metadata: _UntypedDict | None = None - - # Project Params - project_alias: str | None = None - project_metadata: _UntypedDict | None = None - - # Time stamps - last_refreshed_at: float | None = None # last time joint view was pulled from db - - def __init__(self, **kwargs: object) -> None: - # Handle litellm_budget_table_* keys (budget table overrides when key value is None or empty) - for key, value in list(kwargs.items()): - if key.startswith("litellm_budget_table_") and value is not None: - # Extract the corresponding attribute name - attr_name = key.replace("litellm_budget_table_", "") - # Use key's value from kwargs (from DB view), not class default - current = kwargs.get(attr_name) - if current is None: - current = getattr(self, attr_name, None) - # Apply budget value when key has no value, or for model_max_budget when key has empty dict - should_apply = current is None or ( - attr_name == "model_max_budget" and _is_str_keyed_dict(current) and len(current) == 0 - ) - if should_apply: - kwargs[attr_name] = value - if key == "end_user_id" and value is not None and isinstance(value, int): - kwargs[key] = str(value) - - if kwargs.get("organization_id") is not None: - kwargs["org_id"] = kwargs.pop("organization_id") - # Initialize the superclass - super().__init__(**kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # pydantic validates raw values - - -class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response object for user api key auth - """ - Return the row in the db - """ - - api_key: str | None = None - user_role: LitellmUserRoles | None = None - allowed_model_region: AllowedModelRegion | None = None - parent_otel_span: Span | None = None - rpm_limit_per_model: dict[str, int] | None = None - tpm_limit_per_model: dict[str, int] | None = None - user_tpm_limit: int | None = None - user_rpm_limit: int | None = None - user_email: str | None = None - user_spend: float | None = None - user_max_budget: float | None = None - # Values stay `object` rather than BudgetConfig: this is the raw JSON column, - # and validating it here would make one malformed row fail auth outright. - # resolve_model_budget validates the single entry a request actually needs. - user_model_max_budget: Mapping[str, object] | None = None - team_member_model_max_budget: Mapping[str, object] | None = Field(default=None, exclude=True) - request_route: str | None = None - is_session_token: bool = False - # Server-only marker set exclusively by the MCP gateway admission path - # (reload_admitted_user) for a keyless user-subject admitted via a gateway DCR session - # bearer or bridge envelope. Not a DB column and never populated from caller-controlled key - # metadata or JWT claims, so it cannot be forged to gain the team-inherited MCP grant union - # or to escape the caller-Authorization egress scrub. exclude=True keeps it out of serialization. - mcp_admitted_user_subject: bool = Field(default=False, exclude=True) - requires_fresh_policy: bool = Field(default=False, exclude=True) - mcp_explicit_grants_only: bool = Field(default=False, exclude=True) - # team_id -> that team's mcp_rpm_limit map, for a keyless admitted subject that reaches MCP - # servers through several teams at once and therefore has no single team_id for the limiter to - # key off. Server-only and stripped from validated input for the same reason as the marker - # above: a forged entry would let a caller pick which team's rpm bucket it is charged against. - mcp_source_team_rpm_limits: dict[str, dict[str, int]] | None = Field(default=None, exclude=True) - # The single MCP server_id a gateway session bearer was scoped to at authorize time (RFC 8707 - # resource), or None for an aggregate-scope session. A RESTRICTION intersected against the live - # grant resolution, never a grant. Server-only, set exclusively by the MCP gateway admission - # path via post-construction assignment and stripped from validated input like the markers - # above; a forged value could at most narrow, but the stripping keeps the field's provenance - # single-owner so its meaning stays trustworthy. - mcp_session_resource_server_id: str | None = Field(default=None, exclude=True) - mcp_toolset_id: str | None = Field(default=None, exclude=True) - authenticated_by_custom_auth: bool = Field(default=False, exclude=True) - via_virtual_key: bool = Field( - default=False, - exclude=True, - description=( - "Server-only marker set exclusively by the DB virtual-key and master-key auth paths via " - "post-construction assignment. Stripped from validated input so custom auth handlers, JWT " - "claims, or key metadata cannot forge it. Gates overwrite_user_with_key_hash stamping: only " - "a credential the proxy itself validated as a key may be forwarded as the provider-facing " - "user id." - ), - ) - invoked_agent_id: str | None = Field(default=None, exclude=True) - invoked_agent_policy: AgentResponse | None = Field(default=None, exclude=True) - agent_invocation_cost: float | None = Field(default=None, exclude=True) - billing_agent_policy: AgentResponse | None = Field(default=None, exclude=True) - _managed_delegation_verified: bool = PrivateAttr(default=False) - managed_agent_policy: AgentResponse | None = Field(default=None, exclude=True) - managed_agent_context: ManagedAgentContext | None = Field(default=None, exclude=True) - agent_caller: AgentCaller | None = Field( - default=None, - exclude=True, - description=( - "Set per request from the x-litellm-user-id / x-litellm-team-id headers an agent echoes back on " - "calls made with its own key. Every check treats it as a ceiling, so a forged value can only " - "narrow the agent's access." - ), - ) - budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True) - team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True) - user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True) - org_budget_snapshot: OrgBudgetSnapshot | None = Field(default=None, exclude=True) - matched_model_access_groups: list[str] | None = Field(default=None, exclude=True) - budget_throttle_pct: float | None = Field(default=None, exclude=True) - user: Any | None = None # Expanded user object when expand=user is used - created_by_user: Any | None = None # Expanded created_by user when expand=user is used - end_user_object_permission: LiteLLM_ObjectPermissionTable | None = None - # Team object_permission preloaded in auth (e.g. get_team_object) to avoid - # per-request object_permission fetches in downstream checks (vector stores, etc.) - team_object_permission: LiteLLM_ObjectPermissionTable | None = None - # Decoded upstream IdP claims (groups, roles, etc.) propagated by JWT auth machinery - # and forwarded into outbound tokens by guardrails such as MCPJWTSigner. - jwt_claims: _UntypedDict | None = None - - model_config = ConfigDict(arbitrary_types_allowed=True) - - @model_validator(mode="before") - @classmethod - def check_api_key(cls, values: object) -> object: - # If values is already an instance (not a dict), return it as-is - if not _is_str_keyed_dict(values): - return values - # mcp_admitted_user_subject is a server-only marker, set ONLY by the MCP gateway admission - # path via post-construction assignment. Strip it from any validated input (constructor - # kwargs, model_validate, a JWT/key claim splat) so it can never be forged from caller data. - values.pop("mcp_admitted_user_subject", None) - values.pop("requires_fresh_policy", None) - values.pop("mcp_explicit_grants_only", None) - values.pop("mcp_source_team_rpm_limits", None) - values.pop("mcp_session_resource_server_id", None) - values.pop("mcp_toolset_id", None) - values.pop("via_virtual_key", None) - values.pop("authenticated_by_custom_auth", None) - values.pop("agent_caller", None) - values.pop("managed_agent_context", None) - values.pop("managed_agent_policy", None) - values.pop("invoked_agent_id", None) - values.pop("invoked_agent_policy", None) - values.pop("agent_invocation_cost", None) - values.pop("billing_agent_policy", None) - api_key: Final = values.get("api_key") - if api_key is not None: - values.update({"token": cls._safe_hash_litellm_api_key(api_key)}) # pyright: ignore[reportArgumentType] # non-str fails in the hasher - if isinstance(api_key, str): - values.update({"api_key": cls._safe_hash_litellm_api_key(api_key)}) - return values - - @classmethod - def _safe_hash_litellm_api_key(cls, api_key: str) -> str: - """ - Helper to ensure all logged keys are hashed - Covers: - 1. Regular API keys from LiteLLM DB - 2. JWT tokens used for connecting to LiteLLM API - """ - normalized = api_key - if normalized[:7].lower() == "bearer ": - normalized = normalized[7:] - if normalized.startswith("sk-"): - return hash_token(normalized) - if is_jwt(token=normalized): - return f"hashed-jwt-{hash_token(token=normalized)}" - return normalized - - @classmethod - def get_litellm_internal_health_check_user_api_key_auth(cls) -> "UserAPIKeyAuth": - """ - Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. - - This is used to track number of requests/spend for health check calls. - """ - from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME - - return cls( - api_key=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - team_id=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - key_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - team_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, - ) - - @classmethod - def get_litellm_cli_user_api_key_auth(cls) -> "UserAPIKeyAuth": - """ - Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. - - This is used to track number of requests/spend for health check calls. - """ - from litellm.constants import LITTELM_CLI_SERVICE_ACCOUNT_NAME - - return cls( - api_key=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - team_id=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - key_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - team_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, - ) - - @classmethod - def get_litellm_internal_jobs_user_api_key_auth(cls) -> "UserAPIKeyAuth": - """ - Returns a `UserAPIKeyAuth` object for internal LiteLLM jobs like key rotation. - - This is used to track actions performed by automated system jobs. - """ - from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME - - return cls( - api_key=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, - team_id="system", - key_alias=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, - team_alias="system", - user_id="system", - user_role=LitellmUserRoles.PROXY_ADMIN, - ) - - @property - def is_team_service_account(self) -> bool: - return ( - self.user_id is None - and self.team_id is not None - and bool(self.metadata) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped base field - and self.metadata.get("service_account_id") is not None # pyright: ignore[reportUnknownMemberType] # untyped base field - ) - - -def user_api_key_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: - """Return True if the caller's role grants unscoped read access to all - tenant resources (managed files, batches, vector stores, spend rows, etc). - - Lives on _types.py so leaf modules (e.g. litellm.llms.base_llm.managed_resources) - can use it without pulling in litellm.proxy.utils via management_endpoints. - """ - return user_api_key_dict.user_role in ( - LitellmUserRoles.PROXY_ADMIN, - LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, - ) +from litellm.types.proxy.auth.user_api_key_auth import ( # noqa: E402 # re-export after the definitions above + LiteLLM_VerificationTokenView as LiteLLM_VerificationTokenView, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.auth.user_api_key_auth import ( + UserAPIKeyAuth as UserAPIKeyAuth, # noqa: PLC0414 # public re-export +) +from litellm.types.proxy.auth.user_api_key_auth import ( + user_api_key_has_admin_view as user_api_key_has_admin_view, # noqa: PLC0414 # public re-export +) class UserInfoResponse(LiteLLMPydanticObjectBase): diff --git a/litellm/types/proxy/auth/user_api_key_auth.py b/litellm/types/proxy/auth/user_api_key_auth.py index 3d879fdcc43..18dd4e53701 100644 --- a/litellm/types/proxy/auth/user_api_key_auth.py +++ b/litellm/types/proxy/auth/user_api_key_auth.py @@ -1,4 +1,33 @@ -from typing import Final +from collections.abc import Mapping +from typing import ( + TYPE_CHECKING, + Any, # noqa: TID251 # moved UserAPIKeyAuth fields and the Span alias are typed Any + Final, + TypeAlias, +) + +from pydantic import ConfigDict, Field, PrivateAttr, model_validator +from typing_extensions import TypeIs # noqa: TID251 # narrows the before-validator input without copying it + +from litellm.models.object_permission import LiteLLM_ObjectPermissionTable +from litellm.models.team import Member +from litellm.models.verification_token import LiteLLM_VerificationToken +from litellm.types.agents import AgentCaller, AgentResponse +from litellm.types.proxy.agent_identity import ManagedAgentContext +from litellm.types.proxy.auth.user_roles import LitellmUserRoles +from litellm.types.proxy.carried_budget_state import ( + OrgBudgetSnapshot, + TeamBudgetSnapshot, + UserBudgetSnapshot, +) +from litellm.types.router import AllowedModelRegion + +if TYPE_CHECKING: + from opentelemetry.trace import Span as _Span + + Span: TypeAlias = _Span | Any +else: + Span: TypeAlias = Any def is_jwt(token: str | None) -> bool: @@ -6,3 +35,303 @@ def is_jwt(token: str | None) -> bool: return False parts: Final = token.split(".") return len(parts) == 3 + + +def hash_token(token: str) -> str: + import hashlib + + # Hash the string using SHA-256 + hashed_token: Final = hashlib.sha256(token.encode()).hexdigest() + + return hashed_token + + +_UntypedDict: TypeAlias = dict[Any, Any] + + +def _is_str_keyed_dict(value: object) -> TypeIs[dict[str, object]]: # guard-ok: pydantic input and JSON keys are str + return isinstance(value, dict) + + +class LiteLLM_VerificationTokenView(LiteLLM_VerificationToken): + """ + Combined view of litellm verification token + litellm team table (select values) + """ + + team_spend: float | None = None + team_alias: str | None = None + team_tpm_limit: int | None = None + team_rpm_limit: int | None = None + team_tpd_limit: int | None = None + team_max_budget: float | None = None + team_soft_budget: float | None = None + team_model_max_budget: dict[str, object] | None = None + team_models: list[Any] = [] + team_blocked: bool = False + soft_budget: float | None = None + team_model_aliases: _UntypedDict | None = None + team_member: Member | None = None + team_metadata: _UntypedDict | None = None + team_object_permission_id: str | None = None + + # Team Member Specific Params + team_member_spend: float | None = None + team_member_tpm_limit: int | None = None + team_member_rpm_limit: int | None = None + + # End User Params + end_user_id: str | None = None + end_user_tpm_limit: int | None = None + end_user_rpm_limit: int | None = None + end_user_tpd_limit: int | None = None + end_user_max_budget: float | None = None + end_user_model_max_budget: _UntypedDict | None = None + + # Organization Params + organization_alias: str | None = None + organization_max_budget: float | None = None + organization_tpm_limit: int | None = None + organization_rpm_limit: int | None = None + organization_metadata: _UntypedDict | None = None + + # Project Params + project_alias: str | None = None + project_metadata: _UntypedDict | None = None + + # Time stamps + last_refreshed_at: float | None = None # last time joint view was pulled from db + + def __init__(self, **kwargs: object) -> None: + # Handle litellm_budget_table_* keys (budget table overrides when key value is None or empty) + for key, value in list(kwargs.items()): + if key.startswith("litellm_budget_table_") and value is not None: + # Extract the corresponding attribute name + attr_name = key.replace("litellm_budget_table_", "") + # Use key's value from kwargs (from DB view), not class default + current = kwargs.get(attr_name) + if current is None: + current = getattr(self, attr_name, None) + # Apply budget value when key has no value, or for model_max_budget when key has empty dict + should_apply = current is None or ( + attr_name == "model_max_budget" and _is_str_keyed_dict(current) and len(current) == 0 + ) + if should_apply: + kwargs[attr_name] = value + if key == "end_user_id" and value is not None and isinstance(value, int): + kwargs[key] = str(value) + + if kwargs.get("organization_id") is not None: + kwargs["org_id"] = kwargs.pop("organization_id") + # Initialize the superclass + super().__init__(**kwargs) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # pydantic validates raw values + + +class UserAPIKeyAuth(LiteLLM_VerificationTokenView): # the expected response object for user api key auth + """ + Return the row in the db + """ + + api_key: str | None = None + user_role: LitellmUserRoles | None = None + allowed_model_region: AllowedModelRegion | None = None + parent_otel_span: Span | None = None + rpm_limit_per_model: dict[str, int] | None = None + tpm_limit_per_model: dict[str, int] | None = None + user_tpm_limit: int | None = None + user_rpm_limit: int | None = None + user_email: str | None = None + user_spend: float | None = None + user_max_budget: float | None = None + # Values stay `object` rather than BudgetConfig: this is the raw JSON column, + # and validating it here would make one malformed row fail auth outright. + # resolve_model_budget validates the single entry a request actually needs. + user_model_max_budget: Mapping[str, object] | None = None + team_member_model_max_budget: Mapping[str, object] | None = Field(default=None, exclude=True) + request_route: str | None = None + is_session_token: bool = False + # Server-only marker set exclusively by the MCP gateway admission path + # (reload_admitted_user) for a keyless user-subject admitted via a gateway DCR session + # bearer or bridge envelope. Not a DB column and never populated from caller-controlled key + # metadata or JWT claims, so it cannot be forged to gain the team-inherited MCP grant union + # or to escape the caller-Authorization egress scrub. exclude=True keeps it out of serialization. + mcp_admitted_user_subject: bool = Field(default=False, exclude=True) + requires_fresh_policy: bool = Field(default=False, exclude=True) + mcp_explicit_grants_only: bool = Field(default=False, exclude=True) + # team_id -> that team's mcp_rpm_limit map, for a keyless admitted subject that reaches MCP + # servers through several teams at once and therefore has no single team_id for the limiter to + # key off. Server-only and stripped from validated input for the same reason as the marker + # above: a forged entry would let a caller pick which team's rpm bucket it is charged against. + mcp_source_team_rpm_limits: dict[str, dict[str, int]] | None = Field(default=None, exclude=True) + # The single MCP server_id a gateway session bearer was scoped to at authorize time (RFC 8707 + # resource), or None for an aggregate-scope session. A RESTRICTION intersected against the live + # grant resolution, never a grant. Server-only, set exclusively by the MCP gateway admission + # path via post-construction assignment and stripped from validated input like the markers + # above; a forged value could at most narrow, but the stripping keeps the field's provenance + # single-owner so its meaning stays trustworthy. + mcp_session_resource_server_id: str | None = Field(default=None, exclude=True) + mcp_toolset_id: str | None = Field(default=None, exclude=True) + authenticated_by_custom_auth: bool = Field(default=False, exclude=True) + via_virtual_key: bool = Field( + default=False, + exclude=True, + description=( + "Server-only marker set exclusively by the DB virtual-key and master-key auth paths via " + "post-construction assignment. Stripped from validated input so custom auth handlers, JWT " + "claims, or key metadata cannot forge it. Gates overwrite_user_with_key_hash stamping: only " + "a credential the proxy itself validated as a key may be forwarded as the provider-facing " + "user id." + ), + ) + invoked_agent_id: str | None = Field(default=None, exclude=True) + invoked_agent_policy: AgentResponse | None = Field(default=None, exclude=True) + agent_invocation_cost: float | None = Field(default=None, exclude=True) + billing_agent_policy: AgentResponse | None = Field(default=None, exclude=True) + _managed_delegation_verified: bool = PrivateAttr(default=False) + managed_agent_policy: AgentResponse | None = Field(default=None, exclude=True) + managed_agent_context: ManagedAgentContext | None = Field(default=None, exclude=True) + agent_caller: AgentCaller | None = Field( + default=None, + exclude=True, + description=( + "Set per request from the x-litellm-user-id / x-litellm-team-id headers an agent echoes back on " + "calls made with its own key. Every check treats it as a ceiling, so a forged value can only " + "narrow the agent's access." + ), + ) + budget_reservation: dict[str, Any] | None = Field(default=None, exclude=True) + team_budget_snapshot: TeamBudgetSnapshot | None = Field(default=None, exclude=True) + user_budget_snapshot: UserBudgetSnapshot | None = Field(default=None, exclude=True) + org_budget_snapshot: OrgBudgetSnapshot | None = Field(default=None, exclude=True) + matched_model_access_groups: list[str] | None = Field(default=None, exclude=True) + budget_throttle_pct: float | None = Field(default=None, exclude=True) + user: Any | None = None # Expanded user object when expand=user is used + created_by_user: Any | None = None # Expanded created_by user when expand=user is used + end_user_object_permission: LiteLLM_ObjectPermissionTable | None = None + # Team object_permission preloaded in auth (e.g. get_team_object) to avoid + # per-request object_permission fetches in downstream checks (vector stores, etc.) + team_object_permission: LiteLLM_ObjectPermissionTable | None = None + # Decoded upstream IdP claims (groups, roles, etc.) propagated by JWT auth machinery + # and forwarded into outbound tokens by guardrails such as MCPJWTSigner. + jwt_claims: _UntypedDict | None = None + + model_config = ConfigDict(arbitrary_types_allowed=True) + + @model_validator(mode="before") + @classmethod + def check_api_key(cls, values: object) -> object: + # If values is already an instance (not a dict), return it as-is + if not _is_str_keyed_dict(values): + return values + # mcp_admitted_user_subject is a server-only marker, set ONLY by the MCP gateway admission + # path via post-construction assignment. Strip it from any validated input (constructor + # kwargs, model_validate, a JWT/key claim splat) so it can never be forged from caller data. + values.pop("mcp_admitted_user_subject", None) + values.pop("requires_fresh_policy", None) + values.pop("mcp_explicit_grants_only", None) + values.pop("mcp_source_team_rpm_limits", None) + values.pop("mcp_session_resource_server_id", None) + values.pop("mcp_toolset_id", None) + values.pop("via_virtual_key", None) + values.pop("authenticated_by_custom_auth", None) + values.pop("agent_caller", None) + values.pop("managed_agent_context", None) + values.pop("managed_agent_policy", None) + values.pop("invoked_agent_id", None) + values.pop("invoked_agent_policy", None) + values.pop("agent_invocation_cost", None) + values.pop("billing_agent_policy", None) + api_key: Final = values.get("api_key") + if api_key is not None: + values.update({"token": cls._safe_hash_litellm_api_key(api_key)}) # pyright: ignore[reportArgumentType] # non-str fails in the hasher + if isinstance(api_key, str): + values.update({"api_key": cls._safe_hash_litellm_api_key(api_key)}) + return values + + @classmethod + def _safe_hash_litellm_api_key(cls, api_key: str) -> str: + """ + Helper to ensure all logged keys are hashed + Covers: + 1. Regular API keys from LiteLLM DB + 2. JWT tokens used for connecting to LiteLLM API + """ + normalized = api_key + if normalized[:7].lower() == "bearer ": + normalized = normalized[7:] + if normalized.startswith("sk-"): + return hash_token(normalized) + if is_jwt(token=normalized): + return f"hashed-jwt-{hash_token(token=normalized)}" + return normalized + + @classmethod + def get_litellm_internal_health_check_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. + + This is used to track number of requests/spend for health check calls. + """ + from litellm.constants import LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + team_id=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + key_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + team_alias=LITTELM_INTERNAL_HEALTH_SERVICE_ACCOUNT_NAME, + ) + + @classmethod + def get_litellm_cli_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for the litellm internal health check service account. + + This is used to track number of requests/spend for health check calls. + """ + from litellm.constants import LITTELM_CLI_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + team_id=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + key_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + team_alias=LITTELM_CLI_SERVICE_ACCOUNT_NAME, + ) + + @classmethod + def get_litellm_internal_jobs_user_api_key_auth(cls) -> "UserAPIKeyAuth": + """ + Returns a `UserAPIKeyAuth` object for internal LiteLLM jobs like key rotation. + + This is used to track actions performed by automated system jobs. + """ + from litellm.constants import LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME + + return cls( + api_key=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + team_id="system", + key_alias=LITELLM_INTERNAL_JOBS_SERVICE_ACCOUNT_NAME, + team_alias="system", + user_id="system", + user_role=LitellmUserRoles.PROXY_ADMIN, + ) + + @property + def is_team_service_account(self) -> bool: + return ( + self.user_id is None + and self.team_id is not None + and bool(self.metadata) # pyright: ignore[reportUnknownMemberType, reportUnknownArgumentType] # untyped base field + and self.metadata.get("service_account_id") is not None # pyright: ignore[reportUnknownMemberType] # untyped base field + ) + + +def user_api_key_has_admin_view(user_api_key_dict: UserAPIKeyAuth) -> bool: + """Return True if the caller's role grants unscoped read access to all + tenant resources (managed files, batches, vector stores, spend rows, etc). + + Lives on _types.py so leaf modules (e.g. litellm.llms.base_llm.managed_resources) + can use it without pulling in litellm.proxy.utils via management_endpoints. + """ + return user_api_key_dict.user_role in ( + LitellmUserRoles.PROXY_ADMIN, + LitellmUserRoles.PROXY_ADMIN_VIEW_ONLY, + ) diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 9905ee619c2..308558450b1 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -7817,7 +7817,7 @@ async def test_process_single_key_update(): mock_delete_cache.return_value = None # Mock hash_token (imported from litellm.proxy._types) - with patch("litellm.proxy._types.hash_token") as mock_hash: + with patch("litellm.types.proxy.auth.user_api_key_auth.hash_token") as mock_hash: mock_hash.return_value = "hashed-test-key-123" # Mock _hash_token_if_needed @@ -7963,7 +7963,7 @@ async def test_bulk_update_keys_success(monkeypatch): with patch( "litellm.proxy.management_endpoints.key_management_endpoints.delete_cache_key_object" ): - with patch("litellm.proxy._types.hash_token") as mock_hash: + with patch("litellm.types.proxy.auth.user_api_key_auth.hash_token") as mock_hash: mock_hash.side_effect = ["hashed-key-1", "hashed-key-2"] def _hash_for_bulk_success(token: str) -> str: @@ -8091,7 +8091,7 @@ async def test_bulk_update_keys_partial_failures(monkeypatch): with patch( "litellm.proxy.management_endpoints.key_management_endpoints.delete_cache_key_object" ): - with patch("litellm.proxy._types.hash_token") as mock_hash: + with patch("litellm.types.proxy.auth.user_api_key_auth.hash_token") as mock_hash: mock_hash.return_value = "hashed-key-1" def _hash_for_bulk_partial(token: str) -> str: From c582bc6e68a1b78e9d63d6d63f37ee66f8c976c1 Mon Sep 17 00:00:00 2001 From: nate-berri Date: Sat, 10 Oct 2026 14:02:53 -0700 Subject: [PATCH 08/20] refactor(core_utils): move resource_ownership out of the proxy (#45805) * refactor(types): move LitellmUserRoles and KeyManagementRoutes into litellm.types.proxy.auth.user_roles Part of LIT-9423 * refactor(types): move is_jwt into litellm.types.proxy.auth.user_api_key_auth Part of LIT-9423 * refactor(auth): alias is_jwt import so JWTHandler.is_jwt is not read as recursive * refactor(types): type UserAPIKeyAuth, its token view and hash_token in place Part of LIT-9423 * refactor(types): move UserAPIKeyAuth, its token view and hash_token into litellm.types.proxy.auth.user_api_key_auth Part of LIT-9423 * fix(types): keep agent, budget snapshot and is_jwt names importable from proxy._types Part of LIT-9423 * refactor(core_utils): move resource_ownership out of the proxy Part of LIT-9423 * fix(types): restore main's _types.py content alongside the user_api_key_auth move Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * fix(sdk): resolve litellm.proxy lazily on attribute access The layer cleanup stopped import litellm from loading any proxy module, so litellm.proxy was no longer bound and litellm.proxy._types.X raised AttributeError. The module __getattr__ now imports litellm.proxy on first access, which keeps import litellm free of proxy._types and fixes the docs and code-quality checks on main * style(types): drop blank lines between proxy._types re-export imports Part of LIT-9423 * refactor(proxy): keep only user_api_key_auth re-exports in _types.py Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * refactor(proxy): carry main's UserAPIKeyAuth field into moved file Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * style(types): drop blank lines between proxy._types re-export imports Part of LIT-9423 --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../litellm_core_utils/resource_ownership.py | 93 +++++++++++++++ .../base_llm/managed_resources/isolation.py | 2 +- litellm/llms/litellm_proxy/skills/handler.py | 12 +- .../proxy/common_utils/resource_ownership.py | 106 +++--------------- .../common_utils/test_resource_ownership.py | 29 +++++ 5 files changed, 146 insertions(+), 96 deletions(-) create mode 100644 litellm/litellm_core_utils/resource_ownership.py create mode 100644 tests/unit/proxy/common_utils/test_resource_ownership.py diff --git a/litellm/litellm_core_utils/resource_ownership.py b/litellm/litellm_core_utils/resource_ownership.py new file mode 100644 index 00000000000..e0894b825a7 --- /dev/null +++ b/litellm/litellm_core_utils/resource_ownership.py @@ -0,0 +1,93 @@ +from typing import Final + +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.types.proxy.auth.user_roles import LitellmUserRoles + + +def is_proxy_admin(user_api_key_dict: UserAPIKeyAuth | None) -> bool: + if user_api_key_dict is None: + return False + + return ( + user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN + or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value + ) + + +def get_resource_owner_scopes( + user_api_key_dict: UserAPIKeyAuth | None, +) -> list[str]: + """ + Return ownership scopes that may access a user-created proxy resource. + + Raw user_id is included for rows created before scope prefixes existed. + Prefixes avoid collisions when falling back to team/org/key ownership + for keys that do not have a user_id. + + Identity-less callers (no user_id, team_id, org_id, api_key, or token) + return ``[]`` — they share no scope with any other caller, so access + checks against an existing owner always fail and creates that depend + on a primary scope must reject up front. Returning a shared sentinel + here would let any two identity-less callers see each other's data. + """ + if user_api_key_dict is None: + return [] + + scopes: Final[list[str]] = [] + + def _add(scope: str | None) -> None: + if scope and scope not in scopes: + scopes.append(scope) + + if user_api_key_dict.user_id: + _add(user_api_key_dict.user_id) + _add(f"user:{user_api_key_dict.user_id}") + if user_api_key_dict.team_id: + _add(f"team:{user_api_key_dict.team_id}") + if user_api_key_dict.org_id: + _add(f"org:{user_api_key_dict.org_id}") + if user_api_key_dict.api_key: + _add(f"key:{user_api_key_dict.api_key}") + if user_api_key_dict.token: + _add(f"key:{user_api_key_dict.token}") + + return scopes + + +def get_primary_resource_owner_scope( + user_api_key_dict: UserAPIKeyAuth | None, +) -> str | None: + """Return the canonical owner scope to stamp on newly-created rows. + + ``None`` for identity-less callers — callers that depend on a primary + scope to record ownership must surface that as a hard error rather + than fall back to a shared sentinel (which would collapse every + identity-less caller into the same logical owner). + """ + if user_api_key_dict is None: + return None + + if user_api_key_dict.user_id: + return user_api_key_dict.user_id + if user_api_key_dict.team_id: + return f"team:{user_api_key_dict.team_id}" + if user_api_key_dict.org_id: + return f"org:{user_api_key_dict.org_id}" + if user_api_key_dict.api_key: + return f"key:{user_api_key_dict.api_key}" + if user_api_key_dict.token: + return f"key:{user_api_key_dict.token}" + return None + + +def user_can_access_resource_owner( + owner: str | None, + user_api_key_dict: UserAPIKeyAuth | None, +) -> bool: + if user_api_key_dict is None: + return True + if is_proxy_admin(user_api_key_dict): + return True + if owner is None: + return False + return owner in get_resource_owner_scopes(user_api_key_dict) diff --git a/litellm/llms/base_llm/managed_resources/isolation.py b/litellm/llms/base_llm/managed_resources/isolation.py index 6a71e8e9223..0a09de79891 100644 --- a/litellm/llms/base_llm/managed_resources/isolation.py +++ b/litellm/llms/base_llm/managed_resources/isolation.py @@ -29,7 +29,7 @@ def resolve_resource_owner_id( A key with neither a user_id nor a team_id would otherwise stamp ``created_by=None`` and be locked out of its own resources, so it owns them under its hashed token instead, using the ``key:`` scope prefix - already used by ``proxy/common_utils/resource_ownership.py``. ``None`` + already used by ``litellm_core_utils/resource_ownership.py``. ``None`` means the caller has no usable identity of its own and must fall back to team scoping, or be denied. """ diff --git a/litellm/llms/litellm_proxy/skills/handler.py b/litellm/llms/litellm_proxy/skills/handler.py index 0dbe47791fb..d56dca1699b 100644 --- a/litellm/llms/litellm_proxy/skills/handler.py +++ b/litellm/llms/litellm_proxy/skills/handler.py @@ -11,18 +11,18 @@ from typing import Final from litellm._logging import verbose_logger from litellm.caching.in_memory_cache import InMemoryCache +from litellm.litellm_core_utils.resource_ownership import ( + get_primary_resource_owner_scope, + get_resource_owner_scopes, + is_proxy_admin, + user_can_access_resource_owner, +) from litellm.llms.litellm_proxy.skills.constants import ( LITELLM_SKILL_ID_PREFIX, MAX_SKILLS_PER_SEARCH, ) from litellm.models.skills import LiteLLM_SkillsTable, NewSkillRequest from litellm.proxy._types import UserAPIKeyAuth -from litellm.proxy.common_utils.resource_ownership import ( - get_primary_resource_owner_scope, - get_resource_owner_scopes, - is_proxy_admin, - user_can_access_resource_owner, -) from litellm.repositories.table_repositories import SkillsRepository # Skills are looked up on every chat completion that has skills enabled diff --git a/litellm/proxy/common_utils/resource_ownership.py b/litellm/proxy/common_utils/resource_ownership.py index c75a5ba8e6e..985ad8a6f77 100644 --- a/litellm/proxy/common_utils/resource_ownership.py +++ b/litellm/proxy/common_utils/resource_ownership.py @@ -1,92 +1,20 @@ from typing import Final -from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.litellm_core_utils.resource_ownership import ( + get_primary_resource_owner_scope, + get_resource_owner_scopes, + is_proxy_admin, + user_can_access_resource_owner, +) +from litellm.types.proxy.auth.user_api_key_auth import UserAPIKeyAuth +from litellm.types.proxy.auth.user_roles import LitellmUserRoles - -def is_proxy_admin(user_api_key_dict: UserAPIKeyAuth | None) -> bool: - if user_api_key_dict is None: - return False - - return ( - user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN - or user_api_key_dict.user_role == LitellmUserRoles.PROXY_ADMIN.value - ) - - -def get_resource_owner_scopes( - user_api_key_dict: UserAPIKeyAuth | None, -) -> list[str]: - """ - Return ownership scopes that may access a user-created proxy resource. - - Raw user_id is included for rows created before scope prefixes existed. - Prefixes avoid collisions when falling back to team/org/key ownership - for keys that do not have a user_id. - - Identity-less callers (no user_id, team_id, org_id, api_key, or token) - return ``[]`` — they share no scope with any other caller, so access - checks against an existing owner always fail and creates that depend - on a primary scope must reject up front. Returning a shared sentinel - here would let any two identity-less callers see each other's data. - """ - if user_api_key_dict is None: - return [] - - scopes: Final[list[str]] = [] - - def _add(scope: str | None) -> None: - if scope and scope not in scopes: - scopes.append(scope) - - if user_api_key_dict.user_id: - _add(user_api_key_dict.user_id) - _add(f"user:{user_api_key_dict.user_id}") - if user_api_key_dict.team_id: - _add(f"team:{user_api_key_dict.team_id}") - if user_api_key_dict.org_id: - _add(f"org:{user_api_key_dict.org_id}") - if user_api_key_dict.api_key: - _add(f"key:{user_api_key_dict.api_key}") - if user_api_key_dict.token: - _add(f"key:{user_api_key_dict.token}") - - return scopes - - -def get_primary_resource_owner_scope( - user_api_key_dict: UserAPIKeyAuth | None, -) -> str | None: - """Return the canonical owner scope to stamp on newly-created rows. - - ``None`` for identity-less callers — callers that depend on a primary - scope to record ownership must surface that as a hard error rather - than fall back to a shared sentinel (which would collapse every - identity-less caller into the same logical owner). - """ - if user_api_key_dict is None: - return None - - if user_api_key_dict.user_id: - return user_api_key_dict.user_id - if user_api_key_dict.team_id: - return f"team:{user_api_key_dict.team_id}" - if user_api_key_dict.org_id: - return f"org:{user_api_key_dict.org_id}" - if user_api_key_dict.api_key: - return f"key:{user_api_key_dict.api_key}" - if user_api_key_dict.token: - return f"key:{user_api_key_dict.token}" - return None - - -def user_can_access_resource_owner( - owner: str | None, - user_api_key_dict: UserAPIKeyAuth | None, -) -> bool: - if user_api_key_dict is None: - return True - if is_proxy_admin(user_api_key_dict): - return True - if owner is None: - return False - return owner in get_resource_owner_scopes(user_api_key_dict) +__all__ = [ + "Final", + "LitellmUserRoles", + "UserAPIKeyAuth", + "get_primary_resource_owner_scope", + "get_resource_owner_scopes", + "is_proxy_admin", + "user_can_access_resource_owner", +] diff --git a/tests/unit/proxy/common_utils/test_resource_ownership.py b/tests/unit/proxy/common_utils/test_resource_ownership.py new file mode 100644 index 00000000000..a13f66dd59c --- /dev/null +++ b/tests/unit/proxy/common_utils/test_resource_ownership.py @@ -0,0 +1,29 @@ +from typing import Final + +from litellm.litellm_core_utils import resource_ownership as core_resource_ownership +from litellm.proxy._types import LitellmUserRoles, UserAPIKeyAuth +from litellm.proxy.common_utils import resource_ownership as proxy_resource_ownership + +PREVIOUSLY_EXPOSED_NAMES: Final = ( + "Final", + "LitellmUserRoles", + "UserAPIKeyAuth", + "get_primary_resource_owner_scope", + "get_resource_owner_scopes", + "is_proxy_admin", + "user_can_access_resource_owner", +) + + +def test_proxy_path_reexports_core_objects() -> None: + mismatched: Final = [ + name + for name in PREVIOUSLY_EXPOSED_NAMES + if getattr(proxy_resource_ownership, name) is not getattr(core_resource_ownership, name) + ] + assert mismatched == [] + + +def test_moved_module_uses_the_proxy_types_objects() -> None: + assert core_resource_ownership.LitellmUserRoles is LitellmUserRoles + assert core_resource_ownership.UserAPIKeyAuth is UserAPIKeyAuth From 5b6ecab52d16963c7d33047fd3976f94e43b8b1d Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:03:33 -0700 Subject: [PATCH 09/20] fix(proxy): describe rejected characters in key_alias validation error (#41412) * fix(proxy): describe rejected characters in key_alias validation error Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): wrap key_alias validation error message Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Co-authored-by: Yuneng Jiang --- .../proxy/management_endpoints/key_management_endpoints.py | 5 ++++- .../management_endpoints/test_key_management_endpoints.py | 3 +++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/key_management_endpoints.py b/litellm/proxy/management_endpoints/key_management_endpoints.py index 6e226868918..782a7e1db3c 100644 --- a/litellm/proxy/management_endpoints/key_management_endpoints.py +++ b/litellm/proxy/management_endpoints/key_management_endpoints.py @@ -8040,7 +8040,10 @@ def _validate_key_alias_format(key_alias: str | None) -> None: raise_if_unsafe_secret_name(key_alias) except ValueError: raise ProxyException( - message="Invalid key_alias", + message=( + "Invalid key_alias: must not contain control characters (such as tab or newline) " + + 'or a ".." path segment' + ), type=ProxyErrorTypes.bad_request_error, param="key_alias", code=400, diff --git a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py index 308558450b1..ee39df92ef2 100644 --- a/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py +++ b/tests/unit/proxy/management_endpoints/test_key_management_endpoints.py @@ -10733,6 +10733,7 @@ class TestValidateKeyAliasFormat: "foo\n- !grant\n role: !!admin\n member: attacker", "foo\rbar", "foo\x00bar", + "my-test-key\t", ], ) def test_validate_key_alias_format_rejects_traversal_and_control_chars_even_when_flag_disabled( @@ -10751,6 +10752,8 @@ class TestValidateKeyAliasFormat: _validate_key_alias_format(unsafe_alias) assert str(exc.value.code) == "400" assert "Invalid key_alias" in str(exc.value.message) + assert "control characters" in str(exc.value.message) + assert '".." path segment' in str(exc.value.message) def test_validate_key_alias_format_valid(self, monkeypatch): from litellm.proxy.management_endpoints.key_management_endpoints import ( From bc2cc42340a2f363c68011250dd545268d90dfdd Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 21:04:59 +0000 Subject: [PATCH 10/20] docs(proxy): describe /v1/decisions and /v1/systemone in Swagger UI (#45838) * docs(proxy): describe /v1/decisions and /v1/systemone in Swagger UI Add summaries, descriptions, inlined request body schemas with examples and typed 200 responses to the decision routes and their aliases, so the OpenAPI document and the lazy snapshot show both request formats Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(proxy): cap _inlined schema walk depth for the recursion check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- litellm/proxy/_lazy_openapi_snapshot.json | 1805 ++++++++++++++++- .../proxy/common_utils/custom_openapi_spec.py | 38 + .../proxy/decisions_endpoints/endpoints.py | 89 +- litellm/types/decisions.py | 86 +- .../code_coverage_tests/recursive_detector.py | 1 + .../common_utils/test_custom_openapi_spec.py | 92 +- .../decisions_endpoints/test_endpoints.py | 64 +- ui/litellm-dashboard/src/lib/http/schema.d.ts | 866 +++++++- 8 files changed, 2961 insertions(+), 80 deletions(-) diff --git a/litellm/proxy/_lazy_openapi_snapshot.json b/litellm/proxy/_lazy_openapi_snapshot.json index 69db4525a17..534796032c6 100644 --- a/litellm/proxy/_lazy_openapi_snapshot.json +++ b/litellm/proxy/_lazy_openapi_snapshot.json @@ -9494,17 +9494,878 @@ }, "decisions": { "components": { - "schemas": {} + "schemas": { + "ChoiceAnswer": { + "additionalProperties": true, + "properties": { + "choice": { + "title": "Choice", + "type": "string" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "choice", + "confidence", + "probabilities" + ], + "title": "ChoiceAnswer", + "type": "object" + }, + "DecisionsResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "additionalProperties": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/ChoiceAnswer", + "noul": "#/components/schemas/NoulAnswer", + "score": "#/components/schemas/ScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/NoulAnswer" + }, + { + "$ref": "#/components/schemas/ChoiceAnswer" + }, + { + "$ref": "#/components/schemas/ScoreAnswer" + } + ] + }, + "title": "Answers", + "type": "object" + }, + "model": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Model" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/components/schemas/DecisionsUsage" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "answers" + ], + "title": "DecisionsResponse", + "type": "object" + }, + "DecisionsUsage": { + "additionalProperties": true, + "properties": { + "input_tokens": { + "default": 0, + "title": "Input Tokens", + "type": "integer" + }, + "output_tokens": { + "default": 0, + "title": "Output Tokens", + "type": "integer" + } + }, + "title": "DecisionsUsage", + "type": "object" + }, + "NoulAnswer": { + "additionalProperties": true, + "properties": { + "noul": { + "title": "Noul", + "type": "number" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "noul" + ], + "title": "NoulAnswer", + "type": "object" + }, + "OpenAIChoiceAnswer": { + "properties": { + "choice": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Choice" + }, + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIChoiceProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "type": { + "const": "choice", + "default": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "choice", + "probabilities", + "confidence" + ], + "title": "OpenAIChoiceAnswer", + "type": "object" + }, + "OpenAIChoiceProbability": { + "properties": { + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value", + "probability" + ], + "title": "OpenAIChoiceProbability", + "type": "object" + }, + "OpenAIDecisionInputTokensDetails": { + "properties": { + "cache_write_tokens": { + "default": 0, + "title": "Cache Write Tokens", + "type": "integer" + }, + "cached_tokens": { + "default": 0, + "title": "Cached Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionInputTokensDetails", + "type": "object" + }, + "OpenAIDecisionOutputTokensDetails": { + "properties": { + "reasoning_tokens": { + "default": 0, + "title": "Reasoning Tokens", + "type": "integer" + } + }, + "title": "OpenAIDecisionOutputTokensDetails", + "type": "object" + }, + "OpenAIDecisionResponse": { + "additionalProperties": true, + "properties": { + "answers": { + "items": { + "discriminator": { + "mapping": { + "choice": "#/components/schemas/OpenAIChoiceAnswer", + "predicate": "#/components/schemas/OpenAIPredicateAnswer", + "refusal": "#/components/schemas/OpenAIRefusalAnswer", + "score": "#/components/schemas/OpenAIScoreAnswer" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/OpenAIPredicateAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIChoiceAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIScoreAnswer" + }, + { + "$ref": "#/components/schemas/OpenAIRefusalAnswer" + } + ] + }, + "title": "Answers", + "type": "array" + }, + "model": { + "title": "Model", + "type": "string" + }, + "usage": { + "$ref": "#/components/schemas/OpenAIDecisionUsage" + } + }, + "required": [ + "model", + "answers", + "usage" + ], + "title": "OpenAIDecisionResponse", + "type": "object" + }, + "OpenAIDecisionUsage": { + "properties": { + "input_tokens": { + "title": "Input Tokens", + "type": "integer" + }, + "input_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionInputTokensDetails", + "default": { + "cache_write_tokens": 0, + "cached_tokens": 0 + } + }, + "output_tokens": { + "title": "Output Tokens", + "type": "integer" + }, + "output_tokens_details": { + "$ref": "#/components/schemas/OpenAIDecisionOutputTokensDetails", + "default": { + "reasoning_tokens": 0 + } + }, + "total_tokens": { + "title": "Total Tokens", + "type": "integer" + } + }, + "required": [ + "input_tokens", + "output_tokens", + "total_tokens" + ], + "title": "OpenAIDecisionUsage", + "type": "object" + }, + "OpenAIPredicateAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "type": { + "const": "predicate", + "default": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "probability" + ], + "title": "OpenAIPredicateAnswer", + "type": "object" + }, + "OpenAIRefusalAnswer": { + "properties": { + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "type": { + "const": "refusal", + "default": "refusal", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name" + ], + "title": "OpenAIRefusalAnswer", + "type": "object" + }, + "OpenAIScoreAnswer": { + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Name" + }, + "probabilities": { + "items": { + "$ref": "#/components/schemas/OpenAIScoreProbability" + }, + "title": "Probabilities", + "type": "array" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "default": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "name", + "score", + "probabilities", + "confidence" + ], + "title": "OpenAIScoreAnswer", + "type": "object" + }, + "OpenAIScoreProbability": { + "properties": { + "label": { + "title": "Label", + "type": "string" + }, + "probability": { + "title": "Probability", + "type": "number" + }, + "value": { + "title": "Value", + "type": "integer" + } + }, + "required": [ + "value", + "label", + "probability" + ], + "title": "OpenAIScoreProbability", + "type": "object" + }, + "ScoreAnswer": { + "additionalProperties": true, + "properties": { + "confidence": { + "title": "Confidence", + "type": "number" + }, + "legend": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "title": "Legend", + "type": "object" + }, + "probabilities": { + "additionalProperties": { + "type": "number" + }, + "title": "Probabilities", + "type": "object" + }, + "score": { + "title": "Score", + "type": "number" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "score", + "confidence", + "legend", + "probabilities" + ], + "title": "ScoreAnswer", + "type": "object" + } + } }, "paths": { "/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9515,7 +10376,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9523,12 +10384,290 @@ }, "/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9539,7 +10678,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] @@ -9547,12 +10686,382 @@ }, "/v1/decisions": { "post": { + "description": "Judge an `input` against 1 to 128 `questions` and get the answers back in the same order.\n\nQuestion types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with\n`probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these\nrules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy\nmodel list. System One decision models accept this format too, LiteLLM translates the request and the\nanswers. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "decisions_v1_decisions_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "model": "luna", + "questions": [ + { + "instructions": "Is the customer asking for a refund?", + "name": "is_refund_request", + "type": "predicate" + }, + { + "choices": [ + { + "description": "can wait a week", + "value": "low" + }, + { + "description": "needs action today", + "value": "high" + } + ], + "instructions": "How urgent is this?", + "name": "urgency", + "type": "choice" + }, + { + "instructions": "How frustrated is the customer?", + "levels": [ + { + "label": "calm" + }, + { + "label": "mildly annoyed" + }, + { + "label": "angry" + } + ], + "name": "frustration", + "type": "score" + } + ] + }, + "schema": { + "additionalProperties": true, + "properties": { + "input": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "additionalProperties": false, + "properties": { + "content": { + "anyOf": [ + { + "type": "string" + }, + { + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "text": { + "title": "Text", + "type": "string" + }, + "type": { + "const": "input_text", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "text" + ], + "title": "OpenAIDecisionInputText", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "detail": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "image_url": { + "title": "Image Url", + "type": "string" + }, + "type": { + "const": "input_image", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "image_url" + ], + "title": "OpenAIDecisionInputImage", + "type": "object" + } + ] + }, + "type": "array" + } + ], + "title": "Content" + }, + "role": { + "const": "user", + "default": "user", + "title": "Role", + "type": "string" + }, + "type": { + "const": "message", + "default": "message", + "title": "Type", + "type": "string" + } + }, + "required": [ + "content" + ], + "title": "OpenAIDecisionInputMessage", + "type": "object" + }, + "type": "array" + } + ], + "description": "The text being judged, or user messages whose content mixes input_text and input_image parts", + "title": "Input" + }, + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "description": "Predicate, choice and score questions. Answers come back in the same order", + "items": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The yes/no question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "predicate", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions" + ], + "title": "OpenAIPredicateQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "choices": { + "description": "The options the model picks from. Values must be unique", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "value": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "boolean" + } + ], + "title": "Value" + } + }, + "required": [ + "value" + ], + "title": "OpenAIChoiceOption", + "type": "object" + }, + "maxItems": 255, + "minItems": 2, + "title": "Choices", + "type": "array" + }, + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "choices" + ], + "title": "OpenAIChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": false, + "properties": { + "instructions": { + "description": "The question to ask about the input", + "title": "Instructions", + "type": "string" + }, + "levels": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "additionalProperties": false, + "properties": { + "description": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "title": "Description" + }, + "label": { + "title": "Label", + "type": "string" + } + }, + "required": [ + "label" + ], + "title": "OpenAIScoreLevel", + "type": "object" + }, + "maxItems": 10, + "minItems": 2, + "title": "Levels", + "type": "array" + }, + "name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "Echoed in the matching answer", + "title": "Name" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "instructions", + "levels" + ], + "title": "OpenAIScoreQuestion", + "type": "object" + } + ] + }, + "maxItems": 128, + "minItems": 1, + "title": "Questions", + "type": "array" + }, + "safety_identifier": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "description": "A stable id for your end user. Sent to OpenAI and dropped for other providers", + "title": "Safety Identifier" + } + }, + "required": [ + "input", + "questions", + "model" + ], + "title": "OpenAIDecisionRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/OpenAIDecisionResponse" + } } }, "description": "Successful Response" @@ -9563,7 +11072,7 @@ "APIKeyHeader": [] } ], - "summary": "Decisions", + "summary": "Ask questions about an input (OpenAI Decisions format)", "tags": [ "decisions" ] @@ -9571,12 +11080,290 @@ }, "/v1/systemone": { "post": { + "description": "Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name.\n\nQuestion types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with\n`confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`,\n`legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider\nis called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format\ntoo, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported.\n\n[Docs](https://docs.litellm.ai/docs/decisions)", "operationId": "systemone_v1_systemone_post", + "requestBody": { + "content": { + "application/json": { + "example": { + "model": "jev", + "questions": { + "frustration": { + "criteria": [ + "calm", + "mildly annoyed", + "angry" + ], + "instructions": "How frustrated is the customer?", + "type": "score" + }, + "is_refund_request": { + "instructions": "Is the customer asking for a refund?", + "type": "noul" + }, + "urgency": { + "criteria": { + "high": "needs action today", + "low": "can wait a week" + }, + "instructions": "How urgent is this?", + "type": "choice" + } + }, + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + }, + "schema": { + "additionalProperties": true, + "properties": { + "model": { + "description": "A decision model from the proxy model_list", + "title": "Model", + "type": "string" + }, + "questions": { + "additionalProperties": { + "discriminator": { + "propertyName": "type" + }, + "oneOf": [ + { + "additionalProperties": true, + "properties": { + "criteria": { + "anyOf": [ + { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "propertyNames": { + "enum": [ + "true", + "false" + ] + }, + "type": "object" + }, + { + "type": "null" + } + ], + "description": "Optional descriptions of what makes the answer true and what makes it false", + "title": "Criteria" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The yes/no question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "noul", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type" + ], + "title": "NoulQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "additionalProperties": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ] + }, + "description": "Candidate labels mapped to an optional description of each", + "maxProperties": 255, + "minProperties": 1, + "title": "Criteria", + "type": "object" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "choice", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ChoiceQuestion", + "type": "object" + }, + { + "additionalProperties": true, + "properties": { + "criteria": { + "description": "Scale levels from lowest to highest. The score is a level index", + "items": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ] + }, + "maxItems": 10, + "minItems": 1, + "title": "Criteria", + "type": "array" + }, + "instructions": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + }, + { + "type": "null" + } + ], + "description": "The question to ask about the state", + "title": "Instructions" + }, + "type": { + "const": "score", + "title": "Type", + "type": "string" + } + }, + "required": [ + "type", + "criteria" + ], + "title": "ScoreQuestion", + "type": "object" + } + ] + }, + "description": "Named noul, choice and score questions. Each key becomes a key in the answers", + "maxProperties": 128, + "minProperties": 1, + "propertyNames": { + "minLength": 1 + }, + "title": "Questions", + "type": "object" + }, + "state": { + "anyOf": [ + { + "type": "string" + }, + { + "additionalProperties": true, + "type": "object" + }, + { + "items": {}, + "type": "array" + } + ], + "description": "The thing being judged, such as a support ticket, a document or a chat transcript", + "title": "State" + } + }, + "required": [ + "state", + "questions", + "model" + ], + "title": "DecisionsRequest", + "type": "object" + } + } + }, + "required": true + }, "responses": { "200": { "content": { "application/json": { - "schema": {} + "schema": { + "$ref": "#/components/schemas/DecisionsResponse" + } } }, "description": "Successful Response" @@ -9587,7 +11374,7 @@ "APIKeyHeader": [] } ], - "summary": "Systemone", + "summary": "Ask named questions about a state (System One format)", "tags": [ "decisions" ] diff --git a/litellm/proxy/common_utils/custom_openapi_spec.py b/litellm/proxy/common_utils/custom_openapi_spec.py index c1d4442893a..45ca509b7b7 100644 --- a/litellm/proxy/common_utils/custom_openapi_spec.py +++ b/litellm/proxy/common_utils/custom_openapi_spec.py @@ -437,3 +437,41 @@ class CustomOpenAPISpec: # Add responses API request schema return CustomOpenAPISpec.add_responses_api_request_schema(with_embeddings) + + +_MAX_INLINE_DEPTH: Final = 32 + + +def _inlined(node: JsonValue, defs: Mapping[str, JsonValue], depth: int = 0) -> JsonValue: + if depth >= _MAX_INLINE_DEPTH or not isinstance(node, (list, dict)): + return node + if isinstance(node, list): + return [_inlined(item, defs, depth + 1) for item in node] + ref: Final = node.get("$ref") + if isinstance(ref, str) and ref.startswith("#/$defs/"): + target: Final = _inlined(defs[ref.removeprefix("#/$defs/")], defs, depth + 1) + siblings: Final = {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$ref"} + return {**target, **siblings} if isinstance(target, dict) else siblings + if "propertyName" in node and "mapping" in node: + return {"propertyName": node["propertyName"]} + return {key: _inlined(value, defs, depth + 1) for key, value in node.items() if key != "$defs"} + + +def inline_request_body(model_class: type, example: Mapping[str, object]) -> JsonObject: + """ + Build an ``openapi_extra["requestBody"]`` for a route that reads its body with ``request.body()``. + + Swagger UI resolves ``$ref`` against the whole document, so a schema placed inline in an operation + cannot keep Pydantic's ``#/$defs/...`` references. Every reference is expanded in place instead, + which also keeps the operation self-contained inside the lazy OpenAPI snapshot. + """ + schema: Final = CustomOpenAPISpec.get_pydantic_schema(model_class) + media: Final[JsonObject] = {"example": cast(JsonValue, dict(example))} # cast-ok: JSON example literal + if schema is None: + return {"required": True, "content": {"application/json": media}} + raw_defs: Final = schema.get("$defs") + defs: Final[Mapping[str, JsonValue]] = raw_defs if isinstance(raw_defs, dict) else MappingProxyType({}) + return { + "required": True, + "content": {"application/json": {"schema": _inlined(schema, defs), **media}}, + } diff --git a/litellm/proxy/decisions_endpoints/endpoints.py b/litellm/proxy/decisions_endpoints/endpoints.py index 4f4209208ac..c8d4768dad0 100644 --- a/litellm/proxy/decisions_endpoints/endpoints.py +++ b/litellm/proxy/decisions_endpoints/endpoints.py @@ -9,7 +9,15 @@ from pydantic import TypeAdapter, ValidationError from litellm.exceptions import BadRequestError from litellm.proxy.auth.user_api_key_auth import UserAPIKeyAuth, user_api_key_auth from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing -from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody +from litellm.proxy.common_utils.custom_openapi_spec import inline_request_body +from litellm.types.decisions import ( + DecisionsRequest, + DecisionsRequestBody, + DecisionsResponse, + OpenAIDecisionRequest, + OpenAIDecisionRequestBody, + OpenAIDecisionResponse, +) router: Final = APIRouter() _REQUEST_DATA_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) @@ -20,6 +28,51 @@ _OPENAI_DECISION_REQUEST_BODY_ADAPTER: Final[TypeAdapter[OpenAIDecisionRequestBo _GENERAL_SETTINGS_ADAPTER: Final[TypeAdapter[dict[str, object]]] = TypeAdapter(dict[str, object]) _OPTIONAL_STRING_ADAPTER: Final[TypeAdapter[str | None]] = TypeAdapter(str | None) _OPTIONAL_FLOAT_ADAPTER: Final[TypeAdapter[float | None]] = TypeAdapter(float | None) +_SYSTEMONE_REQUEST_BODY: Final = inline_request_body( + DecisionsRequest, + { + "model": "jev", + "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": { + "is_refund_request": {"type": "noul", "instructions": "Is the customer asking for a refund?"}, + "urgency": { + "type": "choice", + "instructions": "How urgent is this?", + "criteria": {"low": "can wait a week", "high": "needs action today"}, + }, + "frustration": { + "type": "score", + "instructions": "How frustrated is the customer?", + "criteria": ["calm", "mildly annoyed", "angry"], + }, + }, + }, +) +_DECISIONS_REQUEST_BODY: Final = inline_request_body( + OpenAIDecisionRequest, + { + "model": "luna", + "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + "questions": [ + {"type": "predicate", "name": "is_refund_request", "instructions": "Is the customer asking for a refund?"}, + { + "type": "choice", + "name": "urgency", + "instructions": "How urgent is this?", + "choices": [ + {"value": "low", "description": "can wait a week"}, + {"value": "high", "description": "needs action today"}, + ], + }, + { + "type": "score", + "name": "frustration", + "instructions": "How frustrated is the customer?", + "levels": [{"label": "calm"}, {"label": "mildly annoyed"}, {"label": "angry"}], + }, + ], + }, +) async def _invalid_request( @@ -121,18 +174,35 @@ async def _process_decisions( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) @router.post( "/systemone", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask named questions about a state (System One format)", + responses={200: {"model": DecisionsResponse}}, + openapi_extra={"requestBody": _SYSTEMONE_REQUEST_BODY}, ) async def systemone( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + + Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, @@ -146,18 +216,35 @@ async def systemone( dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) @router.post( "/decisions", dependencies=[Depends(user_api_key_auth)], response_class=ORJSONResponse, # pyright: ignore[reportDeprecated] # required endpoint contract tags=["decisions"], + summary="Ask questions about an input (OpenAI Decisions format)", + responses={200: {"model": OpenAIDecisionResponse}}, + openapi_extra={"requestBody": _DECISIONS_REQUEST_BODY}, ) async def decisions( request: Request, fastapi_response: Response, user_api_key_dict: Annotated[UserAPIKeyAuth, Depends(user_api_key_auth)], ): + """ + Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + + Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + model list. System One decision models accept this format too, LiteLLM translates the request and the + answers. Streaming is not supported. + + [Docs](https://docs.litellm.ai/docs/decisions) + """ return await _process_decisions( request=request, fastapi_response=fastapi_response, diff --git a/litellm/types/decisions.py b/litellm/types/decisions.py index b24690dbd9d..3bc90a5109a 100644 --- a/litellm/types/decisions.py +++ b/litellm/types/decisions.py @@ -15,8 +15,13 @@ MAX_DECISION_QUESTIONS: Final = 128 class NoulQuestion(LiteLLMPydanticObjectBase): type: Literal["noul"] - instructions: DecisionsJSON | None = None - criteria: NoulCriteria | None = None + instructions: Annotated[DecisionsJSON | None, Field(description="The yes/no question to ask about the state")] = ( + None + ) + criteria: Annotated[ + NoulCriteria | None, + Field(description="Optional descriptions of what makes the answer true and what makes it false"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) @@ -29,16 +34,24 @@ class NoulQuestion(LiteLLMPydanticObjectBase): class ChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Mapping[str, DecisionsJSON | None], Field(min_length=1, max_length=255)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Mapping[str, DecisionsJSON | None], + Field(min_length=1, max_length=255, description="Candidate labels mapped to an optional description of each"), + ] model_config = ConfigDict(extra="allow", frozen=True) class ScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - instructions: DecisionsJSON | None = None - criteria: Annotated[Sequence[DecisionsJSON], Field(min_length=1, max_length=10)] + instructions: Annotated[DecisionsJSON | None, Field(description="The question to ask about the state")] = None + criteria: Annotated[ + Sequence[DecisionsJSON], + Field( + min_length=1, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="allow", frozen=True) @@ -55,14 +68,20 @@ DecisionQuestionMap: TypeAlias = Annotated[ class DecisionsRequestBody(LiteLLMPydanticObjectBase): - state: DecisionsJSON - questions: DecisionQuestionMap + state: Annotated[ + DecisionsJSON, + Field(description="The thing being judged, such as a support ticket, a document or a chat transcript"), + ] + questions: Annotated[ + DecisionQuestionMap, + Field(description="Named noul, choice and score questions. Each key becomes a key in the answers"), + ] model_config = ConfigDict(extra="allow", frozen=True) class DecisionsRequest(DecisionsRequestBody): - model: str + model: Annotated[str, Field(description="A decision model from the proxy model_list")] @with_config(ConfigDict(extra="allow")) @@ -171,8 +190,8 @@ OpenAIDecisionInput: TypeAlias = str | Sequence[OpenAIDecisionInputMessage] class OpenAIPredicateQuestion(LiteLLMPydanticObjectBase): type: Literal["predicate"] - name: str | None = None - instructions: str + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The yes/no question to ask about the input")] model_config = ConfigDict(extra="forbid", frozen=True) @@ -192,9 +211,12 @@ def systemone_choice_key(value: str | bool) -> str: class OpenAIChoiceQuestion(LiteLLMPydanticObjectBase): type: Literal["choice"] - name: str | None = None - instructions: str - choices: Annotated[Sequence[OpenAIChoiceOption], Field(min_length=2, max_length=255)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + choices: Annotated[ + Sequence[OpenAIChoiceOption], + Field(min_length=2, max_length=255, description="The options the model picks from. Values must be unique"), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -215,9 +237,14 @@ class OpenAIScoreLevel(LiteLLMPydanticObjectBase): class OpenAIScoreQuestion(LiteLLMPydanticObjectBase): type: Literal["score"] - name: str | None = None - instructions: str - levels: Annotated[Sequence[OpenAIScoreLevel], Field(min_length=2, max_length=10)] + name: Annotated[str | None, Field(description="Echoed in the matching answer")] = None + instructions: Annotated[str, Field(description="The question to ask about the input")] + levels: Annotated[ + Sequence[OpenAIScoreLevel], + Field( + min_length=2, max_length=10, description="Scale levels from lowest to highest. The score is a level index" + ), + ] model_config = ConfigDict(extra="forbid", frozen=True) @@ -229,13 +256,32 @@ OpenAIDecisionQuestion: TypeAlias = Annotated[ class OpenAIDecisionRequestBody(LiteLLMPydanticObjectBase): - input: OpenAIDecisionInput - questions: Annotated[Sequence[OpenAIDecisionQuestion], Field(min_length=1, max_length=MAX_DECISION_QUESTIONS)] - safety_identifier: str | None = None + input: Annotated[ + OpenAIDecisionInput, + Field( + description="The text being judged, or user messages whose content mixes input_text and input_image parts" + ), + ] + questions: Annotated[ + Sequence[OpenAIDecisionQuestion], + Field( + min_length=1, + max_length=MAX_DECISION_QUESTIONS, + description="Predicate, choice and score questions. Answers come back in the same order", + ), + ] + safety_identifier: Annotated[ + str | None, + Field(description="A stable id for your end user. Sent to OpenAI and dropped for other providers"), + ] = None model_config = ConfigDict(extra="allow", frozen=True) +class OpenAIDecisionRequest(OpenAIDecisionRequestBody): + model: Annotated[str, Field(description="A decision model from the proxy model_list")] + + class OpenAIPredicateAnswer(LiteLLMPydanticObjectBase): type: Literal["predicate"] = "predicate" name: str | None diff --git a/tests/code_coverage_tests/recursive_detector.py b/tests/code_coverage_tests/recursive_detector.py index cea2b4154a7..e1d58ee73d2 100644 --- a/tests/code_coverage_tests/recursive_detector.py +++ b/tests/code_coverage_tests/recursive_detector.py @@ -58,6 +58,7 @@ IGNORE_FUNCTIONS = [ "sanitize_oci_schema", # OCI: bounded by JSON-schema tree depth. "_freeze_for_dedupe", # OTEL: max depth set (default 16, _FREEZE_MAX_DEPTH); fails closed by returning repr(value) at the cap. "apply_json_merge_patch", # max depth set (_MAX_MERGE_DEPTH=64); fails closed by raising ValueError at the cap. + "_inlined", # max depth set (_MAX_INLINE_DEPTH=32); passes the schema node through untouched at the cap. Walks a Pydantic JSON schema once at import time. "_filter_argument_value", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by blocking the tool call at the cap. "_redact_scanned_content", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); fails closed by returning "[REDACTED]" at the cap. "replace_ciphertexts", # max depth set (DEFAULT_MAX_RECURSE_DEPTH); walks stored JSON, which has no cycles, and leaves values below the cap untouched. diff --git a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py index 485dbcef2f6..41ab613e7ea 100644 --- a/tests/unit/proxy/common_utils/test_custom_openapi_spec.py +++ b/tests/unit/proxy/common_utils/test_custom_openapi_spec.py @@ -4,11 +4,14 @@ Simple unit tests for CustomOpenAPISpec class. Tests basic functionality of OpenAPI schema generation. """ -from unittest.mock import Mock, patch +import json +from typing import Annotated, Literal +from unittest.mock import patch import pytest +from pydantic import BaseModel, ConfigDict, Field -from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec +from litellm.proxy.common_utils.custom_openapi_spec import CustomOpenAPISpec, inline_request_body class TestCustomOpenAPISpec: @@ -27,19 +30,13 @@ class TestCustomOpenAPISpec: }, } - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_chat_completion_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_chat_completion_request_schema(self, mock_add_schema, base_openapi_schema): """Test that chat completion schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.proxy._types.ProxyChatCompletionRequest") as mock_model: - result = CustomOpenAPISpec.add_chat_completion_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_chat_completion_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -50,9 +47,7 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") def test_add_embedding_request_schema(self, mock_add_schema, base_openapi_schema): """Test that embedding schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema @@ -69,19 +64,13 @@ class TestCustomOpenAPISpec: ) assert result == base_openapi_schema - @patch( - "litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema" - ) - def test_add_responses_api_request_schema( - self, mock_add_schema, base_openapi_schema - ): + @patch("litellm.proxy.common_utils.custom_openapi_spec.CustomOpenAPISpec.add_request_schema") + def test_add_responses_api_request_schema(self, mock_add_schema, base_openapi_schema): """Test that responses API schema is added correctly.""" mock_add_schema.return_value = base_openapi_schema with patch("litellm.types.llms.openai.ResponsesAPIRequestParams") as mock_model: - result = CustomOpenAPISpec.add_responses_api_request_schema( - base_openapi_schema - ) + result = CustomOpenAPISpec.add_responses_api_request_schema(base_openapi_schema) mock_add_schema.assert_called_once_with( openapi_schema=base_openapi_schema, @@ -123,15 +112,11 @@ def test_defs_rewritten_in_add_schema_to_components(): ) assert "$defs" not in openapi_schema assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][0]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" ) assert ( - openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"][ - "items" - ]["anyOf"][1]["$ref"] + openapi_schema["components"]["schemas"]["SchemaName"]["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" ) @@ -188,14 +173,8 @@ def test_rewrite_defs_refs(): rewritten = CustomOpenAPISpec._rewrite_defs_refs(schema=schema, renames={}) assert "$defs" not in rewritten - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] - == "#/components/schemas/UserMessage" - ) - assert ( - rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] - == "#/components/schemas/AssistantMessage" - ) + assert rewritten["properties"]["messages"]["items"]["anyOf"][0]["$ref"] == "#/components/schemas/UserMessage" + assert rewritten["properties"]["messages"]["items"]["anyOf"][1]["$ref"] == "#/components/schemas/AssistantMessage" def test_get_pydantic_schema_generates_schema_for_responses_request_typed_dict(): @@ -390,3 +369,42 @@ def test_add_schema_to_components_renames_def_with_different_required_set(): assert schemas["Block"]["required"] == ["type", "x", "y"] assert schemas["Req_Block"]["required"] == ["keys", "type", "x", "y"] assert schemas["Req"]["properties"]["b"]["$ref"] == "#/components/schemas/Req_Block" + + +class _Cat(BaseModel): + kind: Literal["cat"] + lives: int = 9 + + model_config = ConfigDict(frozen=True) + + +class _Dog(BaseModel): + kind: Literal["dog"] + good: bool = True + + model_config = ConfigDict(frozen=True) + + +class _Shelter(BaseModel): + pets: dict[str, Annotated[_Cat | _Dog, Field(discriminator="kind")]] + favorite: _Cat | None = None + + model_config = ConfigDict(frozen=True) + + +def test_inline_request_body_expands_every_ref_so_the_operation_stands_alone(): + body = inline_request_body(_Shelter, {"pets": {"tom": {"kind": "cat"}}}) + + media = body["content"]["application/json"] + assert body["required"] is True + assert media["example"] == {"pets": {"tom": {"kind": "cat"}}} + assert "$ref" not in json.dumps(media["schema"]) + assert "$defs" not in media["schema"] + pet_schema = media["schema"]["properties"]["pets"]["additionalProperties"] + variants = {variant["properties"]["kind"]["const"]: variant for variant in pet_schema["oneOf"]} + assert variants["cat"]["properties"]["lives"]["default"] == 9 + assert variants["dog"]["properties"]["good"]["type"] == "boolean" + assert pet_schema["discriminator"] == {"propertyName": "kind"} + favorite = media["schema"]["properties"]["favorite"] + assert favorite["default"] is None + assert any(option.get("properties", {}).get("lives") for option in favorite["anyOf"]) diff --git a/tests/unit/proxy/decisions_endpoints/test_endpoints.py b/tests/unit/proxy/decisions_endpoints/test_endpoints.py index ed78fc7063a..87b2b64ad8d 100644 --- a/tests/unit/proxy/decisions_endpoints/test_endpoints.py +++ b/tests/unit/proxy/decisions_endpoints/test_endpoints.py @@ -12,17 +12,19 @@ import respx from fastapi import FastAPI from fastapi.routing import APIRoute from fastapi.testclient import TestClient +from pydantic import TypeAdapter from starlette.routing import Match import litellm from litellm.proxy._lazy_features import LAZY_FEATURES, LazyFeature, attach_lazy_features -from litellm.proxy.decisions_endpoints.endpoints import decisions, systemone +from litellm.proxy.decisions_endpoints.endpoints import decisions, router, systemone from litellm.proxy.pass_through_endpoints.pass_through_endpoints import SafeRouteAdder from litellm.proxy.proxy_server import ( app, cleanup_router_config_variables, initialize, ) +from litellm.types.decisions import DecisionsRequestBody, OpenAIDecisionRequestBody _INPUT_TOKENS: Final[int] = 367 _OUTPUT_TOKENS: Final[int] = 3 @@ -688,3 +690,63 @@ def test_with_lazy_routes_disabled_a_config_pass_through_at_v1_decisions_still_w assert client.post("/v1/decisions", json={"model": "gpt-6-luna"}).json() == {"served_by": "pass-through"} assert _serving_endpoint(bare, "/v1/decisions") is pass_through assert _serving_endpoint(bare, "/decisions") is decisions + + +@pytest.fixture(scope="module") +def decisions_openapi() -> dict[str, object]: + app = FastAPI() + app.include_router(router) + return app.openapi() + + +@pytest.mark.parametrize( + ("path", "body_adapter", "response_schema", "required"), + ( + ("/v1/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ("/systemone", TypeAdapter(DecisionsRequestBody), "DecisionsResponse", {"model", "state", "questions"}), + ( + "/v1/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ( + "/decisions", + TypeAdapter(OpenAIDecisionRequestBody), + "OpenAIDecisionResponse", + {"model", "input", "questions"}, + ), + ), +) +def test_swagger_documents_the_request_format_and_response_of_each_decisions_route( + decisions_openapi: dict[str, object], + path: str, + body_adapter: TypeAdapter[DecisionsRequestBody] | TypeAdapter[OpenAIDecisionRequestBody], + response_schema: str, + required: set[str], +) -> None: + operation = decisions_openapi["paths"][path]["post"] + media = operation["requestBody"]["content"]["application/json"] + + assert "Decisions" in operation["summary"] or "System One" in operation["summary"] + assert "docs.litellm.ai/docs/decisions" in operation["description"] + assert required <= set(media["schema"]["required"]) + assert required <= set(media["schema"]["properties"]) + assert "$ref" not in json.dumps(media["schema"]) + assert media["schema"]["properties"]["model"]["description"] + + example = {key: value for key, value in media["example"].items() if key != "model"} + assert len(body_adapter.validate_python(example).questions) == 3 + + response_ref = operation["responses"]["200"]["content"]["application/json"]["schema"]["$ref"] + assert response_ref == f"#/components/schemas/{response_schema}" + assert "answers" in decisions_openapi["components"]["schemas"][response_schema]["properties"] + + +def test_the_two_decisions_routes_document_different_request_formats(decisions_openapi: dict[str, object]) -> None: + def properties(path: str) -> set[str]: + operation = decisions_openapi["paths"][path]["post"] + return set(operation["requestBody"]["content"]["application/json"]["schema"]["properties"]) + + assert "state" in properties("/v1/systemone") and "state" not in properties("/v1/decisions") + assert "input" in properties("/v1/decisions") and "input" not in properties("/v1/systemone") diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 47df4593a65..05e80d26e09 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -4703,7 +4703,18 @@ export interface paths { }; get?: never; put?: never; - /** Decisions */ + /** + * Ask questions about an input (OpenAI Decisions format) + * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * + * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * model list. System One decision models accept this format too, LiteLLM translates the request and the + * answers. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["decisions_decisions_post"]; delete?: never; options?: never; @@ -15837,7 +15848,18 @@ export interface paths { }; get?: never; put?: never; - /** Systemone */ + /** + * Ask named questions about a state (System One format) + * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * + * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["systemone_systemone_post"]; delete?: never; options?: never; @@ -19646,7 +19668,18 @@ export interface paths { }; get?: never; put?: never; - /** Decisions */ + /** + * Ask questions about an input (OpenAI Decisions format) + * @description Judge an `input` against 1 to 128 `questions` and get the answers back in the same order. + * + * Question types are `predicate` (yes/no with `probability`), `choice` (one of the `choices`, with + * `probabilities`) and `score` (an index into `levels`, with `probabilities`). Requests that break these + * rules are rejected with a 400 before any provider is called. `model` is any decision model in the proxy + * model list. System One decision models accept this format too, LiteLLM translates the request and the + * answers. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["decisions_v1_decisions_post"]; delete?: never; options?: never; @@ -21991,7 +22024,18 @@ export interface paths { }; get?: never; put?: never; - /** Systemone */ + /** + * Ask named questions about a state (System One format) + * @description Judge a `state` against 1 to 128 named `questions` and get one calibrated answer per question name. + * + * Question types are `noul` (probability the answer is yes), `choice` (one label out of `criteria`, with + * `confidence` and `probabilities`) and `score` (an index into the `criteria` levels, with `confidence`, + * `legend` and `probabilities`). Requests that break these rules are rejected with a 400 before any provider + * is called. `model` is any decision model in the proxy model list. OpenAI decision models accept this format + * too, LiteLLM translates it and keeps the answers keyed by question name. Streaming is not supported. + * + * [Docs](https://docs.litellm.ai/docs/decisions) + */ post: operations["systemone_v1_systemone_post"]; delete?: never; options?: never; @@ -28642,6 +28686,24 @@ export interface components { */ role: "user" | "assistant"; }; + /** ChoiceAnswer */ + ChoiceAnswer: { + /** Choice */ + choice: string; + /** Confidence */ + confidence: number; + /** Probabilities */ + probabilities: { + [key: string]: number; + }; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "choice"; + } & { + [key: string]: unknown; + }; /** ChoiceLogprobs */ ChoiceLogprobs: { /** Content */ @@ -31008,6 +31070,33 @@ export interface components { */ total_tokens: number; }; + /** DecisionsResponse */ + DecisionsResponse: { + /** Answers */ + answers: { + [key: string]: components["schemas"]["NoulAnswer"] | components["schemas"]["ChoiceAnswer"] | components["schemas"]["ScoreAnswer"]; + }; + /** Model */ + model?: string | null; + usage?: components["schemas"]["DecisionsUsage"] | null; + } & { + [key: string]: unknown; + }; + /** DecisionsUsage */ + DecisionsUsage: { + /** + * Input Tokens + * @default 0 + */ + input_tokens: number; + /** + * Output Tokens + * @default 0 + */ + output_tokens: number; + } & { + [key: string]: unknown; + }; /** * DefaultInternalUserParams * @description Default parameters to apply when a new user signs in via SSO or is created on the /user/new API endpoint @@ -38834,6 +38923,18 @@ export interface components { /** User Role */ user_role?: ("proxy_admin" | "proxy_admin_viewer" | "internal_user" | "internal_user_viewer") | null; }; + /** NoulAnswer */ + NoulAnswer: { + /** Noul */ + noul: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "noul"; + } & { + [key: string]: unknown; + }; /** * OAuth2SecurityScheme * @description Defines a security scheme using OAuth 2.0. @@ -38872,6 +38973,129 @@ export interface components { [key: string]: unknown; } | null; }; + /** OpenAIChoiceAnswer */ + OpenAIChoiceAnswer: { + /** Choice */ + choice: string | boolean; + /** Confidence */ + confidence: number; + /** Name */ + name: string | null; + /** Probabilities */ + probabilities: components["schemas"]["OpenAIChoiceProbability"][]; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "choice"; + }; + /** OpenAIChoiceProbability */ + OpenAIChoiceProbability: { + /** Probability */ + probability: number; + /** Value */ + value: string | boolean; + }; + /** OpenAIDecisionInputTokensDetails */ + OpenAIDecisionInputTokensDetails: { + /** + * Cache Write Tokens + * @default 0 + */ + cache_write_tokens: number; + /** + * Cached Tokens + * @default 0 + */ + cached_tokens: number; + }; + /** OpenAIDecisionOutputTokensDetails */ + OpenAIDecisionOutputTokensDetails: { + /** + * Reasoning Tokens + * @default 0 + */ + reasoning_tokens: number; + }; + /** OpenAIDecisionResponse */ + OpenAIDecisionResponse: { + /** Answers */ + answers: (components["schemas"]["OpenAIPredicateAnswer"] | components["schemas"]["OpenAIChoiceAnswer"] | components["schemas"]["OpenAIScoreAnswer"] | components["schemas"]["OpenAIRefusalAnswer"])[]; + /** Model */ + model: string; + usage: components["schemas"]["OpenAIDecisionUsage"]; + } & { + [key: string]: unknown; + }; + /** OpenAIDecisionUsage */ + OpenAIDecisionUsage: { + /** Input Tokens */ + input_tokens: number; + /** + * @default { + * "cache_write_tokens": 0, + * "cached_tokens": 0 + * } + */ + input_tokens_details: components["schemas"]["OpenAIDecisionInputTokensDetails"]; + /** Output Tokens */ + output_tokens: number; + /** + * @default { + * "reasoning_tokens": 0 + * } + */ + output_tokens_details: components["schemas"]["OpenAIDecisionOutputTokensDetails"]; + /** Total Tokens */ + total_tokens: number; + }; + /** OpenAIPredicateAnswer */ + OpenAIPredicateAnswer: { + /** Name */ + name: string | null; + /** Probability */ + probability: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "predicate"; + }; + /** OpenAIRefusalAnswer */ + OpenAIRefusalAnswer: { + /** Name */ + name: string | null; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "refusal"; + }; + /** OpenAIScoreAnswer */ + OpenAIScoreAnswer: { + /** Confidence */ + confidence: number; + /** Name */ + name: string | null; + /** Probabilities */ + probabilities: components["schemas"]["OpenAIScoreProbability"][]; + /** Score */ + score: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "score"; + }; + /** OpenAIScoreProbability */ + OpenAIScoreProbability: { + /** Label */ + label: string; + /** Probability */ + probability: number; + /** Value */ + value: number; + }; /** * OpenIdConnectSecurityScheme * @description Defines a security scheme using OpenID Connect. @@ -43986,6 +44210,30 @@ export interface components { */ window_seconds: number; }; + /** ScoreAnswer */ + ScoreAnswer: { + /** Confidence */ + confidence: number; + /** Legend */ + legend: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[]; + }; + /** Probabilities */ + probabilities: { + [key: string]: number; + }; + /** Score */ + score: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "score"; + } & { + [key: string]: unknown; + }; /** * Screenshot * @description A screenshot action. @@ -56556,7 +56804,179 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + * "model": "luna", + * "questions": [ + * { + * "instructions": "Is the customer asking for a refund?", + * "name": "is_refund_request", + * "type": "predicate" + * }, + * { + * "choices": [ + * { + * "description": "can wait a week", + * "value": "low" + * }, + * { + * "description": "needs action today", + * "value": "high" + * } + * ], + * "instructions": "How urgent is this?", + * "name": "urgency", + * "type": "choice" + * }, + * { + * "instructions": "How frustrated is the customer?", + * "levels": [ + * { + * "label": "calm" + * }, + * { + * "label": "mildly annoyed" + * }, + * { + * "label": "angry" + * } + * ], + * "name": "frustration", + * "type": "score" + * } + * ] + * } + */ + "application/json": { + /** + * Input + * @description The text being judged, or user messages whose content mixes input_text and input_image parts + */ + input: string | { + /** Content */ + content: string | ({ + /** Text */ + text: string; + /** + * Type + * @constant + */ + type: "input_text"; + } | { + /** Detail */ + detail?: string | null; + /** Image Url */ + image_url: string; + /** + * Type + * @constant + */ + type: "input_image"; + })[]; + /** + * Role + * @default user + * @constant + */ + role?: "user"; + /** + * Type + * @default message + * @constant + */ + type?: "message"; + }[]; + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Predicate, choice and score questions. Answers come back in the same order + */ + questions: ({ + /** + * Instructions + * @description The yes/no question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "predicate"; + } | { + /** + * Choices + * @description The options the model picks from. Values must be unique + */ + choices: { + /** Description */ + description?: string | null; + /** Value */ + value: string | boolean; + }[]; + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Levels + * @description Scale levels from lowest to highest. The score is a level index + */ + levels: { + /** Description */ + description?: string | null; + /** Label */ + label: string; + }[]; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "score"; + })[]; + /** + * Safety Identifier + * @description A stable id for your end user. Sent to OpenAI and dropped for other providers + */ + safety_identifier?: string | null; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -56564,7 +56984,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["OpenAIDecisionResponse"]; }; }; }; @@ -70673,7 +71093,132 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "jev", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "urgency": { + * "criteria": { + * "high": "needs action today", + * "low": "can wait a week" + * }, + * "instructions": "How urgent is this?", + * "type": "choice" + * } + * }, + * "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + * } + */ + "application/json": { + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Named noul, choice and score questions. Each key becomes a key in the answers + */ + questions: { + [key: string]: ({ + /** + * Criteria + * @description Optional descriptions of what makes the answer true and what makes it false + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description The yes/no question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Candidate labels mapped to an optional description of each + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Scale levels from lowest to highest. The score is a level index + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[])[]; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + } & { + [key: string]: unknown; + }); + }; + /** + * State + * @description The thing being judged, such as a support ticket, a document or a chat transcript + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -70681,7 +71226,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["DecisionsResponse"]; }; }; }; @@ -75840,7 +76385,179 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "input": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today.", + * "model": "luna", + * "questions": [ + * { + * "instructions": "Is the customer asking for a refund?", + * "name": "is_refund_request", + * "type": "predicate" + * }, + * { + * "choices": [ + * { + * "description": "can wait a week", + * "value": "low" + * }, + * { + * "description": "needs action today", + * "value": "high" + * } + * ], + * "instructions": "How urgent is this?", + * "name": "urgency", + * "type": "choice" + * }, + * { + * "instructions": "How frustrated is the customer?", + * "levels": [ + * { + * "label": "calm" + * }, + * { + * "label": "mildly annoyed" + * }, + * { + * "label": "angry" + * } + * ], + * "name": "frustration", + * "type": "score" + * } + * ] + * } + */ + "application/json": { + /** + * Input + * @description The text being judged, or user messages whose content mixes input_text and input_image parts + */ + input: string | { + /** Content */ + content: string | ({ + /** Text */ + text: string; + /** + * Type + * @constant + */ + type: "input_text"; + } | { + /** Detail */ + detail?: string | null; + /** Image Url */ + image_url: string; + /** + * Type + * @constant + */ + type: "input_image"; + })[]; + /** + * Role + * @default user + * @constant + */ + role?: "user"; + /** + * Type + * @default message + * @constant + */ + type?: "message"; + }[]; + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Predicate, choice and score questions. Answers come back in the same order + */ + questions: ({ + /** + * Instructions + * @description The yes/no question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "predicate"; + } | { + /** + * Choices + * @description The options the model picks from. Values must be unique + */ + choices: { + /** Description */ + description?: string | null; + /** Value */ + value: string | boolean; + }[]; + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "choice"; + } | { + /** + * Instructions + * @description The question to ask about the input + */ + instructions: string; + /** + * Levels + * @description Scale levels from lowest to highest. The score is a level index + */ + levels: { + /** Description */ + description?: string | null; + /** Label */ + label: string; + }[]; + /** + * Name + * @description Echoed in the matching answer + */ + name?: string | null; + /** + * Type + * @constant + */ + type: "score"; + })[]; + /** + * Safety Identifier + * @description A stable id for your end user. Sent to OpenAI and dropped for other providers + */ + safety_identifier?: string | null; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -75848,7 +76565,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["OpenAIDecisionResponse"]; }; }; }; @@ -79013,7 +79730,132 @@ export interface operations { path?: never; cookie?: never; }; - requestBody?: never; + requestBody: { + content: { + /** + * @example { + * "model": "jev", + * "questions": { + * "frustration": { + * "criteria": [ + * "calm", + * "mildly annoyed", + * "angry" + * ], + * "instructions": "How frustrated is the customer?", + * "type": "score" + * }, + * "is_refund_request": { + * "instructions": "Is the customer asking for a refund?", + * "type": "noul" + * }, + * "urgency": { + * "criteria": { + * "high": "needs action today", + * "low": "can wait a week" + * }, + * "instructions": "How urgent is this?", + * "type": "choice" + * } + * }, + * "state": "Customer wrote: I was charged twice for order #4411 and want one charge refunded today." + * } + */ + "application/json": { + /** + * Model + * @description A decision model from the proxy model_list + */ + model: string; + /** + * Questions + * @description Named noul, choice and score questions. Each key becomes a key in the answers + */ + questions: { + [key: string]: ({ + /** + * Criteria + * @description Optional descriptions of what makes the answer true and what makes it false + */ + criteria?: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + } | null; + /** + * Instructions + * @description The yes/no question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "noul"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Candidate labels mapped to an optional description of each + */ + criteria: { + [key: string]: string | { + [key: string]: unknown; + } | unknown[] | null; + }; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "choice"; + } & { + [key: string]: unknown; + }) | ({ + /** + * Criteria + * @description Scale levels from lowest to highest. The score is a level index + */ + criteria: (string | { + [key: string]: unknown; + } | unknown[])[]; + /** + * Instructions + * @description The question to ask about the state + */ + instructions?: string | { + [key: string]: unknown; + } | unknown[] | null; + /** + * Type + * @constant + */ + type: "score"; + } & { + [key: string]: unknown; + }); + }; + /** + * State + * @description The thing being judged, such as a support ticket, a document or a chat transcript + */ + state: string | { + [key: string]: unknown; + } | unknown[]; + } & { + [key: string]: unknown; + }; + }; + }; responses: { /** @description Successful Response */ 200: { @@ -79021,7 +79863,7 @@ export interface operations { [name: string]: unknown; }; content: { - "application/json": unknown; + "application/json": components["schemas"]["DecisionsResponse"]; }; }; }; From dc7738195639c92dc01a1b5ed5ede96be980083f Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:05:03 -0700 Subject: [PATCH 11/20] feat(ui): replace Reset example with an Example preset dropdown and a Clear button on the Decisions tab (#45848) * feat(ui): replace Reset example with an Example preset dropdown and a Clear button on the Decisions tab Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> * fix(ui): match Decisions presets by content and keep an omitted model out of Clear and preset picks Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --------- Co-authored-by: kerry Co-authored-by: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../SystemOneUI.integration.test.tsx | 87 ++++++++- .../components/systemOneUI/SystemOneUI.tsx | 58 ++++-- .../systemOneUI/lib/example.test.ts | 60 ++++++ .../components/systemOneUI/lib/example.ts | 180 +++++++++++++++--- 4 files changed, 340 insertions(+), 45 deletions(-) create mode 100644 ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/lib/example.test.ts diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx index ab0a67dbbab..c5c6023cf27 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.integration.test.tsx @@ -71,6 +71,12 @@ const selectEndpoint = async (endpoint: "/v1/decisions" | "/v1/systemone") => { await userEvent.click(await screen.findByRole("option", { name: endpoint, exact: true })); }; +const selectPreset = async (label: string) => { + screen.getByRole("combobox", { name: "Example" }).focus(); + await userEvent.keyboard("{ArrowDown}"); + await userEvent.click(await screen.findByRole("option", { name: label })); +}; + const createResponse = (body: SystemOneResponse | OpenAIDecisionsResponse, status = 200, errorText = "") => ({ ok: status >= 200 && status < 300, @@ -99,20 +105,81 @@ describe("SystemOneUI integration", () => { document.cookie = "token=; path=/; max-age=0"; }); - it("restores the example after the request is edited", async () => { - const user = userEvent.setup(); + it("switches between example presets and reads Custom once the request is edited", async () => { render(); const editor = await openJsonEditor(); const example = (editor as HTMLTextAreaElement).value; - const resetButton = screen.getByRole("button", { name: "Reset example" }); - expect(resetButton).toBeDisabled(); + const picker = screen.getByRole("combobox", { name: "Example" }); + expect(picker).toHaveTextContent("Triage a bug report"); fireEvent.change(editor, { target: { value: "{}" } }); - expect(resetButton).toBeEnabled(); - await user.click(resetButton); - + expect(picker).toHaveTextContent("Custom"); + await selectPreset("Triage a bug report"); expect(editor).toHaveValue(example); - expect(resetButton).toBeDisabled(); + expect(picker).toHaveTextContent("Triage a bug report"); + + await selectPreset("Score a sales lead"); + expect(picker).toHaveTextContent("Score a sales lead"); + expect(JSON.parse((editor as HTMLTextAreaElement).value)).toMatchObject({ questions: { fit: { type: "score" } } }); + }); + + it("clears the whole request and lets a preset be picked from nothing", async () => { + const user = userEvent.setup(); + render(); + const clear = screen.getByRole("button", { name: "Clear" }); + expect(screen.getByRole("textbox", { name: "Input" })).toHaveDisplayValue(/streaming responses/); + + await user.click(clear); + expect(screen.getByRole("textbox", { name: "Input" })).toHaveValue(""); + expect(screen.queryAllByRole("group", { name: /Question / })).toHaveLength(0); + expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); + expect(clear).toBeDisabled(); + expect(screen.getByRole("combobox", { name: "Example" })).toHaveTextContent("Custom"); + + await selectPreset("Route a support ticket"); + expect(screen.getByRole("textbox", { name: "Input" })).toHaveDisplayValue(/charged twice/); + expect(screen.getAllByRole("group", { name: /Question / })).toHaveLength(1); + expect(screen.getByRole("button", { name: "Send" })).toBeEnabled(); + }); + + it("keeps the preset label after picking a model and leaves an omitted model out when clearing", async () => { + mockModelLookup.mockResolvedValue(modelGroupInfoResponse(DECISION_AND_CHAT_MODELS)); + render(); + await waitFor(() => expect(screen.getByRole("combobox", { name: "Decision model" })).toHaveValue("jev-latest")); + const picker = screen.getByRole("combobox", { name: "Example" }); + const editor = await openJsonEditor(); + const example = JSON.parse((editor as HTMLTextAreaElement).value) as Record; + const { model: _model, ...withoutModel } = example; + + fireEvent.change(editor, { + target: { value: JSON.stringify({ ...withoutModel, model: "pplx-decider" }, null, 2) }, + }); + expect(picker).toHaveTextContent("Triage a bug report"); + + fireEvent.change(editor, { target: { value: JSON.stringify(withoutModel, null, 2) } }); + expect(picker).toHaveTextContent("Triage a bug report"); + await userEvent.click(screen.getByRole("button", { name: "Clear" })); + expect(JSON.parse((editor as HTMLTextAreaElement).value)).toEqual({ state: "", questions: {} }); + await selectPreset("Score a sales lead"); + expect(JSON.parse((editor as HTMLTextAreaElement).value)).not.toHaveProperty("model"); + }); + + it("offers the same presets on /v1/decisions in the OpenAI shape and clears to an empty input", async () => { + const user = userEvent.setup(); + render(); + await selectEndpoint("/v1/decisions"); + await selectPreset("Moderate a forum comment"); + await user.click(screen.getByRole("tab", { name: "JSON" })); + const editor = screen.getByRole("textbox", { name: "Decisions JSON payload" }) as HTMLTextAreaElement; + const payload = JSON.parse(editor.value); + expect(payload.input).toMatch(/stream=false/); + expect(payload.questions).toEqual([ + { type: "predicate", name: "breaks_rules", instructions: "Does this comment break the community guidelines?" }, + ]); + + await user.click(screen.getByRole("button", { name: "Clear" })); + expect(JSON.parse(editor.value)).toEqual({ input: "", questions: [] }); + expect(screen.getByRole("button", { name: "Send" })).toBeDisabled(); }); it("offers both literal proxy endpoints without TypeSafe or product prefixes", async () => { @@ -227,7 +294,7 @@ describe("SystemOneUI integration", () => { await selectEndpoint("/v1/decisions"); expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue(JSON.stringify(payload)); - await user.click(screen.getByRole("button", { name: "Reset example" })); + await selectPreset("Triage a bug report"); expect(screen.getByRole("textbox", { name: "Decisions JSON payload" })).toHaveValue(nativeExample); await selectEndpoint("/v1/systemone"); expect(screen.getByRole("textbox", { name: "System One JSON payload" })).toHaveValue(systemOneDraft); @@ -403,7 +470,7 @@ describe("SystemOneUI integration", () => { ); expect(next).toEqual({ ...payload, questions: [{ ...payload.questions[0], instructions: "Is it urgent?" }] }); await user.click(screen.getByRole("tab", { name: "Form" })); - await user.click(screen.getByRole("button", { name: "Reset example" })); + await selectPreset("Triage a bug report"); expect(screen.getByRole("textbox", { name: "Input" })).toBeInTheDocument(); expect(screen.getAllByRole("group", { name: /Question \d/ })).toHaveLength(3); await selectEndpoint("/v1/systemone"); diff --git a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx index 02d42851b8e..c46238315ff 100644 --- a/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx +++ b/ui/litellm-dashboard/src/app/(dashboard)/playground/components/systemOneUI/SystemOneUI.tsx @@ -9,10 +9,10 @@ import { SearchSelect } from "@/components/shared/SearchSelect"; import { DECISIONS_DOCS_URL } from "@/lib/decisionModels"; import { uiHref } from "@/utils/uiHref"; import { useMutation } from "@tanstack/react-query"; -import { Code, Info, LoaderCircle, RotateCcw, Send } from "lucide-react"; +import { Code, Eraser, Info, LoaderCircle, Send } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { makePlaygroundDecisionRequest, type PlaygroundDecisionRequest } from "../../llm_calls/system_one"; -import { decisionsExample, openAIDecisionsExample } from "./lib/example"; +import { DECISION_PRESETS, emptyPayload, matchPreset, presetPayload, type DecisionPreset } from "./lib/example"; import { payloadModel, withPayloadModel } from "./lib/payloadModel"; import JsonEditor from "./JsonEditor"; import QuestionBreakdown from "./QuestionBreakdown"; @@ -35,8 +35,36 @@ interface SystemOneSendVariables { signal: AbortSignal; } +const CUSTOM_PRESET = "custom"; const DECISION_MODELS_DISCUSSION_URL = "https://github.com/BerriAI/litellm/discussions/44231"; +interface PresetPickerProps { + activePreset: DecisionPreset | undefined; + onPick: (id: string | null) => void; +} + +function PresetPicker({ activePreset, onPick }: PresetPickerProps) { + return ( + + ); +} + function getCustomProxyBaseUrl(): string | undefined { return typeof window === "undefined" ? undefined : window.sessionStorage.getItem("customProxyBaseUrl") || undefined; } @@ -53,9 +81,10 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = const [endpoint, setEndpoint] = useState("/v1/systemone"); const [drafts, setDrafts] = useState>>({}); const [view, setView] = useState("form"); - const example = endpoint === "/v1/decisions" ? openAIDecisionsExample : decisionsExample; - const examplePayload = JSON.stringify(example(decisionModels[0]), null, 2); - const rawPayload = drafts[endpoint] ?? examplePayload; + const rawPayload = drafts[endpoint] ?? presetPayload(DECISION_PRESETS[0], endpoint, decisionModels[0]); + const draftModel = drafts[endpoint] === undefined ? decisionModels[0] : payloadModel(rawPayload); + const activePreset = matchPreset(endpoint, rawPayload); + const clearedPayload = emptyPayload(endpoint, draftModel); const activeController = useRef(null); const validated = useMemo( () => @@ -95,9 +124,11 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = } } - function handleResetExample() { - clearRequestState(); - setDrafts((current) => Object.fromEntries(Object.entries(current).filter(([key]) => key !== endpoint))); + function handlePresetPick(id: string | null) { + const preset = DECISION_PRESETS.find((candidate) => candidate.id === id); + if (preset !== undefined) { + handlePayloadChange(presetPayload(preset, endpoint, draftModel)); + } } function handleModelPick(model: string | null) { @@ -204,9 +235,14 @@ export default function SystemOneUI({ accessToken, disabledPersonalKeyCreation = )}
- {view === "json" && (